diff options
author | ilnaz <ilnaz@ydb.tech> | 2022-12-13 16:01:38 +0300 |
---|---|---|
committer | ilnaz <ilnaz@ydb.tech> | 2022-12-13 16:01:38 +0300 |
commit | f2bea70bea01921ec43846224d100f2c70dd5719 (patch) | |
tree | eead917572063b63adc1c9a76284c8fbd10f25a3 | |
parent | 1ab9ee3dfe0ab4023a3a57bf55de31dff3eac908 (diff) | |
download | ydb-f2bea70bea01921ec43846224d100f2c70dd5719.tar.gz |
Add cross-link
180 files changed, 40428 insertions, 6 deletions
diff --git a/conanfile.txt b/conanfile.txt index dc47a55728..7840be9e81 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -2,7 +2,6 @@ bison/3.5.3 c-ares/1.18.1 libiconv/1.15 -liburing/2.3 openssl/1.1.1l ragel/6.10 yasm/1.3.0 diff --git a/contrib/libs/CMakeLists.linux-aarch64.txt b/contrib/libs/CMakeLists.linux-aarch64.txt index 6a8ef32256..28808866b4 100644 --- a/contrib/libs/CMakeLists.linux-aarch64.txt +++ b/contrib/libs/CMakeLists.linux-aarch64.txt @@ -31,6 +31,7 @@ add_subdirectory(libbz2) add_subdirectory(libc_compat) add_subdirectory(libevent) add_subdirectory(libunwind) +add_subdirectory(liburing) add_subdirectory(libxml) add_subdirectory(linuxvdso) add_subdirectory(llvm12) diff --git a/contrib/libs/CMakeLists.linux.txt b/contrib/libs/CMakeLists.linux.txt index 2c5a43d6f3..1288742ef9 100644 --- a/contrib/libs/CMakeLists.linux.txt +++ b/contrib/libs/CMakeLists.linux.txt @@ -32,6 +32,7 @@ add_subdirectory(libbz2) add_subdirectory(libc_compat) add_subdirectory(libevent) add_subdirectory(libunwind) +add_subdirectory(liburing) add_subdirectory(libxml) add_subdirectory(linuxvdso) add_subdirectory(llvm12) diff --git a/contrib/libs/liburing/CHANGELOG b/contrib/libs/liburing/CHANGELOG new file mode 100644 index 0000000000..09511af427 --- /dev/null +++ b/contrib/libs/liburing/CHANGELOG @@ -0,0 +1,46 @@ +liburing-2.3 release + +- Support non-libc build for aarch64. +- Add io_uring_{enter,enter2,register,setup} syscall functions. +- Add sync cancel interface, io_uring_register_sync_cancel(). +- Fix return value of io_uring_submit_and_wait_timeout() to match the + man page. +- Improvements to the regression tests +- Add support and test case for passthrough IO +- Add recv and recvmsg multishot helpers and support +- Add documentation and support for IORING_SETUP_DEFER_TASKRUN +- Fix potential missing kernel entry with IORING_SETUP_IOPOLL +- Add support and documentation for zero-copy network transmit +- Various optimizations +- Many cleanups +- Many man page additions and updates + +liburing-2.2 release + +- Support non-libc builds. +- Optimized syscall handling for x86-64/x86/aarch64. +- Enable non-lib function calls for fast path functions. +- Add support for multishot accept. +- io_uring_register_files() will set RLIMIT_NOFILE if necessary. +- Add support for registered ring fds, io_uring_register_ring_fd(), + reducing the overhead of an io_uring_enter() system call. +- Add support for the message ring opcode. +- Add support for newer request cancelation features. +- Add support for IORING_SETUP_COOP_TASKRUN, which can help reduce the + overhead of io_uring in general. Most applications should set this flag, + see the io_uring_setup.2 man page for details. +- Add support for registering a sparse buffer and file set. +- Add support for a new buffer provide scheme, see + io_uring_register_buf_ring.3 for details. +- Add io_uring_submit_and_wait_timeout() for submitting IO and waiting + for completions with a timeout. +- Add io_uring_prep_{read,write}v2 prep helpers. +- Add io_uring_prep_close_direct() helper. +- Add support for SQE128 and CQE32, which are doubly sized SQE and CQE + rings. This is needed for some cases of the new IORING_OP_URING_CMD, + notably for NVMe passthrough. +- ~5500 lines of man page additions, including adding ~90 new man pages. +- Synced with the 5.19 kernel release, supporting all the features of + 5.19 and earlier. +- 24 new regression test cases, and ~7000 lines of new tests in general. +- General optimizations and fixes. diff --git a/contrib/libs/liburing/CMakeLists.linux-aarch64.txt b/contrib/libs/liburing/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..beb2d14e87 --- /dev/null +++ b/contrib/libs/liburing/CMakeLists.linux-aarch64.txt @@ -0,0 +1,23 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(contrib-libs-liburing) +target_compile_options(contrib-libs-liburing PRIVATE + -DLIBURING_INTERNAL + -Wno-everything +) +target_include_directories(contrib-libs-liburing PUBLIC + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/include +) +target_sources(contrib-libs-liburing PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/queue.c + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/register.c + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/setup.c + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/syscall.c +) diff --git a/contrib/libs/liburing/CMakeLists.linux.txt b/contrib/libs/liburing/CMakeLists.linux.txt new file mode 100644 index 0000000000..beb2d14e87 --- /dev/null +++ b/contrib/libs/liburing/CMakeLists.linux.txt @@ -0,0 +1,23 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(contrib-libs-liburing) +target_compile_options(contrib-libs-liburing PRIVATE + -DLIBURING_INTERNAL + -Wno-everything +) +target_include_directories(contrib-libs-liburing PUBLIC + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/include +) +target_sources(contrib-libs-liburing PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/queue.c + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/register.c + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/setup.c + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/syscall.c +) diff --git a/contrib/libs/liburing/CMakeLists.txt b/contrib/libs/liburing/CMakeLists.txt new file mode 100644 index 0000000000..fb9f9efce2 --- /dev/null +++ b/contrib/libs/liburing/CMakeLists.txt @@ -0,0 +1,13 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux.txt) +endif() diff --git a/contrib/libs/liburing/COPYING b/contrib/libs/liburing/COPYING new file mode 100644 index 0000000000..e5ab03e123 --- /dev/null +++ b/contrib/libs/liburing/COPYING @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + <one line to give the library's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + <signature of Ty Coon>, 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/contrib/libs/liburing/COPYING.GPL b/contrib/libs/liburing/COPYING.GPL new file mode 100644 index 0000000000..d159169d10 --- /dev/null +++ b/contrib/libs/liburing/COPYING.GPL @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/contrib/libs/liburing/LICENSE b/contrib/libs/liburing/LICENSE new file mode 100644 index 0000000000..d559f33a88 --- /dev/null +++ b/contrib/libs/liburing/LICENSE @@ -0,0 +1,20 @@ +Copyright 2020 Jens Axboe + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/libs/liburing/README b/contrib/libs/liburing/README new file mode 100644 index 0000000000..80d2b3dc5d --- /dev/null +++ b/contrib/libs/liburing/README @@ -0,0 +1,58 @@ +liburing +-------- + +This is the io_uring library, liburing. liburing provides helpers to setup and +teardown io_uring instances, and also a simplified interface for +applications that don't need (or want) to deal with the full kernel +side implementation. + +For more info on io_uring, please see: + +https://kernel.dk/io_uring.pdf + +Subscribe to io-uring@vger.kernel.org for io_uring related discussions +and development for both kernel and userspace. The list is archived here: + +https://lore.kernel.org/io-uring/ + + +kernel version dependency +-------------------------- + +liburing itself is not tied to any specific kernel release, and hence it's +possible to use the newest liburing release even on older kernels (and vice +versa). Newer features may only be available on more recent kernels, +obviously. + + +ulimit settings +--------------- + +io_uring accounts memory it needs under the rlimit memlocked option, which +can be quite low on some setups (64K). The default is usually enough for +most use cases, but bigger rings or things like registered buffers deplete +it quickly. root isn't under this restriction, but regular users are. Going +into detail on how to bump the limit on various systems is beyond the scope +of this little blurb, but check /etc/security/limits.conf for user specific +settings, or /etc/systemd/user.conf and /etc/systemd/system.conf for systemd +setups. This affects 5.11 and earlier, new kernels are less dependent +on RLIMIT_MEMLOCK as it is only used for registering buffers. + + +Regressions tests +----------------- + +The bulk of liburing is actually regression/unit tests for both liburing and +the kernel io_uring support. Please note that this suite isn't expected to +pass on older kernels, and may even crash or hang older kernels! + + +License +------- + +All software contained within this repo is dual licensed LGPL and MIT, see +COPYING and LICENSE, except for a header coming from the kernel which is +dual licensed GPL with a Linux-syscall-note exception and MIT, see +COPYING.GPL and <https://spdx.org/licenses/Linux-syscall-note.html>. + +Jens Axboe 2022-05-19 diff --git a/contrib/libs/liburing/SECURITY.md b/contrib/libs/liburing/SECURITY.md new file mode 100644 index 0000000000..c9c2ffe116 --- /dev/null +++ b/contrib/libs/liburing/SECURITY.md @@ -0,0 +1,6 @@ +# Security Policy + +## Reporting a Vulnerability + +Please report any security issue to axboe@kernel.dk where the issue will be triaged appropriately. +Thank you in advance for helping to keep liburing secure. diff --git a/contrib/libs/liburing/config-host.h b/contrib/libs/liburing/config-host.h new file mode 100644 index 0000000000..1488c383ea --- /dev/null +++ b/contrib/libs/liburing/config-host.h @@ -0,0 +1,13 @@ +/* + * Automatically generated by configure - do not modify + * Configured with: * './configure' * '--prefix=/var/empty/liburing' * '--includedir=/var/empty/tmp/out/include' * '--mandir=/var/empty/tmp/out/share/man' + */ +#define CONFIG_HAVE_KERNEL_RWF_T +#define CONFIG_HAVE_KERNEL_TIMESPEC +#define CONFIG_HAVE_OPEN_HOW +#define CONFIG_HAVE_STATX +#define CONFIG_HAVE_GLIBC_STATX +#define CONFIG_HAVE_CXX +#define CONFIG_HAVE_UCONTEXT +#define CONFIG_HAVE_STRINGOP_OVERFLOW +#define CONFIG_HAVE_ARRAY_BOUNDS diff --git a/contrib/libs/liburing/src/arch/aarch64/lib.h b/contrib/libs/liburing/src/arch/aarch64/lib.h new file mode 100644 index 0000000000..5a75c1a672 --- /dev/null +++ b/contrib/libs/liburing/src/arch/aarch64/lib.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: MIT */ + +#ifndef LIBURING_ARCH_AARCH64_LIB_H +#define LIBURING_ARCH_AARCH64_LIB_H + +#include <elf.h> +#include <sys/auxv.h> +#include "../../syscall.h" + +static inline long __get_page_size(void) +{ + Elf64_Off buf[2]; + long ret = 4096; + int fd; + + fd = __sys_open("/proc/self/auxv", O_RDONLY, 0); + if (fd < 0) + return ret; + + while (1) { + ssize_t x; + + x = __sys_read(fd, buf, sizeof(buf)); + if (x < sizeof(buf)) + break; + + if (buf[0] == AT_PAGESZ) { + ret = buf[1]; + break; + } + } + + __sys_close(fd); + return ret; +} + +static inline long get_page_size(void) +{ + static long cache_val; + + if (cache_val) + return cache_val; + + cache_val = __get_page_size(); + return cache_val; +} + +#endif /* #ifndef LIBURING_ARCH_AARCH64_LIB_H */ diff --git a/contrib/libs/liburing/src/arch/aarch64/syscall.h b/contrib/libs/liburing/src/arch/aarch64/syscall.h new file mode 100644 index 0000000000..3015ac3723 --- /dev/null +++ b/contrib/libs/liburing/src/arch/aarch64/syscall.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: MIT */ + +#ifndef LIBURING_ARCH_AARCH64_SYSCALL_H +#define LIBURING_ARCH_AARCH64_SYSCALL_H + +#if defined(__aarch64__) + +#define __do_syscallN(...) ({ \ + __asm__ volatile ( \ + "svc 0" \ + : "=r"(x0) \ + : __VA_ARGS__ \ + : "memory", "cc"); \ + (long) x0; \ +}) + +#define __do_syscall0(__n) ({ \ + register long x8 __asm__("x8") = __n; \ + register long x0 __asm__("x0"); \ + \ + __do_syscallN("r" (x8)); \ +}) + +#define __do_syscall1(__n, __a) ({ \ + register long x8 __asm__("x8") = __n; \ + register __typeof__(__a) x0 __asm__("x0") = __a; \ + \ + __do_syscallN("r" (x8), "0" (x0)); \ +}) + +#define __do_syscall2(__n, __a, __b) ({ \ + register long x8 __asm__("x8") = __n; \ + register __typeof__(__a) x0 __asm__("x0") = __a; \ + register __typeof__(__b) x1 __asm__("x1") = __b; \ + \ + __do_syscallN("r" (x8), "0" (x0), "r" (x1)); \ +}) + +#define __do_syscall3(__n, __a, __b, __c) ({ \ + register long x8 __asm__("x8") = __n; \ + register __typeof__(__a) x0 __asm__("x0") = __a; \ + register __typeof__(__b) x1 __asm__("x1") = __b; \ + register __typeof__(__c) x2 __asm__("x2") = __c; \ + \ + __do_syscallN("r" (x8), "0" (x0), "r" (x1), "r" (x2)); \ +}) + +#define __do_syscall4(__n, __a, __b, __c, __d) ({ \ + register long x8 __asm__("x8") = __n; \ + register __typeof__(__a) x0 __asm__("x0") = __a; \ + register __typeof__(__b) x1 __asm__("x1") = __b; \ + register __typeof__(__c) x2 __asm__("x2") = __c; \ + register __typeof__(__d) x3 __asm__("x3") = __d; \ + \ + __do_syscallN("r" (x8), "0" (x0), "r" (x1), "r" (x2), "r" (x3));\ +}) + +#define __do_syscall5(__n, __a, __b, __c, __d, __e) ({ \ + register long x8 __asm__("x8") = __n; \ + register __typeof__(__a) x0 __asm__("x0") = __a; \ + register __typeof__(__b) x1 __asm__("x1") = __b; \ + register __typeof__(__c) x2 __asm__("x2") = __c; \ + register __typeof__(__d) x3 __asm__("x3") = __d; \ + register __typeof__(__e) x4 __asm__("x4") = __e; \ + \ + __do_syscallN("r" (x8), "0" (x0), "r" (x1), "r" (x2), "r" (x3), \ + "r"(x4)); \ +}) + +#define __do_syscall6(__n, __a, __b, __c, __d, __e, __f) ({ \ + register long x8 __asm__("x8") = __n; \ + register __typeof__(__a) x0 __asm__("x0") = __a; \ + register __typeof__(__b) x1 __asm__("x1") = __b; \ + register __typeof__(__c) x2 __asm__("x2") = __c; \ + register __typeof__(__d) x3 __asm__("x3") = __d; \ + register __typeof__(__e) x4 __asm__("x4") = __e; \ + register __typeof__(__f) x5 __asm__("x5") = __f; \ + \ + __do_syscallN("r" (x8), "0" (x0), "r" (x1), "r" (x2), "r" (x3), \ + "r" (x4), "r"(x5)); \ +}) + +#include "../syscall-defs.h" + +#else /* #if defined(__aarch64__) */ + +#error #include "../generic/syscall.h" + +#endif /* #if defined(__aarch64__) */ + +#endif /* #ifndef LIBURING_ARCH_AARCH64_SYSCALL_H */ diff --git a/contrib/libs/liburing/src/arch/syscall-defs.h b/contrib/libs/liburing/src/arch/syscall-defs.h new file mode 100644 index 0000000000..d351f8b736 --- /dev/null +++ b/contrib/libs/liburing/src/arch/syscall-defs.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: MIT */ + +#ifndef LIBURING_ARCH_SYSCALL_DEFS_H +#define LIBURING_ARCH_SYSCALL_DEFS_H + +#include <fcntl.h> + +static inline int __sys_open(const char *pathname, int flags, mode_t mode) +{ + /* + * Some architectures don't have __NR_open, but __NR_openat. + */ +#ifdef __NR_open + return (int) __do_syscall3(__NR_open, pathname, flags, mode); +#else + return (int) __do_syscall4(__NR_openat, AT_FDCWD, pathname, flags, mode); +#endif +} + +static inline ssize_t __sys_read(int fd, void *buffer, size_t size) +{ + return (ssize_t) __do_syscall3(__NR_read, fd, buffer, size); +} + +static inline void *__sys_mmap(void *addr, size_t length, int prot, int flags, + int fd, off_t offset) +{ + int nr; + +#if defined(__NR_mmap2) + nr = __NR_mmap2; + offset >>= 12; +#else + nr = __NR_mmap; +#endif + return (void *) __do_syscall6(nr, addr, length, prot, flags, fd, offset); +} + +static inline int __sys_munmap(void *addr, size_t length) +{ + return (int) __do_syscall2(__NR_munmap, addr, length); +} + +static inline int __sys_madvise(void *addr, size_t length, int advice) +{ + return (int) __do_syscall3(__NR_madvise, addr, length, advice); +} + +static inline int __sys_getrlimit(int resource, struct rlimit *rlim) +{ + return (int) __do_syscall2(__NR_getrlimit, resource, rlim); +} + +static inline int __sys_setrlimit(int resource, const struct rlimit *rlim) +{ + return (int) __do_syscall2(__NR_setrlimit, resource, rlim); +} + +static inline int __sys_close(int fd) +{ + return (int) __do_syscall1(__NR_close, fd); +} + +static inline int __sys_io_uring_register(unsigned int fd, unsigned int opcode, + const void *arg, unsigned int nr_args) +{ + return (int) __do_syscall4(__NR_io_uring_register, fd, opcode, arg, + nr_args); +} + +static inline int __sys_io_uring_setup(unsigned int entries, + struct io_uring_params *p) +{ + return (int) __do_syscall2(__NR_io_uring_setup, entries, p); +} + +static inline int __sys_io_uring_enter2(unsigned int fd, unsigned int to_submit, + unsigned int min_complete, + unsigned int flags, sigset_t *sig, + size_t sz) +{ + return (int) __do_syscall6(__NR_io_uring_enter, fd, to_submit, + min_complete, flags, sig, sz); +} + +static inline int __sys_io_uring_enter(unsigned int fd, unsigned int to_submit, + unsigned int min_complete, + unsigned int flags, sigset_t *sig) +{ + return __sys_io_uring_enter2(fd, to_submit, min_complete, flags, sig, + _NSIG / 8); +} + +#endif diff --git a/contrib/libs/liburing/src/arch/x86/lib.h b/contrib/libs/liburing/src/arch/x86/lib.h new file mode 100644 index 0000000000..6ece2d44f2 --- /dev/null +++ b/contrib/libs/liburing/src/arch/x86/lib.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: MIT */ + +#ifndef LIBURING_ARCH_X86_LIB_H +#define LIBURING_ARCH_X86_LIB_H + +static inline long get_page_size(void) +{ + return 4096; +} + +#endif /* #ifndef LIBURING_ARCH_X86_LIB_H */ diff --git a/contrib/libs/liburing/src/arch/x86/syscall.h b/contrib/libs/liburing/src/arch/x86/syscall.h new file mode 100644 index 0000000000..fdd9aecf1a --- /dev/null +++ b/contrib/libs/liburing/src/arch/x86/syscall.h @@ -0,0 +1,296 @@ +/* SPDX-License-Identifier: MIT */ + +#ifndef LIBURING_ARCH_X86_SYSCALL_H +#define LIBURING_ARCH_X86_SYSCALL_H + +#if defined(__x86_64__) +/** + * Note for syscall registers usage (x86-64): + * - %rax is the syscall number. + * - %rax is also the return value. + * - %rdi is the 1st argument. + * - %rsi is the 2nd argument. + * - %rdx is the 3rd argument. + * - %r10 is the 4th argument (**yes it's %r10, not %rcx!**). + * - %r8 is the 5th argument. + * - %r9 is the 6th argument. + * + * `syscall` instruction will clobber %r11 and %rcx. + * + * After the syscall returns to userspace: + * - %r11 will contain %rflags. + * - %rcx will contain the return address. + * + * IOW, after the syscall returns to userspace: + * %r11 == %rflags and %rcx == %rip. + */ + +#define __do_syscall0(NUM) ({ \ + intptr_t rax; \ + \ + __asm__ volatile( \ + "syscall" \ + : "=a"(rax) /* %rax */ \ + : "a"(NUM) /* %rax */ \ + : "rcx", "r11", "memory" \ + ); \ + rax; \ +}) + +#define __do_syscall1(NUM, ARG1) ({ \ + intptr_t rax; \ + \ + __asm__ volatile( \ + "syscall" \ + : "=a"(rax) /* %rax */ \ + : "a"((NUM)), /* %rax */ \ + "D"((ARG1)) /* %rdi */ \ + : "rcx", "r11", "memory" \ + ); \ + rax; \ +}) + +#define __do_syscall2(NUM, ARG1, ARG2) ({ \ + intptr_t rax; \ + \ + __asm__ volatile( \ + "syscall" \ + : "=a"(rax) /* %rax */ \ + : "a"((NUM)), /* %rax */ \ + "D"((ARG1)), /* %rdi */ \ + "S"((ARG2)) /* %rsi */ \ + : "rcx", "r11", "memory" \ + ); \ + rax; \ +}) + +#define __do_syscall3(NUM, ARG1, ARG2, ARG3) ({ \ + intptr_t rax; \ + \ + __asm__ volatile( \ + "syscall" \ + : "=a"(rax) /* %rax */ \ + : "a"((NUM)), /* %rax */ \ + "D"((ARG1)), /* %rdi */ \ + "S"((ARG2)), /* %rsi */ \ + "d"((ARG3)) /* %rdx */ \ + : "rcx", "r11", "memory" \ + ); \ + rax; \ +}) + +#define __do_syscall4(NUM, ARG1, ARG2, ARG3, ARG4) ({ \ + intptr_t rax; \ + register __typeof__(ARG4) __r10 __asm__("r10") = (ARG4); \ + \ + __asm__ volatile( \ + "syscall" \ + : "=a"(rax) /* %rax */ \ + : "a"((NUM)), /* %rax */ \ + "D"((ARG1)), /* %rdi */ \ + "S"((ARG2)), /* %rsi */ \ + "d"((ARG3)), /* %rdx */ \ + "r"(__r10) /* %r10 */ \ + : "rcx", "r11", "memory" \ + ); \ + rax; \ +}) + +#define __do_syscall5(NUM, ARG1, ARG2, ARG3, ARG4, ARG5) ({ \ + intptr_t rax; \ + register __typeof__(ARG4) __r10 __asm__("r10") = (ARG4); \ + register __typeof__(ARG5) __r8 __asm__("r8") = (ARG5); \ + \ + __asm__ volatile( \ + "syscall" \ + : "=a"(rax) /* %rax */ \ + : "a"((NUM)), /* %rax */ \ + "D"((ARG1)), /* %rdi */ \ + "S"((ARG2)), /* %rsi */ \ + "d"((ARG3)), /* %rdx */ \ + "r"(__r10), /* %r10 */ \ + "r"(__r8) /* %r8 */ \ + : "rcx", "r11", "memory" \ + ); \ + rax; \ +}) + +#define __do_syscall6(NUM, ARG1, ARG2, ARG3, ARG4, ARG5, ARG6) ({ \ + intptr_t rax; \ + register __typeof__(ARG4) __r10 __asm__("r10") = (ARG4); \ + register __typeof__(ARG5) __r8 __asm__("r8") = (ARG5); \ + register __typeof__(ARG6) __r9 __asm__("r9") = (ARG6); \ + \ + __asm__ volatile( \ + "syscall" \ + : "=a"(rax) /* %rax */ \ + : "a"((NUM)), /* %rax */ \ + "D"((ARG1)), /* %rdi */ \ + "S"((ARG2)), /* %rsi */ \ + "d"((ARG3)), /* %rdx */ \ + "r"(__r10), /* %r10 */ \ + "r"(__r8), /* %r8 */ \ + "r"(__r9) /* %r9 */ \ + : "rcx", "r11", "memory" \ + ); \ + rax; \ +}) + +#include "../syscall-defs.h" + +#else /* #if defined(__x86_64__) */ + +#ifdef CONFIG_NOLIBC +/** + * Note for syscall registers usage (x86, 32-bit): + * - %eax is the syscall number. + * - %eax is also the return value. + * - %ebx is the 1st argument. + * - %ecx is the 2nd argument. + * - %edx is the 3rd argument. + * - %esi is the 4th argument. + * - %edi is the 5th argument. + * - %ebp is the 6th argument. + */ + +#define __do_syscall0(NUM) ({ \ + intptr_t eax; \ + \ + __asm__ volatile( \ + "int $0x80" \ + : "=a"(eax) /* %eax */ \ + : "a"(NUM) /* %eax */ \ + : "memory" \ + ); \ + eax; \ +}) + +#define __do_syscall1(NUM, ARG1) ({ \ + intptr_t eax; \ + \ + __asm__ volatile( \ + "int $0x80" \ + : "=a"(eax) /* %eax */ \ + : "a"(NUM), /* %eax */ \ + "b"((ARG1)) /* %ebx */ \ + : "memory" \ + ); \ + eax; \ +}) + +#define __do_syscall2(NUM, ARG1, ARG2) ({ \ + intptr_t eax; \ + \ + __asm__ volatile( \ + "int $0x80" \ + : "=a" (eax) /* %eax */ \ + : "a"(NUM), /* %eax */ \ + "b"((ARG1)), /* %ebx */ \ + "c"((ARG2)) /* %ecx */ \ + : "memory" \ + ); \ + eax; \ +}) + +#define __do_syscall3(NUM, ARG1, ARG2, ARG3) ({ \ + intptr_t eax; \ + \ + __asm__ volatile( \ + "int $0x80" \ + : "=a" (eax) /* %eax */ \ + : "a"(NUM), /* %eax */ \ + "b"((ARG1)), /* %ebx */ \ + "c"((ARG2)), /* %ecx */ \ + "d"((ARG3)) /* %edx */ \ + : "memory" \ + ); \ + eax; \ +}) + +#define __do_syscall4(NUM, ARG1, ARG2, ARG3, ARG4) ({ \ + intptr_t eax; \ + \ + __asm__ volatile( \ + "int $0x80" \ + : "=a" (eax) /* %eax */ \ + : "a"(NUM), /* %eax */ \ + "b"((ARG1)), /* %ebx */ \ + "c"((ARG2)), /* %ecx */ \ + "d"((ARG3)), /* %edx */ \ + "S"((ARG4)) /* %esi */ \ + : "memory" \ + ); \ + eax; \ +}) + +#define __do_syscall5(NUM, ARG1, ARG2, ARG3, ARG4, ARG5) ({ \ + intptr_t eax; \ + \ + __asm__ volatile( \ + "int $0x80" \ + : "=a" (eax) /* %eax */ \ + : "a"(NUM), /* %eax */ \ + "b"((ARG1)), /* %ebx */ \ + "c"((ARG2)), /* %ecx */ \ + "d"((ARG3)), /* %edx */ \ + "S"((ARG4)), /* %esi */ \ + "D"((ARG5)) /* %edi */ \ + : "memory" \ + ); \ + eax; \ +}) + + +/* + * On i386, the 6th argument of syscall goes in %ebp. However, both Clang + * and GCC cannot use %ebp in the clobber list and in the "r" constraint + * without using -fomit-frame-pointer. To make it always available for + * any kind of compilation, the below workaround is implemented: + * + * 1) Push the 6-th argument. + * 2) Push %ebp. + * 3) Load the 6-th argument from 4(%esp) to %ebp. + * 4) Do the syscall (int $0x80). + * 5) Pop %ebp (restore the old value of %ebp). + * 6) Add %esp by 4 (undo the stack pointer). + * + * WARNING: + * Don't use register variables for __do_syscall6(), there is a known + * GCC bug that results in an endless loop. + * + * BugLink: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105032 + * + */ +#define __do_syscall6(NUM, ARG1, ARG2, ARG3, ARG4, ARG5, ARG6) ({ \ + intptr_t eax = (intptr_t)(NUM); \ + intptr_t arg6 = (intptr_t)(ARG6); /* Always in memory */ \ + __asm__ volatile ( \ + "pushl %[_arg6]\n\t" \ + "pushl %%ebp\n\t" \ + "movl 4(%%esp),%%ebp\n\t" \ + "int $0x80\n\t" \ + "popl %%ebp\n\t" \ + "addl $4,%%esp" \ + : "+a"(eax) /* %eax */ \ + : "b"(ARG1), /* %ebx */ \ + "c"(ARG2), /* %ecx */ \ + "d"(ARG3), /* %edx */ \ + "S"(ARG4), /* %esi */ \ + "D"(ARG5), /* %edi */ \ + [_arg6]"m"(arg6) /* memory */ \ + : "memory", "cc" \ + ); \ + eax; \ +}) + +#include "../syscall-defs.h" + +#else /* #ifdef CONFIG_NOLIBC */ + +#error #include "../generic/syscall.h" + +#endif /* #ifdef CONFIG_NOLIBC */ + +#endif /* #if defined(__x86_64__) */ + +#endif /* #ifndef LIBURING_ARCH_X86_SYSCALL_H */ diff --git a/contrib/libs/liburing/src/include/liburing.h b/contrib/libs/liburing/src/include/liburing.h new file mode 100644 index 0000000000..12a703fdd2 --- /dev/null +++ b/contrib/libs/liburing/src/include/liburing.h @@ -0,0 +1,1354 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef LIB_URING_H +#define LIB_URING_H + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 500 /* Required for glibc to expose sigset_t */ +#endif + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* Required for musl to expose cpu_set_t */ +#endif + +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/uio.h> +#include <errno.h> +#include <signal.h> +#include <stdbool.h> +#include <inttypes.h> +#include <time.h> +#include <fcntl.h> +#include <sched.h> +#include <linux/swab.h> +#include "liburing/compat.h" +#include "liburing/io_uring.h" +#include "liburing/barrier.h" + +#ifndef uring_unlikely +#define uring_unlikely(cond) __builtin_expect(!!(cond), 0) +#endif + +#ifndef uring_likely +#define uring_likely(cond) __builtin_expect(!!(cond), 1) +#endif + +#ifdef __alpha__ +/* + * alpha and mips are the exceptions, all other architectures have + * common numbers for new system calls. + */ +#ifndef __NR_io_uring_setup +#define __NR_io_uring_setup 535 +#endif +#ifndef __NR_io_uring_enter +#define __NR_io_uring_enter 536 +#endif +#ifndef __NR_io_uring_register +#define __NR_io_uring_register 537 +#endif +#elif defined __mips__ +#ifndef __NR_io_uring_setup +#define __NR_io_uring_setup (__NR_Linux + 425) +#endif +#ifndef __NR_io_uring_enter +#define __NR_io_uring_enter (__NR_Linux + 426) +#endif +#ifndef __NR_io_uring_register +#define __NR_io_uring_register (__NR_Linux + 427) +#endif +#else /* !__alpha__ and !__mips__ */ +#ifndef __NR_io_uring_setup +#define __NR_io_uring_setup 425 +#endif +#ifndef __NR_io_uring_enter +#define __NR_io_uring_enter 426 +#endif +#ifndef __NR_io_uring_register +#define __NR_io_uring_register 427 +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Library interface to io_uring + */ +struct io_uring_sq { + unsigned *khead; + unsigned *ktail; + // Deprecated: use `ring_mask` instead of `*kring_mask` + unsigned *kring_mask; + // Deprecated: use `ring_entries` instead of `*kring_entries` + unsigned *kring_entries; + unsigned *kflags; + unsigned *kdropped; + unsigned *array; + struct io_uring_sqe *sqes; + + unsigned sqe_head; + unsigned sqe_tail; + + size_t ring_sz; + void *ring_ptr; + + unsigned ring_mask; + unsigned ring_entries; + + unsigned pad[2]; +}; + +struct io_uring_cq { + unsigned *khead; + unsigned *ktail; + // Deprecated: use `ring_mask` instead of `*kring_mask` + unsigned *kring_mask; + // Deprecated: use `ring_entries` instead of `*kring_entries` + unsigned *kring_entries; + unsigned *kflags; + unsigned *koverflow; + struct io_uring_cqe *cqes; + + size_t ring_sz; + void *ring_ptr; + + unsigned ring_mask; + unsigned ring_entries; + + unsigned pad[2]; +}; + +struct io_uring { + struct io_uring_sq sq; + struct io_uring_cq cq; + unsigned flags; + int ring_fd; + + unsigned features; + int enter_ring_fd; + __u8 int_flags; + __u8 pad[3]; + unsigned pad2; +}; + +/* + * Library interface + */ + +/* + * return an allocated io_uring_probe structure, or NULL if probe fails (for + * example, if it is not available). The caller is responsible for freeing it + */ +struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring); +/* same as io_uring_get_probe_ring, but takes care of ring init and teardown */ +struct io_uring_probe *io_uring_get_probe(void); + +/* + * frees a probe allocated through io_uring_get_probe() or + * io_uring_get_probe_ring() + */ +void io_uring_free_probe(struct io_uring_probe *probe); + +static inline int io_uring_opcode_supported(const struct io_uring_probe *p, + int op) +{ + if (op > p->last_op) + return 0; + return (p->ops[op].flags & IO_URING_OP_SUPPORTED) != 0; +} + +int io_uring_queue_init_params(unsigned entries, struct io_uring *ring, + struct io_uring_params *p); +int io_uring_queue_init(unsigned entries, struct io_uring *ring, + unsigned flags); +int io_uring_queue_mmap(int fd, struct io_uring_params *p, + struct io_uring *ring); +int io_uring_ring_dontfork(struct io_uring *ring); +void io_uring_queue_exit(struct io_uring *ring); +unsigned io_uring_peek_batch_cqe(struct io_uring *ring, + struct io_uring_cqe **cqes, unsigned count); +int io_uring_wait_cqes(struct io_uring *ring, struct io_uring_cqe **cqe_ptr, + unsigned wait_nr, struct __kernel_timespec *ts, + sigset_t *sigmask); +int io_uring_wait_cqe_timeout(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr, + struct __kernel_timespec *ts); +int io_uring_submit(struct io_uring *ring); +int io_uring_submit_and_wait(struct io_uring *ring, unsigned wait_nr); +int io_uring_submit_and_wait_timeout(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr, + unsigned wait_nr, + struct __kernel_timespec *ts, + sigset_t *sigmask); + +int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs, + unsigned nr_iovecs); +int io_uring_register_buffers_tags(struct io_uring *ring, + const struct iovec *iovecs, + const __u64 *tags, unsigned nr); +int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr); +int io_uring_register_buffers_update_tag(struct io_uring *ring, + unsigned off, + const struct iovec *iovecs, + const __u64 *tags, unsigned nr); +int io_uring_unregister_buffers(struct io_uring *ring); + +int io_uring_register_files(struct io_uring *ring, const int *files, + unsigned nr_files); +int io_uring_register_files_tags(struct io_uring *ring, const int *files, + const __u64 *tags, unsigned nr); +int io_uring_register_files_sparse(struct io_uring *ring, unsigned nr); +int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off, + const int *files, const __u64 *tags, + unsigned nr_files); + +int io_uring_unregister_files(struct io_uring *ring); +int io_uring_register_files_update(struct io_uring *ring, unsigned off, + const int *files, unsigned nr_files); +int io_uring_register_eventfd(struct io_uring *ring, int fd); +int io_uring_register_eventfd_async(struct io_uring *ring, int fd); +int io_uring_unregister_eventfd(struct io_uring *ring); +int io_uring_register_probe(struct io_uring *ring, struct io_uring_probe *p, + unsigned nr); +int io_uring_register_personality(struct io_uring *ring); +int io_uring_unregister_personality(struct io_uring *ring, int id); +int io_uring_register_restrictions(struct io_uring *ring, + struct io_uring_restriction *res, + unsigned int nr_res); +int io_uring_enable_rings(struct io_uring *ring); +int __io_uring_sqring_wait(struct io_uring *ring); +int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz, + const cpu_set_t *mask); +int io_uring_unregister_iowq_aff(struct io_uring *ring); +int io_uring_register_iowq_max_workers(struct io_uring *ring, + unsigned int *values); +int io_uring_register_ring_fd(struct io_uring *ring); +int io_uring_unregister_ring_fd(struct io_uring *ring); +int io_uring_register_buf_ring(struct io_uring *ring, + struct io_uring_buf_reg *reg, unsigned int flags); +int io_uring_unregister_buf_ring(struct io_uring *ring, int bgid); +int io_uring_register_sync_cancel(struct io_uring *ring, + struct io_uring_sync_cancel_reg *reg); + +int io_uring_register_file_alloc_range(struct io_uring *ring, + unsigned off, unsigned len); + +int io_uring_get_events(struct io_uring *ring); +int io_uring_submit_and_get_events(struct io_uring *ring); + +/* + * io_uring syscalls. + */ +int io_uring_enter(unsigned int fd, unsigned int to_submit, + unsigned int min_complete, unsigned int flags, sigset_t *sig); +int io_uring_enter2(unsigned int fd, unsigned int to_submit, + unsigned int min_complete, unsigned int flags, + sigset_t *sig, size_t sz); +int io_uring_setup(unsigned int entries, struct io_uring_params *p); +int io_uring_register(unsigned int fd, unsigned int opcode, const void *arg, + unsigned int nr_args); + +/* + * Helper for the peek/wait single cqe functions. Exported because of that, + * but probably shouldn't be used directly in an application. + */ +int __io_uring_get_cqe(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr, unsigned submit, + unsigned wait_nr, sigset_t *sigmask); + +#define LIBURING_UDATA_TIMEOUT ((__u64) -1) + +/* + * Calculates the step size for CQE iteration. + * For standard CQE's its 1, for big CQE's its two. + */ +#define io_uring_cqe_shift(ring) \ + (!!((ring)->flags & IORING_SETUP_CQE32)) + +#define io_uring_cqe_index(ring,ptr,mask) \ + (((ptr) & (mask)) << io_uring_cqe_shift(ring)) + +#define io_uring_for_each_cqe(ring, head, cqe) \ + /* \ + * io_uring_smp_load_acquire() enforces the order of tail \ + * and CQE reads. \ + */ \ + for (head = *(ring)->cq.khead; \ + (cqe = (head != io_uring_smp_load_acquire((ring)->cq.ktail) ? \ + &(ring)->cq.cqes[io_uring_cqe_index(ring, head, (ring)->cq.ring_mask)] : NULL)); \ + head++) \ + +/* + * Must be called after io_uring_for_each_cqe() + */ +static inline void io_uring_cq_advance(struct io_uring *ring, + unsigned nr) +{ + if (nr) { + struct io_uring_cq *cq = &ring->cq; + + /* + * Ensure that the kernel only sees the new value of the head + * index after the CQEs have been read. + */ + io_uring_smp_store_release(cq->khead, *cq->khead + nr); + } +} + +/* + * Must be called after io_uring_{peek,wait}_cqe() after the cqe has + * been processed by the application. + */ +static inline void io_uring_cqe_seen(struct io_uring *ring, + struct io_uring_cqe *cqe) +{ + if (cqe) + io_uring_cq_advance(ring, 1); +} + +/* + * Command prep helpers + */ + +/* + * Associate pointer @data with the sqe, for later retrieval from the cqe + * at command completion time with io_uring_cqe_get_data(). + */ +static inline void io_uring_sqe_set_data(struct io_uring_sqe *sqe, void *data) +{ + sqe->user_data = (unsigned long) data; +} + +static inline void *io_uring_cqe_get_data(const struct io_uring_cqe *cqe) +{ + return (void *) (uintptr_t) cqe->user_data; +} + +/* + * Assign a 64-bit value to this sqe, which can get retrieved at completion + * time with io_uring_cqe_get_data64. Just like the non-64 variants, except + * these store a 64-bit type rather than a data pointer. + */ +static inline void io_uring_sqe_set_data64(struct io_uring_sqe *sqe, + __u64 data) +{ + sqe->user_data = data; +} + +static inline __u64 io_uring_cqe_get_data64(const struct io_uring_cqe *cqe) +{ + return cqe->user_data; +} + +/* + * Tell the app the have the 64-bit variants of the get/set userdata + */ +#define LIBURING_HAVE_DATA64 + +static inline void io_uring_sqe_set_flags(struct io_uring_sqe *sqe, + unsigned flags) +{ + sqe->flags = (__u8) flags; +} + +static inline void __io_uring_set_target_fixed_file(struct io_uring_sqe *sqe, + unsigned int file_index) +{ + /* 0 means no fixed files, indexes should be encoded as "index + 1" */ + sqe->file_index = file_index + 1; +} + +static inline void io_uring_prep_rw(int op, struct io_uring_sqe *sqe, int fd, + const void *addr, unsigned len, + __u64 offset) +{ + sqe->opcode = (__u8) op; + sqe->flags = 0; + sqe->ioprio = 0; + sqe->fd = fd; + sqe->off = offset; + sqe->addr = (unsigned long) addr; + sqe->len = len; + sqe->rw_flags = 0; + sqe->buf_index = 0; + sqe->personality = 0; + sqe->file_index = 0; + sqe->addr3 = 0; + sqe->__pad2[0] = 0; +} + +/** + * @pre Either fd_in or fd_out must be a pipe. + * @param off_in If fd_in refers to a pipe, off_in must be (int64_t) -1; + * If fd_in does not refer to a pipe and off_in is (int64_t) -1, + * then bytes are read from fd_in starting from the file offset + * and it is adjust appropriately; + * If fd_in does not refer to a pipe and off_in is not + * (int64_t) -1, then the starting offset of fd_in will be + * off_in. + * @param off_out The description of off_in also applied to off_out. + * @param splice_flags see man splice(2) for description of flags. + * + * This splice operation can be used to implement sendfile by splicing to an + * intermediate pipe first, then splice to the final destination. + * In fact, the implementation of sendfile in kernel uses splice internally. + * + * NOTE that even if fd_in or fd_out refers to a pipe, the splice operation + * can still failed with EINVAL if one of the fd doesn't explicitly support + * splice operation, e.g. reading from terminal is unsupported from kernel 5.7 + * to 5.11. + * Check issue #291 for more information. + */ +static inline void io_uring_prep_splice(struct io_uring_sqe *sqe, + int fd_in, int64_t off_in, + int fd_out, int64_t off_out, + unsigned int nbytes, + unsigned int splice_flags) +{ + io_uring_prep_rw(IORING_OP_SPLICE, sqe, fd_out, NULL, nbytes, + (__u64) off_out); + sqe->splice_off_in = (__u64) off_in; + sqe->splice_fd_in = fd_in; + sqe->splice_flags = splice_flags; +} + +static inline void io_uring_prep_tee(struct io_uring_sqe *sqe, + int fd_in, int fd_out, + unsigned int nbytes, + unsigned int splice_flags) +{ + io_uring_prep_rw(IORING_OP_TEE, sqe, fd_out, NULL, nbytes, 0); + sqe->splice_off_in = 0; + sqe->splice_fd_in = fd_in; + sqe->splice_flags = splice_flags; +} + +static inline void io_uring_prep_readv(struct io_uring_sqe *sqe, int fd, + const struct iovec *iovecs, + unsigned nr_vecs, __u64 offset) +{ + io_uring_prep_rw(IORING_OP_READV, sqe, fd, iovecs, nr_vecs, offset); +} + +static inline void io_uring_prep_readv2(struct io_uring_sqe *sqe, int fd, + const struct iovec *iovecs, + unsigned nr_vecs, __u64 offset, + int flags) +{ + io_uring_prep_readv(sqe, fd, iovecs, nr_vecs, offset); + sqe->rw_flags = flags; +} + +static inline void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd, + void *buf, unsigned nbytes, + __u64 offset, int buf_index) +{ + io_uring_prep_rw(IORING_OP_READ_FIXED, sqe, fd, buf, nbytes, offset); + sqe->buf_index = (__u16) buf_index; +} + +static inline void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd, + const struct iovec *iovecs, + unsigned nr_vecs, __u64 offset) +{ + io_uring_prep_rw(IORING_OP_WRITEV, sqe, fd, iovecs, nr_vecs, offset); +} + +static inline void io_uring_prep_writev2(struct io_uring_sqe *sqe, int fd, + const struct iovec *iovecs, + unsigned nr_vecs, __u64 offset, + int flags) +{ + io_uring_prep_writev(sqe, fd, iovecs, nr_vecs, offset); + sqe->rw_flags = flags; +} + +static inline void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd, + const void *buf, unsigned nbytes, + __u64 offset, int buf_index) +{ + io_uring_prep_rw(IORING_OP_WRITE_FIXED, sqe, fd, buf, nbytes, offset); + sqe->buf_index = (__u16) buf_index; +} + +static inline void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd, + struct msghdr *msg, unsigned flags) +{ + io_uring_prep_rw(IORING_OP_RECVMSG, sqe, fd, msg, 1, 0); + sqe->msg_flags = flags; +} + +static inline void io_uring_prep_recvmsg_multishot(struct io_uring_sqe *sqe, int fd, + struct msghdr *msg, unsigned flags) +{ + io_uring_prep_recvmsg(sqe, fd, msg, flags); + sqe->ioprio |= IORING_RECV_MULTISHOT; +} + +static inline void io_uring_prep_sendmsg(struct io_uring_sqe *sqe, int fd, + const struct msghdr *msg, + unsigned flags) +{ + io_uring_prep_rw(IORING_OP_SENDMSG, sqe, fd, msg, 1, 0); + sqe->msg_flags = flags; +} + +static inline unsigned __io_uring_prep_poll_mask(unsigned poll_mask) +{ +#if __BYTE_ORDER == __BIG_ENDIAN + poll_mask = __swahw32(poll_mask); +#endif + return poll_mask; +} + +static inline void io_uring_prep_poll_add(struct io_uring_sqe *sqe, int fd, + unsigned poll_mask) +{ + io_uring_prep_rw(IORING_OP_POLL_ADD, sqe, fd, NULL, 0, 0); + sqe->poll32_events = __io_uring_prep_poll_mask(poll_mask); +} + +static inline void io_uring_prep_poll_multishot(struct io_uring_sqe *sqe, + int fd, unsigned poll_mask) +{ + io_uring_prep_poll_add(sqe, fd, poll_mask); + sqe->len = IORING_POLL_ADD_MULTI; +} + +static inline void io_uring_prep_poll_remove(struct io_uring_sqe *sqe, + __u64 user_data) +{ + io_uring_prep_rw(IORING_OP_POLL_REMOVE, sqe, -1, NULL, 0, 0); + sqe->addr = user_data; +} + +static inline void io_uring_prep_poll_update(struct io_uring_sqe *sqe, + __u64 old_user_data, + __u64 new_user_data, + unsigned poll_mask, unsigned flags) +{ + io_uring_prep_rw(IORING_OP_POLL_REMOVE, sqe, -1, NULL, flags, + new_user_data); + sqe->addr = old_user_data; + sqe->poll32_events = __io_uring_prep_poll_mask(poll_mask); +} + +static inline void io_uring_prep_fsync(struct io_uring_sqe *sqe, int fd, + unsigned fsync_flags) +{ + io_uring_prep_rw(IORING_OP_FSYNC, sqe, fd, NULL, 0, 0); + sqe->fsync_flags = fsync_flags; +} + +static inline void io_uring_prep_nop(struct io_uring_sqe *sqe) +{ + io_uring_prep_rw(IORING_OP_NOP, sqe, -1, NULL, 0, 0); +} + +static inline void io_uring_prep_timeout(struct io_uring_sqe *sqe, + struct __kernel_timespec *ts, + unsigned count, unsigned flags) +{ + io_uring_prep_rw(IORING_OP_TIMEOUT, sqe, -1, ts, 1, count); + sqe->timeout_flags = flags; +} + +static inline void io_uring_prep_timeout_remove(struct io_uring_sqe *sqe, + __u64 user_data, unsigned flags) +{ + io_uring_prep_rw(IORING_OP_TIMEOUT_REMOVE, sqe, -1, NULL, 0, 0); + sqe->addr = user_data; + sqe->timeout_flags = flags; +} + +static inline void io_uring_prep_timeout_update(struct io_uring_sqe *sqe, + struct __kernel_timespec *ts, + __u64 user_data, unsigned flags) +{ + io_uring_prep_rw(IORING_OP_TIMEOUT_REMOVE, sqe, -1, NULL, 0, + (uintptr_t) ts); + sqe->addr = user_data; + sqe->timeout_flags = flags | IORING_TIMEOUT_UPDATE; +} + +static inline void io_uring_prep_accept(struct io_uring_sqe *sqe, int fd, + struct sockaddr *addr, + socklen_t *addrlen, int flags) +{ + io_uring_prep_rw(IORING_OP_ACCEPT, sqe, fd, addr, 0, + (__u64) (unsigned long) addrlen); + sqe->accept_flags = (__u32) flags; +} + +/* accept directly into the fixed file table */ +static inline void io_uring_prep_accept_direct(struct io_uring_sqe *sqe, int fd, + struct sockaddr *addr, + socklen_t *addrlen, int flags, + unsigned int file_index) +{ + io_uring_prep_accept(sqe, fd, addr, addrlen, flags); + __io_uring_set_target_fixed_file(sqe, file_index); +} + +static inline void io_uring_prep_multishot_accept(struct io_uring_sqe *sqe, + int fd, struct sockaddr *addr, + socklen_t *addrlen, int flags) +{ + io_uring_prep_accept(sqe, fd, addr, addrlen, flags); + sqe->ioprio |= IORING_ACCEPT_MULTISHOT; +} + +/* multishot accept directly into the fixed file table */ +static inline void io_uring_prep_multishot_accept_direct(struct io_uring_sqe *sqe, + int fd, + struct sockaddr *addr, + socklen_t *addrlen, + int flags) +{ + io_uring_prep_multishot_accept(sqe, fd, addr, addrlen, flags); + __io_uring_set_target_fixed_file(sqe, IORING_FILE_INDEX_ALLOC - 1); +} + +static inline void io_uring_prep_cancel64(struct io_uring_sqe *sqe, + __u64 user_data, int flags) +{ + io_uring_prep_rw(IORING_OP_ASYNC_CANCEL, sqe, -1, NULL, 0, 0); + sqe->addr = user_data; + sqe->cancel_flags = (__u32) flags; +} + +static inline void io_uring_prep_cancel(struct io_uring_sqe *sqe, + void *user_data, int flags) +{ + io_uring_prep_cancel64(sqe, (__u64) (uintptr_t) user_data, flags); +} + +static inline void io_uring_prep_cancel_fd(struct io_uring_sqe *sqe, int fd, + unsigned int flags) +{ + io_uring_prep_rw(IORING_OP_ASYNC_CANCEL, sqe, fd, NULL, 0, 0); + sqe->cancel_flags = (__u32) flags | IORING_ASYNC_CANCEL_FD; +} + +static inline void io_uring_prep_link_timeout(struct io_uring_sqe *sqe, + struct __kernel_timespec *ts, + unsigned flags) +{ + io_uring_prep_rw(IORING_OP_LINK_TIMEOUT, sqe, -1, ts, 1, 0); + sqe->timeout_flags = flags; +} + +static inline void io_uring_prep_connect(struct io_uring_sqe *sqe, int fd, + const struct sockaddr *addr, + socklen_t addrlen) +{ + io_uring_prep_rw(IORING_OP_CONNECT, sqe, fd, addr, 0, addrlen); +} + +static inline void io_uring_prep_files_update(struct io_uring_sqe *sqe, + int *fds, unsigned nr_fds, + int offset) +{ + io_uring_prep_rw(IORING_OP_FILES_UPDATE, sqe, -1, fds, nr_fds, + (__u64) offset); +} + +static inline void io_uring_prep_fallocate(struct io_uring_sqe *sqe, int fd, + int mode, off_t offset, off_t len) +{ + io_uring_prep_rw(IORING_OP_FALLOCATE, sqe, fd, + 0, (unsigned int) mode, (__u64) offset); + sqe->addr = (__u64) len; +} + +static inline void io_uring_prep_openat(struct io_uring_sqe *sqe, int dfd, + const char *path, int flags, + mode_t mode) +{ + io_uring_prep_rw(IORING_OP_OPENAT, sqe, dfd, path, mode, 0); + sqe->open_flags = (__u32) flags; +} + +/* open directly into the fixed file table */ +static inline void io_uring_prep_openat_direct(struct io_uring_sqe *sqe, + int dfd, const char *path, + int flags, mode_t mode, + unsigned file_index) +{ + io_uring_prep_openat(sqe, dfd, path, flags, mode); + __io_uring_set_target_fixed_file(sqe, file_index); +} + +static inline void io_uring_prep_close(struct io_uring_sqe *sqe, int fd) +{ + io_uring_prep_rw(IORING_OP_CLOSE, sqe, fd, NULL, 0, 0); +} + +static inline void io_uring_prep_close_direct(struct io_uring_sqe *sqe, + unsigned file_index) +{ + io_uring_prep_close(sqe, 0); + __io_uring_set_target_fixed_file(sqe, file_index); +} + +static inline void io_uring_prep_read(struct io_uring_sqe *sqe, int fd, + void *buf, unsigned nbytes, __u64 offset) +{ + io_uring_prep_rw(IORING_OP_READ, sqe, fd, buf, nbytes, offset); +} + +static inline void io_uring_prep_write(struct io_uring_sqe *sqe, int fd, + const void *buf, unsigned nbytes, + __u64 offset) +{ + io_uring_prep_rw(IORING_OP_WRITE, sqe, fd, buf, nbytes, offset); +} + +struct statx; +static inline void io_uring_prep_statx(struct io_uring_sqe *sqe, int dfd, + const char *path, int flags, unsigned mask, + struct statx *statxbuf) +{ + io_uring_prep_rw(IORING_OP_STATX, sqe, dfd, path, mask, + (__u64) (unsigned long) statxbuf); + sqe->statx_flags = (__u32) flags; +} + +static inline void io_uring_prep_fadvise(struct io_uring_sqe *sqe, int fd, + __u64 offset, off_t len, int advice) +{ + io_uring_prep_rw(IORING_OP_FADVISE, sqe, fd, NULL, (__u32) len, offset); + sqe->fadvise_advice = (__u32) advice; +} + +static inline void io_uring_prep_madvise(struct io_uring_sqe *sqe, void *addr, + off_t length, int advice) +{ + io_uring_prep_rw(IORING_OP_MADVISE, sqe, -1, addr, (__u32) length, 0); + sqe->fadvise_advice = (__u32) advice; +} + +static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd, + const void *buf, size_t len, int flags) +{ + io_uring_prep_rw(IORING_OP_SEND, sqe, sockfd, buf, (__u32) len, 0); + sqe->msg_flags = (__u32) flags; +} + +static inline void io_uring_prep_send_zc(struct io_uring_sqe *sqe, int sockfd, + const void *buf, size_t len, int flags, + unsigned zc_flags) +{ + io_uring_prep_rw(IORING_OP_SEND_ZC, sqe, sockfd, buf, (__u32) len, 0); + sqe->msg_flags = (__u32) flags; + sqe->ioprio = zc_flags; +} + +static inline void io_uring_prep_send_zc_fixed(struct io_uring_sqe *sqe, + int sockfd, const void *buf, + size_t len, int flags, + unsigned zc_flags, + unsigned buf_index) +{ + io_uring_prep_send_zc(sqe, sockfd, buf, len, flags, zc_flags); + sqe->ioprio |= IORING_RECVSEND_FIXED_BUF; + sqe->buf_index = buf_index; +} + +static inline void io_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd, + const struct msghdr *msg, + unsigned flags) +{ + io_uring_prep_sendmsg(sqe, fd, msg, flags); + sqe->opcode = IORING_OP_SENDMSG_ZC; +} + +static inline void io_uring_prep_send_set_addr(struct io_uring_sqe *sqe, + const struct sockaddr *dest_addr, + __u16 addr_len) +{ + sqe->addr2 = (unsigned long)(const void *)dest_addr; + sqe->addr_len = addr_len; +} + +static inline void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd, + void *buf, size_t len, int flags) +{ + io_uring_prep_rw(IORING_OP_RECV, sqe, sockfd, buf, (__u32) len, 0); + sqe->msg_flags = (__u32) flags; +} + +static inline void io_uring_prep_recv_multishot(struct io_uring_sqe *sqe, + int sockfd, void *buf, + size_t len, int flags) +{ + io_uring_prep_recv(sqe, sockfd, buf, len, flags); + sqe->ioprio |= IORING_RECV_MULTISHOT; +} + +static inline struct io_uring_recvmsg_out * +io_uring_recvmsg_validate(void *buf, int buf_len, struct msghdr *msgh) +{ + unsigned long header = msgh->msg_controllen + msgh->msg_namelen + + sizeof(struct io_uring_recvmsg_out); + if (buf_len < 0 || (unsigned long)buf_len < header) + return NULL; + return (struct io_uring_recvmsg_out *)buf; +} + +static inline void *io_uring_recvmsg_name(struct io_uring_recvmsg_out *o) +{ + return (void *) &o[1]; +} + +static inline struct cmsghdr * +io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out *o, + struct msghdr *msgh) +{ + if (o->controllen < sizeof(struct cmsghdr)) + return NULL; + + return (struct cmsghdr *)((unsigned char *) io_uring_recvmsg_name(o) + + msgh->msg_namelen); +} + +static inline struct cmsghdr * +io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msgh, + struct cmsghdr *cmsg) +{ + unsigned char *end; + + if (cmsg->cmsg_len < sizeof(struct cmsghdr)) + return NULL; + end = (unsigned char *) io_uring_recvmsg_cmsg_firsthdr(o, msgh) + + o->controllen; + cmsg = (struct cmsghdr *)((unsigned char *) cmsg + + CMSG_ALIGN(cmsg->cmsg_len)); + + if ((unsigned char *) (cmsg + 1) > end) + return NULL; + if (((unsigned char *) cmsg) + CMSG_ALIGN(cmsg->cmsg_len) > end) + return NULL; + + return cmsg; +} + +static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o, + struct msghdr *msgh) +{ + return (void *)((unsigned char *)io_uring_recvmsg_name(o) + + msgh->msg_namelen + msgh->msg_controllen); +} + +static inline unsigned int +io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o, + int buf_len, struct msghdr *msgh) +{ + unsigned long payload_start, payload_end; + + payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh); + payload_end = (unsigned long) o + buf_len; + return (unsigned int) (payload_end - payload_start); +} + +static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd, + const char *path, struct open_how *how) +{ + io_uring_prep_rw(IORING_OP_OPENAT2, sqe, dfd, path, sizeof(*how), + (uint64_t) (uintptr_t) how); +} + +/* open directly into the fixed file table */ +static inline void io_uring_prep_openat2_direct(struct io_uring_sqe *sqe, + int dfd, const char *path, + struct open_how *how, + unsigned file_index) +{ + io_uring_prep_openat2(sqe, dfd, path, how); + __io_uring_set_target_fixed_file(sqe, file_index); +} + +struct epoll_event; +static inline void io_uring_prep_epoll_ctl(struct io_uring_sqe *sqe, int epfd, + int fd, int op, + struct epoll_event *ev) +{ + io_uring_prep_rw(IORING_OP_EPOLL_CTL, sqe, epfd, ev, + (__u32) op, (__u32) fd); +} + +static inline void io_uring_prep_provide_buffers(struct io_uring_sqe *sqe, + void *addr, int len, int nr, + int bgid, int bid) +{ + io_uring_prep_rw(IORING_OP_PROVIDE_BUFFERS, sqe, nr, addr, (__u32) len, + (__u64) bid); + sqe->buf_group = (__u16) bgid; +} + +static inline void io_uring_prep_remove_buffers(struct io_uring_sqe *sqe, + int nr, int bgid) +{ + io_uring_prep_rw(IORING_OP_REMOVE_BUFFERS, sqe, nr, NULL, 0, 0); + sqe->buf_group = (__u16) bgid; +} + +static inline void io_uring_prep_shutdown(struct io_uring_sqe *sqe, int fd, + int how) +{ + io_uring_prep_rw(IORING_OP_SHUTDOWN, sqe, fd, NULL, (__u32) how, 0); +} + +static inline void io_uring_prep_unlinkat(struct io_uring_sqe *sqe, int dfd, + const char *path, int flags) +{ + io_uring_prep_rw(IORING_OP_UNLINKAT, sqe, dfd, path, 0, 0); + sqe->unlink_flags = (__u32) flags; +} + +static inline void io_uring_prep_unlink(struct io_uring_sqe *sqe, + const char *path, int flags) +{ + io_uring_prep_unlinkat(sqe, AT_FDCWD, path, flags); +} + +static inline void io_uring_prep_renameat(struct io_uring_sqe *sqe, int olddfd, + const char *oldpath, int newdfd, + const char *newpath, unsigned int flags) +{ + io_uring_prep_rw(IORING_OP_RENAMEAT, sqe, olddfd, oldpath, + (__u32) newdfd, + (uint64_t) (uintptr_t) newpath); + sqe->rename_flags = (__u32) flags; +} + +static inline void io_uring_prep_rename(struct io_uring_sqe *sqe, + const char *oldpath, const char *newpath) +{ + io_uring_prep_renameat(sqe, AT_FDCWD, oldpath, AT_FDCWD, newpath, 0); +} + +static inline void io_uring_prep_sync_file_range(struct io_uring_sqe *sqe, + int fd, unsigned len, + __u64 offset, int flags) +{ + io_uring_prep_rw(IORING_OP_SYNC_FILE_RANGE, sqe, fd, NULL, len, offset); + sqe->sync_range_flags = (__u32) flags; +} + +static inline void io_uring_prep_mkdirat(struct io_uring_sqe *sqe, int dfd, + const char *path, mode_t mode) +{ + io_uring_prep_rw(IORING_OP_MKDIRAT, sqe, dfd, path, mode, 0); +} + +static inline void io_uring_prep_mkdir(struct io_uring_sqe *sqe, + const char *path, mode_t mode) +{ + io_uring_prep_mkdirat(sqe, AT_FDCWD, path, mode); +} + +static inline void io_uring_prep_symlinkat(struct io_uring_sqe *sqe, + const char *target, int newdirfd, + const char *linkpath) +{ + io_uring_prep_rw(IORING_OP_SYMLINKAT, sqe, newdirfd, target, 0, + (uint64_t) (uintptr_t) linkpath); +} + +static inline void io_uring_prep_symlink(struct io_uring_sqe *sqe, + const char *target, const char *linkpath) +{ + io_uring_prep_symlinkat(sqe, target, AT_FDCWD, linkpath); +} + +static inline void io_uring_prep_linkat(struct io_uring_sqe *sqe, int olddfd, + const char *oldpath, int newdfd, + const char *newpath, int flags) +{ + io_uring_prep_rw(IORING_OP_LINKAT, sqe, olddfd, oldpath, (__u32) newdfd, + (uint64_t) (uintptr_t) newpath); + sqe->hardlink_flags = (__u32) flags; +} + +static inline void io_uring_prep_link(struct io_uring_sqe *sqe, + const char *oldpath, const char *newpath, int flags) +{ + io_uring_prep_linkat(sqe, AT_FDCWD, oldpath, AT_FDCWD, newpath, flags); +} + +static inline void io_uring_prep_msg_ring(struct io_uring_sqe *sqe, int fd, + unsigned int len, __u64 data, + unsigned int flags) +{ + io_uring_prep_rw(IORING_OP_MSG_RING, sqe, fd, NULL, len, data); + sqe->rw_flags = flags; +} + +static inline void io_uring_prep_getxattr(struct io_uring_sqe *sqe, + const char *name, + char *value, + const char *path, + unsigned int len) +{ + io_uring_prep_rw(IORING_OP_GETXATTR, sqe, 0, name, len, + (__u64) (uintptr_t) value); + sqe->addr3 = (__u64) (uintptr_t) path; + sqe->xattr_flags = 0; +} + +static inline void io_uring_prep_setxattr(struct io_uring_sqe *sqe, + const char *name, + const char *value, + const char *path, + int flags, + unsigned int len) +{ + io_uring_prep_rw(IORING_OP_SETXATTR, sqe, 0, name, len, + (__u64) (uintptr_t) value); + sqe->addr3 = (__u64) (uintptr_t) path; + sqe->xattr_flags = flags; +} + +static inline void io_uring_prep_fgetxattr(struct io_uring_sqe *sqe, + int fd, + const char *name, + char *value, + unsigned int len) +{ + io_uring_prep_rw(IORING_OP_FGETXATTR, sqe, fd, name, len, + (__u64) (uintptr_t) value); + sqe->xattr_flags = 0; +} + +static inline void io_uring_prep_fsetxattr(struct io_uring_sqe *sqe, + int fd, + const char *name, + const char *value, + int flags, + unsigned int len) +{ + io_uring_prep_rw(IORING_OP_FSETXATTR, sqe, fd, name, len, + (__u64) (uintptr_t) value); + sqe->xattr_flags = flags; +} + +static inline void io_uring_prep_socket(struct io_uring_sqe *sqe, int domain, + int type, int protocol, + unsigned int flags) +{ + io_uring_prep_rw(IORING_OP_SOCKET, sqe, domain, NULL, protocol, type); + sqe->rw_flags = flags; +} + +static inline void io_uring_prep_socket_direct(struct io_uring_sqe *sqe, + int domain, int type, + int protocol, + unsigned file_index, + unsigned int flags) +{ + io_uring_prep_rw(IORING_OP_SOCKET, sqe, domain, NULL, protocol, type); + sqe->rw_flags = flags; + __io_uring_set_target_fixed_file(sqe, file_index); +} + +static inline void io_uring_prep_socket_direct_alloc(struct io_uring_sqe *sqe, + int domain, int type, int protocol, + unsigned int flags) +{ + io_uring_prep_rw(IORING_OP_SOCKET, sqe, domain, NULL, protocol, type); + sqe->rw_flags = flags; + __io_uring_set_target_fixed_file(sqe, IORING_FILE_INDEX_ALLOC - 1); +} + +/* + * Returns number of unconsumed (if SQPOLL) or unsubmitted entries exist in + * the SQ ring + */ +static inline unsigned io_uring_sq_ready(const struct io_uring *ring) +{ + unsigned khead = *ring->sq.khead; + + /* + * Without a barrier, we could miss an update and think the SQ wasn't + * ready. We don't need the load acquire for non-SQPOLL since then we + * drive updates. + */ + if (ring->flags & IORING_SETUP_SQPOLL) + khead = io_uring_smp_load_acquire(ring->sq.khead); + + /* always use real head, to avoid losing sync for short submit */ + return ring->sq.sqe_tail - khead; +} + +/* + * Returns how much space is left in the SQ ring. + */ +static inline unsigned io_uring_sq_space_left(const struct io_uring *ring) +{ + return ring->sq.ring_entries - io_uring_sq_ready(ring); +} + +/* + * Only applicable when using SQPOLL - allows the caller to wait for space + * to free up in the SQ ring, which happens when the kernel side thread has + * consumed one or more entries. If the SQ ring is currently non-full, no + * action is taken. Note: may return -EINVAL if the kernel doesn't support + * this feature. + */ +static inline int io_uring_sqring_wait(struct io_uring *ring) +{ + if (!(ring->flags & IORING_SETUP_SQPOLL)) + return 0; + if (io_uring_sq_space_left(ring)) + return 0; + + return __io_uring_sqring_wait(ring); +} + +/* + * Returns how many unconsumed entries are ready in the CQ ring + */ +static inline unsigned io_uring_cq_ready(const struct io_uring *ring) +{ + return io_uring_smp_load_acquire(ring->cq.ktail) - *ring->cq.khead; +} + +/* + * Returns true if there are overflow entries waiting to be flushed onto + * the CQ ring + */ +static inline bool io_uring_cq_has_overflow(const struct io_uring *ring) +{ + return IO_URING_READ_ONCE(*ring->sq.kflags) & IORING_SQ_CQ_OVERFLOW; +} + +/* + * Returns true if the eventfd notification is currently enabled + */ +static inline bool io_uring_cq_eventfd_enabled(const struct io_uring *ring) +{ + if (!ring->cq.kflags) + return true; + + return !(*ring->cq.kflags & IORING_CQ_EVENTFD_DISABLED); +} + +/* + * Toggle eventfd notification on or off, if an eventfd is registered with + * the ring. + */ +static inline int io_uring_cq_eventfd_toggle(struct io_uring *ring, + bool enabled) +{ + uint32_t flags; + + if (!!enabled == io_uring_cq_eventfd_enabled(ring)) + return 0; + + if (!ring->cq.kflags) + return -EOPNOTSUPP; + + flags = *ring->cq.kflags; + + if (enabled) + flags &= ~IORING_CQ_EVENTFD_DISABLED; + else + flags |= IORING_CQ_EVENTFD_DISABLED; + + IO_URING_WRITE_ONCE(*ring->cq.kflags, flags); + + return 0; +} + +/* + * Return an IO completion, waiting for 'wait_nr' completions if one isn't + * readily available. Returns 0 with cqe_ptr filled in on success, -errno on + * failure. + */ +static inline int io_uring_wait_cqe_nr(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr, + unsigned wait_nr) +{ + return __io_uring_get_cqe(ring, cqe_ptr, 0, wait_nr, NULL); +} + +/* + * Internal helper, don't use directly in applications. Use one of the + * "official" versions of this, io_uring_peek_cqe(), io_uring_wait_cqe(), + * or io_uring_wait_cqes*(). + */ +static inline int __io_uring_peek_cqe(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr, + unsigned *nr_available) +{ + struct io_uring_cqe *cqe; + int err = 0; + unsigned available; + unsigned mask = ring->cq.ring_mask; + int shift = 0; + + if (ring->flags & IORING_SETUP_CQE32) + shift = 1; + + do { + unsigned tail = io_uring_smp_load_acquire(ring->cq.ktail); + unsigned head = *ring->cq.khead; + + cqe = NULL; + available = tail - head; + if (!available) + break; + + cqe = &ring->cq.cqes[(head & mask) << shift]; + if (!(ring->features & IORING_FEAT_EXT_ARG) && + cqe->user_data == LIBURING_UDATA_TIMEOUT) { + if (cqe->res < 0) + err = cqe->res; + io_uring_cq_advance(ring, 1); + if (!err) + continue; + cqe = NULL; + } + + break; + } while (1); + + *cqe_ptr = cqe; + if (nr_available) + *nr_available = available; + return err; +} + +/* + * Return an IO completion, if one is readily available. Returns 0 with + * cqe_ptr filled in on success, -errno on failure. + */ +static inline int io_uring_peek_cqe(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr) +{ + if (!__io_uring_peek_cqe(ring, cqe_ptr, NULL) && *cqe_ptr) + return 0; + + return io_uring_wait_cqe_nr(ring, cqe_ptr, 0); +} + +/* + * Return an IO completion, waiting for it if necessary. Returns 0 with + * cqe_ptr filled in on success, -errno on failure. + */ +static inline int io_uring_wait_cqe(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr) +{ + if (!__io_uring_peek_cqe(ring, cqe_ptr, NULL) && *cqe_ptr) + return 0; + + return io_uring_wait_cqe_nr(ring, cqe_ptr, 1); +} + +/* + * Return an sqe to fill. Application must later call io_uring_submit() + * when it's ready to tell the kernel about it. The caller may call this + * function multiple times before calling io_uring_submit(). + * + * Returns a vacant sqe, or NULL if we're full. + */ +static inline struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring) +{ + struct io_uring_sq *sq = &ring->sq; + unsigned int head, next = sq->sqe_tail + 1; + int shift = 0; + + if (ring->flags & IORING_SETUP_SQE128) + shift = 1; + if (!(ring->flags & IORING_SETUP_SQPOLL)) + head = IO_URING_READ_ONCE(*sq->khead); + else + head = io_uring_smp_load_acquire(sq->khead); + + if (next - head <= sq->ring_entries) { + struct io_uring_sqe *sqe; + + sqe = &sq->sqes[(sq->sqe_tail & sq->ring_mask) << shift]; + sq->sqe_tail = next; + return sqe; + } + + return NULL; +} + +/* + * Return the appropriate mask for a buffer ring of size 'ring_entries' + */ +static inline int io_uring_buf_ring_mask(__u32 ring_entries) +{ + return ring_entries - 1; +} + +static inline void io_uring_buf_ring_init(struct io_uring_buf_ring *br) +{ + br->tail = 0; +} + +/* + * Assign 'buf' with the addr/len/buffer ID supplied + */ +static inline void io_uring_buf_ring_add(struct io_uring_buf_ring *br, + void *addr, unsigned int len, + unsigned short bid, int mask, + int buf_offset) +{ + struct io_uring_buf *buf = &br->bufs[(br->tail + buf_offset) & mask]; + + buf->addr = (unsigned long) (uintptr_t) addr; + buf->len = len; + buf->bid = bid; +} + +/* + * Make 'count' new buffers visible to the kernel. Called after + * io_uring_buf_ring_add() has been called 'count' times to fill in new + * buffers. + */ +static inline void io_uring_buf_ring_advance(struct io_uring_buf_ring *br, + int count) +{ + unsigned short new_tail = br->tail + count; + + io_uring_smp_store_release(&br->tail, new_tail); +} + +/* + * Make 'count' new buffers visible to the kernel while at the same time + * advancing the CQ ring seen entries. This can be used when the application + * is using ring provided buffers and returns buffers while processing CQEs, + * avoiding an extra atomic when needing to increment both the CQ ring and + * the ring buffer index at the same time. + */ +static inline void io_uring_buf_ring_cq_advance(struct io_uring *ring, + struct io_uring_buf_ring *br, + int count) +{ + br->tail += count; + io_uring_cq_advance(ring, count); +} + +#ifndef LIBURING_INTERNAL +static inline struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring) +{ + return _io_uring_get_sqe(ring); +} +#else +struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring); +#endif + +ssize_t io_uring_mlock_size(unsigned entries, unsigned flags); +ssize_t io_uring_mlock_size_params(unsigned entries, struct io_uring_params *p); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/libs/liburing/src/include/liburing/barrier.h b/contrib/libs/liburing/src/include/liburing/barrier.h new file mode 100644 index 0000000000..aedeb47663 --- /dev/null +++ b/contrib/libs/liburing/src/include/liburing/barrier.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef LIBURING_BARRIER_H +#define LIBURING_BARRIER_H + +/* +From the kernel documentation file refcount-vs-atomic.rst: + +A RELEASE memory ordering guarantees that all prior loads and +stores (all po-earlier instructions) on the same CPU are completed +before the operation. It also guarantees that all po-earlier +stores on the same CPU and all propagated stores from other CPUs +must propagate to all other CPUs before the release operation +(A-cumulative property). This is implemented using +:c:func:`smp_store_release`. + +An ACQUIRE memory ordering guarantees that all post loads and +stores (all po-later instructions) on the same CPU are +completed after the acquire operation. It also guarantees that all +po-later stores on the same CPU must propagate to all other CPUs +after the acquire operation executes. This is implemented using +:c:func:`smp_acquire__after_ctrl_dep`. +*/ + +#ifdef __cplusplus +#include <atomic> + +template <typename T> +static inline void IO_URING_WRITE_ONCE(T &var, T val) +{ + std::atomic_store_explicit(reinterpret_cast<std::atomic<T> *>(&var), + val, std::memory_order_relaxed); +} +template <typename T> +static inline T IO_URING_READ_ONCE(const T &var) +{ + return std::atomic_load_explicit( + reinterpret_cast<const std::atomic<T> *>(&var), + std::memory_order_relaxed); +} + +template <typename T> +static inline void io_uring_smp_store_release(T *p, T v) +{ + std::atomic_store_explicit(reinterpret_cast<std::atomic<T> *>(p), v, + std::memory_order_release); +} + +template <typename T> +static inline T io_uring_smp_load_acquire(const T *p) +{ + return std::atomic_load_explicit( + reinterpret_cast<const std::atomic<T> *>(p), + std::memory_order_acquire); +} + +static inline void io_uring_smp_mb() +{ + std::atomic_thread_fence(std::memory_order_seq_cst); +} +#else +#include <stdatomic.h> + +#define IO_URING_WRITE_ONCE(var, val) \ + atomic_store_explicit((_Atomic __typeof__(var) *)&(var), \ + (val), memory_order_relaxed) +#define IO_URING_READ_ONCE(var) \ + atomic_load_explicit((_Atomic __typeof__(var) *)&(var), \ + memory_order_relaxed) + +#define io_uring_smp_store_release(p, v) \ + atomic_store_explicit((_Atomic __typeof__(*(p)) *)(p), (v), \ + memory_order_release) +#define io_uring_smp_load_acquire(p) \ + atomic_load_explicit((_Atomic __typeof__(*(p)) *)(p), \ + memory_order_acquire) + +#define io_uring_smp_mb() \ + atomic_thread_fence(memory_order_seq_cst) +#endif + +#endif /* defined(LIBURING_BARRIER_H) */ diff --git a/contrib/libs/liburing/src/include/liburing/compat.h b/contrib/libs/liburing/src/include/liburing/compat.h new file mode 100644 index 0000000000..e579439578 --- /dev/null +++ b/contrib/libs/liburing/src/include/liburing/compat.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef LIBURING_COMPAT_H +#define LIBURING_COMPAT_H + +#include <linux/time_types.h> + +#include <linux/openat2.h> + +#endif diff --git a/contrib/libs/liburing/src/include/liburing/io_uring.h b/contrib/libs/liburing/src/include/liburing/io_uring.h new file mode 100644 index 0000000000..a3e09208df --- /dev/null +++ b/contrib/libs/liburing/src/include/liburing/io_uring.h @@ -0,0 +1,682 @@ +/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */ +/* + * Header file for the io_uring interface. + * + * Copyright (C) 2019 Jens Axboe + * Copyright (C) 2019 Christoph Hellwig + */ +#ifndef LINUX_IO_URING_H +#define LINUX_IO_URING_H + +#include <linux/fs.h> +#include <linux/types.h> +/* + * this file is shared with liburing and that has to autodetect + * if linux/time_types.h is available + */ +#ifdef __KERNEL__ +#define HAVE_LINUX_TIME_TYPES_H 1 +#endif +#ifdef HAVE_LINUX_TIME_TYPES_H +#include <linux/time_types.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * IO submission data structure (Submission Queue Entry) + */ +struct io_uring_sqe { + __u8 opcode; /* type of operation for this sqe */ + __u8 flags; /* IOSQE_ flags */ + __u16 ioprio; /* ioprio for the request */ + __s32 fd; /* file descriptor to do IO on */ + union { + __u64 off; /* offset into file */ + __u64 addr2; + struct { + __u32 cmd_op; + __u32 __pad1; + }; + }; + union { + __u64 addr; /* pointer to buffer or iovecs */ + __u64 splice_off_in; + }; + __u32 len; /* buffer size or number of iovecs */ + union { + __kernel_rwf_t rw_flags; + __u32 fsync_flags; + __u16 poll_events; /* compatibility */ + __u32 poll32_events; /* word-reversed for BE */ + __u32 sync_range_flags; + __u32 msg_flags; + __u32 timeout_flags; + __u32 accept_flags; + __u32 cancel_flags; + __u32 open_flags; + __u32 statx_flags; + __u32 fadvise_advice; + __u32 splice_flags; + __u32 rename_flags; + __u32 unlink_flags; + __u32 hardlink_flags; + __u32 xattr_flags; + __u32 msg_ring_flags; + __u32 uring_cmd_flags; + }; + __u64 user_data; /* data to be passed back at completion time */ + /* pack this to avoid bogus arm OABI complaints */ + union { + /* index into fixed buffers, if used */ + __u16 buf_index; + /* for grouped buffer selection */ + __u16 buf_group; + } __attribute__((packed)); + /* personality to use, if used */ + __u16 personality; + union { + __s32 splice_fd_in; + __u32 file_index; + struct { + __u16 addr_len; + __u16 __pad3[1]; + }; + }; + union { + struct { + __u64 addr3; + __u64 __pad2[1]; + }; + /* + * If the ring is initialized with IORING_SETUP_SQE128, then + * this field is used for 80 bytes of arbitrary command data + */ + __u8 cmd[0]; + }; +}; + +/* + * If sqe->file_index is set to this for opcodes that instantiate a new + * direct descriptor (like openat/openat2/accept), then io_uring will allocate + * an available direct descriptor instead of having the application pass one + * in. The picked direct descriptor will be returned in cqe->res, or -ENFILE + * if the space is full. + */ +#define IORING_FILE_INDEX_ALLOC (~0U) + +enum { + IOSQE_FIXED_FILE_BIT, + IOSQE_IO_DRAIN_BIT, + IOSQE_IO_LINK_BIT, + IOSQE_IO_HARDLINK_BIT, + IOSQE_ASYNC_BIT, + IOSQE_BUFFER_SELECT_BIT, + IOSQE_CQE_SKIP_SUCCESS_BIT, +}; + +/* + * sqe->flags + */ +/* use fixed fileset */ +#define IOSQE_FIXED_FILE (1U << IOSQE_FIXED_FILE_BIT) +/* issue after inflight IO */ +#define IOSQE_IO_DRAIN (1U << IOSQE_IO_DRAIN_BIT) +/* links next sqe */ +#define IOSQE_IO_LINK (1U << IOSQE_IO_LINK_BIT) +/* like LINK, but stronger */ +#define IOSQE_IO_HARDLINK (1U << IOSQE_IO_HARDLINK_BIT) +/* always go async */ +#define IOSQE_ASYNC (1U << IOSQE_ASYNC_BIT) +/* select buffer from sqe->buf_group */ +#define IOSQE_BUFFER_SELECT (1U << IOSQE_BUFFER_SELECT_BIT) +/* don't post CQE if request succeeded */ +#define IOSQE_CQE_SKIP_SUCCESS (1U << IOSQE_CQE_SKIP_SUCCESS_BIT) + +/* + * io_uring_setup() flags + */ +#define IORING_SETUP_IOPOLL (1U << 0) /* io_context is polled */ +#define IORING_SETUP_SQPOLL (1U << 1) /* SQ poll thread */ +#define IORING_SETUP_SQ_AFF (1U << 2) /* sq_thread_cpu is valid */ +#define IORING_SETUP_CQSIZE (1U << 3) /* app defines CQ size */ +#define IORING_SETUP_CLAMP (1U << 4) /* clamp SQ/CQ ring sizes */ +#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */ +#define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */ +#define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */ +/* + * Cooperative task running. When requests complete, they often require + * forcing the submitter to transition to the kernel to complete. If this + * flag is set, work will be done when the task transitions anyway, rather + * than force an inter-processor interrupt reschedule. This avoids interrupting + * a task running in userspace, and saves an IPI. + */ +#define IORING_SETUP_COOP_TASKRUN (1U << 8) +/* + * If COOP_TASKRUN is set, get notified if task work is available for + * running and a kernel transition would be needed to run it. This sets + * IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN. + */ +#define IORING_SETUP_TASKRUN_FLAG (1U << 9) +#define IORING_SETUP_SQE128 (1U << 10) /* SQEs are 128 byte */ +#define IORING_SETUP_CQE32 (1U << 11) /* CQEs are 32 byte */ +/* + * Only one task is allowed to submit requests + */ +#define IORING_SETUP_SINGLE_ISSUER (1U << 12) + +/* + * Defer running task work to get events. + * Rather than running bits of task work whenever the task transitions + * try to do it just before it is needed. + */ +#define IORING_SETUP_DEFER_TASKRUN (1U << 13) + +enum io_uring_op { + IORING_OP_NOP, + IORING_OP_READV, + IORING_OP_WRITEV, + IORING_OP_FSYNC, + IORING_OP_READ_FIXED, + IORING_OP_WRITE_FIXED, + IORING_OP_POLL_ADD, + IORING_OP_POLL_REMOVE, + IORING_OP_SYNC_FILE_RANGE, + IORING_OP_SENDMSG, + IORING_OP_RECVMSG, + IORING_OP_TIMEOUT, + IORING_OP_TIMEOUT_REMOVE, + IORING_OP_ACCEPT, + IORING_OP_ASYNC_CANCEL, + IORING_OP_LINK_TIMEOUT, + IORING_OP_CONNECT, + IORING_OP_FALLOCATE, + IORING_OP_OPENAT, + IORING_OP_CLOSE, + IORING_OP_FILES_UPDATE, + IORING_OP_STATX, + IORING_OP_READ, + IORING_OP_WRITE, + IORING_OP_FADVISE, + IORING_OP_MADVISE, + IORING_OP_SEND, + IORING_OP_RECV, + IORING_OP_OPENAT2, + IORING_OP_EPOLL_CTL, + IORING_OP_SPLICE, + IORING_OP_PROVIDE_BUFFERS, + IORING_OP_REMOVE_BUFFERS, + IORING_OP_TEE, + IORING_OP_SHUTDOWN, + IORING_OP_RENAMEAT, + IORING_OP_UNLINKAT, + IORING_OP_MKDIRAT, + IORING_OP_SYMLINKAT, + IORING_OP_LINKAT, + IORING_OP_MSG_RING, + IORING_OP_FSETXATTR, + IORING_OP_SETXATTR, + IORING_OP_FGETXATTR, + IORING_OP_GETXATTR, + IORING_OP_SOCKET, + IORING_OP_URING_CMD, + IORING_OP_SEND_ZC, + IORING_OP_SENDMSG_ZC, + + /* this goes last, obviously */ + IORING_OP_LAST, +}; + +/* + * sqe->uring_cmd_flags + * IORING_URING_CMD_FIXED use registered buffer; pass thig flag + * along with setting sqe->buf_index. + */ +#define IORING_URING_CMD_FIXED (1U << 0) + + +/* + * sqe->fsync_flags + */ +#define IORING_FSYNC_DATASYNC (1U << 0) + +/* + * sqe->timeout_flags + */ +#define IORING_TIMEOUT_ABS (1U << 0) +#define IORING_TIMEOUT_UPDATE (1U << 1) +#define IORING_TIMEOUT_BOOTTIME (1U << 2) +#define IORING_TIMEOUT_REALTIME (1U << 3) +#define IORING_LINK_TIMEOUT_UPDATE (1U << 4) +#define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5) +#define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME) +#define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE) +/* + * sqe->splice_flags + * extends splice(2) flags + */ +#define SPLICE_F_FD_IN_FIXED (1U << 31) /* the last bit of __u32 */ + +/* + * POLL_ADD flags. Note that since sqe->poll_events is the flag space, the + * command flags for POLL_ADD are stored in sqe->len. + * + * IORING_POLL_ADD_MULTI Multishot poll. Sets IORING_CQE_F_MORE if + * the poll handler will continue to report + * CQEs on behalf of the same SQE. + * + * IORING_POLL_UPDATE Update existing poll request, matching + * sqe->addr as the old user_data field. + * + * IORING_POLL_LEVEL Level triggered poll. + */ +#define IORING_POLL_ADD_MULTI (1U << 0) +#define IORING_POLL_UPDATE_EVENTS (1U << 1) +#define IORING_POLL_UPDATE_USER_DATA (1U << 2) +#define IORING_POLL_ADD_LEVEL (1U << 3) + +/* + * ASYNC_CANCEL flags. + * + * IORING_ASYNC_CANCEL_ALL Cancel all requests that match the given key + * IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the + * request 'user_data' + * IORING_ASYNC_CANCEL_ANY Match any request + * IORING_ASYNC_CANCEL_FD_FIXED 'fd' passed in is a fixed descriptor + */ +#define IORING_ASYNC_CANCEL_ALL (1U << 0) +#define IORING_ASYNC_CANCEL_FD (1U << 1) +#define IORING_ASYNC_CANCEL_ANY (1U << 2) +#define IORING_ASYNC_CANCEL_FD_FIXED (1U << 3) + +/* + * send/sendmsg and recv/recvmsg flags (sqe->ioprio) + * + * IORING_RECVSEND_POLL_FIRST If set, instead of first attempting to send + * or receive and arm poll if that yields an + * -EAGAIN result, arm poll upfront and skip + * the initial transfer attempt. + * + * IORING_RECV_MULTISHOT Multishot recv. Sets IORING_CQE_F_MORE if + * the handler will continue to report + * CQEs on behalf of the same SQE. + * + * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in + * the buf_index field. + */ +#define IORING_RECVSEND_POLL_FIRST (1U << 0) +#define IORING_RECV_MULTISHOT (1U << 1) +#define IORING_RECVSEND_FIXED_BUF (1U << 2) + +/* + * accept flags stored in sqe->ioprio + */ +#define IORING_ACCEPT_MULTISHOT (1U << 0) + +/* + * IORING_OP_MSG_RING command types, stored in sqe->addr + */ +enum { + IORING_MSG_DATA, /* pass sqe->len as 'res' and off as user_data */ + IORING_MSG_SEND_FD, /* send a registered fd to another ring */ +}; + +/* + * IORING_OP_MSG_RING flags (sqe->msg_ring_flags) + * + * IORING_MSG_RING_CQE_SKIP Don't post a CQE to the target ring. Not + * applicable for IORING_MSG_DATA, obviously. + */ +#define IORING_MSG_RING_CQE_SKIP (1U << 0) + +/* + * IO completion data structure (Completion Queue Entry) + */ +struct io_uring_cqe { + __u64 user_data; /* sqe->data submission passed back */ + __s32 res; /* result code for this event */ + __u32 flags; + + /* + * If the ring is initialized with IORING_SETUP_CQE32, then this field + * contains 16-bytes of padding, doubling the size of the CQE. + */ + __u64 big_cqe[]; +}; + +/* + * cqe->flags + * + * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID + * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries + * IORING_CQE_F_SOCK_NONEMPTY If set, more data to read after socket recv + * IORING_CQE_F_NOTIF Set for notification CQEs. Can be used to distinct + * them from sends. + */ +#define IORING_CQE_F_BUFFER (1U << 0) +#define IORING_CQE_F_MORE (1U << 1) +#define IORING_CQE_F_SOCK_NONEMPTY (1U << 2) +#define IORING_CQE_F_NOTIF (1U << 3) + +enum { + IORING_CQE_BUFFER_SHIFT = 16, +}; + +/* + * Magic offsets for the application to mmap the data it needs + */ +#define IORING_OFF_SQ_RING 0ULL +#define IORING_OFF_CQ_RING 0x8000000ULL +#define IORING_OFF_SQES 0x10000000ULL + +/* + * Filled with the offset for mmap(2) + */ +struct io_sqring_offsets { + __u32 head; + __u32 tail; + __u32 ring_mask; + __u32 ring_entries; + __u32 flags; + __u32 dropped; + __u32 array; + __u32 resv1; + __u64 resv2; +}; + +/* + * sq_ring->flags + */ +#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */ +#define IORING_SQ_CQ_OVERFLOW (1U << 1) /* CQ ring is overflown */ +#define IORING_SQ_TASKRUN (1U << 2) /* task should enter the kernel */ + +struct io_cqring_offsets { + __u32 head; + __u32 tail; + __u32 ring_mask; + __u32 ring_entries; + __u32 overflow; + __u32 cqes; + __u32 flags; + __u32 resv1; + __u64 resv2; +}; + +/* + * cq_ring->flags + */ + +/* disable eventfd notifications */ +#define IORING_CQ_EVENTFD_DISABLED (1U << 0) + +/* + * io_uring_enter(2) flags + */ +#define IORING_ENTER_GETEVENTS (1U << 0) +#define IORING_ENTER_SQ_WAKEUP (1U << 1) +#define IORING_ENTER_SQ_WAIT (1U << 2) +#define IORING_ENTER_EXT_ARG (1U << 3) +#define IORING_ENTER_REGISTERED_RING (1U << 4) + +/* + * Passed in for io_uring_setup(2). Copied back with updated info on success + */ +struct io_uring_params { + __u32 sq_entries; + __u32 cq_entries; + __u32 flags; + __u32 sq_thread_cpu; + __u32 sq_thread_idle; + __u32 features; + __u32 wq_fd; + __u32 resv[3]; + struct io_sqring_offsets sq_off; + struct io_cqring_offsets cq_off; +}; + +/* + * io_uring_params->features flags + */ +#define IORING_FEAT_SINGLE_MMAP (1U << 0) +#define IORING_FEAT_NODROP (1U << 1) +#define IORING_FEAT_SUBMIT_STABLE (1U << 2) +#define IORING_FEAT_RW_CUR_POS (1U << 3) +#define IORING_FEAT_CUR_PERSONALITY (1U << 4) +#define IORING_FEAT_FAST_POLL (1U << 5) +#define IORING_FEAT_POLL_32BITS (1U << 6) +#define IORING_FEAT_SQPOLL_NONFIXED (1U << 7) +#define IORING_FEAT_EXT_ARG (1U << 8) +#define IORING_FEAT_NATIVE_WORKERS (1U << 9) +#define IORING_FEAT_RSRC_TAGS (1U << 10) +#define IORING_FEAT_CQE_SKIP (1U << 11) +#define IORING_FEAT_LINKED_FILE (1U << 12) + +/* + * io_uring_register(2) opcodes and arguments + */ +enum { + IORING_REGISTER_BUFFERS = 0, + IORING_UNREGISTER_BUFFERS = 1, + IORING_REGISTER_FILES = 2, + IORING_UNREGISTER_FILES = 3, + IORING_REGISTER_EVENTFD = 4, + IORING_UNREGISTER_EVENTFD = 5, + IORING_REGISTER_FILES_UPDATE = 6, + IORING_REGISTER_EVENTFD_ASYNC = 7, + IORING_REGISTER_PROBE = 8, + IORING_REGISTER_PERSONALITY = 9, + IORING_UNREGISTER_PERSONALITY = 10, + IORING_REGISTER_RESTRICTIONS = 11, + IORING_REGISTER_ENABLE_RINGS = 12, + + /* extended with tagging */ + IORING_REGISTER_FILES2 = 13, + IORING_REGISTER_FILES_UPDATE2 = 14, + IORING_REGISTER_BUFFERS2 = 15, + IORING_REGISTER_BUFFERS_UPDATE = 16, + + /* set/clear io-wq thread affinities */ + IORING_REGISTER_IOWQ_AFF = 17, + IORING_UNREGISTER_IOWQ_AFF = 18, + + /* set/get max number of io-wq workers */ + IORING_REGISTER_IOWQ_MAX_WORKERS = 19, + + /* register/unregister io_uring fd with the ring */ + IORING_REGISTER_RING_FDS = 20, + IORING_UNREGISTER_RING_FDS = 21, + + /* register ring based provide buffer group */ + IORING_REGISTER_PBUF_RING = 22, + IORING_UNREGISTER_PBUF_RING = 23, + + /* sync cancelation API */ + IORING_REGISTER_SYNC_CANCEL = 24, + + /* register a range of fixed file slots for automatic slot allocation */ + IORING_REGISTER_FILE_ALLOC_RANGE = 25, + + /* this goes last */ + IORING_REGISTER_LAST +}; + +/* io-wq worker categories */ +enum { + IO_WQ_BOUND, + IO_WQ_UNBOUND, +}; + +/* deprecated, see struct io_uring_rsrc_update */ +struct io_uring_files_update { + __u32 offset; + __u32 resv; + __aligned_u64 /* __s32 * */ fds; +}; + +/* + * Register a fully sparse file space, rather than pass in an array of all + * -1 file descriptors. + */ +#define IORING_RSRC_REGISTER_SPARSE (1U << 0) + +struct io_uring_rsrc_register { + __u32 nr; + __u32 flags; + __u64 resv2; + __aligned_u64 data; + __aligned_u64 tags; +}; + +struct io_uring_rsrc_update { + __u32 offset; + __u32 resv; + __aligned_u64 data; +}; + +struct io_uring_rsrc_update2 { + __u32 offset; + __u32 resv; + __aligned_u64 data; + __aligned_u64 tags; + __u32 nr; + __u32 resv2; +}; + +struct io_uring_notification_slot { + __u64 tag; + __u64 resv[3]; +}; + +struct io_uring_notification_register { + __u32 nr_slots; + __u32 resv; + __u64 resv2; + __u64 data; + __u64 resv3; +}; + +/* Skip updating fd indexes set to this value in the fd table */ +#define IORING_REGISTER_FILES_SKIP (-2) + +#define IO_URING_OP_SUPPORTED (1U << 0) + +struct io_uring_probe_op { + __u8 op; + __u8 resv; + __u16 flags; /* IO_URING_OP_* flags */ + __u32 resv2; +}; + +struct io_uring_probe { + __u8 last_op; /* last opcode supported */ + __u8 ops_len; /* length of ops[] array below */ + __u16 resv; + __u32 resv2[3]; + struct io_uring_probe_op ops[]; +}; + +struct io_uring_restriction { + __u16 opcode; + union { + __u8 register_op; /* IORING_RESTRICTION_REGISTER_OP */ + __u8 sqe_op; /* IORING_RESTRICTION_SQE_OP */ + __u8 sqe_flags; /* IORING_RESTRICTION_SQE_FLAGS_* */ + }; + __u8 resv; + __u32 resv2[3]; +}; + +struct io_uring_buf { + __u64 addr; + __u32 len; + __u16 bid; + __u16 resv; +}; + +struct io_uring_buf_ring { + union { + /* + * To avoid spilling into more pages than we need to, the + * ring tail is overlaid with the io_uring_buf->resv field. + */ + struct { + __u64 resv1; + __u32 resv2; + __u16 resv3; + __u16 tail; + }; + struct io_uring_buf bufs[0]; + }; +}; + +/* argument for IORING_(UN)REGISTER_PBUF_RING */ +struct io_uring_buf_reg { + __u64 ring_addr; + __u32 ring_entries; + __u16 bgid; + __u16 pad; + __u64 resv[3]; +}; + +/* + * io_uring_restriction->opcode values + */ +enum { + /* Allow an io_uring_register(2) opcode */ + IORING_RESTRICTION_REGISTER_OP = 0, + + /* Allow an sqe opcode */ + IORING_RESTRICTION_SQE_OP = 1, + + /* Allow sqe flags */ + IORING_RESTRICTION_SQE_FLAGS_ALLOWED = 2, + + /* Require sqe flags (these flags must be set on each submission) */ + IORING_RESTRICTION_SQE_FLAGS_REQUIRED = 3, + + IORING_RESTRICTION_LAST +}; + +struct io_uring_getevents_arg { + __u64 sigmask; + __u32 sigmask_sz; + __u32 pad; + __u64 ts; +}; + +/* + * Argument for IORING_REGISTER_SYNC_CANCEL + */ +struct io_uring_sync_cancel_reg { + __u64 addr; + __s32 fd; + __u32 flags; + struct __kernel_timespec timeout; + __u64 pad[4]; +}; + +/* + * Argument for IORING_REGISTER_FILE_ALLOC_RANGE + * The range is specified as [off, off + len) + */ +struct io_uring_file_index_range { + __u32 off; + __u32 len; + __u64 resv; +}; + +struct io_uring_recvmsg_out { + __u32 namelen; + __u32 controllen; + __u32 payloadlen; + __u32 flags; +}; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/libs/liburing/src/int_flags.h b/contrib/libs/liburing/src/int_flags.h new file mode 100644 index 0000000000..90505ec340 --- /dev/null +++ b/contrib/libs/liburing/src/int_flags.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef LIBURING_INT_FLAGS +#define LIBURING_INT_FLAGS + +enum { + INT_FLAG_REG_RING = 1, +}; + +#endif diff --git a/contrib/libs/liburing/src/lib.h b/contrib/libs/liburing/src/lib.h new file mode 100644 index 0000000000..82c0ab10d3 --- /dev/null +++ b/contrib/libs/liburing/src/lib.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef LIBURING_LIB_H +#define LIBURING_LIB_H + +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#if defined(__x86_64__) || defined(__i386__) +#include "arch/x86/lib.h" +#elif defined(__aarch64__) +#include "arch/aarch64/lib.h" +#else +/* + * We don't have nolibc support for this arch. Must use libc! + */ +#ifdef CONFIG_NOLIBC +#error "This arch doesn't support building liburing without libc" +#endif +/* libc wrappers. */ +#error #include "arch/generic/lib.h" +#endif + + +#ifndef offsetof +#define offsetof(TYPE, FIELD) ((size_t) &((TYPE *)0)->FIELD) +#endif + +#ifndef container_of +#define container_of(PTR, TYPE, FIELD) ({ \ + __typeof__(((TYPE *)0)->FIELD) *__FIELD_PTR = (PTR); \ + (TYPE *)((char *) __FIELD_PTR - offsetof(TYPE, FIELD)); \ +}) +#endif + +#define __maybe_unused __attribute__((__unused__)) +#define __hot __attribute__((__hot__)) +#define __cold __attribute__((__cold__)) + +void *__uring_malloc(size_t len); +void __uring_free(void *p); + +static inline void *uring_malloc(size_t len) +{ +#ifdef CONFIG_NOLIBC + return __uring_malloc(len); +#else + return malloc(len); +#endif +} + +static inline void uring_free(void *ptr) +{ +#ifdef CONFIG_NOLIBC + __uring_free(ptr); +#else + free(ptr); +#endif +} + +#endif /* #ifndef LIBURING_LIB_H */ diff --git a/contrib/libs/liburing/src/queue.c b/contrib/libs/liburing/src/queue.c new file mode 100644 index 0000000000..4b4220782d --- /dev/null +++ b/contrib/libs/liburing/src/queue.c @@ -0,0 +1,436 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#define _POSIX_C_SOURCE 200112L + +#include "lib.h" +#include "syscall.h" +#include "liburing.h" +#include "int_flags.h" +#include "liburing/compat.h" +#include "liburing/io_uring.h" + +/* + * Returns true if we're not using SQ thread (thus nobody submits but us) + * or if IORING_SQ_NEED_WAKEUP is set, so submit thread must be explicitly + * awakened. For the latter case, we set the thread wakeup flag. + * If no SQEs are ready for submission, returns false. + */ +static inline bool sq_ring_needs_enter(struct io_uring *ring, + unsigned submit, + unsigned *flags) +{ + if (!submit) + return false; + + if (!(ring->flags & IORING_SETUP_SQPOLL)) + return true; + + /* + * Ensure the kernel can see the store to the SQ tail before we read + * the flags. + */ + io_uring_smp_mb(); + + if (uring_unlikely(IO_URING_READ_ONCE(*ring->sq.kflags) & + IORING_SQ_NEED_WAKEUP)) { + *flags |= IORING_ENTER_SQ_WAKEUP; + return true; + } + + return false; +} + +static inline bool cq_ring_needs_flush(struct io_uring *ring) +{ + return IO_URING_READ_ONCE(*ring->sq.kflags) & + (IORING_SQ_CQ_OVERFLOW | IORING_SQ_TASKRUN); +} + +static inline bool cq_ring_needs_enter(struct io_uring *ring) +{ + return (ring->flags & IORING_SETUP_IOPOLL) || cq_ring_needs_flush(ring); +} + +struct get_data { + unsigned submit; + unsigned wait_nr; + unsigned get_flags; + int sz; + int has_ts; + void *arg; +}; + +static int _io_uring_get_cqe(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr, + struct get_data *data) +{ + struct io_uring_cqe *cqe = NULL; + bool looped = false; + int err = 0; + + do { + bool need_enter = false; + unsigned flags = 0; + unsigned nr_available; + int ret; + + ret = __io_uring_peek_cqe(ring, &cqe, &nr_available); + if (ret) { + if (!err) + err = ret; + break; + } + if (!cqe && !data->wait_nr && !data->submit) { + /* + * If we already looped once, we already entererd + * the kernel. Since there's nothing to submit or + * wait for, don't keep retrying. + */ + if (looped || !cq_ring_needs_enter(ring)) { + if (!err) + err = -EAGAIN; + break; + } + need_enter = true; + } + if (data->wait_nr > nr_available || need_enter) { + flags = IORING_ENTER_GETEVENTS | data->get_flags; + need_enter = true; + } + if (sq_ring_needs_enter(ring, data->submit, &flags)) + need_enter = true; + if (!need_enter) + break; + if (looped && data->has_ts) { + struct io_uring_getevents_arg *arg = data->arg; + + if (!cqe && arg->ts && !err) + err = -ETIME; + break; + } + + if (ring->int_flags & INT_FLAG_REG_RING) + flags |= IORING_ENTER_REGISTERED_RING; + ret = __sys_io_uring_enter2(ring->enter_ring_fd, data->submit, + data->wait_nr, flags, data->arg, + data->sz); + if (ret < 0) { + if (!err) + err = ret; + break; + } + + data->submit -= ret; + if (cqe) + break; + if (!looped) { + looped = true; + err = ret; + } + } while (1); + + *cqe_ptr = cqe; + return err; +} + +int __io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr, + unsigned submit, unsigned wait_nr, sigset_t *sigmask) +{ + struct get_data data = { + .submit = submit, + .wait_nr = wait_nr, + .get_flags = 0, + .sz = _NSIG / 8, + .arg = sigmask, + }; + + return _io_uring_get_cqe(ring, cqe_ptr, &data); +} + +int io_uring_get_events(struct io_uring *ring) +{ + int flags = IORING_ENTER_GETEVENTS; + + if (ring->int_flags & INT_FLAG_REG_RING) + flags |= IORING_ENTER_REGISTERED_RING; + return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL); +} + +/* + * Fill in an array of IO completions up to count, if any are available. + * Returns the amount of IO completions filled. + */ +unsigned io_uring_peek_batch_cqe(struct io_uring *ring, + struct io_uring_cqe **cqes, unsigned count) +{ + unsigned ready; + bool overflow_checked = false; + int shift = 0; + + if (ring->flags & IORING_SETUP_CQE32) + shift = 1; + +again: + ready = io_uring_cq_ready(ring); + if (ready) { + unsigned head = *ring->cq.khead; + unsigned mask = ring->cq.ring_mask; + unsigned last; + int i = 0; + + count = count > ready ? ready : count; + last = head + count; + for (;head != last; head++, i++) + cqes[i] = &ring->cq.cqes[(head & mask) << shift]; + + return count; + } + + if (overflow_checked) + return 0; + + if (cq_ring_needs_flush(ring)) { + io_uring_get_events(ring); + overflow_checked = true; + goto again; + } + + return 0; +} + +/* + * Sync internal state with kernel ring state on the SQ side. Returns the + * number of pending items in the SQ ring, for the shared ring. + */ +unsigned __io_uring_flush_sq(struct io_uring *ring) +{ + struct io_uring_sq *sq = &ring->sq; + unsigned tail = sq->sqe_tail; + + if (sq->sqe_head != tail) { + sq->sqe_head = tail; + /* + * Ensure kernel sees the SQE updates before the tail update. + */ + if (!(ring->flags & IORING_SETUP_SQPOLL)) + IO_URING_WRITE_ONCE(*sq->ktail, tail); + else + io_uring_smp_store_release(sq->ktail, tail); + } + /* + * This _may_ look problematic, as we're not supposed to be reading + * SQ->head without acquire semantics. When we're in SQPOLL mode, the + * kernel submitter could be updating this right now. For non-SQPOLL, + * task itself does it, and there's no potential race. But even for + * SQPOLL, the load is going to be potentially out-of-date the very + * instant it's done, regardless or whether or not it's done + * atomically. Worst case, we're going to be over-estimating what + * we can submit. The point is, we need to be able to deal with this + * situation regardless of any perceived atomicity. + */ + return tail - *sq->khead; +} + +/* + * If we have kernel support for IORING_ENTER_EXT_ARG, then we can use that + * more efficiently than queueing an internal timeout command. + */ +static int io_uring_wait_cqes_new(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr, + unsigned wait_nr, + struct __kernel_timespec *ts, + sigset_t *sigmask) +{ + struct io_uring_getevents_arg arg = { + .sigmask = (unsigned long) sigmask, + .sigmask_sz = _NSIG / 8, + .ts = (unsigned long) ts + }; + struct get_data data = { + .wait_nr = wait_nr, + .get_flags = IORING_ENTER_EXT_ARG, + .sz = sizeof(arg), + .has_ts = ts != NULL, + .arg = &arg + }; + + return _io_uring_get_cqe(ring, cqe_ptr, &data); +} + +/* + * Like io_uring_wait_cqe(), except it accepts a timeout value as well. Note + * that an sqe is used internally to handle the timeout. For kernel doesn't + * support IORING_FEAT_EXT_ARG, applications using this function must never + * set sqe->user_data to LIBURING_UDATA_TIMEOUT! + * + * For kernels without IORING_FEAT_EXT_ARG (5.10 and older), if 'ts' is + * specified, the application need not call io_uring_submit() before + * calling this function, as we will do that on its behalf. From this it also + * follows that this function isn't safe to use for applications that split SQ + * and CQ handling between two threads and expect that to work without + * synchronization, as this function manipulates both the SQ and CQ side. + * + * For kernels with IORING_FEAT_EXT_ARG, no implicit submission is done and + * hence this function is safe to use for applications that split SQ and CQ + * handling between two threads. + */ +static int __io_uring_submit_timeout(struct io_uring *ring, unsigned wait_nr, + struct __kernel_timespec *ts) +{ + struct io_uring_sqe *sqe; + int ret; + + /* + * If the SQ ring is full, we may need to submit IO first + */ + sqe = io_uring_get_sqe(ring); + if (!sqe) { + ret = io_uring_submit(ring); + if (ret < 0) + return ret; + sqe = io_uring_get_sqe(ring); + if (!sqe) + return -EAGAIN; + } + io_uring_prep_timeout(sqe, ts, wait_nr, 0); + sqe->user_data = LIBURING_UDATA_TIMEOUT; + return __io_uring_flush_sq(ring); +} + +int io_uring_wait_cqes(struct io_uring *ring, struct io_uring_cqe **cqe_ptr, + unsigned wait_nr, struct __kernel_timespec *ts, + sigset_t *sigmask) +{ + int to_submit = 0; + + if (ts) { + if (ring->features & IORING_FEAT_EXT_ARG) + return io_uring_wait_cqes_new(ring, cqe_ptr, wait_nr, + ts, sigmask); + to_submit = __io_uring_submit_timeout(ring, wait_nr, ts); + if (to_submit < 0) + return to_submit; + } + + return __io_uring_get_cqe(ring, cqe_ptr, to_submit, wait_nr, sigmask); +} + +int io_uring_submit_and_wait_timeout(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr, + unsigned wait_nr, + struct __kernel_timespec *ts, + sigset_t *sigmask) +{ + int to_submit; + + if (ts) { + if (ring->features & IORING_FEAT_EXT_ARG) { + struct io_uring_getevents_arg arg = { + .sigmask = (unsigned long) sigmask, + .sigmask_sz = _NSIG / 8, + .ts = (unsigned long) ts + }; + struct get_data data = { + .submit = __io_uring_flush_sq(ring), + .wait_nr = wait_nr, + .get_flags = IORING_ENTER_EXT_ARG, + .sz = sizeof(arg), + .has_ts = ts != NULL, + .arg = &arg + }; + + return _io_uring_get_cqe(ring, cqe_ptr, &data); + } + to_submit = __io_uring_submit_timeout(ring, wait_nr, ts); + if (to_submit < 0) + return to_submit; + } else + to_submit = __io_uring_flush_sq(ring); + + return __io_uring_get_cqe(ring, cqe_ptr, to_submit, wait_nr, sigmask); +} + +/* + * See io_uring_wait_cqes() - this function is the same, it just always uses + * '1' as the wait_nr. + */ +int io_uring_wait_cqe_timeout(struct io_uring *ring, + struct io_uring_cqe **cqe_ptr, + struct __kernel_timespec *ts) +{ + return io_uring_wait_cqes(ring, cqe_ptr, 1, ts, NULL); +} + +/* + * Submit sqes acquired from io_uring_get_sqe() to the kernel. + * + * Returns number of sqes submitted + */ +static int __io_uring_submit(struct io_uring *ring, unsigned submitted, + unsigned wait_nr, bool getevents) +{ + bool cq_needs_enter = getevents || wait_nr || cq_ring_needs_enter(ring); + unsigned flags; + int ret; + + flags = 0; + if (sq_ring_needs_enter(ring, submitted, &flags) || cq_needs_enter) { + if (cq_needs_enter) + flags |= IORING_ENTER_GETEVENTS; + if (ring->int_flags & INT_FLAG_REG_RING) + flags |= IORING_ENTER_REGISTERED_RING; + + ret = __sys_io_uring_enter(ring->enter_ring_fd, submitted, + wait_nr, flags, NULL); + } else + ret = submitted; + + return ret; +} + +static int __io_uring_submit_and_wait(struct io_uring *ring, unsigned wait_nr) +{ + return __io_uring_submit(ring, __io_uring_flush_sq(ring), wait_nr, false); +} + +/* + * Submit sqes acquired from io_uring_get_sqe() to the kernel. + * + * Returns number of sqes submitted + */ +int io_uring_submit(struct io_uring *ring) +{ + return __io_uring_submit_and_wait(ring, 0); +} + +/* + * Like io_uring_submit(), but allows waiting for events as well. + * + * Returns number of sqes submitted + */ +int io_uring_submit_and_wait(struct io_uring *ring, unsigned wait_nr) +{ + return __io_uring_submit_and_wait(ring, wait_nr); +} + +int io_uring_submit_and_get_events(struct io_uring *ring) +{ + return __io_uring_submit(ring, __io_uring_flush_sq(ring), 0, true); +} + +#ifdef LIBURING_INTERNAL +struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring) +{ + return _io_uring_get_sqe(ring); +} +#endif + +int __io_uring_sqring_wait(struct io_uring *ring) +{ + int flags = IORING_ENTER_SQ_WAIT; + + if (ring->int_flags & INT_FLAG_REG_RING) + flags |= IORING_ENTER_REGISTERED_RING; + + return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL); +} diff --git a/contrib/libs/liburing/src/register.c b/contrib/libs/liburing/src/register.c new file mode 100644 index 0000000000..13e80a5811 --- /dev/null +++ b/contrib/libs/liburing/src/register.c @@ -0,0 +1,370 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#define _POSIX_C_SOURCE 200112L + +#include "lib.h" +#include "syscall.h" +#include "liburing.h" +#include "int_flags.h" +#include "liburing/compat.h" +#include "liburing/io_uring.h" + +int io_uring_register_buffers_update_tag(struct io_uring *ring, unsigned off, + const struct iovec *iovecs, + const __u64 *tags, + unsigned nr) +{ + struct io_uring_rsrc_update2 up = { + .offset = off, + .data = (unsigned long)iovecs, + .tags = (unsigned long)tags, + .nr = nr, + }; + + return __sys_io_uring_register(ring->ring_fd,IORING_REGISTER_BUFFERS_UPDATE, &up, + sizeof(up)); +} + +int io_uring_register_buffers_tags(struct io_uring *ring, + const struct iovec *iovecs, + const __u64 *tags, + unsigned nr) +{ + struct io_uring_rsrc_register reg = { + .nr = nr, + .data = (unsigned long)iovecs, + .tags = (unsigned long)tags, + }; + + return __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_BUFFERS2, ®, + sizeof(reg)); +} + +int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr) +{ + struct io_uring_rsrc_register reg = { + .flags = IORING_RSRC_REGISTER_SPARSE, + .nr = nr, + }; + + return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS2, + ®, sizeof(reg)); +} + +int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs, + unsigned nr_iovecs) +{ + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS, + iovecs, nr_iovecs); + return (ret < 0) ? ret : 0; +} + +int io_uring_unregister_buffers(struct io_uring *ring) +{ + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_BUFFERS, + NULL, 0); + return (ret < 0) ? ret : 0; +} + +int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off, + const int *files, const __u64 *tags, + unsigned nr_files) +{ + struct io_uring_rsrc_update2 up = { + .offset = off, + .data = (unsigned long)files, + .tags = (unsigned long)tags, + .nr = nr_files, + }; + + return __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_FILES_UPDATE2, &up, + sizeof(up)); +} + +/* + * Register an update for an existing file set. The updates will start at + * 'off' in the original array, and 'nr_files' is the number of files we'll + * update. + * + * Returns number of files updated on success, -ERROR on failure. + */ +int io_uring_register_files_update(struct io_uring *ring, unsigned off, + const int *files, unsigned nr_files) +{ + struct io_uring_files_update up = { + .offset = off, + .fds = (unsigned long) files, + }; + + return __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_FILES_UPDATE, &up, + nr_files); +} + +static int increase_rlimit_nofile(unsigned nr) +{ + int ret; + struct rlimit rlim; + + ret = __sys_getrlimit(RLIMIT_NOFILE, &rlim); + if (ret < 0) + return ret; + + if (rlim.rlim_cur < nr) { + rlim.rlim_cur += nr; + __sys_setrlimit(RLIMIT_NOFILE, &rlim); + } + + return 0; +} + +int io_uring_register_files_sparse(struct io_uring *ring, unsigned nr) +{ + struct io_uring_rsrc_register reg = { + .flags = IORING_RSRC_REGISTER_SPARSE, + .nr = nr, + }; + int ret, did_increase = 0; + + do { + ret = __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_FILES2, ®, + sizeof(reg)); + if (ret >= 0) + break; + if (ret == -EMFILE && !did_increase) { + did_increase = 1; + increase_rlimit_nofile(nr); + continue; + } + break; + } while (1); + + return ret; +} + +int io_uring_register_files_tags(struct io_uring *ring, const int *files, + const __u64 *tags, unsigned nr) +{ + struct io_uring_rsrc_register reg = { + .nr = nr, + .data = (unsigned long)files, + .tags = (unsigned long)tags, + }; + int ret, did_increase = 0; + + do { + ret = __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_FILES2, ®, + sizeof(reg)); + if (ret >= 0) + break; + if (ret == -EMFILE && !did_increase) { + did_increase = 1; + increase_rlimit_nofile(nr); + continue; + } + break; + } while (1); + + return ret; +} + +int io_uring_register_files(struct io_uring *ring, const int *files, + unsigned nr_files) +{ + int ret, did_increase = 0; + + do { + ret = __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_FILES, files, + nr_files); + if (ret >= 0) + break; + if (ret == -EMFILE && !did_increase) { + did_increase = 1; + increase_rlimit_nofile(nr_files); + continue; + } + break; + } while (1); + + return ret; +} + +int io_uring_unregister_files(struct io_uring *ring) +{ + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_FILES, + NULL, 0); + return (ret < 0) ? ret : 0; +} + +int io_uring_register_eventfd(struct io_uring *ring, int event_fd) +{ + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_EVENTFD, + &event_fd, 1); + return (ret < 0) ? ret : 0; +} + +int io_uring_unregister_eventfd(struct io_uring *ring) +{ + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_EVENTFD, + NULL, 0); + return (ret < 0) ? ret : 0; +} + +int io_uring_register_eventfd_async(struct io_uring *ring, int event_fd) +{ + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_EVENTFD_ASYNC, &event_fd, + 1); + return (ret < 0) ? ret : 0; +} + +int io_uring_register_probe(struct io_uring *ring, struct io_uring_probe *p, + unsigned int nr_ops) +{ + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PROBE, p, + nr_ops); + return (ret < 0) ? ret : 0; +} + +int io_uring_register_personality(struct io_uring *ring) +{ + return __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_PERSONALITY, NULL, 0); +} + +int io_uring_unregister_personality(struct io_uring *ring, int id) +{ + return __sys_io_uring_register(ring->ring_fd, + IORING_UNREGISTER_PERSONALITY, NULL, id); +} + +int io_uring_register_restrictions(struct io_uring *ring, + struct io_uring_restriction *res, + unsigned int nr_res) +{ + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_RESTRICTIONS, res, + nr_res); + return (ret < 0) ? ret : 0; +} + +int io_uring_enable_rings(struct io_uring *ring) +{ + return __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_ENABLE_RINGS, NULL, 0); +} + +int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz, + const cpu_set_t *mask) +{ + if (cpusz >= (1U << 31)) + return -EINVAL; + + return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_IOWQ_AFF, + mask, (int) cpusz); +} + +int io_uring_unregister_iowq_aff(struct io_uring *ring) +{ + return __sys_io_uring_register(ring->ring_fd, + IORING_UNREGISTER_IOWQ_AFF, NULL, 0); +} + +int io_uring_register_iowq_max_workers(struct io_uring *ring, unsigned int *val) +{ + return __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_IOWQ_MAX_WORKERS, val, + 2); +} + +int io_uring_register_ring_fd(struct io_uring *ring) +{ + struct io_uring_rsrc_update up = { + .data = ring->ring_fd, + .offset = -1U, + }; + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_RING_FDS, + &up, 1); + if (ret == 1) { + ring->enter_ring_fd = up.offset; + ring->int_flags |= INT_FLAG_REG_RING; + } + return ret; +} + + +int io_uring_unregister_ring_fd(struct io_uring *ring) +{ + struct io_uring_rsrc_update up = { + .offset = ring->enter_ring_fd, + }; + int ret; + + ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_RING_FDS, + &up, 1); + if (ret == 1) { + ring->enter_ring_fd = ring->ring_fd; + ring->int_flags &= ~INT_FLAG_REG_RING; + } + return ret; +} + +int io_uring_register_buf_ring(struct io_uring *ring, + struct io_uring_buf_reg *reg, + unsigned int __maybe_unused flags) +{ + return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PBUF_RING, + reg, 1); +} + +int io_uring_unregister_buf_ring(struct io_uring *ring, int bgid) +{ + struct io_uring_buf_reg reg = { .bgid = bgid }; + + return __sys_io_uring_register(ring->ring_fd, + IORING_UNREGISTER_PBUF_RING, ®, 1); +} + +int io_uring_register_sync_cancel(struct io_uring *ring, + struct io_uring_sync_cancel_reg *reg) +{ + return __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_SYNC_CANCEL, reg, 1); +} + +int io_uring_register_file_alloc_range(struct io_uring *ring, + unsigned off, unsigned len) +{ + struct io_uring_file_index_range range; + + memset(&range, 0, sizeof(range)); + range.off = off; + range.len = len; + + return __sys_io_uring_register(ring->ring_fd, + IORING_REGISTER_FILE_ALLOC_RANGE, &range, + 0); +} diff --git a/contrib/libs/liburing/src/setup.c b/contrib/libs/liburing/src/setup.c new file mode 100644 index 0000000000..0c15e75c1c --- /dev/null +++ b/contrib/libs/liburing/src/setup.c @@ -0,0 +1,370 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#define _DEFAULT_SOURCE + +#include "lib.h" +#include "syscall.h" +#include "liburing.h" +#include "int_flags.h" +#include "liburing/compat.h" +#include "liburing/io_uring.h" + +static void io_uring_unmap_rings(struct io_uring_sq *sq, struct io_uring_cq *cq) +{ + __sys_munmap(sq->ring_ptr, sq->ring_sz); + if (cq->ring_ptr && cq->ring_ptr != sq->ring_ptr) + __sys_munmap(cq->ring_ptr, cq->ring_sz); +} + +static int io_uring_mmap(int fd, struct io_uring_params *p, + struct io_uring_sq *sq, struct io_uring_cq *cq) +{ + size_t size; + int ret; + + size = sizeof(struct io_uring_cqe); + if (p->flags & IORING_SETUP_CQE32) + size += sizeof(struct io_uring_cqe); + + sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned); + cq->ring_sz = p->cq_off.cqes + p->cq_entries * size; + + if (p->features & IORING_FEAT_SINGLE_MMAP) { + if (cq->ring_sz > sq->ring_sz) + sq->ring_sz = cq->ring_sz; + cq->ring_sz = sq->ring_sz; + } + sq->ring_ptr = __sys_mmap(0, sq->ring_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, fd, + IORING_OFF_SQ_RING); + if (IS_ERR(sq->ring_ptr)) + return PTR_ERR(sq->ring_ptr); + + if (p->features & IORING_FEAT_SINGLE_MMAP) { + cq->ring_ptr = sq->ring_ptr; + } else { + cq->ring_ptr = __sys_mmap(0, cq->ring_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, fd, + IORING_OFF_CQ_RING); + if (IS_ERR(cq->ring_ptr)) { + ret = PTR_ERR(cq->ring_ptr); + cq->ring_ptr = NULL; + goto err; + } + } + + sq->khead = sq->ring_ptr + p->sq_off.head; + sq->ktail = sq->ring_ptr + p->sq_off.tail; + sq->kring_mask = sq->ring_ptr + p->sq_off.ring_mask; + sq->kring_entries = sq->ring_ptr + p->sq_off.ring_entries; + sq->kflags = sq->ring_ptr + p->sq_off.flags; + sq->kdropped = sq->ring_ptr + p->sq_off.dropped; + sq->array = sq->ring_ptr + p->sq_off.array; + + size = sizeof(struct io_uring_sqe); + if (p->flags & IORING_SETUP_SQE128) + size += 64; + sq->sqes = __sys_mmap(0, size * p->sq_entries, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES); + if (IS_ERR(sq->sqes)) { + ret = PTR_ERR(sq->sqes); +err: + io_uring_unmap_rings(sq, cq); + return ret; + } + + cq->khead = cq->ring_ptr + p->cq_off.head; + cq->ktail = cq->ring_ptr + p->cq_off.tail; + cq->kring_mask = cq->ring_ptr + p->cq_off.ring_mask; + cq->kring_entries = cq->ring_ptr + p->cq_off.ring_entries; + cq->koverflow = cq->ring_ptr + p->cq_off.overflow; + cq->cqes = cq->ring_ptr + p->cq_off.cqes; + if (p->cq_off.flags) + cq->kflags = cq->ring_ptr + p->cq_off.flags; + + sq->ring_mask = *sq->kring_mask; + sq->ring_entries = *sq->kring_entries; + cq->ring_mask = *cq->kring_mask; + cq->ring_entries = *cq->kring_entries; + return 0; +} + +/* + * For users that want to specify sq_thread_cpu or sq_thread_idle, this + * interface is a convenient helper for mmap()ing the rings. + * Returns -errno on error, or zero on success. On success, 'ring' + * contains the necessary information to read/write to the rings. + */ +__cold int io_uring_queue_mmap(int fd, struct io_uring_params *p, + struct io_uring *ring) +{ + int ret; + + memset(ring, 0, sizeof(*ring)); + ret = io_uring_mmap(fd, p, &ring->sq, &ring->cq); + if (!ret) { + ring->flags = p->flags; + ring->ring_fd = ring->enter_ring_fd = fd; + ring->int_flags = 0; + } + return ret; +} + +/* + * Ensure that the mmap'ed rings aren't available to a child after a fork(2). + * This uses madvise(..., MADV_DONTFORK) on the mmap'ed ranges. + */ +__cold int io_uring_ring_dontfork(struct io_uring *ring) +{ + size_t len; + int ret; + + if (!ring->sq.ring_ptr || !ring->sq.sqes || !ring->cq.ring_ptr) + return -EINVAL; + + len = sizeof(struct io_uring_sqe); + if (ring->flags & IORING_SETUP_SQE128) + len += 64; + len *= ring->sq.ring_entries; + ret = __sys_madvise(ring->sq.sqes, len, MADV_DONTFORK); + if (ret < 0) + return ret; + + len = ring->sq.ring_sz; + ret = __sys_madvise(ring->sq.ring_ptr, len, MADV_DONTFORK); + if (ret < 0) + return ret; + + if (ring->cq.ring_ptr != ring->sq.ring_ptr) { + len = ring->cq.ring_sz; + ret = __sys_madvise(ring->cq.ring_ptr, len, MADV_DONTFORK); + if (ret < 0) + return ret; + } + + return 0; +} + +__cold int io_uring_queue_init_params(unsigned entries, struct io_uring *ring, + struct io_uring_params *p) +{ + int fd, ret; + unsigned *sq_array; + unsigned sq_entries, index; + + fd = __sys_io_uring_setup(entries, p); + if (fd < 0) + return fd; + + ret = io_uring_queue_mmap(fd, p, ring); + if (ret) { + __sys_close(fd); + return ret; + } + + /* + * Directly map SQ slots to SQEs + */ + sq_array = ring->sq.array; + sq_entries = ring->sq.ring_entries; + for (index = 0; index < sq_entries; index++) + sq_array[index] = index; + + ring->features = p->features; + return 0; +} + +/* + * Returns -errno on error, or zero on success. On success, 'ring' + * contains the necessary information to read/write to the rings. + */ +__cold int io_uring_queue_init(unsigned entries, struct io_uring *ring, + unsigned flags) +{ + struct io_uring_params p; + + memset(&p, 0, sizeof(p)); + p.flags = flags; + + return io_uring_queue_init_params(entries, ring, &p); +} + +__cold void io_uring_queue_exit(struct io_uring *ring) +{ + struct io_uring_sq *sq = &ring->sq; + struct io_uring_cq *cq = &ring->cq; + size_t sqe_size; + + sqe_size = sizeof(struct io_uring_sqe); + if (ring->flags & IORING_SETUP_SQE128) + sqe_size += 64; + __sys_munmap(sq->sqes, sqe_size * sq->ring_entries); + io_uring_unmap_rings(sq, cq); + /* + * Not strictly required, but frees up the slot we used now rather + * than at process exit time. + */ + if (ring->int_flags & INT_FLAG_REG_RING) + io_uring_unregister_ring_fd(ring); + __sys_close(ring->ring_fd); +} + +__cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring) +{ + struct io_uring_probe *probe; + size_t len; + int r; + + len = sizeof(*probe) + 256 * sizeof(struct io_uring_probe_op); + probe = uring_malloc(len); + if (!probe) + return NULL; + memset(probe, 0, len); + + r = io_uring_register_probe(ring, probe, 256); + if (r >= 0) + return probe; + + uring_free(probe); + return NULL; +} + +__cold struct io_uring_probe *io_uring_get_probe(void) +{ + struct io_uring ring; + struct io_uring_probe *probe; + int r; + + r = io_uring_queue_init(2, &ring, 0); + if (r < 0) + return NULL; + + probe = io_uring_get_probe_ring(&ring); + io_uring_queue_exit(&ring); + return probe; +} + +__cold void io_uring_free_probe(struct io_uring_probe *probe) +{ + uring_free(probe); +} + +static inline int __fls(unsigned long x) +{ + if (!x) + return 0; + return 8 * sizeof(x) - __builtin_clzl(x); +} + +static unsigned roundup_pow2(unsigned depth) +{ + return 1U << __fls(depth - 1); +} + +static size_t npages(size_t size, long page_size) +{ + size--; + size /= page_size; + return __fls((int) size); +} + +#define KRING_SIZE 320 + +static size_t rings_size(struct io_uring_params *p, unsigned entries, + unsigned cq_entries, long page_size) +{ + size_t pages, sq_size, cq_size; + + cq_size = sizeof(struct io_uring_cqe); + if (p->flags & IORING_SETUP_CQE32) + cq_size += sizeof(struct io_uring_cqe); + cq_size *= cq_entries; + cq_size += KRING_SIZE; + cq_size = (cq_size + 63) & ~63UL; + pages = (size_t) 1 << npages(cq_size, page_size); + + sq_size = sizeof(struct io_uring_sqe); + if (p->flags & IORING_SETUP_SQE128) + sq_size += 64; + sq_size *= entries; + pages += (size_t) 1 << npages(sq_size, page_size); + return pages * page_size; +} + +#define KERN_MAX_ENTRIES 32768 +#define KERN_MAX_CQ_ENTRIES (2 * KERN_MAX_ENTRIES) + +/* + * Return the required ulimit -l memlock memory required for a given ring + * setup, in bytes. May return -errno on error. On newer (5.12+) kernels, + * io_uring no longer requires any memlock memory, and hence this function + * will return 0 for that case. On older (5.11 and prior) kernels, this will + * return the required memory so that the caller can ensure that enough space + * is available before setting up a ring with the specified parameters. + */ +__cold ssize_t io_uring_mlock_size_params(unsigned entries, + struct io_uring_params *p) +{ + struct io_uring_params lp = { }; + struct io_uring ring; + unsigned cq_entries; + long page_size; + ssize_t ret; + + /* + * We only really use this inited ring to see if the kernel is newer + * or not. Newer kernels don't require memlocked memory. If we fail, + * it's most likely because it's an older kernel and we have no + * available memlock space. Just continue on, lp.features will still + * be zeroed at this point and we'll do the right thing. + */ + ret = io_uring_queue_init_params(entries, &ring, &lp); + if (!ret) + io_uring_queue_exit(&ring); + + /* + * Native workers imply using cgroup memory accounting, and hence no + * memlock memory is needed for the ring allocations. + */ + if (lp.features & IORING_FEAT_NATIVE_WORKERS) + return 0; + + if (!entries) + return -EINVAL; + if (entries > KERN_MAX_ENTRIES) { + if (!(p->flags & IORING_SETUP_CLAMP)) + return -EINVAL; + entries = KERN_MAX_ENTRIES; + } + + entries = roundup_pow2(entries); + if (p->flags & IORING_SETUP_CQSIZE) { + if (!p->cq_entries) + return -EINVAL; + cq_entries = p->cq_entries; + if (cq_entries > KERN_MAX_CQ_ENTRIES) { + if (!(p->flags & IORING_SETUP_CLAMP)) + return -EINVAL; + cq_entries = KERN_MAX_CQ_ENTRIES; + } + cq_entries = roundup_pow2(cq_entries); + if (cq_entries < entries) + return -EINVAL; + } else { + cq_entries = 2 * entries; + } + + page_size = get_page_size(); + return rings_size(p, entries, cq_entries, page_size); +} + +/* + * Return required ulimit -l memory space for a given ring setup. See + * @io_uring_mlock_size_params(). + */ +__cold ssize_t io_uring_mlock_size(unsigned entries, unsigned flags) +{ + struct io_uring_params p = { .flags = flags, }; + + return io_uring_mlock_size_params(entries, &p); +} diff --git a/contrib/libs/liburing/src/syscall.c b/contrib/libs/liburing/src/syscall.c new file mode 100644 index 0000000000..0e38a4eac7 --- /dev/null +++ b/contrib/libs/liburing/src/syscall.c @@ -0,0 +1,30 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ + +#include "syscall.h" +#include <liburing.h> + +int io_uring_enter(unsigned int fd, unsigned int to_submit, + unsigned int min_complete, unsigned int flags, sigset_t *sig) +{ + return __sys_io_uring_enter(fd, to_submit, min_complete, flags, sig); +} + +int io_uring_enter2(unsigned int fd, unsigned int to_submit, + unsigned int min_complete, unsigned int flags, + sigset_t *sig, size_t sz) +{ + return __sys_io_uring_enter2(fd, to_submit, min_complete, flags, sig, + sz); +} + +int io_uring_setup(unsigned int entries, struct io_uring_params *p) +{ + return __sys_io_uring_setup(entries, p); +} + +int io_uring_register(unsigned int fd, unsigned int opcode, const void *arg, + unsigned int nr_args) +{ + return __sys_io_uring_register(fd, opcode, arg, nr_args); +} diff --git a/contrib/libs/liburing/src/syscall.h b/contrib/libs/liburing/src/syscall.h new file mode 100644 index 0000000000..4fa77e60ed --- /dev/null +++ b/contrib/libs/liburing/src/syscall.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef LIBURING_SYSCALL_H +#define LIBURING_SYSCALL_H + +#include <errno.h> +#include <signal.h> +#include <stdint.h> +#include <unistd.h> +#include <stdbool.h> +#include <sys/mman.h> +#include <sys/syscall.h> +#include <sys/resource.h> +#include <liburing.h> + +/* + * Don't put this below the #include "arch/$arch/syscall.h", that + * file may need it. + */ +struct io_uring_params; + +static inline void *ERR_PTR(intptr_t n) +{ + return (void *) n; +} + +static inline int PTR_ERR(const void *ptr) +{ + return (int) (intptr_t) ptr; +} + +static inline bool IS_ERR(const void *ptr) +{ + return uring_unlikely((uintptr_t) ptr >= (uintptr_t) -4095UL); +} + +#if defined(__x86_64__) || defined(__i386__) +#include "arch/x86/syscall.h" +#elif defined(__aarch64__) +#include "arch/aarch64/syscall.h" +#else +/* + * We don't have native syscall wrappers + * for this arch. Must use libc! + */ +#ifdef CONFIG_NOLIBC + #error "This arch doesn't support building liburing without libc" +#endif +/* libc syscall wrappers. */ +#error #include "arch/generic/syscall.h" +#endif +#endif diff --git a/contrib/libs/liburing/test/232c93d07b74.c b/contrib/libs/liburing/test/232c93d07b74.c new file mode 100644 index 0000000000..ab28adab17 --- /dev/null +++ b/contrib/libs/liburing/test/232c93d07b74.c @@ -0,0 +1,306 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test case for socket read/write through IORING_OP_READV and + * IORING_OP_WRITEV, using both TCP and sockets and blocking and + * non-blocking IO. + * + * Heavily based on a test case from Hrvoje Zeba <zeba.hrvoje@gmail.com> + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <assert.h> + +#include <pthread.h> +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <netinet/tcp.h> +#include <netinet/in.h> +#include <arpa/inet.h> + +#include "helpers.h" +#include "liburing.h" + +#define RECV_BUFF_SIZE 2 +#define SEND_BUFF_SIZE 3 + +struct params { + int tcp; + int non_blocking; + __be16 bind_port; +}; + +pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +int rcv_ready = 0; + +static void set_rcv_ready(void) +{ + pthread_mutex_lock(&mutex); + + rcv_ready = 1; + pthread_cond_signal(&cond); + + pthread_mutex_unlock(&mutex); +} + +static void wait_for_rcv_ready(void) +{ + pthread_mutex_lock(&mutex); + + while (!rcv_ready) + pthread_cond_wait(&cond, &mutex); + + pthread_mutex_unlock(&mutex); +} + +static void *rcv(void *arg) +{ + struct params *p = arg; + int s0; + int res; + + if (p->tcp) { + int val = 1; + + + s0 = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + res = setsockopt(s0, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + assert(res != -1); + res = setsockopt(s0, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + assert(res != -1); + + struct sockaddr_in addr; + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr("127.0.0.1"); + assert(t_bind_ephemeral_port(s0, &addr) == 0); + p->bind_port = addr.sin_port; + } else { + s0 = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + assert(s0 != -1); + + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + + addr.sun_family = AF_UNIX; + memcpy(addr.sun_path, "\0sock", 6); + res = bind(s0, (struct sockaddr *) &addr, sizeof(addr)); + assert(res != -1); + } + res = listen(s0, 128); + assert(res != -1); + + set_rcv_ready(); + + int s1 = accept(s0, NULL, NULL); + assert(s1 != -1); + + if (p->non_blocking) { + int flags = fcntl(s1, F_GETFL, 0); + assert(flags != -1); + + flags |= O_NONBLOCK; + res = fcntl(s1, F_SETFL, flags); + assert(res != -1); + } + + struct io_uring m_io_uring; + void *ret = NULL; + + res = io_uring_queue_init(32, &m_io_uring, 0); + assert(res >= 0); + + int bytes_read = 0; + int expected_byte = 0; + int done = 0; + + while (!done && bytes_read != 33) { + char buff[RECV_BUFF_SIZE]; + struct iovec iov; + + iov.iov_base = buff; + iov.iov_len = sizeof(buff); + + struct io_uring_sqe *sqe = io_uring_get_sqe(&m_io_uring); + assert(sqe != NULL); + + io_uring_prep_readv(sqe, s1, &iov, 1, 0); + + res = io_uring_submit(&m_io_uring); + assert(res != -1); + + struct io_uring_cqe *cqe; + unsigned head; + unsigned count = 0; + + while (!done && count != 1) { + io_uring_for_each_cqe(&m_io_uring, head, cqe) { + if (cqe->res < 0) + assert(cqe->res == -EAGAIN); + else { + int i; + + for (i = 0; i < cqe->res; i++) { + if (buff[i] != expected_byte) { + fprintf(stderr, + "Received %d, wanted %d\n", + buff[i], expected_byte); + ret++; + done = 1; + } + expected_byte++; + } + bytes_read += cqe->res; + } + + count++; + } + + assert(count <= 1); + io_uring_cq_advance(&m_io_uring, count); + } + } + + shutdown(s1, SHUT_RDWR); + close(s1); + close(s0); + io_uring_queue_exit(&m_io_uring); + return ret; +} + +static void *snd(void *arg) +{ + struct params *p = arg; + int s0; + int ret; + + wait_for_rcv_ready(); + + if (p->tcp) { + int val = 1; + + s0 = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + ret = setsockopt(s0, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)); + assert(ret != -1); + + struct sockaddr_in addr; + + addr.sin_family = AF_INET; + addr.sin_port = p->bind_port; + addr.sin_addr.s_addr = inet_addr("127.0.0.1"); + ret = connect(s0, (struct sockaddr*) &addr, sizeof(addr)); + assert(ret != -1); + } else { + s0 = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + assert(s0 != -1); + + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + + addr.sun_family = AF_UNIX; + memcpy(addr.sun_path, "\0sock", 6); + ret = connect(s0, (struct sockaddr*) &addr, sizeof(addr)); + assert(ret != -1); + } + + if (p->non_blocking) { + int flags = fcntl(s0, F_GETFL, 0); + assert(flags != -1); + + flags |= O_NONBLOCK; + ret = fcntl(s0, F_SETFL, flags); + assert(ret != -1); + } + + struct io_uring m_io_uring; + + ret = io_uring_queue_init(32, &m_io_uring, 0); + assert(ret >= 0); + + int bytes_written = 0; + int done = 0; + + while (!done && bytes_written != 33) { + char buff[SEND_BUFF_SIZE]; + int i; + + for (i = 0; i < SEND_BUFF_SIZE; i++) + buff[i] = i + bytes_written; + + struct iovec iov; + + iov.iov_base = buff; + iov.iov_len = sizeof(buff); + + struct io_uring_sqe *sqe = io_uring_get_sqe(&m_io_uring); + assert(sqe != NULL); + + io_uring_prep_writev(sqe, s0, &iov, 1, 0); + + ret = io_uring_submit(&m_io_uring); + assert(ret != -1); + + struct io_uring_cqe *cqe; + unsigned head; + unsigned count = 0; + + while (!done && count != 1) { + io_uring_for_each_cqe(&m_io_uring, head, cqe) { + if (cqe->res < 0) { + if (cqe->res == -EPIPE) { + done = 1; + break; + } + assert(cqe->res == -EAGAIN); + } else { + bytes_written += cqe->res; + } + + count++; + } + + assert(count <= 1); + io_uring_cq_advance(&m_io_uring, count); + } + usleep(100000); + } + + shutdown(s0, SHUT_RDWR); + close(s0); + io_uring_queue_exit(&m_io_uring); + return NULL; +} + +int main(int argc, char *argv[]) +{ + struct params p; + pthread_t t1, t2; + void *res1, *res2; + int i, exit_val = T_EXIT_PASS; + + if (argc > 1) + return T_EXIT_SKIP; + + for (i = 0; i < 4; i++) { + p.tcp = i & 1; + p.non_blocking = (i & 2) >> 1; + + rcv_ready = 0; + + pthread_create(&t1, NULL, rcv, &p); + pthread_create(&t2, NULL, snd, &p); + pthread_join(t1, &res1); + pthread_join(t2, &res2); + if (res1 || res2) { + fprintf(stderr, "Failed tcp=%d, non_blocking=%d\n", p.tcp, p.non_blocking); + exit_val = T_EXIT_FAIL; + } + } + + return exit_val; +} diff --git a/contrib/libs/liburing/test/35fa71a030ca.c b/contrib/libs/liburing/test/35fa71a030ca.c new file mode 100644 index 0000000000..cf8e3ff605 --- /dev/null +++ b/contrib/libs/liburing/test/35fa71a030ca.c @@ -0,0 +1,330 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <dirent.h> +#include <endian.h> +#include <errno.h> +#include <fcntl.h> +#include <pthread.h> +#include <signal.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/prctl.h> +#include <sys/stat.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <time.h> +#include <unistd.h> +#include <sys/mman.h> + +#include <linux/futex.h> + +#include "liburing.h" +#include "helpers.h" +#include "../src/syscall.h" + +#if !defined(SYS_futex) && defined(SYS_futex_time64) +# define SYS_futex SYS_futex_time64 +#endif + +static void sleep_ms(uint64_t ms) +{ + usleep(ms * 1000); +} + +static uint64_t current_time_ms(void) +{ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts)) + exit(1); + return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000; +} + +static void thread_start(void* (*fn)(void*), void* arg) +{ + pthread_t th; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 128 << 10); + int i; + for (i = 0; i < 100; i++) { + if (pthread_create(&th, &attr, fn, arg) == 0) { + pthread_attr_destroy(&attr); + return; + } + if (errno == EAGAIN) { + usleep(50); + continue; + } + break; + } + exit(1); +} + +typedef struct { + int state; +} event_t; + +static void event_init(event_t* ev) +{ + ev->state = 0; +} + +static void event_reset(event_t* ev) +{ + ev->state = 0; +} + +static void event_set(event_t* ev) +{ + if (ev->state) + exit(1); + __atomic_store_n(&ev->state, 1, __ATOMIC_RELEASE); + syscall(SYS_futex, &ev->state, FUTEX_WAKE | FUTEX_PRIVATE_FLAG); +} + +static void event_wait(event_t* ev) +{ + while (!__atomic_load_n(&ev->state, __ATOMIC_ACQUIRE)) + syscall(SYS_futex, &ev->state, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, 0, 0); +} + +static int event_isset(event_t* ev) +{ + return __atomic_load_n(&ev->state, __ATOMIC_ACQUIRE); +} + +static int event_timedwait(event_t* ev, uint64_t timeout) +{ + uint64_t start = current_time_ms(); + uint64_t now = start; + for (;;) { + uint64_t remain = timeout - (now - start); + struct timespec ts; + ts.tv_sec = remain / 1000; + ts.tv_nsec = (remain % 1000) * 1000 * 1000; + syscall(SYS_futex, &ev->state, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, 0, &ts); + if (__atomic_load_n(&ev->state, __ATOMIC_RELAXED)) + return 1; + now = current_time_ms(); + if (now - start > timeout) + return 0; + } +} + +static bool write_file(const char* file, const char* what, ...) +{ + char buf[1024]; + va_list args; + va_start(args, what); + vsnprintf(buf, sizeof(buf), what, args); + va_end(args); + buf[sizeof(buf) - 1] = 0; + int len = strlen(buf); + int fd = open(file, O_WRONLY | O_CLOEXEC); + if (fd == -1) + return false; + if (write(fd, buf, len) != len) { + int err = errno; + close(fd); + errno = err; + return false; + } + close(fd); + return true; +} + +static void kill_and_wait(int pid, int* status) +{ + kill(-pid, SIGKILL); + kill(pid, SIGKILL); + int i; + for (i = 0; i < 100; i++) { + if (waitpid(-1, status, WNOHANG | __WALL) == pid) + return; + usleep(1000); + } + DIR* dir = opendir("/sys/fs/fuse/connections"); + if (dir) { + for (;;) { + struct dirent* ent = readdir(dir); + if (!ent) + break; + if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) + continue; + char abort[300]; + snprintf(abort, sizeof(abort), "/sys/fs/fuse/connections/%s/abort", + ent->d_name); + int fd = open(abort, O_WRONLY); + if (fd == -1) { + continue; + } + if (write(fd, abort, 1) < 0) { + } + close(fd); + } + closedir(dir); + } else { + } + while (waitpid(-1, status, __WALL) != pid) { + } +} + +#define SYZ_HAVE_SETUP_TEST 1 +static void setup_test() +{ + prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); + setpgrp(); + write_file("/proc/self/oom_score_adj", "1000"); +} + +struct thread_t { + int created, call; + event_t ready, done; +}; + +static struct thread_t threads[16]; +static void execute_call(int call); +static int running; + +static void* thr(void* arg) +{ + struct thread_t* th = (struct thread_t*)arg; + for (;;) { + event_wait(&th->ready); + event_reset(&th->ready); + execute_call(th->call); + __atomic_fetch_sub(&running, 1, __ATOMIC_RELAXED); + event_set(&th->done); + } + return 0; +} + +static void execute_one(void) +{ + int i, call, thread; + for (call = 0; call < 3; call++) { + for (thread = 0; thread < (int)(sizeof(threads) / sizeof(threads[0])); + thread++) { + struct thread_t* th = &threads[thread]; + if (!th->created) { + th->created = 1; + event_init(&th->ready); + event_init(&th->done); + event_set(&th->done); + thread_start(thr, th); + } + if (!event_isset(&th->done)) + continue; + event_reset(&th->done); + th->call = call; + __atomic_fetch_add(&running, 1, __ATOMIC_RELAXED); + event_set(&th->ready); + event_timedwait(&th->done, 45); + break; + } + } + for (i = 0; i < 100 && __atomic_load_n(&running, __ATOMIC_RELAXED); i++) + sleep_ms(1); +} + +static void execute_one(void); + +#define WAIT_FLAGS __WALL + +static void loop(void) +{ + for (;;) { + int pid = fork(); + if (pid < 0) + exit(1); + if (pid == 0) { + setup_test(); + execute_one(); + exit(0); + } + int status = 0; + uint64_t start = current_time_ms(); + for (;;) { + if (waitpid(-1, &status, WNOHANG | WAIT_FLAGS) == pid) + break; + sleep_ms(1); + if (current_time_ms() - start < 5 * 1000) + continue; + kill_and_wait(pid, &status); + break; + } + } +} + +uint64_t r[1] = {0xffffffffffffffff}; + +void execute_call(int call) +{ + long res; + switch (call) { + case 0: + *(uint32_t*)0x20000040 = 0; + *(uint32_t*)0x20000044 = 0; + *(uint32_t*)0x20000048 = 0; + *(uint32_t*)0x2000004c = 0; + *(uint32_t*)0x20000050 = 0; + *(uint32_t*)0x20000054 = 0; + *(uint32_t*)0x20000058 = 0; + *(uint32_t*)0x2000005c = 0; + *(uint32_t*)0x20000060 = 0; + *(uint32_t*)0x20000064 = 0; + *(uint32_t*)0x20000068 = 0; + *(uint32_t*)0x2000006c = 0; + *(uint32_t*)0x20000070 = 0; + *(uint32_t*)0x20000074 = 0; + *(uint32_t*)0x20000078 = 0; + *(uint32_t*)0x2000007c = 0; + *(uint32_t*)0x20000080 = 0; + *(uint32_t*)0x20000084 = 0; + *(uint64_t*)0x20000088 = 0; + *(uint32_t*)0x20000090 = 0; + *(uint32_t*)0x20000094 = 0; + *(uint32_t*)0x20000098 = 0; + *(uint32_t*)0x2000009c = 0; + *(uint32_t*)0x200000a0 = 0; + *(uint32_t*)0x200000a4 = 0; + *(uint32_t*)0x200000a8 = 0; + *(uint32_t*)0x200000ac = 0; + *(uint64_t*)0x200000b0 = 0; + res = __sys_io_uring_setup(0x64, (struct io_uring_params *) 0x20000040UL); + if (res != -1) + r[0] = res; + break; + case 1: + __sys_io_uring_register((long)r[0], 0, 0, 0); + break; + case 2: + __sys_io_uring_register((long)r[0], 0, 0, 0); + break; + } +} + +static void sig_int(int sig) +{ + exit(0); +} + +int main(int argc, char *argv[]) +{ + if (argc > 1) + return T_EXIT_SKIP; + signal(SIGINT, sig_int); + mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + signal(SIGALRM, sig_int); + alarm(5); + + loop(); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/500f9fbadef8.c b/contrib/libs/liburing/test/500f9fbadef8.c new file mode 100644 index 0000000000..c9e4397820 --- /dev/null +++ b/contrib/libs/liburing/test/500f9fbadef8.c @@ -0,0 +1,90 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: Single depth submit+wait poll hang test + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "helpers.h" +#include "liburing.h" + +#define BLOCKS 4096 + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct iovec iov; + char buf[32]; + off_t offset; + unsigned blocks; + int ret, fd; + + if (argc > 1) + return T_EXIT_SKIP; + + t_posix_memalign(&iov.iov_base, 4096, 4096); + iov.iov_len = 4096; + + ret = io_uring_queue_init(1, &ring, IORING_SETUP_IOPOLL); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return T_EXIT_FAIL; + + } + + sprintf(buf, "./XXXXXX"); + fd = mkostemp(buf, O_WRONLY | O_DIRECT | O_CREAT); + if (fd < 0) { + perror("mkostemp"); + return T_EXIT_FAIL; + } + + offset = 0; + blocks = BLOCKS; + do { + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_writev(sqe, fd, &iov, 1, offset); + ret = io_uring_submit_and_wait(&ring, 1); + if (ret < 0) { + fprintf(stderr, "submit_and_wait: %d\n", ret); + goto err; + } + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion: %d\n", ret); + goto err; + } + if (cqe->res != 4096) { + if (cqe->res == -EOPNOTSUPP) + goto skipped; + goto err; + } + io_uring_cqe_seen(&ring, cqe); + offset += 4096; + } while (--blocks); + + close(fd); + unlink(buf); + return T_EXIT_PASS; +err: + close(fd); + unlink(buf); + return T_EXIT_FAIL; +skipped: + fprintf(stderr, "Polling not supported in current dir, test skipped\n"); + close(fd); + unlink(buf); + return T_EXIT_SKIP; +} diff --git a/contrib/libs/liburing/test/7ad0e4b2f83c.c b/contrib/libs/liburing/test/7ad0e4b2f83c.c new file mode 100644 index 0000000000..067def169a --- /dev/null +++ b/contrib/libs/liburing/test/7ad0e4b2f83c.c @@ -0,0 +1,95 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <stdio.h> +#include <time.h> +#include <sys/time.h> +#include "liburing.h" +#include "helpers.h" + +static unsigned long long mtime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000; + usec /= 1000; + return sec + usec; +} + +static unsigned long long mtime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return mtime_since(tv, &end); +} + +int main(int argc, char *argv[]) +{ + struct __kernel_timespec ts1, ts2; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring ring; + unsigned long msec; + struct timeval tv; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(32, &ring, 0); + if (ret) { + fprintf(stderr, "io_uring_queue_init=%d\n", ret); + return T_EXIT_FAIL; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "io_uring_submit1=%d\n", ret); + return T_EXIT_FAIL; + } + + + ts1.tv_sec = 5, + ts1.tv_nsec = 0; + ret = io_uring_wait_cqe_timeout(&ring, &cqe, &ts1); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe_timeout=%d\n", ret); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(&ring, cqe); + gettimeofday(&tv, NULL); + + ts2.tv_sec = 1; + ts2.tv_nsec = 0; + sqe = io_uring_get_sqe(&ring); + io_uring_prep_timeout(sqe, &ts2, 0, 0); + sqe->user_data = 89; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "io_uring_submit2=%d\n", ret); + return T_EXIT_FAIL; + } + + io_uring_wait_cqe(&ring, &cqe); + io_uring_cqe_seen(&ring, cqe); + msec = mtime_since_now(&tv); + if (msec >= 900 && msec <= 1100) { + io_uring_queue_exit(&ring); + return T_EXIT_PASS; + } + + fprintf(stderr, "%s: Timeout seems wonky (got %lu)\n", __FUNCTION__, + msec); + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/8a9973408177.c b/contrib/libs/liburing/test/8a9973408177.c new file mode 100644 index 0000000000..ab9a033749 --- /dev/null +++ b/contrib/libs/liburing/test/8a9973408177.c @@ -0,0 +1,108 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +static int register_file(struct io_uring *ring) +{ + char buf[32]; + int ret, fd; + + sprintf(buf, "./XXXXXX"); + fd = mkstemp(buf); + if (fd < 0) { + perror("open"); + return 1; + } + + ret = io_uring_register_files(ring, &fd, 1); + if (ret) { + fprintf(stderr, "file register %d\n", ret); + return 1; + } + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "file register %d\n", ret); + return 1; + } + + unlink(buf); + close(fd); + return 0; +} + +static int test_single_fsync(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + char buf[32]; + int fd, ret; + + sprintf(buf, "./XXXXXX"); + fd = mkstemp(buf); + if (fd < 0) { + perror("open"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_fsync(sqe, fd, 0); + + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + unlink(buf); + return 0; +err: + unlink(buf); + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return T_EXIT_FAIL; + } + + ret = register_file(&ring); + if (ret) + return ret; + ret = test_single_fsync(&ring); + if (ret) { + printf("test_single_fsync failed\n"); + return ret; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/917257daa0fe.c b/contrib/libs/liburing/test/917257daa0fe.c new file mode 100644 index 0000000000..b24d140d83 --- /dev/null +++ b/contrib/libs/liburing/test/917257daa0fe.c @@ -0,0 +1,55 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <endian.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/mman.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" +#include "../src/syscall.h" + +int main(int argc, char *argv[]) +{ + if (argc > 1) + return T_EXIT_SKIP; + + mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + + *(uint32_t*)0x20000000 = 0; + *(uint32_t*)0x20000004 = 0; + *(uint32_t*)0x20000008 = 6; + *(uint32_t*)0x2000000c = 0; + *(uint32_t*)0x20000010 = 0x3af; + *(uint32_t*)0x20000014 = 0; + *(uint32_t*)0x20000018 = 0; + *(uint32_t*)0x2000001c = 0; + *(uint32_t*)0x20000020 = 0; + *(uint32_t*)0x20000024 = 0; + *(uint32_t*)0x20000028 = 0; + *(uint32_t*)0x2000002c = 0; + *(uint32_t*)0x20000030 = 0; + *(uint32_t*)0x20000034 = 0; + *(uint32_t*)0x20000038 = 0; + *(uint32_t*)0x2000003c = 0; + *(uint32_t*)0x20000040 = 0; + *(uint32_t*)0x20000044 = 0; + *(uint64_t*)0x20000048 = 0; + *(uint32_t*)0x20000050 = 0; + *(uint32_t*)0x20000054 = 0; + *(uint32_t*)0x20000058 = 0; + *(uint32_t*)0x2000005c = 0; + *(uint32_t*)0x20000060 = 0; + *(uint32_t*)0x20000064 = 0; + *(uint32_t*)0x20000068 = 0; + *(uint32_t*)0x2000006c = 0; + *(uint64_t*)0x20000070 = 0; + __sys_io_uring_setup(0x7a6, (struct io_uring_params *) 0x20000000UL); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/a0908ae19763.c b/contrib/libs/liburing/test/a0908ae19763.c new file mode 100644 index 0000000000..dbe9c5100a --- /dev/null +++ b/contrib/libs/liburing/test/a0908ae19763.c @@ -0,0 +1,60 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <endian.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/mman.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" +#include "../src/syscall.h" + +uint64_t r[1] = {0xffffffffffffffff}; + +int main(int argc, char *argv[]) +{ + if (argc > 1) + return T_EXIT_SKIP; + mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + intptr_t res = 0; + *(uint32_t*)0x20000080 = 0; + *(uint32_t*)0x20000084 = 0; + *(uint32_t*)0x20000088 = 0; + *(uint32_t*)0x2000008c = 0; + *(uint32_t*)0x20000090 = 0; + *(uint32_t*)0x20000094 = 0; + *(uint32_t*)0x20000098 = 0; + *(uint32_t*)0x2000009c = 0; + *(uint32_t*)0x200000a0 = 0; + *(uint32_t*)0x200000a4 = 0; + *(uint32_t*)0x200000a8 = 0; + *(uint32_t*)0x200000ac = 0; + *(uint32_t*)0x200000b0 = 0; + *(uint32_t*)0x200000b4 = 0; + *(uint32_t*)0x200000b8 = 0; + *(uint32_t*)0x200000bc = 0; + *(uint32_t*)0x200000c0 = 0; + *(uint32_t*)0x200000c4 = 0; + *(uint64_t*)0x200000c8 = 0; + *(uint32_t*)0x200000d0 = 0; + *(uint32_t*)0x200000d4 = 0; + *(uint32_t*)0x200000d8 = 0; + *(uint32_t*)0x200000dc = 0; + *(uint32_t*)0x200000e0 = 0; + *(uint32_t*)0x200000e4 = 0; + *(uint32_t*)0x200000e8 = 0; + *(uint32_t*)0x200000ec = 0; + *(uint64_t*)0x200000f0 = 0; + res = __sys_io_uring_setup(0xa4, (struct io_uring_params *) 0x20000080); + if (res != -1) + r[0] = res; + *(uint32_t*)0x20000280 = -1; + __sys_io_uring_register(r[0], 2, (const void *) 0x20000280, 1); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/a4c0b3decb33.c b/contrib/libs/liburing/test/a4c0b3decb33.c new file mode 100644 index 0000000000..376ecb3d87 --- /dev/null +++ b/contrib/libs/liburing/test/a4c0b3decb33.c @@ -0,0 +1,182 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <dirent.h> +#include <endian.h> +#include <errno.h> +#include <fcntl.h> +#include <signal.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/prctl.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/mman.h> +#include <time.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" +#include "../src/syscall.h" + +static void sleep_ms(uint64_t ms) +{ + usleep(ms * 1000); +} + +static uint64_t current_time_ms(void) +{ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts)) + exit(1); + return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000; +} + +static bool write_file(const char* file, const char* what, ...) +{ + char buf[1024]; + va_list args; + va_start(args, what); + vsnprintf(buf, sizeof(buf), what, args); + va_end(args); + buf[sizeof(buf) - 1] = 0; + int len = strlen(buf); + int fd = open(file, O_WRONLY | O_CLOEXEC); + if (fd == -1) + return false; + if (write(fd, buf, len) != len) { + int err = errno; + close(fd); + errno = err; + return false; + } + close(fd); + return true; +} + +static void kill_and_wait(int pid, int* status) +{ + kill(-pid, SIGKILL); + kill(pid, SIGKILL); + int i; + for (i = 0; i < 100; i++) { + if (waitpid(-1, status, WNOHANG | __WALL) == pid) + return; + usleep(1000); + } + DIR* dir = opendir("/sys/fs/fuse/connections"); + if (dir) { + for (;;) { + struct dirent* ent = readdir(dir); + if (!ent) + break; + if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) + continue; + char abort[300]; + snprintf(abort, sizeof(abort), "/sys/fs/fuse/connections/%s/abort", + ent->d_name); + int fd = open(abort, O_WRONLY); + if (fd == -1) { + continue; + } + if (write(fd, abort, 1) < 0) { + } + close(fd); + } + closedir(dir); + } else { + } + while (waitpid(-1, status, __WALL) != pid) { + } +} + +static void setup_test() +{ + prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); + setpgrp(); + write_file("/proc/self/oom_score_adj", "1000"); +} + +static void execute_one(void); + +#define WAIT_FLAGS __WALL + +static void loop(void) +{ + int iter; + for (iter = 0; iter < 5000; iter++) { + int pid = fork(); + if (pid < 0) + exit(1); + if (pid == 0) { + setup_test(); + execute_one(); + exit(0); + } + int status = 0; + uint64_t start = current_time_ms(); + for (;;) { + if (waitpid(-1, &status, WNOHANG | WAIT_FLAGS) == pid) + break; + sleep_ms(1); + if (current_time_ms() - start < 5 * 1000) + continue; + kill_and_wait(pid, &status); + break; + } + } +} + +void execute_one(void) +{ + *(uint32_t*)0x20000080 = 0; + *(uint32_t*)0x20000084 = 0; + *(uint32_t*)0x20000088 = 3; + *(uint32_t*)0x2000008c = 3; + *(uint32_t*)0x20000090 = 0x175; + *(uint32_t*)0x20000094 = 0; + *(uint32_t*)0x20000098 = 0; + *(uint32_t*)0x2000009c = 0; + *(uint32_t*)0x200000a0 = 0; + *(uint32_t*)0x200000a4 = 0; + *(uint32_t*)0x200000a8 = 0; + *(uint32_t*)0x200000ac = 0; + *(uint32_t*)0x200000b0 = 0; + *(uint32_t*)0x200000b4 = 0; + *(uint32_t*)0x200000b8 = 0; + *(uint32_t*)0x200000bc = 0; + *(uint32_t*)0x200000c0 = 0; + *(uint32_t*)0x200000c4 = 0; + *(uint64_t*)0x200000c8 = 0; + *(uint32_t*)0x200000d0 = 0; + *(uint32_t*)0x200000d4 = 0; + *(uint32_t*)0x200000d8 = 0; + *(uint32_t*)0x200000dc = 0; + *(uint32_t*)0x200000e0 = 0; + *(uint32_t*)0x200000e4 = 0; + *(uint32_t*)0x200000e8 = 0; + *(uint32_t*)0x200000ec = 0; + *(uint64_t*)0x200000f0 = 0; + __sys_io_uring_setup(0x983, (struct io_uring_params *) 0x20000080); +} + +static void sig_int(int sig) +{ + exit(0); +} + +int main(int argc, char *argv[]) +{ + if (argc > 1) + return T_EXIT_SKIP; + signal(SIGINT, sig_int); + mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + loop(); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/accept-link.c b/contrib/libs/liburing/test/accept-link.c new file mode 100644 index 0000000000..af47e0f4f0 --- /dev/null +++ b/contrib/libs/liburing/test/accept-link.c @@ -0,0 +1,256 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <assert.h> +#include <pthread.h> +#include <sys/socket.h> +#include <netinet/tcp.h> +#include <netinet/in.h> +#include <poll.h> +#include <arpa/inet.h> + +#include "liburing.h" +#include "helpers.h" + +pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + +static int recv_thread_ready = 0; +static int recv_thread_done = 0; + +static void signal_var(int *var) +{ + pthread_mutex_lock(&mutex); + *var = 1; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&mutex); +} + +static void wait_for_var(int *var) +{ + pthread_mutex_lock(&mutex); + + while (!*var) + pthread_cond_wait(&cond, &mutex); + + pthread_mutex_unlock(&mutex); +} + +struct data { + unsigned expected[2]; + unsigned just_positive[2]; + unsigned long timeout; + unsigned short port; + unsigned int addr; + int stop; +}; + +static void *send_thread(void *arg) +{ + struct data *data = arg; + int ret; + + wait_for_var(&recv_thread_ready); + + if (data->stop) + return NULL; + + int s0 = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + assert(s0 != -1); + + struct sockaddr_in addr; + + addr.sin_family = AF_INET; + addr.sin_port = data->port; + addr.sin_addr.s_addr = data->addr; + + ret = connect(s0, (struct sockaddr*)&addr, sizeof(addr)); + assert(ret != -1); + + wait_for_var(&recv_thread_done); + + close(s0); + return NULL; +} + +void *recv_thread(void *arg) +{ + struct data *data = arg; + struct io_uring ring; + int i, ret; + + ret = io_uring_queue_init(8, &ring, 0); + assert(ret == 0); + + int s0 = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + assert(s0 != -1); + + int32_t val = 1; + ret = setsockopt(s0, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + assert(ret != -1); + ret = setsockopt(s0, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + assert(ret != -1); + + struct sockaddr_in addr; + + addr.sin_family = AF_INET; + data->addr = inet_addr("127.0.0.1"); + addr.sin_addr.s_addr = data->addr; + + i = 0; + do { + data->port = htons(1025 + (rand() % 64510)); + addr.sin_port = data->port; + + if (bind(s0, (struct sockaddr*)&addr, sizeof(addr)) != -1) + break; + } while (++i < 100); + + if (i >= 100) { + printf("Can't find good port, skipped\n"); + data->stop = 1; + signal_var(&recv_thread_ready); + goto out; + } + + ret = listen(s0, 128); + assert(ret != -1); + + signal_var(&recv_thread_ready); + + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(&ring); + assert(sqe != NULL); + + io_uring_prep_accept(sqe, s0, NULL, NULL, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring); + assert(sqe != NULL); + + struct __kernel_timespec ts; + ts.tv_sec = data->timeout / 1000000000; + ts.tv_nsec = data->timeout % 1000000000; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 2; + + ret = io_uring_submit(&ring); + assert(ret == 2); + + for (i = 0; i < 2; i++) { + struct io_uring_cqe *cqe; + int idx; + + if (io_uring_wait_cqe(&ring, &cqe)) { + fprintf(stderr, "wait cqe failed\n"); + goto err; + } + idx = cqe->user_data - 1; + if (cqe->res != data->expected[idx]) { + if (cqe->res > 0 && data->just_positive[idx]) + goto ok; + if (cqe->res == -EBADF) { + fprintf(stdout, "Accept not supported, skipping\n"); + data->stop = 1; + goto out; + } + fprintf(stderr, "cqe %" PRIu64 " got %d, wanted %d\n", + (uint64_t) cqe->user_data, cqe->res, + data->expected[idx]); + goto err; + } +ok: + if (cqe->user_data == 1 && cqe->res > 0) + close(cqe->res); + + io_uring_cqe_seen(&ring, cqe); + } + + signal_var(&recv_thread_done); + +out: + close(s0); + return NULL; +err: + close(s0); + return (void *) 1; +} + +static int test_accept_timeout(int do_connect, unsigned long timeout) +{ + struct io_uring ring; + struct io_uring_params p = {}; + pthread_t t1, t2; + struct data d; + void *tret; + int ret, fast_poll; + + ret = io_uring_queue_init_params(1, &ring, &p); + if (ret) { + fprintf(stderr, "queue_init: %d\n", ret); + return 1; + }; + + fast_poll = (p.features & IORING_FEAT_FAST_POLL) != 0; + io_uring_queue_exit(&ring); + + recv_thread_ready = 0; + recv_thread_done = 0; + + memset(&d, 0, sizeof(d)); + d.timeout = timeout; + if (!do_connect) { + if (fast_poll) { + d.expected[0] = -ECANCELED; + d.expected[1] = -ETIME; + } else { + d.expected[0] = -EINTR; + d.expected[1] = -EALREADY; + } + } else { + d.expected[0] = -1U; + d.just_positive[0] = 1; + d.expected[1] = -ECANCELED; + } + + pthread_create(&t1, NULL, recv_thread, &d); + + if (do_connect) + pthread_create(&t2, NULL, send_thread, &d); + + pthread_join(t1, &tret); + if (tret) + ret++; + + if (do_connect) { + pthread_join(t2, &tret); + if (tret) + ret++; + } + + return ret; +} + +int main(int argc, char *argv[]) +{ + if (argc > 1) + return T_EXIT_SKIP; + if (test_accept_timeout(0, 200000000)) { + fprintf(stderr, "accept timeout 0 failed\n"); + return T_EXIT_FAIL; + } + + if (test_accept_timeout(1, 1000000000)) { + fprintf(stderr, "accept and connect timeout 0 failed\n"); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/accept-reuse.c b/contrib/libs/liburing/test/accept-reuse.c new file mode 100644 index 0000000000..fb15ade840 --- /dev/null +++ b/contrib/libs/liburing/test/accept-reuse.c @@ -0,0 +1,166 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <liburing.h> +#include <netdb.h> +#include <string.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <unistd.h> +#include <stdio.h> +#include <errno.h> +#include "liburing.h" +#include "helpers.h" +#include "../src/syscall.h" + +struct io_uring io_uring; + +int sys_io_uring_enter(const int fd, + const unsigned to_submit, + const unsigned min_complete, + const unsigned flags, sigset_t * const sig) +{ + return __sys_io_uring_enter(fd, to_submit, min_complete, flags, sig); +} + +int submit_sqe(void) +{ + struct io_uring_sq *sq = &io_uring.sq; + const unsigned tail = *sq->ktail; + + sq->array[tail & sq->ring_mask] = 0; + io_uring_smp_store_release(sq->ktail, tail + 1); + + return sys_io_uring_enter(io_uring.ring_fd, 1, 0, 0, NULL); +} + +int main(int argc, char **argv) +{ + struct addrinfo *addr_info_list = NULL; + struct addrinfo *ai, *addr_info = NULL; + struct io_uring_params params; + struct io_uring_sqe *sqe; + struct addrinfo hints; + struct sockaddr sa; + socklen_t sa_size = sizeof(sa); + int ret, listen_fd, connect_fd, val, i; + + if (argc > 1) + return T_EXIT_SKIP; + + memset(¶ms, 0, sizeof(params)); + ret = io_uring_queue_init_params(4, &io_uring, ¶ms); + if (ret) { + fprintf(stderr, "io_uring_init_failed: %d\n", ret); + return T_EXIT_FAIL; + } + if (!(params.features & IORING_FEAT_SUBMIT_STABLE)) { + fprintf(stdout, "FEAT_SUBMIT_STABLE not there, skipping\n"); + return T_EXIT_SKIP; + } + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE | AI_NUMERICSERV; + + ret = getaddrinfo(NULL, "12345", &hints, &addr_info_list); + if (ret < 0) { + perror("getaddrinfo"); + return T_EXIT_FAIL; + } + + for (ai = addr_info_list; ai; ai = ai->ai_next) { + if (ai->ai_family == AF_INET || ai->ai_family == AF_INET6) { + addr_info = ai; + break; + } + } + if (!addr_info) { + fprintf(stderr, "addrinfo not found\n"); + return T_EXIT_FAIL; + } + + sqe = &io_uring.sq.sqes[0]; + listen_fd = -1; + + ret = socket(addr_info->ai_family, SOCK_STREAM, + addr_info->ai_protocol); + if (ret < 0) { + perror("socket"); + return T_EXIT_FAIL; + } + listen_fd = ret; + + val = 1; + setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(int)); + setsockopt(listen_fd, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(int)); + + ret = bind(listen_fd, addr_info->ai_addr, addr_info->ai_addrlen); + if (ret < 0) { + perror("bind"); + return T_EXIT_FAIL; + } + + ret = listen(listen_fd, SOMAXCONN); + if (ret < 0) { + perror("listen"); + return T_EXIT_FAIL; + } + + memset(&sa, 0, sizeof(sa)); + + io_uring_prep_accept(sqe, listen_fd, &sa, &sa_size, 0); + sqe->user_data = 1; + ret = submit_sqe(); + if (ret != 1) { + fprintf(stderr, "submit failed: %d\n", ret); + return T_EXIT_FAIL; + } + + connect_fd = -1; + ret = socket(addr_info->ai_family, SOCK_STREAM, addr_info->ai_protocol); + if (ret < 0) { + perror("socket"); + return T_EXIT_FAIL; + } + connect_fd = ret; + + io_uring_prep_connect(sqe, connect_fd, addr_info->ai_addr, + addr_info->ai_addrlen); + sqe->user_data = 2; + ret = submit_sqe(); + if (ret != 1) { + fprintf(stderr, "submit failed: %d\n", ret); + return T_EXIT_FAIL; + } + + for (i = 0; i < 2; i++) { + struct io_uring_cqe *cqe = NULL; + + ret = io_uring_wait_cqe(&io_uring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe: %d\n", ret); + return T_EXIT_FAIL; + } + + switch (cqe->user_data) { + case 1: + if (cqe->res < 0) { + fprintf(stderr, "accept failed: %d\n", cqe->res); + return T_EXIT_FAIL; + } + break; + case 2: + if (cqe->res) { + fprintf(stderr, "connect failed: %d\n", cqe->res); + return T_EXIT_FAIL; + } + break; + } + io_uring_cq_advance(&io_uring, 1); + } + + freeaddrinfo(addr_info_list); + io_uring_queue_exit(&io_uring); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/accept-test.c b/contrib/libs/liburing/test/accept-test.c new file mode 100644 index 0000000000..0c76bfea40 --- /dev/null +++ b/contrib/libs/liburing/test/accept-test.c @@ -0,0 +1,84 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: Check to see if accept handles addr and addrlen + */ +#include <stdio.h> +#include <errno.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <assert.h> +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + struct sockaddr_un addr; + socklen_t addrlen = sizeof(addr); + int ret, fd; + struct __kernel_timespec ts = { + .tv_sec = 0, + .tv_nsec = 1000000 + }; + + if (argc > 1) + return T_EXIT_SKIP; + + if (io_uring_queue_init(4, &ring, 0) != 0) { + fprintf(stderr, "ring setup failed\n"); + return T_EXIT_FAIL; + } + + fd = socket(AF_UNIX, SOCK_STREAM, 0); + assert(fd != -1); + + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + memcpy(addr.sun_path, "\0sock2", 7); + + ret = bind(fd, (struct sockaddr *)&addr, addrlen); + assert(ret != -1); + ret = listen(fd, 128); + assert(ret != -1); + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return T_EXIT_FAIL; + } + io_uring_prep_accept(sqe, fd, (struct sockaddr*)&addr, &addrlen, 0); + sqe->user_data = 1; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "Got submit %d, expected 1\n", ret); + return T_EXIT_FAIL; + } + + ret = io_uring_wait_cqe_timeout(&ring, &cqe, &ts); + if (!ret) { + if (cqe->res == -EBADF || cqe->res == -EINVAL) { + fprintf(stdout, "Accept not supported, skipping\n"); + goto skip; + } else if (cqe->res < 0) { + fprintf(stderr, "cqe error %d\n", cqe->res); + goto err; + } + } else if (ret != -ETIME) { + fprintf(stderr, "accept() failed to use addr & addrlen parameters!\n"); + return T_EXIT_FAIL; + } + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; + +skip: + io_uring_queue_exit(&ring); + return T_EXIT_SKIP; +err: + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/accept.c b/contrib/libs/liburing/test/accept.c new file mode 100644 index 0000000000..cde1542a20 --- /dev/null +++ b/contrib/libs/liburing/test/accept.c @@ -0,0 +1,900 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Check that IORING_OP_ACCEPT works, and send some data across to verify we + * didn't get a junk fd. + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <assert.h> +#include <limits.h> + +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/un.h> +#include <netinet/tcp.h> +#include <netinet/in.h> +#include <arpa/inet.h> + +#include "helpers.h" +#include "liburing.h" + +#define MAX_FDS 32 +#define NOP_USER_DATA (1LLU << 50) +#define INITIAL_USER_DATA 1000 + +static int no_accept; +static int no_accept_multi; + +struct data { + char buf[128]; + struct iovec iov; +}; + +struct accept_test_args { + int accept_should_error; + bool fixed; + bool nonblock; + bool queue_accept_before_connect; + bool multishot; + int extra_loops; + bool overflow; +}; + +static void close_fds(int fds[], int nr) +{ + int i; + + for (i = 0; i < nr; i++) + close(fds[i]); +} + +static void close_sock_fds(int s_fd[], int c_fd[], int nr, bool fixed) +{ + if (!fixed) + close_fds(s_fd, nr); + close_fds(c_fd, nr); +} + +static void queue_send(struct io_uring *ring, int fd) +{ + struct io_uring_sqe *sqe; + struct data *d; + + d = t_malloc(sizeof(*d)); + d->iov.iov_base = d->buf; + d->iov.iov_len = sizeof(d->buf); + + sqe = io_uring_get_sqe(ring); + io_uring_prep_writev(sqe, fd, &d->iov, 1, 0); + sqe->user_data = 1; +} + +static void queue_recv(struct io_uring *ring, int fd, bool fixed) +{ + struct io_uring_sqe *sqe; + struct data *d; + + d = t_malloc(sizeof(*d)); + d->iov.iov_base = d->buf; + d->iov.iov_len = sizeof(d->buf); + + sqe = io_uring_get_sqe(ring); + io_uring_prep_readv(sqe, fd, &d->iov, 1, 0); + sqe->user_data = 2; + if (fixed) + sqe->flags |= IOSQE_FIXED_FILE; +} + +static void queue_accept_multishot(struct io_uring *ring, int fd, + int idx, bool fixed) +{ + struct io_uring_sqe *sqe = io_uring_get_sqe(ring); + int ret; + + if (fixed) + io_uring_prep_multishot_accept_direct(sqe, fd, + NULL, NULL, + 0); + else + io_uring_prep_multishot_accept(sqe, fd, NULL, NULL, 0); + + io_uring_sqe_set_data64(sqe, idx); + ret = io_uring_submit(ring); + assert(ret != -1); +} + +static void queue_accept_conn(struct io_uring *ring, int fd, + struct accept_test_args args) +{ + struct io_uring_sqe *sqe; + int ret; + int fixed_idx = args.fixed ? 0 : -1; + int count = 1 + args.extra_loops; + + if (args.multishot) { + queue_accept_multishot(ring, fd, INITIAL_USER_DATA, args.fixed); + return; + } + + while (count--) { + sqe = io_uring_get_sqe(ring); + if (fixed_idx < 0) { + io_uring_prep_accept(sqe, fd, NULL, NULL, 0); + } else { + io_uring_prep_accept_direct(sqe, fd, NULL, NULL, + 0, fixed_idx); + } + ret = io_uring_submit(ring); + assert(ret != -1); + } +} + +static int accept_conn(struct io_uring *ring, int fixed_idx, int *multishot, int fd) +{ + struct io_uring_cqe *pcqe; + struct io_uring_cqe cqe; + int ret; + + do { + ret = io_uring_wait_cqe(ring, &pcqe); + assert(!ret); + cqe = *pcqe; + io_uring_cqe_seen(ring, pcqe); + } while (cqe.user_data == NOP_USER_DATA); + + if (*multishot) { + if (!(cqe.flags & IORING_CQE_F_MORE)) { + (*multishot)++; + queue_accept_multishot(ring, fd, *multishot, fixed_idx == 0); + } else { + if (cqe.user_data != *multishot) { + fprintf(stderr, "received multishot after told done!\n"); + return -ECANCELED; + } + } + } + + ret = cqe.res; + + if (fixed_idx >= 0) { + if (ret > 0) { + if (!multishot) { + close(ret); + return -EINVAL; + } + } else if (!ret) { + ret = fixed_idx; + } + } + return ret; +} + +static int start_accept_listen(struct sockaddr_in *addr, int port_off, + int extra_flags) +{ + int fd, ret; + + fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC | extra_flags, + IPPROTO_TCP); + + int32_t val = 1; + ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + assert(ret != -1); + ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + assert(ret != -1); + + struct sockaddr_in laddr; + + if (!addr) + addr = &laddr; + + addr->sin_family = AF_INET; + addr->sin_addr.s_addr = inet_addr("127.0.0.1"); + assert(!t_bind_ephemeral_port(fd, addr)); + ret = listen(fd, 128); + assert(ret != -1); + + return fd; +} + +static int set_client_fd(struct sockaddr_in *addr) +{ + int32_t val; + int fd, ret; + + fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + + val = 1; + ret = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)); + assert(ret != -1); + + int32_t flags = fcntl(fd, F_GETFL, 0); + assert(flags != -1); + + flags |= O_NONBLOCK; + ret = fcntl(fd, F_SETFL, flags); + assert(ret != -1); + + ret = connect(fd, (struct sockaddr *)addr, sizeof(*addr)); + assert(ret == -1); + + flags = fcntl(fd, F_GETFL, 0); + assert(flags != -1); + + flags &= ~O_NONBLOCK; + ret = fcntl(fd, F_SETFL, flags); + assert(ret != -1); + + return fd; +} + +static void cause_overflow(struct io_uring *ring) +{ + int i, ret; + + for (i = 0; i < ring->cq.ring_entries; i++) { + struct io_uring_sqe *sqe = io_uring_get_sqe(ring); + + io_uring_prep_nop(sqe); + io_uring_sqe_set_data64(sqe, NOP_USER_DATA); + ret = io_uring_submit(ring); + assert(ret != -1); + } + +} + +static void clear_overflow(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + + while (!io_uring_peek_cqe(ring, &cqe)) { + if (cqe->user_data != NOP_USER_DATA) + break; + io_uring_cqe_seen(ring, cqe); + } +} + +static int test_loop(struct io_uring *ring, + struct accept_test_args args, + int recv_s0, + struct sockaddr_in *addr) +{ + struct io_uring_cqe *cqe; + uint32_t head, count = 0; + int i, ret, s_fd[MAX_FDS], c_fd[MAX_FDS], done = 0; + bool fixed = args.fixed; + bool multishot = args.multishot; + uint32_t multishot_mask = 0; + int nr_fds = multishot ? MAX_FDS : 1; + int multishot_idx = multishot ? INITIAL_USER_DATA : 0; + + if (args.overflow) + cause_overflow(ring); + + for (i = 0; i < nr_fds; i++) { + c_fd[i] = set_client_fd(addr); + if (args.overflow && i == nr_fds / 2) + clear_overflow(ring); + } + + if (!args.queue_accept_before_connect) + queue_accept_conn(ring, recv_s0, args); + + for (i = 0; i < nr_fds; i++) { + s_fd[i] = accept_conn(ring, fixed ? 0 : -1, &multishot_idx, recv_s0); + if (s_fd[i] == -EINVAL) { + if (args.accept_should_error) + goto out; + fprintf(stdout, + "%s %s Accept not supported, skipping\n", + fixed ? "Fixed" : "", + multishot ? "Multishot" : ""); + if (multishot) + no_accept_multi = 1; + else + no_accept = 1; + goto out; + } else if (s_fd[i] < 0) { + if (args.accept_should_error && + (s_fd[i] == -EBADF || s_fd[i] == -EINVAL)) + goto out; + fprintf(stderr, "%s %s Accept[%d] got %d\n", + fixed ? "Fixed" : "", + multishot ? "Multishot" : "", + i, s_fd[i]); + goto err; + } + + if (multishot && fixed) { + if (s_fd[i] >= MAX_FDS) { + fprintf(stderr, + "Fixed Multishot Accept[%d] got outbound index: %d\n", + i, s_fd[i]); + goto err; + } + /* + * for fixed multishot accept test, the file slots + * allocated are [0, 32), this means we finally end up + * with each bit of a u32 being 1. + */ + multishot_mask |= (1U << s_fd[i]); + } + } + + if (multishot) { + if (fixed && (~multishot_mask != 0U)) { + fprintf(stderr, "Fixed Multishot Accept misses events\n"); + goto err; + } + goto out; + } + + queue_send(ring, c_fd[0]); + queue_recv(ring, s_fd[0], fixed); + + ret = io_uring_submit_and_wait(ring, 2); + assert(ret != -1); + + while (count < 2) { + io_uring_for_each_cqe(ring, head, cqe) { + if (cqe->res < 0) { + fprintf(stderr, "Got cqe res %d, user_data %i\n", + cqe->res, (int)cqe->user_data); + done = 1; + break; + } + assert(cqe->res == 128); + count++; + } + + assert(count <= 2); + io_uring_cq_advance(ring, count); + if (done) + goto err; + } + +out: + close_sock_fds(s_fd, c_fd, nr_fds, fixed); + return 0; +err: + close_sock_fds(s_fd, c_fd, nr_fds, fixed); + return 1; +} + +static int test(struct io_uring *ring, struct accept_test_args args) +{ + struct sockaddr_in addr; + int ret = 0; + int loop; + int32_t recv_s0 = start_accept_listen(&addr, 0, + args.nonblock ? O_NONBLOCK : 0); + if (args.queue_accept_before_connect) + queue_accept_conn(ring, recv_s0, args); + for (loop = 0; loop < 1 + args.extra_loops; loop++) { + ret = test_loop(ring, args, recv_s0, &addr); + if (ret) + break; + } + + close(recv_s0); + return ret; +} + +static void sig_alrm(int sig) +{ + exit(0); +} + +static int test_accept_pending_on_exit(void) +{ + struct io_uring m_io_uring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fd, ret; + + ret = io_uring_queue_init(32, &m_io_uring, 0); + assert(ret >= 0); + + fd = start_accept_listen(NULL, 0, 0); + + sqe = io_uring_get_sqe(&m_io_uring); + io_uring_prep_accept(sqe, fd, NULL, NULL, 0); + ret = io_uring_submit(&m_io_uring); + assert(ret != -1); + + signal(SIGALRM, sig_alrm); + alarm(1); + ret = io_uring_wait_cqe(&m_io_uring, &cqe); + assert(!ret); + io_uring_cqe_seen(&m_io_uring, cqe); + + io_uring_queue_exit(&m_io_uring); + return 0; +} + +struct test_accept_many_args { + unsigned int usecs; + bool nonblock; + bool single_sock; + bool close_fds; +}; + +/* + * Test issue many accepts and see if we handle cancellation on exit + */ +static int test_accept_many(struct test_accept_many_args args) +{ + struct io_uring m_io_uring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + unsigned long cur_lim; + struct rlimit rlim; + int *fds, i, ret; + unsigned int nr = 128; + int nr_socks = args.single_sock ? 1 : nr; + + if (getrlimit(RLIMIT_NPROC, &rlim) < 0) { + perror("getrlimit"); + return 1; + } + + cur_lim = rlim.rlim_cur; + rlim.rlim_cur = nr / 4; + + if (setrlimit(RLIMIT_NPROC, &rlim) < 0) { + perror("setrlimit"); + return 1; + } + + ret = io_uring_queue_init(2 * nr, &m_io_uring, 0); + assert(ret >= 0); + + fds = t_calloc(nr_socks, sizeof(int)); + + for (i = 0; i < nr_socks; i++) + fds[i] = start_accept_listen(NULL, i, + args.nonblock ? O_NONBLOCK : 0); + + for (i = 0; i < nr; i++) { + int sock_idx = args.single_sock ? 0 : i; + sqe = io_uring_get_sqe(&m_io_uring); + io_uring_prep_accept(sqe, fds[sock_idx], NULL, NULL, 0); + sqe->user_data = 1 + i; + ret = io_uring_submit(&m_io_uring); + assert(ret == 1); + } + + if (args.usecs) + usleep(args.usecs); + + if (args.close_fds) + for (i = 0; i < nr_socks; i++) + close(fds[i]); + + for (i = 0; i < nr; i++) { + if (io_uring_peek_cqe(&m_io_uring, &cqe)) + break; + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Expected cqe to be cancelled %d\n", cqe->res); + ret = 1; + goto out; + } + io_uring_cqe_seen(&m_io_uring, cqe); + } + ret = 0; +out: + rlim.rlim_cur = cur_lim; + if (setrlimit(RLIMIT_NPROC, &rlim) < 0) { + perror("setrlimit"); + return 1; + } + + free(fds); + io_uring_queue_exit(&m_io_uring); + return ret; +} + +static int test_accept_cancel(unsigned usecs, unsigned int nr, bool multishot) +{ + struct io_uring m_io_uring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fd, i, ret; + + if (multishot && no_accept_multi) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(32, &m_io_uring, 0); + assert(ret >= 0); + + fd = start_accept_listen(NULL, 0, 0); + + for (i = 1; i <= nr; i++) { + sqe = io_uring_get_sqe(&m_io_uring); + if (!multishot) + io_uring_prep_accept(sqe, fd, NULL, NULL, 0); + else + io_uring_prep_multishot_accept(sqe, fd, NULL, NULL, 0); + sqe->user_data = i; + ret = io_uring_submit(&m_io_uring); + assert(ret == 1); + } + + if (usecs) + usleep(usecs); + + for (i = 1; i <= nr; i++) { + sqe = io_uring_get_sqe(&m_io_uring); + io_uring_prep_cancel64(sqe, i, 0); + sqe->user_data = nr + i; + ret = io_uring_submit(&m_io_uring); + assert(ret == 1); + } + for (i = 0; i < nr * 2; i++) { + ret = io_uring_wait_cqe(&m_io_uring, &cqe); + assert(!ret); + /* + * Two cases here: + * + * 1) We cancel the accept4() before it got started, we should + * get '0' for the cancel request and '-ECANCELED' for the + * accept request. + * 2) We cancel the accept4() after it's already running, we + * should get '-EALREADY' for the cancel request and + * '-EINTR' for the accept request. + */ + if (cqe->user_data == 0) { + fprintf(stderr, "unexpected 0 user data\n"); + goto err; + } else if (cqe->user_data <= nr) { + if (cqe->res != -EINTR && cqe->res != -ECANCELED) { + fprintf(stderr, "Cancelled accept got %d\n", cqe->res); + goto err; + } + } else if (cqe->user_data <= nr * 2) { + if (cqe->res != -EALREADY && cqe->res != 0) { + fprintf(stderr, "Cancel got %d\n", cqe->res); + goto err; + } + } + io_uring_cqe_seen(&m_io_uring, cqe); + } + + io_uring_queue_exit(&m_io_uring); + close(fd); + return 0; +err: + io_uring_queue_exit(&m_io_uring); + close(fd); + return 1; +} + +static int test_accept(int count, bool before) +{ + struct io_uring m_io_uring; + int ret; + struct accept_test_args args = { + .queue_accept_before_connect = before, + .extra_loops = count - 1 + }; + + ret = io_uring_queue_init(32, &m_io_uring, 0); + assert(ret >= 0); + ret = test(&m_io_uring, args); + io_uring_queue_exit(&m_io_uring); + return ret; +} + +static int test_multishot_accept(int count, bool before, bool overflow) +{ + struct io_uring m_io_uring; + int ret; + struct accept_test_args args = { + .queue_accept_before_connect = before, + .multishot = true, + .extra_loops = count - 1, + .overflow = overflow + }; + + if (no_accept_multi) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(MAX_FDS + 10, &m_io_uring, 0); + assert(ret >= 0); + ret = test(&m_io_uring, args); + io_uring_queue_exit(&m_io_uring); + return ret; +} + +static int test_accept_multishot_wrong_arg() +{ + struct io_uring m_io_uring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fd, ret; + + ret = io_uring_queue_init(4, &m_io_uring, 0); + assert(ret >= 0); + + fd = start_accept_listen(NULL, 0, 0); + + sqe = io_uring_get_sqe(&m_io_uring); + io_uring_prep_multishot_accept_direct(sqe, fd, NULL, NULL, 0); + sqe->file_index = 1; + ret = io_uring_submit(&m_io_uring); + assert(ret == 1); + + ret = io_uring_wait_cqe(&m_io_uring, &cqe); + assert(!ret); + if (cqe->res != -EINVAL) { + fprintf(stderr, "file index should be IORING_FILE_INDEX_ALLOC \ + if its accept in multishot direct mode\n"); + goto err; + } + io_uring_cqe_seen(&m_io_uring, cqe); + + io_uring_queue_exit(&m_io_uring); + close(fd); + return 0; +err: + io_uring_queue_exit(&m_io_uring); + close(fd); + return 1; +} + + +static int test_accept_nonblock(bool queue_before_connect, int count) +{ + struct io_uring m_io_uring; + int ret; + struct accept_test_args args = { + .nonblock = true, + .queue_accept_before_connect = queue_before_connect, + .extra_loops = count - 1 + }; + + ret = io_uring_queue_init(32, &m_io_uring, 0); + assert(ret >= 0); + ret = test(&m_io_uring, args); + io_uring_queue_exit(&m_io_uring); + return ret; +} + +static int test_accept_fixed(void) +{ + struct io_uring m_io_uring; + int ret, fd = -1; + struct accept_test_args args = { + .fixed = true + }; + + ret = io_uring_queue_init(32, &m_io_uring, 0); + assert(ret >= 0); + ret = io_uring_register_files(&m_io_uring, &fd, 1); + assert(ret == 0); + ret = test(&m_io_uring, args); + io_uring_queue_exit(&m_io_uring); + return ret; +} + +static int test_multishot_fixed_accept(void) +{ + struct io_uring m_io_uring; + int ret, fd[MAX_FDS]; + struct accept_test_args args = { + .fixed = true, + .multishot = true + }; + + if (no_accept_multi) + return T_EXIT_SKIP; + + memset(fd, -1, sizeof(fd)); + ret = io_uring_queue_init(MAX_FDS + 10, &m_io_uring, 0); + assert(ret >= 0); + ret = io_uring_register_files(&m_io_uring, fd, MAX_FDS); + assert(ret == 0); + ret = test(&m_io_uring, args); + io_uring_queue_exit(&m_io_uring); + return ret; +} + +static int test_accept_sqpoll(void) +{ + struct io_uring m_io_uring; + struct io_uring_params p = { }; + int ret; + struct accept_test_args args = { }; + + p.flags = IORING_SETUP_SQPOLL; + ret = t_create_ring_params(32, &m_io_uring, &p); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret < 0) + return ret; + + args.accept_should_error = 1; + if (p.features & IORING_FEAT_SQPOLL_NONFIXED) + args.accept_should_error = 0; + + ret = test(&m_io_uring, args); + io_uring_queue_exit(&m_io_uring); + return ret; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + ret = test_accept(1, false); + if (ret) { + fprintf(stderr, "test_accept failed\n"); + return ret; + } + if (no_accept) + return T_EXIT_SKIP; + + ret = test_accept(2, false); + if (ret) { + fprintf(stderr, "test_accept(2) failed\n"); + return ret; + } + + ret = test_accept(2, true); + if (ret) { + fprintf(stderr, "test_accept(2, true) failed\n"); + return ret; + } + + ret = test_accept_nonblock(false, 1); + if (ret) { + fprintf(stderr, "test_accept_nonblock failed\n"); + return ret; + } + + ret = test_accept_nonblock(true, 1); + if (ret) { + fprintf(stderr, "test_accept_nonblock(before, 1) failed\n"); + return ret; + } + + ret = test_accept_nonblock(true, 3); + if (ret) { + fprintf(stderr, "test_accept_nonblock(before,3) failed\n"); + return ret; + } + + ret = test_accept_fixed(); + if (ret) { + fprintf(stderr, "test_accept_fixed failed\n"); + return ret; + } + + ret = test_multishot_fixed_accept(); + if (ret) { + fprintf(stderr, "test_multishot_fixed_accept failed\n"); + return ret; + } + + ret = test_accept_multishot_wrong_arg(); + if (ret) { + fprintf(stderr, "test_accept_multishot_wrong_arg failed\n"); + return ret; + } + + ret = test_accept_sqpoll(); + if (ret) { + fprintf(stderr, "test_accept_sqpoll failed\n"); + return ret; + } + + ret = test_accept_cancel(0, 1, false); + if (ret) { + fprintf(stderr, "test_accept_cancel nodelay failed\n"); + return ret; + } + + ret = test_accept_cancel(10000, 1, false); + if (ret) { + fprintf(stderr, "test_accept_cancel delay failed\n"); + return ret; + } + + ret = test_accept_cancel(0, 4, false); + if (ret) { + fprintf(stderr, "test_accept_cancel nodelay failed\n"); + return ret; + } + + ret = test_accept_cancel(10000, 4, false); + if (ret) { + fprintf(stderr, "test_accept_cancel delay failed\n"); + return ret; + } + + ret = test_accept_cancel(0, 1, true); + if (ret) { + fprintf(stderr, "test_accept_cancel multishot nodelay failed\n"); + return ret; + } + + ret = test_accept_cancel(10000, 1, true); + if (ret) { + fprintf(stderr, "test_accept_cancel multishot delay failed\n"); + return ret; + } + + ret = test_accept_cancel(0, 4, true); + if (ret) { + fprintf(stderr, "test_accept_cancel multishot nodelay failed\n"); + return ret; + } + + ret = test_accept_cancel(10000, 4, true); + if (ret) { + fprintf(stderr, "test_accept_cancel multishot delay failed\n"); + return ret; + } + + ret = test_multishot_accept(1, true, true); + if (ret) { + fprintf(stderr, "test_multishot_accept(1, false, true) failed\n"); + return ret; + } + + ret = test_multishot_accept(1, false, false); + if (ret) { + fprintf(stderr, "test_multishot_accept(1, false, false) failed\n"); + return ret; + } + + ret = test_multishot_accept(1, true, false); + if (ret) { + fprintf(stderr, "test_multishot_accept(1, true, false) failed\n"); + return ret; + } + + ret = test_accept_many((struct test_accept_many_args) {}); + if (ret) { + fprintf(stderr, "test_accept_many failed\n"); + return ret; + } + + ret = test_accept_many((struct test_accept_many_args) { + .usecs = 100000 }); + if (ret) { + fprintf(stderr, "test_accept_many(sleep) failed\n"); + return ret; + } + + ret = test_accept_many((struct test_accept_many_args) { + .nonblock = true }); + if (ret) { + fprintf(stderr, "test_accept_many(nonblock) failed\n"); + return ret; + } + + ret = test_accept_many((struct test_accept_many_args) { + .nonblock = true, + .single_sock = true, + .close_fds = true }); + if (ret) { + fprintf(stderr, "test_accept_many(nonblock,close) failed\n"); + return ret; + } + + ret = test_accept_pending_on_exit(); + if (ret) { + fprintf(stderr, "test_accept_pending_on_exit failed\n"); + return ret; + } + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/across-fork.c b/contrib/libs/liburing/test/across-fork.c new file mode 100644 index 0000000000..8d6d9a784f --- /dev/null +++ b/contrib/libs/liburing/test/across-fork.c @@ -0,0 +1,285 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test sharing a ring across a fork + */ +#include <fcntl.h> +#include <pthread.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" + + +struct forktestmem +{ + struct io_uring ring; + pthread_barrier_t barrier; + pthread_barrierattr_t barrierattr; +}; + +static int open_tempfile(const char *dir, const char *fname) +{ + int fd; + char buf[32]; + + snprintf(buf, sizeof(buf), "%s/%s", + dir, fname); + fd = open(buf, O_RDWR | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR); + if (fd < 0) { + perror("open"); + exit(1); + } + + return fd; +} + +static int submit_write(struct io_uring *ring, int fd, const char *str, + int wait) +{ + struct io_uring_sqe *sqe; + struct iovec iovec; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "could not get sqe\n"); + return 1; + } + + iovec.iov_base = (char *) str; + iovec.iov_len = strlen(str); + io_uring_prep_writev(sqe, fd, &iovec, 1, 0); + ret = io_uring_submit_and_wait(ring, wait); + if (ret < 0) { + fprintf(stderr, "submit failed: %s\n", strerror(-ret)); + return 1; + } + + return 0; +} + +static int wait_cqe(struct io_uring *ring, const char *stage) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "%s wait_cqe failed %d\n", stage, ret); + return 1; + } + if (cqe->res < 0) { + fprintf(stderr, "%s cqe failed %d\n", stage, cqe->res); + return 1; + } + + io_uring_cqe_seen(ring, cqe); + return 0; +} + +static int verify_file(const char *tmpdir, const char *fname, const char* expect) +{ + int fd; + char buf[512]; + int err = 0; + + memset(buf, 0, sizeof(buf)); + + fd = open_tempfile(tmpdir, fname); + if (fd < 0) + return 1; + + if (read(fd, buf, sizeof(buf) - 1) < 0) + return 1; + + if (strcmp(buf, expect) != 0) { + fprintf(stderr, "content mismatch for %s\n" + "got:\n%s\n" + "expected:\n%s\n", + fname, buf, expect); + err = 1; + } + + close(fd); + return err; +} + +static void cleanup(const char *tmpdir) +{ + char buf[32]; + + /* don't check errors, called during partial runs */ + + snprintf(buf, sizeof(buf), "%s/%s", tmpdir, "shared"); + unlink(buf); + + snprintf(buf, sizeof(buf), "%s/%s", tmpdir, "parent1"); + unlink(buf); + + snprintf(buf, sizeof(buf), "%s/%s", tmpdir, "parent2"); + unlink(buf); + + snprintf(buf, sizeof(buf), "%s/%s", tmpdir, "child"); + unlink(buf); + + rmdir(tmpdir); +} + +int main(int argc, char *argv[]) +{ + struct forktestmem *shmem; + char tmpdir[] = "forktmpXXXXXX"; + int shared_fd; + int ret; + pid_t p; + + if (argc > 1) + return T_EXIT_SKIP; + + shmem = mmap(0, sizeof(struct forktestmem), PROT_READ|PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, 0, 0); + if (!shmem) { + fprintf(stderr, "mmap failed\n"); + exit(T_EXIT_FAIL); + } + + pthread_barrierattr_init(&shmem->barrierattr); + pthread_barrierattr_setpshared(&shmem->barrierattr, 1); + pthread_barrier_init(&shmem->barrier, &shmem->barrierattr, 2); + + ret = io_uring_queue_init(10, &shmem->ring, 0); + if (ret < 0) { + fprintf(stderr, "queue init failed\n"); + exit(T_EXIT_FAIL); + } + + if (mkdtemp(tmpdir) == NULL) { + fprintf(stderr, "temp directory creation failed\n"); + exit(T_EXIT_FAIL); + } + + shared_fd = open_tempfile(tmpdir, "shared"); + + /* + * First do a write before the fork, to test whether child can + * reap that + */ + if (submit_write(&shmem->ring, shared_fd, "before fork: write shared fd\n", 0)) + goto errcleanup; + + p = fork(); + switch (p) { + case -1: + fprintf(stderr, "fork failed\n"); + goto errcleanup; + + default: { + /* parent */ + int parent_fd1; + int parent_fd2; + int wstatus; + + /* wait till fork is started up */ + pthread_barrier_wait(&shmem->barrier); + + parent_fd1 = open_tempfile(tmpdir, "parent1"); + parent_fd2 = open_tempfile(tmpdir, "parent2"); + + /* do a parent write to the shared fd */ + if (submit_write(&shmem->ring, shared_fd, "parent: write shared fd\n", 0)) + goto errcleanup; + + /* do a parent write to an fd where same numbered fd exists in child */ + if (submit_write(&shmem->ring, parent_fd1, "parent: write parent fd 1\n", 0)) + goto errcleanup; + + /* do a parent write to an fd where no same numbered fd exists in child */ + if (submit_write(&shmem->ring, parent_fd2, "parent: write parent fd 2\n", 0)) + goto errcleanup; + + /* wait to switch read/writ roles with child */ + pthread_barrier_wait(&shmem->barrier); + + /* now wait for child to exit, to ensure we still can read completion */ + waitpid(p, &wstatus, 0); + if (WEXITSTATUS(wstatus) != 0) { + fprintf(stderr, "child failed\n"); + goto errcleanup; + } + + if (wait_cqe(&shmem->ring, "p cqe 1")) + goto errcleanup; + + if (wait_cqe(&shmem->ring, "p cqe 2")) + goto errcleanup; + + /* check that IO can still be submitted after child exited */ + if (submit_write(&shmem->ring, shared_fd, "parent: write shared fd after child exit\n", 0)) + goto errcleanup; + + if (wait_cqe(&shmem->ring, "p cqe 3")) + goto errcleanup; + + break; + } + case 0: { + /* child */ + int child_fd; + + /* wait till fork is started up */ + pthread_barrier_wait(&shmem->barrier); + + child_fd = open_tempfile(tmpdir, "child"); + + if (wait_cqe(&shmem->ring, "c cqe shared")) + exit(1); + + if (wait_cqe(&shmem->ring, "c cqe parent 1")) + exit(1); + + if (wait_cqe(&shmem->ring, "c cqe parent 2")) + exit(1); + + if (wait_cqe(&shmem->ring, "c cqe parent 3")) + exit(1); + + /* wait to switch read/writ roles with parent */ + pthread_barrier_wait(&shmem->barrier); + + if (submit_write(&shmem->ring, child_fd, "child: write child fd\n", 0)) + exit(1); + + /* ensure both writes have finished before child exits */ + if (submit_write(&shmem->ring, shared_fd, "child: write shared fd\n", 2)) + exit(1); + + exit(0); + } + } + + if (verify_file(tmpdir, "shared", + "before fork: write shared fd\n" + "parent: write shared fd\n" + "child: write shared fd\n" + "parent: write shared fd after child exit\n") || + verify_file(tmpdir, "parent1", "parent: write parent fd 1\n") || + verify_file(tmpdir, "parent2", "parent: write parent fd 2\n") || + verify_file(tmpdir, "child", "child: write child fd\n")) + goto errcleanup; + + cleanup(tmpdir); + exit(T_EXIT_PASS); + +errcleanup: + cleanup(tmpdir); + exit(T_EXIT_FAIL); +} diff --git a/contrib/libs/liburing/test/b19062a56726.c b/contrib/libs/liburing/test/b19062a56726.c new file mode 100644 index 0000000000..e065e30f6e --- /dev/null +++ b/contrib/libs/liburing/test/b19062a56726.c @@ -0,0 +1,55 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <endian.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/mman.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" +#include "../src/syscall.h" + +int main(int argc, char *argv[]) +{ + if (argc > 1) + return T_EXIT_SKIP; + + mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + + *(uint32_t*)0x20000200 = 0; + *(uint32_t*)0x20000204 = 0; + *(uint32_t*)0x20000208 = 5; + *(uint32_t*)0x2000020c = 0x400; + *(uint32_t*)0x20000210 = 0; + *(uint32_t*)0x20000214 = 0; + *(uint32_t*)0x20000218 = 0; + *(uint32_t*)0x2000021c = 0; + *(uint32_t*)0x20000220 = 0; + *(uint32_t*)0x20000224 = 0; + *(uint32_t*)0x20000228 = 0; + *(uint32_t*)0x2000022c = 0; + *(uint32_t*)0x20000230 = 0; + *(uint32_t*)0x20000234 = 0; + *(uint32_t*)0x20000238 = 0; + *(uint32_t*)0x2000023c = 0; + *(uint32_t*)0x20000240 = 0; + *(uint32_t*)0x20000244 = 0; + *(uint64_t*)0x20000248 = 0; + *(uint32_t*)0x20000250 = 0; + *(uint32_t*)0x20000254 = 0; + *(uint32_t*)0x20000258 = 0; + *(uint32_t*)0x2000025c = 0; + *(uint32_t*)0x20000260 = 0; + *(uint32_t*)0x20000264 = 0; + *(uint32_t*)0x20000268 = 0; + *(uint32_t*)0x2000026c = 0; + *(uint64_t*)0x20000270 = 0; + __sys_io_uring_setup(0xc9f, (struct io_uring_params *) 0x20000200); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/b5837bd5311d.c b/contrib/libs/liburing/test/b5837bd5311d.c new file mode 100644 index 0000000000..1c332078ea --- /dev/null +++ b/contrib/libs/liburing/test/b5837bd5311d.c @@ -0,0 +1,79 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: Check to see if wait_nr is being honored. + */ +#include <stdio.h> +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret; + struct __kernel_timespec ts = { + .tv_sec = 0, + .tv_nsec = 10000000 + }; + + if (argc > 1) + return T_EXIT_SKIP; + + if (io_uring_queue_init(4, &ring, 0) != 0) { + fprintf(stderr, "ring setup failed\n"); + return T_EXIT_FAIL; + } + + /* + * First, submit the timeout sqe so we can actually finish the test + * if everything is in working order. + */ + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return T_EXIT_FAIL; + } + io_uring_prep_timeout(sqe, &ts, (unsigned)-1, 0); + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "Got submit %d, expected 1\n", ret); + return T_EXIT_FAIL; + } + + /* + * Next, submit a nop and wait for two events. If everything is working + * as it should, we should be waiting for more than a millisecond and we + * should see two cqes. Otherwise, execution continues immediately + * and we see only one cqe. + */ + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return T_EXIT_FAIL; + } + io_uring_prep_nop(sqe); + + ret = io_uring_submit_and_wait(&ring, 2); + if (ret != 1) { + fprintf(stderr, "Got submit %d, expected 1\n", ret); + return T_EXIT_FAIL; + } + + if (io_uring_peek_cqe(&ring, &cqe) != 0) { + fprintf(stderr, "Unable to peek cqe!\n"); + return T_EXIT_FAIL; + } + + io_uring_cqe_seen(&ring, cqe); + + if (io_uring_peek_cqe(&ring, &cqe) != 0) { + fprintf(stderr, "Unable to peek cqe!\n"); + return T_EXIT_FAIL; + } + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/buf-ring.c b/contrib/libs/liburing/test/buf-ring.c new file mode 100644 index 0000000000..569639a347 --- /dev/null +++ b/contrib/libs/liburing/test/buf-ring.c @@ -0,0 +1,421 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various shared buffer ring sanity checks + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +static int no_buf_ring; + +/* test trying to register classic group when ring group exists */ +static int test_mixed_reg2(int bgid) +{ + struct io_uring_buf_reg reg = { }; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + void *ptr, *bufs; + int ret; + + ret = t_create_ring(1, &ring, 0); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret != T_SETUP_OK) + return 1; + + if (posix_memalign(&ptr, 4096, 4096)) + return 1; + + reg.ring_addr = (unsigned long) ptr; + reg.ring_entries = 32; + reg.bgid = bgid; + + ret = io_uring_register_buf_ring(&ring, ®, 0); + if (ret) { + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + /* provide classic buffers, group 1 */ + bufs = malloc(8 * 1024); + sqe = io_uring_get_sqe(&ring); + io_uring_prep_provide_buffers(sqe, bufs, 1024, 8, bgid, 0); + io_uring_submit(&ring); + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe %d\n", ret); + return 1; + } + if (cqe->res != -EEXIST && cqe->res != -EINVAL) { + fprintf(stderr, "cqe res %d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + + io_uring_queue_exit(&ring); + return 0; +} + +/* test trying to register ring group when classic group exists */ +static int test_mixed_reg(int bgid) +{ + struct io_uring_buf_reg reg = { }; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + void *ptr, *bufs; + int ret; + + ret = t_create_ring(1, &ring, 0); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret != T_SETUP_OK) + return 1; + + /* provide classic buffers, group 1 */ + bufs = malloc(8 * 1024); + sqe = io_uring_get_sqe(&ring); + io_uring_prep_provide_buffers(sqe, bufs, 1024, 8, bgid, 0); + io_uring_submit(&ring); + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe %d\n", ret); + return 1; + } + if (cqe->res) { + fprintf(stderr, "cqe res %d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + + if (posix_memalign(&ptr, 4096, 4096)) + return 1; + + reg.ring_addr = (unsigned long) ptr; + reg.ring_entries = 32; + reg.bgid = bgid; + + ret = io_uring_register_buf_ring(&ring, ®, 0); + if (ret != -EEXIST) { + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_double_reg_unreg(int bgid) +{ + struct io_uring_buf_reg reg = { }; + struct io_uring ring; + void *ptr; + int ret; + + ret = t_create_ring(1, &ring, 0); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret != T_SETUP_OK) + return 1; + + if (posix_memalign(&ptr, 4096, 4096)) + return 1; + + reg.ring_addr = (unsigned long) ptr; + reg.ring_entries = 32; + reg.bgid = bgid; + + ret = io_uring_register_buf_ring(&ring, ®, 0); + if (ret) { + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + /* check that 2nd register with same bgid fails */ + reg.ring_addr = (unsigned long) ptr; + reg.ring_entries = 32; + reg.bgid = bgid; + + ret = io_uring_register_buf_ring(&ring, ®, 0); + if (ret != -EEXIST) { + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + ret = io_uring_unregister_buf_ring(&ring, bgid); + if (ret) { + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + ret = io_uring_unregister_buf_ring(&ring, bgid); + if (ret != -EINVAL && ret != -ENOENT) { + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_reg_unreg(int bgid) +{ + struct io_uring_buf_reg reg = { }; + struct io_uring ring; + void *ptr; + int ret; + + ret = t_create_ring(1, &ring, 0); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret != T_SETUP_OK) + return 1; + + if (posix_memalign(&ptr, 4096, 4096)) + return 1; + + reg.ring_addr = (unsigned long) ptr; + reg.ring_entries = 32; + reg.bgid = bgid; + + ret = io_uring_register_buf_ring(&ring, ®, 0); + if (ret) { + if (ret == -EINVAL) { + no_buf_ring = 1; + return 0; + } + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + ret = io_uring_unregister_buf_ring(&ring, bgid); + if (ret) { + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_bad_reg(int bgid) +{ + struct io_uring ring; + int ret; + struct io_uring_buf_reg reg = { }; + + ret = t_create_ring(1, &ring, 0); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret != T_SETUP_OK) + return 1; + + reg.ring_addr = 4096; + reg.ring_entries = 32; + reg.bgid = bgid; + + ret = io_uring_register_buf_ring(&ring, ®, 0); + if (!ret) + fprintf(stderr, "Buffer ring register worked unexpectedly\n"); + + io_uring_queue_exit(&ring); + return !ret; +} + +static int test_one_read(int fd, int bgid, struct io_uring *ring) +{ + int ret; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return -1; + } + + io_uring_prep_read(sqe, fd, NULL, 1, 0); + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = bgid; + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return -1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + return -1; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + + if (ret == -ENOBUFS) + return ret; + + if (ret != 1) { + fprintf(stderr, "read result %d\n", ret); + return -1; + } + + return cqe->flags >> 16; +} + +static int test_running(int bgid, int entries, int loops) +{ + struct io_uring_buf_reg reg = { }; + struct io_uring ring; + void *ptr; + char buffer[8]; + int ret; + int ring_size = (entries * sizeof(struct io_uring_buf) + 4095) & (~4095); + int ring_mask = io_uring_buf_ring_mask(entries); + + int loop, idx; + bool *buffers; + struct io_uring_buf_ring *br; + int read_fd; + + ret = t_create_ring(1, &ring, 0); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret != T_SETUP_OK) + return 1; + + if (posix_memalign(&ptr, 4096, ring_size)) + return 1; + + br = (struct io_uring_buf_ring *)ptr; + io_uring_buf_ring_init(br); + + buffers = malloc(sizeof(bool) * entries); + if (!buffers) + return 1; + + read_fd = open("/dev/zero", O_RDONLY); + if (read_fd < 0) + return 1; + + reg.ring_addr = (unsigned long) ptr; + reg.ring_entries = entries; + reg.bgid = bgid; + + ret = io_uring_register_buf_ring(&ring, ®, 0); + if (ret) { + /* by now should have checked if this is supported or not */ + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + for (loop = 0; loop < loops; loop++) { + memset(buffers, 0, sizeof(bool) * entries); + for (idx = 0; idx < entries; idx++) + io_uring_buf_ring_add(br, buffer, sizeof(buffer), idx, ring_mask, idx); + io_uring_buf_ring_advance(br, entries); + + for (idx = 0; idx < entries; idx++) { + memset(buffer, 1, sizeof(buffer)); + ret = test_one_read(read_fd, bgid, &ring); + if (ret < 0) { + fprintf(stderr, "bad run %d/%d = %d\n", loop, idx, ret); + return ret; + } + if (buffers[ret]) { + fprintf(stderr, "reused buffer %d/%d = %d!\n", loop, idx, ret); + return 1; + } + if (buffer[0] != 0) { + fprintf(stderr, "unexpected read %d %d/%d = %d!\n", + (int)buffer[0], loop, idx, ret); + return 1; + } + if (buffer[1] != 1) { + fprintf(stderr, "unexpected spilled read %d %d/%d = %d!\n", + (int)buffer[1], loop, idx, ret); + return 1; + } + buffers[ret] = true; + } + ret = test_one_read(read_fd, bgid, &ring); + if (ret != -ENOBUFS) { + fprintf(stderr, "expected enobufs run %d = %d\n", loop, ret); + return 1; + } + + } + + ret = io_uring_unregister_buf_ring(&ring, bgid); + if (ret) { + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + close(read_fd); + io_uring_queue_exit(&ring); + free(buffers); + return 0; +} + +int main(int argc, char *argv[]) +{ + int bgids[] = { 1, 127, -1 }; + int entries[] = {1, 32768, 4096, -1 }; + int ret, i; + + if (argc > 1) + return T_EXIT_SKIP; + + for (i = 0; bgids[i] != -1; i++) { + ret = test_reg_unreg(bgids[i]); + if (ret) { + fprintf(stderr, "test_reg_unreg failed\n"); + return T_EXIT_FAIL; + } + if (no_buf_ring) + break; + + ret = test_bad_reg(bgids[i]); + if (ret) { + fprintf(stderr, "test_bad_reg failed\n"); + return T_EXIT_FAIL; + } + + ret = test_double_reg_unreg(bgids[i]); + if (ret) { + fprintf(stderr, "test_double_reg_unreg failed\n"); + return T_EXIT_FAIL; + } + + ret = test_mixed_reg(bgids[i]); + if (ret) { + fprintf(stderr, "test_mixed_reg failed\n"); + return T_EXIT_FAIL; + } + + ret = test_mixed_reg2(bgids[i]); + if (ret) { + fprintf(stderr, "test_mixed_reg2 failed\n"); + return T_EXIT_FAIL; + } + } + + for (i = 0; !no_buf_ring && entries[i] != -1; i++) { + ret = test_running(2, entries[i], 3); + if (ret) { + fprintf(stderr, "test_running(%d) failed\n", entries[i]); + return T_EXIT_FAIL; + } + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/ce593a6c480a.c b/contrib/libs/liburing/test/ce593a6c480a.c new file mode 100644 index 0000000000..7b6fc438ae --- /dev/null +++ b/contrib/libs/liburing/test/ce593a6c480a.c @@ -0,0 +1,140 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test 5.7 regression with task_work not being run while a task is + * waiting on another event in the kernel. + */ +#include <errno.h> +#include <poll.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/eventfd.h> +#include <unistd.h> +#include <pthread.h> +#include "liburing.h" +#include "helpers.h" + +static int use_sqpoll = 0; + +void notify_fd(int fd) +{ + char buf[8] = {0, 0, 0, 0, 0, 0, 1}; + int ret; + + ret = write(fd, &buf, 8); + if (ret < 0) + perror("write"); +} + +void *delay_set_fd_from_thread(void *data) +{ + int fd = (intptr_t) data; + + sleep(1); + notify_fd(fd); + return NULL; +} + +int main(int argc, char *argv[]) +{ + struct io_uring_params p = {}; + struct io_uring ring; + int loop_fd, other_fd; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe = NULL; + int ret, use_fd; + char buf[8] = {0, 0, 0, 0, 0, 0, 1}; + pthread_t tid; + + if (argc > 1) + return T_EXIT_SKIP; + + /* Create an eventfd to be registered with the loop to be + * notified of events being ready + */ + loop_fd = eventfd(0, EFD_CLOEXEC); + if (loop_fd == -1) { + fprintf(stderr, "eventfd errno=%d\n", errno); + return T_EXIT_FAIL; + } + + /* Create an eventfd that can create events */ + use_fd = other_fd = eventfd(0, EFD_CLOEXEC); + if (other_fd == -1) { + fprintf(stderr, "eventfd errno=%d\n", errno); + return T_EXIT_FAIL; + } + + if (use_sqpoll) + p.flags = IORING_SETUP_SQPOLL; + + /* Setup the ring with a registered event fd to be notified on events */ + ret = t_create_ring_params(8, &ring, &p); + if (ret == T_SETUP_SKIP) + return T_EXIT_PASS; + else if (ret < 0) + return ret; + + ret = io_uring_register_eventfd(&ring, loop_fd); + if (ret < 0) { + fprintf(stderr, "register_eventfd=%d\n", ret); + return T_EXIT_FAIL; + } + + if (use_sqpoll) { + ret = io_uring_register_files(&ring, &other_fd, 1); + if (ret < 0) { + fprintf(stderr, "register_files=%d\n", ret); + return T_EXIT_FAIL; + } + use_fd = 0; + } + + /* Submit a poll operation to wait on an event in other_fd */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, use_fd, POLLIN); + sqe->user_data = 1; + if (use_sqpoll) + sqe->flags |= IOSQE_FIXED_FILE; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit=%d\n", ret); + return T_EXIT_FAIL; + } + + /* + * CASE 3: Hangs forever in Linux 5.7.5; Works in Linux 5.6.0 When this + * code is uncommented, we don't se a notification on other_fd until + * _after_ we have started the read on loop_fd. In that case, the read() on + * loop_fd seems to hang forever. + */ + pthread_create(&tid, NULL, delay_set_fd_from_thread, + (void*) (intptr_t) other_fd); + + /* Wait on the event fd for an event to be ready */ + do { + ret = read(loop_fd, buf, 8); + } while (ret < 0 && errno == EINTR); + + if (ret < 0) { + perror("read"); + return T_EXIT_FAIL; + } else if (ret != 8) { + fprintf(stderr, "Odd-sized eventfd read: %d\n", ret); + return T_EXIT_FAIL; + } + + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return ret; + } + if (cqe->res < 0) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + return T_EXIT_FAIL; + } + + io_uring_cqe_seen(&ring, cqe); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/close-opath.c b/contrib/libs/liburing/test/close-opath.c new file mode 100644 index 0000000000..85fe30d9df --- /dev/null +++ b/contrib/libs/liburing/test/close-opath.c @@ -0,0 +1,123 @@ +#include "../config-host.h" +// SPDX-License-Identifier: MIT + +#define _GNU_SOURCE 1 +#define _FILE_OFFSET_BITS 64 + +// Test program for io_uring IORING_OP_CLOSE with O_PATH file. +// Author: Clayton Harris <bugs@claycon.org>, 2020-06-07 + +// linux 5.6.14-300.fc32.x86_64 +// gcc 10.1.1-1.fc32 +// liburing.x86_64 0.5-1.fc32 + +// gcc -O2 -Wall -Wextra -std=c11 -o close_opath close_opath.c -luring +// ./close_opath testfilepath + +#include <errno.h> +#include <fcntl.h> +#include <liburing.h> +#include <sys/stat.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> + +typedef struct +{ + const char *const flnames; + const int oflags; +} oflgs_t; + +static int test_io_uring_close(struct io_uring *ring, int fd) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "io_uring_get_sqe() failed\n"); + return -ENOENT; + } + + io_uring_prep_close(sqe, fd); + + ret = io_uring_submit(ring); + if (ret < 0) { + fprintf(stderr, "io_uring_submit() failed, errno %d: %s\n", + -ret, strerror(-ret)); + return ret; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "io_uring_wait_cqe() failed, errno %d: %s\n", + -ret, strerror(-ret)); + return ret; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + + if (ret < 0 && ret != -EOPNOTSUPP && ret != -EINVAL && ret != -EBADF) { + fprintf(stderr, "io_uring close() failed, errno %d: %s\n", + -ret, strerror(-ret)); + return ret; + } + + return 0; +} + +static int open_file(const char *path, const oflgs_t *oflgs) +{ + int fd; + + fd = openat(AT_FDCWD, path, oflgs->oflags, 0); + if (fd < 0) { + int err = errno; + fprintf(stderr, "openat(%s, %s) failed, errno %d: %s\n", + path, oflgs->flnames, err, strerror(err)); + return -err; + } + + return fd; +} + +int main(int argc, char *argv[]) +{ + const char *fname = "."; + struct io_uring ring; + int ret, i; + static const oflgs_t oflgs[] = { + { "O_RDONLY", O_RDONLY }, + { "O_PATH", O_PATH } + }; + + ret = io_uring_queue_init(2, &ring, 0); + if (ret < 0) { + fprintf(stderr, "io_uring_queue_init() failed, errno %d: %s\n", + -ret, strerror(-ret)); + return 0x02; + } + +#define OFLGS_SIZE (sizeof(oflgs) / sizeof(oflgs[0])) + + ret = 0; + for (i = 0; i < OFLGS_SIZE; i++) { + int fd; + + fd = open_file(fname, &oflgs[i]); + if (fd < 0) { + ret |= 0x02; + break; + } + + /* Should always succeed */ + if (test_io_uring_close(&ring, fd) < 0) + ret |= 0x04 << i; + } +#undef OFLGS_SIZE + + io_uring_queue_exit(&ring); + return ret; +} diff --git a/contrib/libs/liburing/test/connect.c b/contrib/libs/liburing/test/connect.c new file mode 100644 index 0000000000..30f3ac1c3f --- /dev/null +++ b/contrib/libs/liburing/test/connect.c @@ -0,0 +1,400 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Check that IORING_OP_CONNECT works, with and without other side + * being open. + */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <poll.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <arpa/inet.h> + +#include "liburing.h" +#include "helpers.h" + +static int no_connect; +static unsigned short use_port; +static unsigned int use_addr; + +static int create_socket(void) +{ + int fd; + + fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (fd == -1) { + perror("socket()"); + return -1; + } + + return fd; +} + +static int submit_and_wait(struct io_uring *ring, int *res) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_submit_and_wait(ring, 1); + if (ret != 1) { + fprintf(stderr, "io_using_submit: got %d\n", ret); + return 1; + } + + ret = io_uring_peek_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_peek_cqe(): no cqe returned"); + return 1; + } + + *res = cqe->res; + io_uring_cqe_seen(ring, cqe); + return 0; +} + +static int wait_for(struct io_uring *ring, int fd, int mask) +{ + struct io_uring_sqe *sqe; + int ret, res; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "unable to get sqe\n"); + return -1; + } + + io_uring_prep_poll_add(sqe, fd, mask); + sqe->user_data = 2; + + ret = submit_and_wait(ring, &res); + if (ret) + return -1; + + if (res < 0) { + fprintf(stderr, "poll(): failed with %d\n", res); + return -1; + } + + return res; +} + +static int listen_on_socket(int fd) +{ + struct sockaddr_in addr; + int ret; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = use_port; + addr.sin_addr.s_addr = use_addr; + + ret = bind(fd, (struct sockaddr*)&addr, sizeof(addr)); + if (ret == -1) { + perror("bind()"); + return -1; + } + + ret = listen(fd, 128); + if (ret == -1) { + perror("listen()"); + return -1; + } + + return 0; +} + +static int configure_connect(int fd, struct sockaddr_in* addr) +{ + int ret, val = 1; + + ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + if (ret == -1) { + perror("setsockopt()"); + return -1; + } + + ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + if (ret == -1) { + perror("setsockopt()"); + return -1; + } + + memset(addr, 0, sizeof(*addr)); + addr->sin_family = AF_INET; + addr->sin_port = use_port; + ret = inet_aton("127.0.0.1", &addr->sin_addr); + return ret; +} + +static int connect_socket(struct io_uring *ring, int fd, int *code) +{ + struct sockaddr_in addr; + int ret, res; + socklen_t code_len = sizeof(*code); + struct io_uring_sqe *sqe; + + if (configure_connect(fd, &addr) == -1) + return -1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "unable to get sqe\n"); + return -1; + } + + io_uring_prep_connect(sqe, fd, (struct sockaddr*)&addr, sizeof(addr)); + sqe->user_data = 1; + + ret = submit_and_wait(ring, &res); + if (ret) + return -1; + + if (res == -EINPROGRESS) { + ret = wait_for(ring, fd, POLLOUT | POLLHUP | POLLERR); + if (ret == -1) + return -1; + + int ev = (ret & POLLOUT) || (ret & POLLHUP) || (ret & POLLERR); + if (!ev) { + fprintf(stderr, "poll(): returned invalid value %#x\n", ret); + return -1; + } + + ret = getsockopt(fd, SOL_SOCKET, SO_ERROR, code, &code_len); + if (ret == -1) { + perror("getsockopt()"); + return -1; + } + } else + *code = res; + return 0; +} + +static int test_connect_with_no_peer(struct io_uring *ring) +{ + int connect_fd; + int ret, code; + + connect_fd = create_socket(); + if (connect_fd == -1) + return -1; + + ret = connect_socket(ring, connect_fd, &code); + if (ret == -1) + goto err; + + if (code != -ECONNREFUSED) { + if (code == -EINVAL || code == -EBADF || code == -EOPNOTSUPP) { + fprintf(stdout, "No connect support, skipping\n"); + no_connect = 1; + goto out; + } + fprintf(stderr, "connect failed with %d\n", code); + goto err; + } + +out: + close(connect_fd); + return 0; + +err: + close(connect_fd); + return -1; +} + +static int test_connect(struct io_uring *ring) +{ + int accept_fd; + int connect_fd; + int ret, code; + + accept_fd = create_socket(); + if (accept_fd == -1) + return -1; + + ret = listen_on_socket(accept_fd); + if (ret == -1) + goto err1; + + connect_fd = create_socket(); + if (connect_fd == -1) + goto err1; + + ret = connect_socket(ring, connect_fd, &code); + if (ret == -1) + goto err2; + + if (code != 0) { + fprintf(stderr, "connect failed with %d\n", code); + goto err2; + } + + close(connect_fd); + close(accept_fd); + + return 0; + +err2: + close(connect_fd); + +err1: + close(accept_fd); + return -1; +} + +static int test_connect_timeout(struct io_uring *ring) +{ + int connect_fd[2] = {-1, -1}; + int accept_fd = -1; + int ret, code; + struct sockaddr_in addr; + struct io_uring_sqe *sqe; + struct __kernel_timespec ts = {.tv_sec = 0, .tv_nsec = 100000}; + + connect_fd[0] = create_socket(); + if (connect_fd[0] == -1) + return -1; + + connect_fd[1] = create_socket(); + if (connect_fd[1] == -1) + goto err; + + accept_fd = create_socket(); + if (accept_fd == -1) + goto err; + + if (configure_connect(connect_fd[0], &addr) == -1) + goto err; + + if (configure_connect(connect_fd[1], &addr) == -1) + goto err; + + ret = bind(accept_fd, (struct sockaddr*)&addr, sizeof(addr)); + if (ret == -1) { + perror("bind()"); + goto err; + } + + ret = listen(accept_fd, 0); // no backlog in order to block connect_fd[1] + if (ret == -1) { + perror("listen()"); + goto err; + } + + // We first connect with one client socket in order to fill the accept queue. + ret = connect_socket(ring, connect_fd[0], &code); + if (ret == -1 || code != 0) { + fprintf(stderr, "unable to connect\n"); + goto err; + } + + // We do not offload completion events from listening socket on purpose. + // This way we create a state where the second connect request being stalled by OS. + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "unable to get sqe\n"); + goto err; + } + + io_uring_prep_connect(sqe, connect_fd[1], (struct sockaddr*)&addr, sizeof(addr)); + sqe->user_data = 1; + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "unable to get sqe\n"); + goto err; + } + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret != 2) { + fprintf(stderr, "submitted %d\n", ret); + return -1; + } + + for (int i = 0; i < 2; i++) { + int expected; + struct io_uring_cqe *cqe; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return -1; + } + + expected = (cqe->user_data == 1) ? -ECANCELED : -ETIME; + if (expected != cqe->res) { + fprintf(stderr, "cqe %d, res %d, wanted %d\n", + (int)cqe->user_data, cqe->res, expected); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + close(connect_fd[0]); + close(connect_fd[1]); + close(accept_fd); + return 0; + +err: + if (connect_fd[0] != -1) + close(connect_fd[0]); + if (connect_fd[1] != -1) + close(connect_fd[1]); + + if (accept_fd != -1) + close(accept_fd); + return -1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "io_uring_queue_setup() = %d\n", ret); + return T_EXIT_FAIL; + } + + srand(getpid()); + use_port = (rand() % 61440) + 4096; + use_port = htons(use_port); + use_addr = inet_addr("127.0.0.1"); + + ret = test_connect_with_no_peer(&ring); + if (ret == -1) { + fprintf(stderr, "test_connect_with_no_peer(): failed\n"); + return T_EXIT_FAIL; + } + if (no_connect) + return T_EXIT_SKIP; + + ret = test_connect(&ring); + if (ret == -1) { + fprintf(stderr, "test_connect(): failed\n"); + return T_EXIT_FAIL; + } + + ret = test_connect_timeout(&ring); + if (ret == -1) { + fprintf(stderr, "test_connect_timeout(): failed\n"); + return T_EXIT_FAIL; + } + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/cq-full.c b/contrib/libs/liburing/test/cq-full.c new file mode 100644 index 0000000000..a14b22d1ed --- /dev/null +++ b/contrib/libs/liburing/test/cq-full.c @@ -0,0 +1,98 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test CQ ring overflow + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +static int queue_n_nops(struct io_uring *ring, int n) +{ + struct io_uring_sqe *sqe; + int i, ret; + + for (i = 0; i < n; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + } + + ret = io_uring_submit(ring); + if (ret < n) { + printf("Submitted only %d\n", ret); + goto err; + } else if (ret < 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring_cqe *cqe; + struct io_uring_params p; + struct io_uring ring; + int i, ret; + + if (argc > 1) + return T_EXIT_SKIP; + + memset(&p, 0, sizeof(p)); + ret = io_uring_queue_init_params(4, &ring, &p); + if (ret) { + printf("ring setup failed\n"); + return T_EXIT_FAIL; + + } + + if (queue_n_nops(&ring, 4)) + goto err; + if (queue_n_nops(&ring, 4)) + goto err; + if (queue_n_nops(&ring, 4)) + goto err; + + i = 0; + do { + ret = io_uring_peek_cqe(&ring, &cqe); + if (ret < 0) { + if (ret == -EAGAIN) + break; + printf("wait completion %d\n", ret); + goto err; + } + io_uring_cqe_seen(&ring, cqe); + if (!cqe) + break; + i++; + } while (1); + + if (i < 8 || + ((*ring.cq.koverflow != 4) && !(p.features & IORING_FEAT_NODROP))) { + printf("CQ overflow fail: %d completions, %u overflow\n", i, + *ring.cq.koverflow); + goto err; + } + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +err: + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/cq-overflow.c b/contrib/libs/liburing/test/cq-overflow.c new file mode 100644 index 0000000000..f43e19c013 --- /dev/null +++ b/contrib/libs/liburing/test/cq-overflow.c @@ -0,0 +1,525 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various CQ ring overflow tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <assert.h> + +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE (256 * 1024) +#define BS 4096 +#define BUFFERS (FILE_SIZE / BS) + +static struct iovec *vecs; + +#define ENTRIES 8 + +/* + * io_uring has rare cases where CQEs are lost. + * This happens when there is no space in the CQ ring, and also there is no + * GFP_ATOMIC memory available. In reality this probably means that the process + * is about to be killed as many other things might start failing, but we still + * want to test that liburing and the kernel deal with this properly. The fault + * injection framework allows us to test this scenario. Unfortunately this + * requires some system wide changes and so we do not enable this by default. + * The tests in this file should work in both cases (where overflows are queued + * and where they are dropped) on recent kernels. + * + * In order to test dropped CQEs you should enable fault injection in the kernel + * config: + * + * CONFIG_FAULT_INJECTION=y + * CONFIG_FAILSLAB=y + * CONFIG_FAULT_INJECTION_DEBUG_FS=y + * + * and then run the test as follows: + * echo Y > /sys/kernel/debug/failslab/task-filter + * echo 100 > /sys/kernel/debug/failslab/probability + * echo 0 > /sys/kernel/debug/failslab/verbose + * echo 100000 > /sys/kernel/debug/failslab/times + * bash -c "echo 1 > /proc/self/make-it-fail && exec ./cq-overflow.t" + */ + +static int test_io(const char *file, unsigned long usecs, unsigned *drops, int fault) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring_params p; + unsigned reaped, total; + struct io_uring ring; + int nodrop, i, fd, ret; + bool cqe_dropped = false; + + fd = open(file, O_RDONLY | O_DIRECT); + if (fd < 0) { + perror("file open"); + return 1; + } + + memset(&p, 0, sizeof(p)); + ret = io_uring_queue_init_params(ENTRIES, &ring, &p); + if (ret) { + close(fd); + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + nodrop = 0; + if (p.features & IORING_FEAT_NODROP) + nodrop = 1; + + total = 0; + for (i = 0; i < BUFFERS / 2; i++) { + off_t offset; + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + offset = BS * (rand() % BUFFERS); + if (fault && i == ENTRIES + 4) + vecs[i].iov_base = NULL; + io_uring_prep_readv(sqe, fd, &vecs[i], 1, offset); + + ret = io_uring_submit(&ring); + if (nodrop && ret == -EBUSY) { + *drops = 1; + total = i; + break; + } else if (ret != 1) { + fprintf(stderr, "submit got %d, wanted %d\n", ret, 1); + total = i; + break; + } + total++; + } + + if (*drops) + goto reap_it; + + usleep(usecs); + + for (i = total; i < BUFFERS; i++) { + off_t offset; + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + offset = BS * (rand() % BUFFERS); + io_uring_prep_readv(sqe, fd, &vecs[i], 1, offset); + + ret = io_uring_submit(&ring); + if (nodrop && ret == -EBUSY) { + *drops = 1; + break; + } else if (ret != 1) { + fprintf(stderr, "submit got %d, wanted %d\n", ret, 1); + break; + } + total++; + } + +reap_it: + reaped = 0; + do { + if (nodrop && !cqe_dropped) { + /* nodrop should never lose events unless cqe_dropped */ + if (reaped == total) + break; + } else { + if (reaped + *ring.cq.koverflow == total) + break; + } + ret = io_uring_wait_cqe(&ring, &cqe); + if (nodrop && ret == -EBADR) { + cqe_dropped = true; + continue; + } else if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + goto err; + } + if (cqe->res != BS) { + if (!(fault && cqe->res == -EFAULT)) { + fprintf(stderr, "cqe res %d, wanted %d\n", + cqe->res, BS); + goto err; + } + } + io_uring_cqe_seen(&ring, cqe); + reaped++; + } while (1); + + if (!io_uring_peek_cqe(&ring, &cqe)) { + fprintf(stderr, "found unexpected completion\n"); + goto err; + } + + if (!nodrop || cqe_dropped) { + *drops = *ring.cq.koverflow; + } else if (*ring.cq.koverflow) { + fprintf(stderr, "Found %u overflows\n", *ring.cq.koverflow); + goto err; + } + + io_uring_queue_exit(&ring); + close(fd); + return 0; +err: + if (fd != -1) + close(fd); + io_uring_queue_exit(&ring); + return 1; +} + +static int reap_events(struct io_uring *ring, unsigned nr_events, int do_wait) +{ + struct io_uring_cqe *cqe; + int i, ret = 0, seq = 0; + unsigned int start_overflow = *ring->cq.koverflow; + bool dropped = false; + + for (i = 0; i < nr_events; i++) { + if (do_wait) + ret = io_uring_wait_cqe(ring, &cqe); + else + ret = io_uring_peek_cqe(ring, &cqe); + if (do_wait && ret == -EBADR) { + unsigned int this_drop = *ring->cq.koverflow - + start_overflow; + + dropped = true; + start_overflow = *ring->cq.koverflow; + assert(this_drop > 0); + i += (this_drop - 1); + continue; + } else if (ret) { + if (ret != -EAGAIN) + fprintf(stderr, "cqe peek failed: %d\n", ret); + break; + } + if (!dropped && cqe->user_data != seq) { + fprintf(stderr, "cqe sequence out-of-order\n"); + fprintf(stderr, "got %d, wanted %d\n", (int) cqe->user_data, + seq); + return -EINVAL; + } + seq++; + io_uring_cqe_seen(ring, cqe); + } + + return i ? i : ret; +} + +/* + * Submit some NOPs and watch if the overflow is correct + */ +static int test_overflow(void) +{ + struct io_uring ring; + struct io_uring_params p; + struct io_uring_sqe *sqe; + unsigned pending; + int ret, i, j; + + memset(&p, 0, sizeof(p)); + ret = io_uring_queue_init_params(4, &ring, &p); + if (ret) { + fprintf(stderr, "io_uring_queue_init failed %d\n", ret); + return 1; + } + + /* submit 4x4 SQEs, should overflow the ring by 8 */ + pending = 0; + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + sqe->user_data = (i * 4) + j; + } + + ret = io_uring_submit(&ring); + if (ret == 4) { + pending += 4; + continue; + } + if (p.features & IORING_FEAT_NODROP) { + if (ret == -EBUSY) + break; + } + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + /* we should now have 8 completions ready */ + ret = reap_events(&ring, pending, 0); + if (ret < 0) + goto err; + + if (!(p.features & IORING_FEAT_NODROP)) { + if (*ring.cq.koverflow != 8) { + fprintf(stderr, "cq ring overflow %d, expected 8\n", + *ring.cq.koverflow); + goto err; + } + } + io_uring_queue_exit(&ring); + return 0; +err: + io_uring_queue_exit(&ring); + return 1; +} + + +static void submit_one_nop(struct io_uring *ring, int ud) +{ + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + assert(sqe); + io_uring_prep_nop(sqe); + sqe->user_data = ud; + ret = io_uring_submit(ring); + assert(ret == 1); +} + +/* + * Create an overflow condition and ensure that SQEs are still processed + */ +static int test_overflow_handling(bool batch, int cqe_multiple, bool poll, + bool defer) +{ + struct io_uring ring; + struct io_uring_params p; + int ret, i, j, ud, cqe_count; + unsigned int count; + int const N = 8; + int const LOOPS = 128; + int const QUEUE_LENGTH = 1024; + int completions[N]; + int queue[QUEUE_LENGTH]; + int queued = 0; + int outstanding = 0; + bool cqe_dropped = false; + + memset(&completions, 0, sizeof(int) * N); + memset(&p, 0, sizeof(p)); + p.cq_entries = 2 * cqe_multiple; + p.flags |= IORING_SETUP_CQSIZE; + + if (poll) + p.flags |= IORING_SETUP_IOPOLL; + + if (defer) + p.flags |= IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN; + + ret = io_uring_queue_init_params(2, &ring, &p); + if (ret) { + fprintf(stderr, "io_uring_queue_init failed %d\n", ret); + return 1; + } + + assert(p.cq_entries < N); + /* submit N SQEs, some should overflow */ + for (i = 0; i < N; i++) { + submit_one_nop(&ring, i); + outstanding++; + } + + for (i = 0; i < LOOPS; i++) { + struct io_uring_cqe *cqes[N]; + + if (io_uring_cq_has_overflow(&ring)) { + /* + * Flush any overflowed CQEs and process those. Actively + * flush these to make sure CQEs arrive in vague order + * of being sent. + */ + ret = io_uring_get_events(&ring); + if (ret != 0) { + fprintf(stderr, + "io_uring_get_events returned %d\n", + ret); + goto err; + } + } else if (!cqe_dropped) { + for (j = 0; j < queued; j++) { + submit_one_nop(&ring, queue[j]); + outstanding++; + } + queued = 0; + } + + /* We have lost some random cqes, stop if no remaining. */ + if (cqe_dropped && outstanding == *ring.cq.koverflow) + break; + + ret = io_uring_wait_cqe(&ring, &cqes[0]); + if (ret == -EBADR) { + cqe_dropped = true; + fprintf(stderr, "CQE dropped\n"); + continue; + } else if (ret != 0) { + fprintf(stderr, "io_uring_wait_cqes failed %d\n", ret); + goto err; + } + cqe_count = 1; + if (batch) { + ret = io_uring_peek_batch_cqe(&ring, &cqes[0], 2); + if (ret < 0) { + fprintf(stderr, + "io_uring_peek_batch_cqe failed %d\n", + ret); + goto err; + } + cqe_count = ret; + } + for (j = 0; j < cqe_count; j++) { + assert(cqes[j]->user_data < N); + ud = cqes[j]->user_data; + completions[ud]++; + assert(queued < QUEUE_LENGTH); + queue[queued++] = (int)ud; + } + io_uring_cq_advance(&ring, cqe_count); + outstanding -= cqe_count; + } + + /* See if there were any drops by flushing the CQ ring *and* overflow */ + do { + struct io_uring_cqe *cqe; + + ret = io_uring_get_events(&ring); + if (ret < 0) { + if (ret == -EBADR) { + fprintf(stderr, "CQE dropped\n"); + cqe_dropped = true; + break; + } + goto err; + } + if (outstanding && !io_uring_cq_ready(&ring)) + ret = io_uring_wait_cqe_timeout(&ring, &cqe, NULL); + + if (ret && ret != -ETIME) { + if (ret == -EBADR) { + fprintf(stderr, "CQE dropped\n"); + cqe_dropped = true; + break; + } + fprintf(stderr, "wait_cqe_timeout = %d\n", ret); + goto err; + } + count = io_uring_cq_ready(&ring); + io_uring_cq_advance(&ring, count); + outstanding -= count; + } while (count); + + io_uring_queue_exit(&ring); + + /* Make sure that completions come back in the same order they were + * sent. If they come back unfairly then this will concentrate on a + * couple of indices. + */ + for (i = 1; !cqe_dropped && i < N; i++) { + if (abs(completions[i] - completions[i - 1]) > 1) { + fprintf(stderr, "bad completion size %d %d\n", + completions[i], completions[i - 1]); + goto err; + } + } + return 0; +err: + io_uring_queue_exit(&ring); + return 1; +} + +int main(int argc, char *argv[]) +{ + const char *fname = ".cq-overflow"; + unsigned iters, drops; + unsigned long usecs; + int ret; + int i; + bool can_defer; + + if (argc > 1) + return T_EXIT_SKIP; + + can_defer = t_probe_defer_taskrun(); + for (i = 0; i < 16; i++) { + bool batch = i & 1; + int mult = (i & 2) ? 1 : 2; + bool poll = i & 4; + bool defer = i & 8; + + if (defer && !can_defer) + continue; + + ret = test_overflow_handling(batch, mult, poll, defer); + if (ret) { + fprintf(stderr, "test_overflow_handling(" + "batch=%d, mult=%d, poll=%d, defer=%d) failed\n", + batch, mult, poll, defer); + goto err; + } + } + + ret = test_overflow(); + if (ret) { + fprintf(stderr, "test_overflow failed\n"); + return ret; + } + + t_create_file(fname, FILE_SIZE); + + vecs = t_create_buffers(BUFFERS, BS); + + iters = 0; + usecs = 1000; + do { + drops = 0; + + if (test_io(fname, usecs, &drops, 0)) { + fprintf(stderr, "test_io nofault failed\n"); + goto err; + } + if (drops) + break; + usecs = (usecs * 12) / 10; + iters++; + } while (iters < 40); + + if (test_io(fname, usecs, &drops, 0)) { + fprintf(stderr, "test_io nofault failed\n"); + goto err; + } + + if (test_io(fname, usecs, &drops, 1)) { + fprintf(stderr, "test_io fault failed\n"); + goto err; + } + + unlink(fname); + return T_EXIT_PASS; +err: + unlink(fname); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/cq-peek-batch.c b/contrib/libs/liburing/test/cq-peek-batch.c new file mode 100644 index 0000000000..1dc32a417c --- /dev/null +++ b/contrib/libs/liburing/test/cq-peek-batch.c @@ -0,0 +1,104 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test CQ peek-batch + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +static int queue_n_nops(struct io_uring *ring, int n, int offset) +{ + struct io_uring_sqe *sqe; + int i, ret; + + for (i = 0; i < n; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + sqe->user_data = i + offset; + } + + ret = io_uring_submit(ring); + if (ret < n) { + printf("Submitted only %d\n", ret); + goto err; + } else if (ret < 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + return 0; +err: + return 1; +} + +#define CHECK_BATCH(ring, got, cqes, count, expected) do {\ + got = io_uring_peek_batch_cqe((ring), cqes, count);\ + if (got != expected) {\ + printf("Got %d CQs, expected %d\n", got, expected);\ + goto err;\ + }\ +} while(0) + +int main(int argc, char *argv[]) +{ + struct io_uring_cqe *cqes[8]; + struct io_uring ring; + int ret, i; + unsigned got; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(4, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return T_EXIT_FAIL; + + } + + CHECK_BATCH(&ring, got, cqes, 4, 0); + if (queue_n_nops(&ring, 4, 0)) + goto err; + + CHECK_BATCH(&ring, got, cqes, 4, 4); + for (i=0;i<4;i++) { + if (i != cqes[i]->user_data) { + printf("Got user_data %" PRIu64 ", expected %d\n", + (uint64_t) cqes[i]->user_data, i); + goto err; + } + } + + if (queue_n_nops(&ring, 4, 4)) + goto err; + + io_uring_cq_advance(&ring, 4); + CHECK_BATCH(&ring, got, cqes, 4, 4); + for (i=0;i<4;i++) { + if (i + 4 != cqes[i]->user_data) { + printf("Got user_data %" PRIu64 ", expected %d\n", + (uint64_t) cqes[i]->user_data, i + 4); + goto err; + } + } + + io_uring_cq_advance(&ring, 8); + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +err: + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/cq-ready.c b/contrib/libs/liburing/test/cq-ready.c new file mode 100644 index 0000000000..bcb15f0f33 --- /dev/null +++ b/contrib/libs/liburing/test/cq-ready.c @@ -0,0 +1,96 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test CQ ready + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +static int queue_n_nops(struct io_uring *ring, int n) +{ + struct io_uring_sqe *sqe; + int i, ret; + + for (i = 0; i < n; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + } + + ret = io_uring_submit(ring); + if (ret < n) { + printf("Submitted only %d\n", ret); + goto err; + } else if (ret < 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + return 0; +err: + return 1; +} + +#define CHECK_READY(ring, expected) do {\ + ready = io_uring_cq_ready((ring));\ + if (ready != expected) {\ + printf("Got %d CQs ready, expected %d\n", ready, expected);\ + goto err;\ + }\ +} while(0) + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + unsigned ready; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(4, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return T_EXIT_FAIL; + + } + + CHECK_READY(&ring, 0); + if (queue_n_nops(&ring, 4)) + goto err; + + CHECK_READY(&ring, 4); + io_uring_cq_advance(&ring, 4); + CHECK_READY(&ring, 0); + if (queue_n_nops(&ring, 4)) + goto err; + + CHECK_READY(&ring, 4); + + io_uring_cq_advance(&ring, 1); + CHECK_READY(&ring, 3); + + io_uring_cq_advance(&ring, 2); + CHECK_READY(&ring, 1); + + io_uring_cq_advance(&ring, 1); + CHECK_READY(&ring, 0); + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +err: + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/cq-size.c b/contrib/libs/liburing/test/cq-size.c new file mode 100644 index 0000000000..08aa587830 --- /dev/null +++ b/contrib/libs/liburing/test/cq-size.c @@ -0,0 +1,66 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test CQ ring sizing + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_params p; + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + memset(&p, 0, sizeof(p)); + p.flags = IORING_SETUP_CQSIZE; + p.cq_entries = 64; + + ret = io_uring_queue_init_params(4, &ring, &p); + if (ret) { + if (ret == -EINVAL) { + printf("Skipped, not supported on this kernel\n"); + goto done; + } + printf("ring setup failed\n"); + return T_EXIT_FAIL; + } + + if (p.cq_entries < 64) { + printf("cq entries invalid (%d)\n", p.cq_entries); + goto err; + } + io_uring_queue_exit(&ring); + + memset(&p, 0, sizeof(p)); + p.flags = IORING_SETUP_CQSIZE; + p.cq_entries = 0; + + ret = io_uring_queue_init_params(4, &ring, &p); + if (ret >= 0) { + printf("zero sized cq ring succeeded\n"); + io_uring_queue_exit(&ring); + goto err; + } + + if (ret != -EINVAL) { + printf("io_uring_queue_init_params failed, but not with -EINVAL" + ", returned error %d (%s)\n", ret, strerror(-ret)); + goto err; + } + +done: + return T_EXIT_PASS; +err: + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/d4ae271dfaae.c b/contrib/libs/liburing/test/d4ae271dfaae.c new file mode 100644 index 0000000000..1a7886ed6b --- /dev/null +++ b/contrib/libs/liburing/test/d4ae271dfaae.c @@ -0,0 +1,97 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test case for SQPOLL missing a 'ret' clear in case of busy. + * + * Heavily based on a test case from + * Xiaoguang Wang <xiaoguang.wang@linux.alibaba.com> + */ +#include <stdio.h> +#include <stdlib.h> +#include <fcntl.h> +#include <unistd.h> +#include <string.h> + +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE (128 * 1024) + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int i, fd, ret; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct iovec *iovecs; + struct io_uring_params p; + char *fname; + void *buf; + + memset(&p, 0, sizeof(p)); + p.flags = IORING_SETUP_SQPOLL; + ret = t_create_ring_params(4, &ring, &p); + if (ret == T_SETUP_SKIP) + return T_EXIT_SKIP; + else if (ret < 0) + return T_EXIT_FAIL; + + if (argc > 1) { + fname = argv[1]; + } else { + fname = ".sqpoll.tmp"; + t_create_file(fname, FILE_SIZE); + } + + fd = open(fname, O_RDONLY | O_DIRECT); + if (fname != argv[1]) + unlink(fname); + if (fd < 0) { + perror("open"); + goto out; + } + + iovecs = t_calloc(10, sizeof(struct iovec)); + for (i = 0; i < 10; i++) { + t_posix_memalign(&buf, 4096, 4096); + iovecs[i].iov_base = buf; + iovecs[i].iov_len = 4096; + } + + ret = io_uring_register_files(&ring, &fd, 1); + if (ret < 0) { + fprintf(stderr, "register files %d\n", ret); + goto out; + } + + for (i = 0; i < 10; i++) { + sqe = io_uring_get_sqe(&ring); + if (!sqe) + break; + + io_uring_prep_readv(sqe, 0, &iovecs[i], 1, 0); + sqe->flags |= IOSQE_FIXED_FILE; + + ret = io_uring_submit(&ring); + usleep(1000); + } + + for (i = 0; i < 10; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + break; + } + if (cqe->res != 4096) { + fprintf(stderr, "ret=%d, wanted 4096\n", cqe->res); + ret = 1; + break; + } + io_uring_cqe_seen(&ring, cqe); + } + + close(fd); +out: + io_uring_queue_exit(&ring); + return ret; +} diff --git a/contrib/libs/liburing/test/d77a67ed5f27.c b/contrib/libs/liburing/test/d77a67ed5f27.c new file mode 100644 index 0000000000..6f35168161 --- /dev/null +++ b/contrib/libs/liburing/test/d77a67ed5f27.c @@ -0,0 +1,66 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <stdio.h> +#include <unistd.h> +#include <string.h> +#include <signal.h> +#include <stdlib.h> +#include "liburing.h" +#include "helpers.h" + +static void sig_alrm(int sig) +{ + fprintf(stderr, "Timed out!\n"); + exit(1); +} + +int main(int argc, char *argv[]) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring_params p; + struct io_uring ring; + int ret, data; + + if (argc > 1) + return T_EXIT_SKIP; + + signal(SIGALRM, sig_alrm); + + memset(&p, 0, sizeof(p)); + p.sq_thread_idle = 100; + p.flags = IORING_SETUP_SQPOLL; + ret = t_create_ring_params(4, &ring, &p); + if (ret == T_SETUP_SKIP) + return T_EXIT_SKIP; + else if (ret < 0) + return T_EXIT_FAIL; + + /* make sure sq thread is sleeping at this point */ + usleep(150000); + alarm(1); + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + return T_EXIT_FAIL; + } + + io_uring_prep_nop(sqe); + io_uring_sqe_set_data(sqe, (void *) (unsigned long) 42); + io_uring_submit_and_wait(&ring, 1); + + ret = io_uring_peek_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "cqe get failed\n"); + return 1; + } + + data = (unsigned long) io_uring_cqe_get_data(cqe); + if (data != 42) { + fprintf(stderr, "invalid data: %d\n", data); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/defer-taskrun.c b/contrib/libs/liburing/test/defer-taskrun.c new file mode 100644 index 0000000000..b07ac2f778 --- /dev/null +++ b/contrib/libs/liburing/test/defer-taskrun.c @@ -0,0 +1,337 @@ +#include "../config-host.h" +// SPDX-License-Identifier: MIT +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <error.h> +#include <sys/eventfd.h> +#include <signal.h> +#include <poll.h> +#include <assert.h> +#include <pthread.h> +#include <sys/types.h> +#include <sys/wait.h> + +#include "liburing.h" +#include "test.h" +#include "helpers.h" + +#define EXEC_FILENAME ".defer-taskrun" +#define EXEC_FILESIZE (1U<<20) + +static bool can_read_t(int fd, int time) +{ + int ret; + struct pollfd p = { + .fd = fd, + .events = POLLIN, + }; + + ret = poll(&p, 1, time); + + return ret == 1; +} + +static bool can_read(int fd) +{ + return can_read_t(fd, 0); +} + +static void eventfd_clear(int fd) +{ + uint64_t val; + int ret; + + assert(can_read(fd)); + ret = read(fd, &val, 8); + assert(ret == 8); +} + +static void eventfd_trigger(int fd) +{ + uint64_t val = 1; + int ret; + + ret = write(fd, &val, sizeof(val)); + assert(ret == sizeof(val)); +} + +#define CHECK(x) if (!(x)) { \ + fprintf(stderr, "%s:%d %s failed\n", __FILE__, __LINE__, #x); \ + return -1; } + +static int test_eventfd(void) +{ + struct io_uring ring; + int ret; + int fda, fdb; + struct io_uring_cqe *cqe; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN); + if (ret) + return ret; + + fda = eventfd(0, EFD_NONBLOCK); + fdb = eventfd(0, EFD_NONBLOCK); + + CHECK(fda >= 0 && fdb >= 0); + + ret = io_uring_register_eventfd(&ring, fda); + if (ret) + return ret; + + CHECK(!can_read(fda)); + CHECK(!can_read(fdb)); + + io_uring_prep_poll_add(io_uring_get_sqe(&ring), fdb, POLLIN); + io_uring_submit(&ring); + CHECK(!can_read(fda)); /* poll should not have completed */ + + io_uring_prep_nop(io_uring_get_sqe(&ring)); + io_uring_submit(&ring); + CHECK(can_read(fda)); /* nop should have */ + + CHECK(io_uring_peek_cqe(&ring, &cqe) == 0); + CHECK(cqe->res == 0); + io_uring_cqe_seen(&ring, cqe); + eventfd_clear(fda); + + eventfd_trigger(fdb); + /* can take time due to rcu_call */ + CHECK(can_read_t(fda, 1000)); + + /* should not have processed the cqe yet */ + CHECK(io_uring_cq_ready(&ring) == 0); + + io_uring_get_events(&ring); + CHECK(io_uring_cq_ready(&ring) == 1); + + + io_uring_queue_exit(&ring); + return 0; +} + +struct thread_data { + struct io_uring ring; + int efd; + char buff[8]; +}; + +void *thread(void *t) +{ + struct thread_data *td = t; + + io_uring_enable_rings(&td->ring); + io_uring_prep_read(io_uring_get_sqe(&td->ring), td->efd, td->buff, sizeof(td->buff), 0); + io_uring_submit(&td->ring); + + return NULL; +} + +static int test_thread_shutdown(void) +{ + pthread_t t1; + int ret; + struct thread_data td; + struct io_uring_cqe *cqe; + uint64_t val = 1; + + ret = io_uring_queue_init(8, &td.ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN | + IORING_SETUP_R_DISABLED); + if (ret) + return ret; + + CHECK(io_uring_get_events(&td.ring) == -EBADFD); + + td.efd = eventfd(0, 0); + CHECK(td.efd >= 0); + + CHECK(pthread_create(&t1, NULL, thread, &td) == 0); + CHECK(pthread_join(t1, NULL) == 0); + + CHECK(io_uring_get_events(&td.ring) == -EEXIST); + + CHECK(write(td.efd, &val, sizeof(val)) == sizeof(val)); + CHECK(io_uring_wait_cqe(&td.ring, &cqe) == -EEXIST); + + close(td.efd); + io_uring_queue_exit(&td.ring); + return 0; +} + +static int test_exec(const char *filename) +{ + int ret; + int fd; + struct io_uring ring; + pid_t fork_pid; + static char * const new_argv[] = {"1", "2", "3", NULL}; + static char * const new_env[] = {NULL}; + char *buff; + + fork_pid = fork(); + CHECK(fork_pid >= 0); + if (fork_pid > 0) { + int wstatus; + + CHECK(waitpid(fork_pid, &wstatus, 0) != (pid_t)-1); + if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus) != T_EXIT_SKIP) { + fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus)); + return -1; + } + return 0; + } + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN); + if (ret) + return ret; + + if (filename) { + fd = open(filename, O_RDONLY | O_DIRECT); + } else { + t_create_file(EXEC_FILENAME, EXEC_FILESIZE); + fd = open(EXEC_FILENAME, O_RDONLY | O_DIRECT); + unlink(EXEC_FILENAME); + } + buff = (char*)malloc(EXEC_FILESIZE); + CHECK(posix_memalign((void **)&buff, 4096, EXEC_FILESIZE) == 0); + CHECK(buff); + + CHECK(fd >= 0); + io_uring_prep_read(io_uring_get_sqe(&ring), fd, buff, EXEC_FILESIZE, 0); + io_uring_submit(&ring); + ret = execve("/proc/self/exe", new_argv, new_env); + /* if we get here it failed anyway */ + fprintf(stderr, "execve failed %d\n", ret); + return -1; +} + +static int test_flag(void) +{ + struct io_uring ring; + int ret; + int fd; + struct io_uring_cqe *cqe; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN | + IORING_SETUP_TASKRUN_FLAG); + CHECK(!ret); + + fd = eventfd(0, EFD_NONBLOCK); + CHECK(fd >= 0); + + io_uring_prep_poll_add(io_uring_get_sqe(&ring), fd, POLLIN); + io_uring_submit(&ring); + CHECK(!can_read(fd)); /* poll should not have completed */ + + eventfd_trigger(fd); + CHECK(can_read(fd)); + + /* should not have processed the poll cqe yet */ + CHECK(io_uring_cq_ready(&ring) == 0); + + /* flag should be set */ + CHECK(IO_URING_READ_ONCE(*ring.sq.kflags) & IORING_SQ_TASKRUN); + + /* Specifically peek, knowing we have only no cqe + * but because the flag is set, liburing should try and get more + */ + ret = io_uring_peek_cqe(&ring, &cqe); + + CHECK(ret == 0 && cqe); + CHECK(!(IO_URING_READ_ONCE(*ring.sq.kflags) & IORING_SQ_TASKRUN)); + + close(fd); + io_uring_queue_exit(&ring); + return 0; +} + +static int test_ring_shutdown(void) +{ + struct io_uring ring; + int ret; + int fd[2]; + char buff = '\0'; + char send = 'X'; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN | + IORING_SETUP_TASKRUN_FLAG); + CHECK(!ret); + + ret = t_create_socket_pair(fd, true); + CHECK(!ret); + + io_uring_prep_recv(io_uring_get_sqe(&ring), fd[0], &buff, 1, 0); + io_uring_submit(&ring); + + ret = write(fd[1], &send, 1); + CHECK(ret == 1); + + /* should not have processed the poll cqe yet */ + CHECK(io_uring_cq_ready(&ring) == 0); + io_uring_queue_exit(&ring); + + /* task work should have been processed by now */ + CHECK(buff = 'X'); + + return 0; +} + +int main(int argc, char *argv[]) +{ + int ret; + const char *filename = NULL; + + if (argc > 2) + return T_EXIT_SKIP; + if (argc == 2) { + /* This test exposes interesting behaviour with a null-blk + * device configured like: + * $ modprobe null-blk completion_nsec=100000000 irqmode=2 + * and then run with $ defer-taskrun.t /dev/nullb0 + */ + filename = argv[1]; + } + + if (!t_probe_defer_taskrun()) + return T_EXIT_SKIP; + + ret = test_thread_shutdown(); + if (ret) { + fprintf(stderr, "test_thread_shutdown failed\n"); + return T_EXIT_FAIL; + } + + ret = test_exec(filename); + if (ret) { + fprintf(stderr, "test_exec failed\n"); + return T_EXIT_FAIL; + } + + ret = test_eventfd(); + if (ret) { + fprintf(stderr, "eventfd failed\n"); + return T_EXIT_FAIL; + } + + ret = test_flag(); + if (ret) { + fprintf(stderr, "flag failed\n"); + return T_EXIT_FAIL; + } + + ret = test_ring_shutdown(); + if (ret) { + fprintf(stderr, "test_ring_shutdown failed\n"); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/defer.c b/contrib/libs/liburing/test/defer.c new file mode 100644 index 0000000000..4e7cd23135 --- /dev/null +++ b/contrib/libs/liburing/test/defer.c @@ -0,0 +1,320 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/uio.h> +#include <stdbool.h> + +#include "helpers.h" +#include "liburing.h" + +#define RING_SIZE 128 +enum { + OP_NOP, + OP_REMOVE_BUFFERS +}; + +struct test_context { + struct io_uring *ring; + struct io_uring_sqe **sqes; + struct io_uring_cqe *cqes; + int nr; +}; + +static void free_context(struct test_context *ctx) +{ + free(ctx->sqes); + free(ctx->cqes); + memset(ctx, 0, sizeof(*ctx)); +} + +static int init_context(struct test_context *ctx, struct io_uring *ring, int nr, + int op) +{ + struct io_uring_sqe *sqe; + int i; + + memset(ctx, 0, sizeof(*ctx)); + ctx->nr = nr; + ctx->ring = ring; + ctx->sqes = t_malloc(nr * sizeof(*ctx->sqes)); + ctx->cqes = t_malloc(nr * sizeof(*ctx->cqes)); + + if (!ctx->sqes || !ctx->cqes) + goto err; + + for (i = 0; i < nr; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) + goto err; + switch (op) { + case OP_NOP: + io_uring_prep_nop(sqe); + break; + case OP_REMOVE_BUFFERS: + io_uring_prep_remove_buffers(sqe, 10, 1); + break; + }; + sqe->user_data = i; + ctx->sqes[i] = sqe; + } + + return 0; +err: + free_context(ctx); + printf("init context failed\n"); + return 1; +} + +static int wait_cqes(struct test_context *ctx) +{ + int ret, i; + struct io_uring_cqe *cqe; + + for (i = 0; i < ctx->nr; i++) { + ret = io_uring_wait_cqe(ctx->ring, &cqe); + + if (ret < 0) { + printf("wait_cqes: wait completion %d\n", ret); + return 1; + } + memcpy(&ctx->cqes[i], cqe, sizeof(*cqe)); + io_uring_cqe_seen(ctx->ring, cqe); + } + + return 0; +} + +static int test_cancelled_userdata(struct io_uring *ring) +{ + struct test_context ctx; + int ret, i, nr = 100; + + if (init_context(&ctx, ring, nr, OP_NOP)) + return 1; + + for (i = 0; i < nr; i++) + ctx.sqes[i]->flags |= IOSQE_IO_LINK; + + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + if (wait_cqes(&ctx)) + goto err; + + for (i = 0; i < nr; i++) { + if (i != ctx.cqes[i].user_data) { + printf("invalid user data\n"); + goto err; + } + } + + free_context(&ctx); + return 0; +err: + free_context(&ctx); + return 1; +} + +static int test_thread_link_cancel(struct io_uring *ring) +{ + struct test_context ctx; + int ret, i, nr = 100; + + if (init_context(&ctx, ring, nr, OP_REMOVE_BUFFERS)) + return 1; + + for (i = 0; i < nr; i++) + ctx.sqes[i]->flags |= IOSQE_IO_LINK; + + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + if (wait_cqes(&ctx)) + goto err; + + for (i = 0; i < nr; i++) { + bool fail = false; + + if (i == 0) + fail = (ctx.cqes[i].res != -ENOENT); + else + fail = (ctx.cqes[i].res != -ECANCELED); + + if (fail) { + printf("invalid status %d\n", ctx.cqes[i].res); + goto err; + } + } + + free_context(&ctx); + return 0; +err: + free_context(&ctx); + return 1; +} + +static int test_drain_with_linked_timeout(struct io_uring *ring) +{ + const int nr = 3; + struct __kernel_timespec ts = { .tv_sec = 1, .tv_nsec = 0, }; + struct test_context ctx; + int ret, i; + + if (init_context(&ctx, ring, nr * 2, OP_NOP)) + return 1; + + for (i = 0; i < nr; i++) { + io_uring_prep_timeout(ctx.sqes[2 * i], &ts, 0, 0); + ctx.sqes[2 * i]->flags |= IOSQE_IO_LINK | IOSQE_IO_DRAIN; + io_uring_prep_link_timeout(ctx.sqes[2 * i + 1], &ts, 0); + } + + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + if (wait_cqes(&ctx)) + goto err; + + free_context(&ctx); + return 0; +err: + free_context(&ctx); + return 1; +} + +static int run_drained(struct io_uring *ring, int nr) +{ + struct test_context ctx; + int ret, i; + + if (init_context(&ctx, ring, nr, OP_NOP)) + return 1; + + for (i = 0; i < nr; i++) + ctx.sqes[i]->flags |= IOSQE_IO_DRAIN; + + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + if (wait_cqes(&ctx)) + goto err; + + free_context(&ctx); + return 0; +err: + free_context(&ctx); + return 1; +} + +static int test_overflow_hung(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + int ret, nr = 10; + + while (*ring->cq.koverflow != 1000) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + return 1; + } + + io_uring_prep_nop(sqe); + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + return 1; + } + } + + return run_drained(ring, nr); +} + +static int test_dropped_hung(struct io_uring *ring) +{ + int nr = 10; + + *ring->sq.kdropped = 1000; + return run_drained(ring, nr); +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring, poll_ring, sqthread_ring; + struct io_uring_params p; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + memset(&p, 0, sizeof(p)); + ret = io_uring_queue_init_params(RING_SIZE, &ring, &p); + if (ret) { + printf("ring setup failed %i\n", ret); + return T_EXIT_FAIL; + } + + ret = io_uring_queue_init(RING_SIZE, &poll_ring, IORING_SETUP_IOPOLL); + if (ret) { + printf("poll_ring setup failed\n"); + return T_EXIT_FAIL; + } + + + ret = test_cancelled_userdata(&poll_ring); + if (ret) { + printf("test_cancelled_userdata failed\n"); + return ret; + } + + if (!(p.features & IORING_FEAT_NODROP)) { + ret = test_overflow_hung(&ring); + if (ret) { + printf("test_overflow_hung failed\n"); + return ret; + } + } + + ret = test_dropped_hung(&ring); + if (ret) { + printf("test_dropped_hung failed\n"); + return ret; + } + + ret = test_drain_with_linked_timeout(&ring); + if (ret) { + printf("test_drain_with_linked_timeout failed\n"); + return ret; + } + + ret = t_create_ring(RING_SIZE, &sqthread_ring, + IORING_SETUP_SQPOLL | IORING_SETUP_IOPOLL); + if (ret == T_SETUP_SKIP) + return T_EXIT_SKIP; + else if (ret < 0) + return T_EXIT_FAIL; + + ret = test_thread_link_cancel(&sqthread_ring); + if (ret) { + printf("test_thread_link_cancel failed\n"); + return ret; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/double-poll-crash.c b/contrib/libs/liburing/test/double-poll-crash.c new file mode 100644 index 0000000000..c24ad86661 --- /dev/null +++ b/contrib/libs/liburing/test/double-poll-crash.c @@ -0,0 +1,196 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// https://syzkaller.appspot.com/bug?id=5c9918d20f771265ad0ffae3c8f3859d24850692 +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <endian.h> +#include <fcntl.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/ioctl.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" +#include "../src/syscall.h" + +#define SIZEOF_IO_URING_SQE 64 +#define SIZEOF_IO_URING_CQE 16 +#define SQ_HEAD_OFFSET 0 +#define SQ_TAIL_OFFSET 64 +#define SQ_RING_MASK_OFFSET 256 +#define SQ_RING_ENTRIES_OFFSET 264 +#define SQ_FLAGS_OFFSET 276 +#define SQ_DROPPED_OFFSET 272 +#define CQ_HEAD_OFFSET 128 +#define CQ_TAIL_OFFSET 192 +#define CQ_RING_MASK_OFFSET 260 +#define CQ_RING_ENTRIES_OFFSET 268 +#define CQ_RING_OVERFLOW_OFFSET 284 +#define CQ_FLAGS_OFFSET 280 +#define CQ_CQES_OFFSET 320 + +static long syz_io_uring_setup(volatile long a0, volatile long a1, + volatile long a2, volatile long a3, + volatile long a4, volatile long a5) +{ + uint32_t entries = (uint32_t)a0; + struct io_uring_params* setup_params = (struct io_uring_params*)a1; + void* vma1 = (void*)a2; + void* vma2 = (void*)a3; + void** ring_ptr_out = (void**)a4; + void** sqes_ptr_out = (void**)a5; + uint32_t fd_io_uring = __sys_io_uring_setup(entries, setup_params); + uint32_t sq_ring_sz = + setup_params->sq_off.array + setup_params->sq_entries * sizeof(uint32_t); + uint32_t cq_ring_sz = setup_params->cq_off.cqes + + setup_params->cq_entries * SIZEOF_IO_URING_CQE; + uint32_t ring_sz = sq_ring_sz > cq_ring_sz ? sq_ring_sz : cq_ring_sz; + *ring_ptr_out = mmap(vma1, ring_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, + IORING_OFF_SQ_RING); + if (*ring_ptr_out == MAP_FAILED) + exit(0); + uint32_t sqes_sz = setup_params->sq_entries * SIZEOF_IO_URING_SQE; + *sqes_ptr_out = + mmap(vma2, sqes_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, IORING_OFF_SQES); + if (*sqes_ptr_out == MAP_FAILED) + exit(0); + return fd_io_uring; +} + +static long syz_io_uring_submit(volatile long a0, volatile long a1, + volatile long a2, volatile long a3) +{ + char* ring_ptr = (char*)a0; + char* sqes_ptr = (char*)a1; + char* sqe = (char*)a2; + uint32_t sqes_index = (uint32_t)a3; + uint32_t sq_ring_entries = *(uint32_t*)(ring_ptr + SQ_RING_ENTRIES_OFFSET); + uint32_t cq_ring_entries = *(uint32_t*)(ring_ptr + CQ_RING_ENTRIES_OFFSET); + uint32_t sq_array_off = + (CQ_CQES_OFFSET + cq_ring_entries * SIZEOF_IO_URING_CQE + 63) & ~63; + if (sq_ring_entries) + sqes_index %= sq_ring_entries; + char* sqe_dest = sqes_ptr + sqes_index * SIZEOF_IO_URING_SQE; + memcpy(sqe_dest, sqe, SIZEOF_IO_URING_SQE); + uint32_t sq_ring_mask = *(uint32_t*)(ring_ptr + SQ_RING_MASK_OFFSET); + uint32_t* sq_tail_ptr = (uint32_t*)(ring_ptr + SQ_TAIL_OFFSET); + uint32_t sq_tail = *sq_tail_ptr & sq_ring_mask; + uint32_t sq_tail_next = *sq_tail_ptr + 1; + uint32_t* sq_array = (uint32_t*)(ring_ptr + sq_array_off); + *(sq_array + sq_tail) = sqes_index; + __atomic_store_n(sq_tail_ptr, sq_tail_next, __ATOMIC_RELEASE); + return 0; +} + +static long syz_open_dev(volatile long a0, volatile long a1, volatile long a2) +{ + if (a0 == 0xc || a0 == 0xb) { + char buf[128]; + sprintf(buf, "/dev/%s/%d:%d", a0 == 0xc ? "char" : "block", (uint8_t)a1, + (uint8_t)a2); + return open(buf, O_RDWR, 0); + } else { + char buf[1024]; + char* hash; + strncpy(buf, (char*)a0, sizeof(buf) - 1); + buf[sizeof(buf) - 1] = 0; + while ((hash = strchr(buf, '#'))) { + *hash = '0' + (char)(a1 % 10); + a1 /= 10; + } + return open(buf, a2, 0); + } +} + +uint64_t r[4] = {0xffffffffffffffff, 0x0, 0x0, 0xffffffffffffffff}; + +int main(int argc, char *argv[]) +{ + void *mmap_ret; +#if !defined(__i386) && !defined(__x86_64__) + return T_EXIT_SKIP; +#endif + + if (argc > 1) + return T_EXIT_SKIP; + + mmap_ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul); + if (mmap_ret == MAP_FAILED) + return T_EXIT_SKIP; + mmap_ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); + if (mmap_ret == MAP_FAILED) + return T_EXIT_SKIP; + intptr_t res = 0; + *(uint32_t*)0x20000484 = 0; + *(uint32_t*)0x20000488 = 0; + *(uint32_t*)0x2000048c = 0; + *(uint32_t*)0x20000490 = 0; + *(uint32_t*)0x20000498 = -1; + *(uint32_t*)0x2000049c = 0; + *(uint32_t*)0x200004a0 = 0; + *(uint32_t*)0x200004a4 = 0; + res = -1; + res = syz_io_uring_setup(0x6ad4, 0x20000480, 0x20ee7000, 0x20ffb000, + 0x20000180, 0x20000040); + if (res != -1) { + r[0] = res; + r[1] = *(uint64_t*)0x20000180; + r[2] = *(uint64_t*)0x20000040; + } + res = -1; + res = syz_open_dev(0xc, 4, 0x15); + if (res != -1) + r[3] = res; + *(uint8_t*)0x20000000 = 6; + *(uint8_t*)0x20000001 = 0; + *(uint16_t*)0x20000002 = 0; + *(uint32_t*)0x20000004 = r[3]; + *(uint64_t*)0x20000008 = 0; + *(uint64_t*)0x20000010 = 0; + *(uint32_t*)0x20000018 = 0; + *(uint16_t*)0x2000001c = 0; + *(uint16_t*)0x2000001e = 0; + *(uint64_t*)0x20000020 = 0; + *(uint16_t*)0x20000028 = 0; + *(uint16_t*)0x2000002a = 0; + *(uint8_t*)0x2000002c = 0; + *(uint8_t*)0x2000002d = 0; + *(uint8_t*)0x2000002e = 0; + *(uint8_t*)0x2000002f = 0; + *(uint8_t*)0x20000030 = 0; + *(uint8_t*)0x20000031 = 0; + *(uint8_t*)0x20000032 = 0; + *(uint8_t*)0x20000033 = 0; + *(uint8_t*)0x20000034 = 0; + *(uint8_t*)0x20000035 = 0; + *(uint8_t*)0x20000036 = 0; + *(uint8_t*)0x20000037 = 0; + *(uint8_t*)0x20000038 = 0; + *(uint8_t*)0x20000039 = 0; + *(uint8_t*)0x2000003a = 0; + *(uint8_t*)0x2000003b = 0; + *(uint8_t*)0x2000003c = 0; + *(uint8_t*)0x2000003d = 0; + *(uint8_t*)0x2000003e = 0; + *(uint8_t*)0x2000003f = 0; + syz_io_uring_submit(r[1], r[2], 0x20000000, 0); + __sys_io_uring_enter(r[0], 0x20450c, 0, 0ul, 0ul); + *(uint32_t*)0x20000080 = 0x7ff; + *(uint32_t*)0x20000084 = 0x8b7; + *(uint32_t*)0x20000088 = 3; + *(uint32_t*)0x2000008c = 0x101; + *(uint8_t*)0x20000090 = 9; + memcpy((void*)0x20000091, "\xaf\x09\x01\xbc\xf9\xc6\xe4\x92\x86\x51\x7d\x7f" + "\xbd\x43\x7d\x16\x69\x3e\x05", + 19); + ioctl(r[3], 0x5404, 0x20000080ul); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/drop-submit.c b/contrib/libs/liburing/test/drop-submit.c new file mode 100644 index 0000000000..daa7d7683a --- /dev/null +++ b/contrib/libs/liburing/test/drop-submit.c @@ -0,0 +1,95 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test IORING_SETUP_SUBMIT_ALL + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> + +#include "liburing.h" +#include "helpers.h" + +static int test(struct io_uring *ring, int expect_drops) +{ + struct io_uring_sqe *sqe; + char buf[32]; + int ret, i; + + for (i = 0; i < 4; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + } + + /* prep two invalid reads, these will fail */ + for (i = 0; i < 2; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_read(sqe, 128, buf, sizeof(buf), 0); + sqe->ioprio = (short) -1; + } + + + ret = io_uring_submit(ring); + if (expect_drops) { + if (ret != 5) { + fprintf(stderr, "drops submit failed: %d\n", ret); + goto err; + } + } else { + if (ret != 6) { + fprintf(stderr, "no drops submit failed: %d\n", ret); + goto err; + } + } + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SUBMIT_ALL); + if (ret) + return 0; + + ret = test(&ring, 0); + if (ret) { + fprintf(stderr, "test no drops failed\n"); + return ret; + } + + io_uring_queue_exit(&ring); + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return T_EXIT_FAIL; + } + + ret = test(&ring, 1); + if (ret) { + fprintf(stderr, "test drops failed\n"); + return ret; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/eeed8b54e0df.c b/contrib/libs/liburing/test/eeed8b54e0df.c new file mode 100644 index 0000000000..c4118a2d51 --- /dev/null +++ b/contrib/libs/liburing/test/eeed8b54e0df.c @@ -0,0 +1,116 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: -EAGAIN handling + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "helpers.h" +#include "liburing.h" + +#define BLOCK 4096 + +#ifndef RWF_NOWAIT +#define RWF_NOWAIT 8 +#endif + +static int get_file_fd(void) +{ + ssize_t ret; + char *buf; + int fd; + + fd = open("testfile", O_RDWR | O_CREAT, 0644); + unlink("testfile"); + if (fd < 0) { + perror("open file"); + return -1; + } + + buf = t_malloc(BLOCK); + memset(buf, 0, BLOCK); + ret = write(fd, buf, BLOCK); + if (ret != BLOCK) { + if (ret < 0) + perror("write"); + else + printf("Short write\n"); + goto err; + } + fsync(fd); + + if (posix_fadvise(fd, 0, 4096, POSIX_FADV_DONTNEED)) { + perror("fadvise"); +err: + close(fd); + free(buf); + return -1; + } + + free(buf); + return fd; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct iovec iov; + int ret, fd; + + if (argc > 1) + return T_EXIT_SKIP; + + iov.iov_base = t_malloc(4096); + iov.iov_len = 4096; + + ret = io_uring_queue_init(2, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return T_EXIT_FAIL; + + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + printf("get sqe failed\n"); + return T_EXIT_FAIL; + } + + fd = get_file_fd(); + if (fd < 0) + return T_EXIT_FAIL; + + io_uring_prep_readv(sqe, fd, &iov, 1, 0); + sqe->rw_flags = RWF_NOWAIT; + + ret = io_uring_submit(&ring); + if (ret != 1) { + printf("Got submit %d, expected 1\n", ret); + goto err; + } + + ret = io_uring_peek_cqe(&ring, &cqe); + if (ret) { + printf("Ring peek got %d\n", ret); + goto err; + } + + if (cqe->res != -EAGAIN && cqe->res != 4096) { + printf("cqe error: %d\n", cqe->res); + goto err; + } + + close(fd); + return T_EXIT_PASS; +err: + close(fd); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/empty-eownerdead.c b/contrib/libs/liburing/test/empty-eownerdead.c new file mode 100644 index 0000000000..b7722ab152 --- /dev/null +++ b/contrib/libs/liburing/test/empty-eownerdead.c @@ -0,0 +1,46 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test if entering with nothing to submit/wait for SQPOLL returns an error. + */ +#include <stdio.h> +#include <errno.h> +#include <string.h> + +#include "liburing.h" +#include "helpers.h" +#include "../src/syscall.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_params p = {}; + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + p.flags = IORING_SETUP_SQPOLL; + p.sq_thread_idle = 100; + + ret = t_create_ring_params(1, &ring, &p); + if (ret == T_SETUP_SKIP) + return T_EXIT_SKIP; + else if (ret < 0) + goto err; + + ret = __sys_io_uring_enter(ring.ring_fd, 0, 0, 0, NULL); + if (ret < 0) { + int __e = errno; + + if (__e == EOWNERDEAD) + fprintf(stderr, "sqe submit unexpected failure due old kernel bug: %s\n", strerror(__e)); + else + fprintf(stderr, "sqe submit unexpected failure: %s\n", strerror(__e)); + goto err; + } + + return T_EXIT_PASS; +err: + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/eventfd-disable.c b/contrib/libs/liburing/test/eventfd-disable.c new file mode 100644 index 0000000000..9ed81d3d14 --- /dev/null +++ b/contrib/libs/liburing/test/eventfd-disable.c @@ -0,0 +1,180 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test disable/enable notifications through eventfd + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <poll.h> +#include <sys/eventfd.h> + +#include "liburing.h" +#include "helpers.h" + +static int test(bool defer) +{ + struct io_uring_params p = {}; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + uint64_t ptr; + struct iovec vec = { + .iov_base = &ptr, + .iov_len = sizeof(ptr) + }; + int ret, evfd, i; + + if (defer) + p.flags |= IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN; + + ret = io_uring_queue_init_params(64, &ring, &p); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + + evfd = eventfd(0, EFD_CLOEXEC); + if (evfd < 0) { + perror("eventfd"); + return T_EXIT_FAIL; + } + + ret = io_uring_register_eventfd(&ring, evfd); + if (ret) { + fprintf(stderr, "failed to register evfd: %d\n", ret); + return T_EXIT_FAIL; + } + + if (!io_uring_cq_eventfd_enabled(&ring)) { + fprintf(stderr, "eventfd disabled\n"); + return T_EXIT_FAIL; + } + + ret = io_uring_cq_eventfd_toggle(&ring, false); + if (ret) { + fprintf(stdout, "Skipping, CQ flags not available!\n"); + return T_EXIT_SKIP; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, evfd, &vec, 1, 0); + sqe->user_data = 1; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return T_EXIT_FAIL; + } + + for (i = 0; i < 63; i++) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + sqe->user_data = 2; + } + + ret = io_uring_submit(&ring); + if (ret != 63) { + fprintf(stderr, "submit: %d\n", ret); + return T_EXIT_FAIL; + } + + for (i = 0; i < 63; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return T_EXIT_FAIL; + } + + switch (cqe->user_data) { + case 1: /* eventfd */ + fprintf(stderr, "eventfd unexpected: %d\n", (int)ptr); + return T_EXIT_FAIL; + case 2: + if (cqe->res) { + fprintf(stderr, "nop: %d\n", cqe->res); + return T_EXIT_FAIL; + } + break; + } + io_uring_cqe_seen(&ring, cqe); + } + + ret = io_uring_cq_eventfd_toggle(&ring, true); + if (ret) { + fprintf(stderr, "io_uring_cq_eventfd_toggle: %d\n", ret); + return T_EXIT_FAIL; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + sqe->user_data = 2; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return T_EXIT_FAIL; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return T_EXIT_FAIL; + } + + switch (cqe->user_data) { + case 1: /* eventfd */ + if (cqe->res != sizeof(ptr)) { + fprintf(stderr, "read res: %d\n", cqe->res); + return T_EXIT_FAIL; + } + + if (ptr != 1) { + fprintf(stderr, "eventfd: %d\n", (int)ptr); + return T_EXIT_FAIL; + } + break; + case 2: + if (cqe->res) { + fprintf(stderr, "nop: %d\n", cqe->res); + return T_EXIT_FAIL; + } + break; + } + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + close(evfd); + return T_EXIT_PASS; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = test(false); + if (ret != T_EXIT_PASS) { + fprintf(stderr, "%s: test(false) failed\n", argv[0]); + return ret; + } + + if (t_probe_defer_taskrun()) { + ret = test(true); + if (ret != T_EXIT_PASS) { + fprintf(stderr, "%s: test(true) failed\n", argv[0]); + return ret; + } + } + + return ret; +} diff --git a/contrib/libs/liburing/test/eventfd-reg.c b/contrib/libs/liburing/test/eventfd-reg.c new file mode 100644 index 0000000000..a1655267ab --- /dev/null +++ b/contrib/libs/liburing/test/eventfd-reg.c @@ -0,0 +1,78 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test eventfd registration+unregistration + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <poll.h> +#include <sys/eventfd.h> + +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_params p = {}; + struct io_uring ring; + int ret, evfd[2], i; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init_params(8, &ring, &p); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + + evfd[0] = eventfd(0, EFD_CLOEXEC); + evfd[1] = eventfd(0, EFD_CLOEXEC); + if (evfd[0] < 0 || evfd[1] < 0) { + perror("eventfd"); + return T_EXIT_FAIL; + } + + ret = io_uring_register_eventfd(&ring, evfd[0]); + if (ret) { + fprintf(stderr, "failed to register evfd: %d\n", ret); + return T_EXIT_FAIL; + } + + /* Check that registrering again will get -EBUSY */ + ret = io_uring_register_eventfd(&ring, evfd[1]); + if (ret != -EBUSY) { + fprintf(stderr, "unexpected 2nd register: %d\n", ret); + return T_EXIT_FAIL; + } + close(evfd[1]); + + ret = io_uring_unregister_eventfd(&ring); + if (ret) { + fprintf(stderr, "unexpected unregister: %d\n", ret); + return T_EXIT_FAIL; + } + + /* loop 100 registers/unregister */ + for (i = 0; i < 100; i++) { + ret = io_uring_register_eventfd(&ring, evfd[0]); + if (ret) { + fprintf(stderr, "failed to register evfd: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = io_uring_unregister_eventfd(&ring); + if (ret) { + fprintf(stderr, "unexpected unregister: %d\n", ret); + return T_EXIT_FAIL; + } + } + + close(evfd[0]); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/eventfd-ring.c b/contrib/libs/liburing/test/eventfd-ring.c new file mode 100644 index 0000000000..d4bed86526 --- /dev/null +++ b/contrib/libs/liburing/test/eventfd-ring.c @@ -0,0 +1,99 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various nop tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <poll.h> +#include <sys/eventfd.h> + +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_params p = {}; + struct io_uring ring1, ring2; + struct io_uring_sqe *sqe; + int ret, evfd1, evfd2; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init_params(8, &ring1, &p); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + if (!(p.features & IORING_FEAT_CUR_PERSONALITY)) { + fprintf(stdout, "Skipping\n"); + return T_EXIT_SKIP; + } + ret = io_uring_queue_init(8, &ring2, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + + evfd1 = eventfd(0, EFD_CLOEXEC); + if (evfd1 < 0) { + perror("eventfd"); + return T_EXIT_FAIL; + } + + evfd2 = eventfd(0, EFD_CLOEXEC); + if (evfd2 < 0) { + perror("eventfd"); + return T_EXIT_FAIL; + } + + ret = io_uring_register_eventfd(&ring1, evfd1); + if (ret) { + fprintf(stderr, "failed to register evfd: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = io_uring_register_eventfd(&ring2, evfd2); + if (ret) { + fprintf(stderr, "failed to register evfd: %d\n", ret); + return T_EXIT_FAIL; + } + + sqe = io_uring_get_sqe(&ring1); + io_uring_prep_poll_add(sqe, evfd2, POLLIN); + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring2); + io_uring_prep_poll_add(sqe, evfd1, POLLIN); + sqe->user_data = 1; + + ret = io_uring_submit(&ring1); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = io_uring_submit(&ring2); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return T_EXIT_FAIL; + } + + sqe = io_uring_get_sqe(&ring1); + io_uring_prep_nop(sqe); + sqe->user_data = 3; + + ret = io_uring_submit(&ring1); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/eventfd.c b/contrib/libs/liburing/test/eventfd.c new file mode 100644 index 0000000000..317de29fb4 --- /dev/null +++ b/contrib/libs/liburing/test/eventfd.c @@ -0,0 +1,114 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various nop tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <poll.h> +#include <sys/eventfd.h> + +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_params p = {}; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + uint64_t ptr; + struct iovec vec = { + .iov_base = &ptr, + .iov_len = sizeof(ptr) + }; + int ret, evfd, i; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init_params(8, &ring, &p); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + if (!(p.features & IORING_FEAT_CUR_PERSONALITY)) { + fprintf(stdout, "Skipping\n"); + return T_EXIT_SKIP; + } + + evfd = eventfd(0, EFD_CLOEXEC); + if (evfd < 0) { + perror("eventfd"); + return T_EXIT_FAIL; + } + + ret = io_uring_register_eventfd(&ring, evfd); + if (ret) { + fprintf(stderr, "failed to register evfd: %d\n", ret); + return T_EXIT_FAIL; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, evfd, POLLIN); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, evfd, &vec, 1, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 2; + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "submit: %d\n", ret); + return T_EXIT_FAIL; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + sqe->user_data = 3; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return T_EXIT_FAIL; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return T_EXIT_FAIL; + } + switch (cqe->user_data) { + case 1: + /* POLLIN */ + if (cqe->res != 1) { + fprintf(stderr, "poll: %d\n", cqe->res); + return T_EXIT_FAIL; + } + break; + case 2: + if (cqe->res != sizeof(ptr)) { + fprintf(stderr, "read: %d\n", cqe->res); + return T_EXIT_FAIL; + } + break; + case 3: + if (cqe->res) { + fprintf(stderr, "nop: %d\n", cqe->res); + return T_EXIT_FAIL; + } + break; + } + io_uring_cqe_seen(&ring, cqe); + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/exec-target.c b/contrib/libs/liburing/test/exec-target.c new file mode 100644 index 0000000000..21ef58ef8a --- /dev/null +++ b/contrib/libs/liburing/test/exec-target.c @@ -0,0 +1,7 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ + +int main(int argc, char *argv[]) +{ + return 0; +} diff --git a/contrib/libs/liburing/test/exit-no-cleanup.c b/contrib/libs/liburing/test/exit-no-cleanup.c new file mode 100644 index 0000000000..2fe3dc7ddd --- /dev/null +++ b/contrib/libs/liburing/test/exit-no-cleanup.c @@ -0,0 +1,118 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test case testing exit without cleanup and io-wq work pending or queued. + * + * From Florian Fischer <florian.fl.fischer@fau.de> + * Link: https://lore.kernel.org/io-uring/20211202165606.mqryio4yzubl7ms5@pasture/ + * + */ +#include <assert.h> +#include <err.h> +#include <errno.h> +#include <pthread.h> +#include <semaphore.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/sysinfo.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" + +#define IORING_ENTRIES 8 + +static pthread_t *threads; +static pthread_barrier_t init_barrier; +static int sleep_fd, notify_fd; +static sem_t sem; + +void *thread_func(void *arg) +{ + struct io_uring ring; + int res; + + res = io_uring_queue_init(IORING_ENTRIES, &ring, 0); + if (res) + err(EXIT_FAILURE, "io_uring_queue_init failed"); + + pthread_barrier_wait(&init_barrier); + + for(;;) { + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + uint64_t buf; + int res; + + sqe = io_uring_get_sqe(&ring); + assert(sqe); + + io_uring_prep_read(sqe, sleep_fd, &buf, sizeof(buf), 0); + + res = io_uring_submit_and_wait(&ring, 1); + if (res < 0) + err(EXIT_FAILURE, "io_uring_submit_and_wait failed"); + + res = io_uring_peek_cqe(&ring, &cqe); + assert(!res); + if (cqe->res < 0) { + errno = -cqe->res; + err(EXIT_FAILURE, "read failed"); + } + assert(cqe->res == sizeof(buf)); + + sem_post(&sem); + + io_uring_cqe_seen(&ring, cqe); + } + + return NULL; +} + +int main(int argc, char *argv[]) +{ + int res, fds[2], i, cpus; + const uint64_t n = 0x42; + + if (argc > 1) + return T_EXIT_SKIP; + + cpus = get_nprocs(); + res = pthread_barrier_init(&init_barrier, NULL, cpus); + if (res) + err(EXIT_FAILURE, "pthread_barrier_init failed"); + + res = sem_init(&sem, 0, 0); + if (res) + err(EXIT_FAILURE, "sem_init failed"); + + threads = t_malloc(sizeof(pthread_t) * cpus); + + res = pipe(fds); + if (res) + err(EXIT_FAILURE, "pipe failed"); + + sleep_fd = fds[0]; + notify_fd = fds[1]; + + for (i = 0; i < cpus; i++) { + errno = pthread_create(&threads[i], NULL, thread_func, NULL); + if (errno) + err(EXIT_FAILURE, "pthread_create failed"); + } + + // Write #cpus notifications + for (i = 0; i < cpus; i++) { + res = write(notify_fd, &n, sizeof(n)); + if (res < 0) + err(EXIT_FAILURE, "write failed"); + assert(res == sizeof(n)); + } + + // Await that all notifications were received + for (i = 0; i < cpus; i++) + sem_wait(&sem); + + // Exit without resource cleanup + exit(EXIT_SUCCESS); +} diff --git a/contrib/libs/liburing/test/fadvise.c b/contrib/libs/liburing/test/fadvise.c new file mode 100644 index 0000000000..86670b80f4 --- /dev/null +++ b/contrib/libs/liburing/test/fadvise.c @@ -0,0 +1,203 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: basic fadvise test + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/time.h> + +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE (128 * 1024) +#define LOOPS 100 +#define MIN_LOOPS 10 + +static unsigned long long utime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000000; + return sec + usec; +} + +static unsigned long long utime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return utime_since(tv, &end); +} + +static int do_fadvise(struct io_uring *ring, int fd, off_t offset, off_t len, + int advice) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "failed to get sqe\n"); + return 1; + } + + io_uring_prep_fadvise(sqe, fd, offset, len, advice); + sqe->user_data = advice; + ret = io_uring_submit_and_wait(ring, 1); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return 1; + } + + ret = cqe->res; + if (ret == -EINVAL || ret == -EBADF) { + fprintf(stdout, "Fadvise not supported, skipping\n"); + unlink(".fadvise.tmp"); + exit(T_EXIT_SKIP); + } else if (ret) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + } + io_uring_cqe_seen(ring, cqe); + return ret; +} + +static long do_read(int fd, char *buf) +{ + struct timeval tv; + int ret; + long t; + + ret = lseek(fd, 0, SEEK_SET); + if (ret) { + perror("lseek"); + return -1; + } + + gettimeofday(&tv, NULL); + ret = read(fd, buf, FILE_SIZE); + t = utime_since_now(&tv); + if (ret < 0) { + perror("read"); + return -1; + } else if (ret != FILE_SIZE) { + fprintf(stderr, "short read1: %d\n", ret); + return -1; + } + + return t; +} + +static int test_fadvise(struct io_uring *ring, const char *filename) +{ + unsigned long cached_read, uncached_read, cached_read2; + int fd, ret; + char *buf; + + fd = open(filename, O_RDONLY); + if (fd < 0) { + perror("open"); + return 1; + } + + buf = t_malloc(FILE_SIZE); + + cached_read = do_read(fd, buf); + if (cached_read == -1) + return 1; + + ret = do_fadvise(ring, fd, 0, FILE_SIZE, POSIX_FADV_DONTNEED); + if (ret) + return 1; + + uncached_read = do_read(fd, buf); + if (uncached_read == -1) + return 1; + + ret = do_fadvise(ring, fd, 0, FILE_SIZE, POSIX_FADV_DONTNEED); + if (ret) + return 1; + + ret = do_fadvise(ring, fd, 0, FILE_SIZE, POSIX_FADV_WILLNEED); + if (ret) + return 1; + + fsync(fd); + + cached_read2 = do_read(fd, buf); + if (cached_read2 == -1) + return 1; + + if (cached_read < uncached_read && + cached_read2 < uncached_read) + return 0; + + return 2; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret, i, good, bad; + char *fname; + + if (argc > 1) { + fname = argv[1]; + } else { + fname = ".fadvise.tmp"; + t_create_file(fname, FILE_SIZE); + } + if (io_uring_queue_init(8, &ring, 0)) { + fprintf(stderr, "ring creation failed\n"); + goto err; + } + + good = bad = 0; + for (i = 0; i < LOOPS; i++) { + ret = test_fadvise(&ring, fname); + if (ret == 1) { + fprintf(stderr, "read_fadvise failed\n"); + goto err; + } else if (!ret) + good++; + else if (ret == 2) + bad++; + if (i >= MIN_LOOPS && !bad) + break; + } + + /* too hard to reliably test, just ignore */ + if (0 && bad > good) { + fprintf(stderr, "Suspicious timings\n"); + goto err; + } + + if (fname != argv[1]) + unlink(fname); + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +err: + if (fname != argv[1]) + unlink(fname); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/fallocate.c b/contrib/libs/liburing/test/fallocate.c new file mode 100644 index 0000000000..a546922f96 --- /dev/null +++ b/contrib/libs/liburing/test/fallocate.c @@ -0,0 +1,257 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring fallocate + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/resource.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +static int no_fallocate; + +static int test_fallocate_rlimit(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct rlimit rlim; + char buf[32]; + int fd, ret; + + if (getrlimit(RLIMIT_FSIZE, &rlim) < 0) { + perror("getrlimit"); + return 1; + } + rlim.rlim_cur = 64 * 1024; + rlim.rlim_max = 64 * 1024; + if (setrlimit(RLIMIT_FSIZE, &rlim) < 0) { + perror("setrlimit"); + return 1; + } + + sprintf(buf, "./XXXXXX"); + fd = mkstemp(buf); + if (fd < 0) { + perror("open"); + return 1; + } + unlink(buf); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_fallocate(sqe, fd, 0, 0, 128*1024); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + + if (cqe->res == -EINVAL) { + fprintf(stdout, "Fallocate not supported, skipping\n"); + no_fallocate = 1; + goto skip; + } else if (cqe->res != -EFBIG) { + fprintf(stderr, "Expected -EFBIG: %d\n", cqe->res); + goto err; + } + io_uring_cqe_seen(ring, cqe); + return 0; +skip: + return T_EXIT_SKIP; +err: + return 1; +} + +static int test_fallocate(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct stat st; + char buf[32]; + int fd, ret; + + sprintf(buf, "./XXXXXX"); + fd = mkstemp(buf); + if (fd < 0) { + perror("open"); + return 1; + } + unlink(buf); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_fallocate(sqe, fd, 0, 0, 128*1024); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + + if (cqe->res == -EINVAL) { + fprintf(stdout, "Fallocate not supported, skipping\n"); + no_fallocate = 1; + goto skip; + } + if (cqe->res) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + goto err; + } + io_uring_cqe_seen(ring, cqe); + + if (fstat(fd, &st) < 0) { + perror("stat"); + goto err; + } + + if (st.st_size != 128*1024) { + fprintf(stderr, "Size mismatch: %llu\n", + (unsigned long long) st.st_size); + goto err; + } + + return 0; +skip: + return T_EXIT_SKIP; +err: + return 1; +} + +static int test_fallocate_fsync(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct stat st; + char buf[32]; + int fd, ret, i; + + if (no_fallocate) + return 0; + + sprintf(buf, "./XXXXXX"); + fd = mkstemp(buf); + if (fd < 0) { + perror("open"); + return 1; + } + unlink(buf); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_fallocate(sqe, fd, 0, 0, 128*1024); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_fsync(sqe, fd, 0); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res) { + fprintf(stderr, "cqe->res=%d,data=%" PRIu64 "\n", cqe->res, + (uint64_t) cqe->user_data); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + if (fstat(fd, &st) < 0) { + perror("stat"); + goto err; + } + + if (st.st_size != 128*1024) { + fprintf(stderr, "Size mismatch: %llu\n", + (unsigned long long) st.st_size); + goto err; + } + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return T_EXIT_FAIL; + } + + ret = test_fallocate(&ring); + if (ret) { + if (ret != T_EXIT_SKIP) { + fprintf(stderr, "test_fallocate failed\n"); + } + return ret; + } + + ret = test_fallocate_fsync(&ring); + if (ret) { + fprintf(stderr, "test_fallocate_fsync failed\n"); + return ret; + } + + ret = test_fallocate_rlimit(&ring); + if (ret) { + if (ret != T_EXIT_SKIP) { + fprintf(stderr, "test_fallocate_rlimit failed\n"); + } + return ret; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/fc2a85cb02ef.c b/contrib/libs/liburing/test/fc2a85cb02ef.c new file mode 100644 index 0000000000..3c61374845 --- /dev/null +++ b/contrib/libs/liburing/test/fc2a85cb02ef.c @@ -0,0 +1,133 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// https://syzkaller.appspot.com/bug?id=1f2ecd7a23dba87e5ca3505ec44514a462cfe8c0 +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <errno.h> +#include <fcntl.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <sys/mman.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" +#include "../src/syscall.h" + +static bool write_file(const char* file, const char* what, ...) +{ + char buf[1024]; + va_list args; + va_start(args, what); + vsnprintf(buf, sizeof(buf), what, args); + va_end(args); + buf[sizeof(buf) - 1] = 0; + int len = strlen(buf); + int fd = open(file, O_WRONLY | O_CLOEXEC); + if (fd == -1) + return false; + if (write(fd, buf, len) != len) { + int err = errno; + close(fd); + errno = err; + return false; + } + close(fd); + return true; +} + +static int inject_fault(int nth) +{ + int fd; + fd = open("/proc/thread-self/fail-nth", O_RDWR); + if (fd == -1) + exit(1); + char buf[16]; + sprintf(buf, "%d", nth + 1); + if (write(fd, buf, strlen(buf)) != (ssize_t)strlen(buf)) + exit(1); + return fd; +} + +static int setup_fault() +{ + static struct { + const char* file; + const char* val; + bool fatal; + } files[] = { + {"/sys/kernel/debug/failslab/ignore-gfp-wait", "N", true}, + {"/sys/kernel/debug/failslab/verbose", "0", false}, + {"/sys/kernel/debug/fail_futex/ignore-private", "N", false}, + {"/sys/kernel/debug/fail_page_alloc/verbose", "0", false}, + {"/sys/kernel/debug/fail_page_alloc/ignore-gfp-highmem", "N", false}, + {"/sys/kernel/debug/fail_page_alloc/ignore-gfp-wait", "N", false}, + {"/sys/kernel/debug/fail_page_alloc/min-order", "0", false}, + }; + unsigned i; + for (i = 0; i < sizeof(files) / sizeof(files[0]); i++) { + if (!write_file(files[i].file, files[i].val)) { + if (files[i].fatal) + return 1; + } + } + return 0; +} + +uint64_t r[2] = {0xffffffffffffffff, 0xffffffffffffffff}; + +int main(int argc, char *argv[]) +{ + if (argc > 1) + return T_EXIT_SKIP; + mmap((void *) 0x20000000ul, 0x1000000ul, 3ul, 0x32ul, -1, 0); + if (setup_fault()) { + printf("Test needs failslab/fail_futex/fail_page_alloc enabled, skipped\n"); + return T_EXIT_SKIP; + } + intptr_t res = 0; + *(uint32_t*)0x20000000 = 0; + *(uint32_t*)0x20000004 = 0; + *(uint32_t*)0x20000008 = 0; + *(uint32_t*)0x2000000c = 0; + *(uint32_t*)0x20000010 = 0; + *(uint32_t*)0x20000014 = 0; + *(uint32_t*)0x20000018 = 0; + *(uint32_t*)0x2000001c = 0; + *(uint32_t*)0x20000020 = 0; + *(uint32_t*)0x20000024 = 0; + *(uint32_t*)0x20000028 = 0; + *(uint32_t*)0x2000002c = 0; + *(uint32_t*)0x20000030 = 0; + *(uint32_t*)0x20000034 = 0; + *(uint32_t*)0x20000038 = 0; + *(uint32_t*)0x2000003c = 0; + *(uint32_t*)0x20000040 = 0; + *(uint32_t*)0x20000044 = 0; + *(uint64_t*)0x20000048 = 0; + *(uint32_t*)0x20000050 = 0; + *(uint32_t*)0x20000054 = 0; + *(uint32_t*)0x20000058 = 0; + *(uint32_t*)0x2000005c = 0; + *(uint32_t*)0x20000060 = 0; + *(uint32_t*)0x20000064 = 0; + *(uint32_t*)0x20000068 = 0; + *(uint32_t*)0x2000006c = 0; + *(uint64_t*)0x20000070 = 0; + res = __sys_io_uring_setup(0x6a6, (struct io_uring_params *) 0x20000000ul); + if (res != -1) + r[0] = res; + res = socket(0x11ul, 2ul, 0x300ul); + if (res != -1) + r[1] = res; + *(uint32_t*)0x20000080 = r[1]; + inject_fault(1); + __sys_io_uring_register(r[0], 2ul, (const void *) 0x20000080ul, 1ul); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/fd-pass.c b/contrib/libs/liburing/test/fd-pass.c new file mode 100644 index 0000000000..9549d418e5 --- /dev/null +++ b/contrib/libs/liburing/test/fd-pass.c @@ -0,0 +1,188 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various fixed file fd passing tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +#define FSIZE 128 +#define PAT 0x9a + +static int no_fd_pass; + +static int verify_fixed_read(struct io_uring *ring, int fixed_fd, int fail) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + unsigned char buf[FSIZE]; + int i; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_read(sqe, fixed_fd, buf, FSIZE, 0); + sqe->flags |= IOSQE_FIXED_FILE; + io_uring_submit(ring); + + io_uring_wait_cqe(ring, &cqe); + if (cqe->res != FSIZE) { + if (fail && cqe->res == -EBADF) + return 0; + fprintf(stderr, "Read: %d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + + for (i = 0; i < FSIZE; i++) { + if (buf[i] != PAT) { + fprintf(stderr, "got %x, wanted %x\n", buf[i], PAT); + return 1; + } + } + + return 0; +} + +static int test(const char *filename) +{ + struct io_uring sring, dring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_queue_init(8, &sring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + ret = io_uring_queue_init(8, &dring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = io_uring_register_files_sparse(&sring, 8); + if (ret) { + if (ret == -EINVAL) + return T_EXIT_SKIP; + fprintf(stderr, "register files failed %d\n", ret); + return T_EXIT_FAIL; + } + ret = io_uring_register_files_sparse(&dring, 8); + if (ret) { + fprintf(stderr, "register files failed %d\n", ret); + return T_EXIT_FAIL; + } + + /* open direct descriptor */ + sqe = io_uring_get_sqe(&sring); + io_uring_prep_openat_direct(sqe, AT_FDCWD, filename, 0, 0644, 0); + io_uring_submit(&sring); + ret = io_uring_wait_cqe(&sring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe failed %d\n", ret); + return T_EXIT_FAIL; + } + if (cqe->res) { + fprintf(stderr, "cqe res %d\n", cqe->res); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(&sring, cqe); + + /* verify data is sane for source ring */ + if (verify_fixed_read(&sring, 0, 0)) + return T_EXIT_FAIL; + + /* send direct descriptor to destination ring */ + sqe = io_uring_get_sqe(&sring); + io_uring_prep_msg_ring(sqe, dring.ring_fd, 0, 0x89, 0); + sqe->addr = 1; + sqe->addr3 = 0; + sqe->file_index = 1; + io_uring_submit(&sring); + + ret = io_uring_wait_cqe(&sring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe failed %d\n", ret); + return T_EXIT_FAIL; + } + if (cqe->res) { + if (cqe->res == -EINVAL && !no_fd_pass) { + no_fd_pass = 1; + return T_EXIT_SKIP; + } + fprintf(stderr, "msg_ring failed %d\n", cqe->res); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(&sring, cqe); + + /* get posted completion for the passing */ + ret = io_uring_wait_cqe(&dring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe failed %d\n", ret); + return T_EXIT_FAIL; + } + if (cqe->user_data != 0x89) { + fprintf(stderr, "bad user_data %ld\n", (long) cqe->res); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(&dring, cqe); + + /* now verify we can read the sane data from the destination ring */ + if (verify_fixed_read(&dring, 0, 0)) + return T_EXIT_FAIL; + + /* close descriptor in source ring */ + sqe = io_uring_get_sqe(&sring); + io_uring_prep_close_direct(sqe, 0); + io_uring_submit(&sring); + + ret = io_uring_wait_cqe(&sring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe failed %d\n", ret); + return T_EXIT_FAIL; + } + if (cqe->res) { + fprintf(stderr, "direct close failed %d\n", cqe->res); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(&sring, cqe); + + /* check that source ring fails after close */ + if (verify_fixed_read(&sring, 0, 1)) + return T_EXIT_FAIL; + + /* check we can still read from destination ring */ + if (verify_fixed_read(&dring, 0, 0)) + return T_EXIT_FAIL; + + return T_EXIT_PASS; +} + +int main(int argc, char *argv[]) +{ + char fname[80]; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + sprintf(fname, ".fd-pass.%d", getpid()); + t_create_file_pattern(fname, FSIZE, PAT); + + ret = test(fname); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test failed\n"); + ret = T_EXIT_FAIL; + } + + unlink(fname); + return ret; +} diff --git a/contrib/libs/liburing/test/file-register.c b/contrib/libs/liburing/test/file-register.c new file mode 100644 index 0000000000..707cd5422f --- /dev/null +++ b/contrib/libs/liburing/test/file-register.c @@ -0,0 +1,1125 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various file registration tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <limits.h> +#include <sys/resource.h> + +#include "helpers.h" +#include "liburing.h" + +static int no_update = 0; + +static void close_files(int *files, int nr_files, int add) +{ + char fname[32]; + int i; + + for (i = 0; i < nr_files; i++) { + if (files) + close(files[i]); + if (!add) + sprintf(fname, ".reg.%d", i); + else + sprintf(fname, ".add.%d", i + add); + unlink(fname); + } + if (files) + free(files); +} + +static int *open_files(int nr_files, int extra, int add) +{ + char fname[32]; + int *files; + int i; + + files = t_calloc(nr_files + extra, sizeof(int)); + + for (i = 0; i < nr_files; i++) { + if (!add) + sprintf(fname, ".reg.%d", i); + else + sprintf(fname, ".add.%d", i + add); + files[i] = open(fname, O_RDWR | O_CREAT, 0644); + if (files[i] < 0) { + perror("open"); + free(files); + files = NULL; + break; + } + } + if (extra) { + for (i = nr_files; i < nr_files + extra; i++) + files[i] = -1; + } + + return files; +} + +static int test_shrink(struct io_uring *ring) +{ + int ret, off, fd; + int *files; + + files = open_files(50, 0, 0); + ret = io_uring_register_files(ring, files, 50); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + + off = 0; + do { + fd = -1; + ret = io_uring_register_files_update(ring, off, &fd, 1); + if (ret != 1) { + if (off == 50 && ret == -EINVAL) + break; + fprintf(stderr, "%s: update ret=%d\n", __FUNCTION__, ret); + break; + } + off++; + } while (1); + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } + + close_files(files, 50, 0); + return 0; +err: + close_files(files, 50, 0); + return 1; +} + + +static int test_grow(struct io_uring *ring) +{ + int ret, off; + int *files, *fds = NULL; + + files = open_files(50, 250, 0); + ret = io_uring_register_files(ring, files, 300); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + + off = 50; + do { + fds = open_files(1, 0, off); + ret = io_uring_register_files_update(ring, off, fds, 1); + if (ret != 1) { + if (off == 300 && ret == -EINVAL) + break; + fprintf(stderr, "%s: update ret=%d\n", __FUNCTION__, ret); + break; + } + if (off >= 300) { + fprintf(stderr, "%s: Succeeded beyond end-of-list?\n", __FUNCTION__); + goto err; + } + off++; + } while (1); + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } + + close_files(files, 100, 0); + close_files(NULL, 251, 50); + return 0; +err: + close_files(files, 100, 0); + close_files(NULL, 251, 50); + return 1; +} + +static int test_replace_all(struct io_uring *ring) +{ + int *files, *fds = NULL; + int ret, i; + + files = open_files(100, 0, 0); + ret = io_uring_register_files(ring, files, 100); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + + fds = t_malloc(100 * sizeof(int)); + for (i = 0; i < 100; i++) + fds[i] = -1; + + ret = io_uring_register_files_update(ring, 0, fds, 100); + if (ret != 100) { + fprintf(stderr, "%s: update ret=%d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } + + close_files(files, 100, 0); + if (fds) + free(fds); + return 0; +err: + close_files(files, 100, 0); + if (fds) + free(fds); + return 1; +} + +static int test_replace(struct io_uring *ring) +{ + int *files, *fds = NULL; + int ret; + + files = open_files(100, 0, 0); + ret = io_uring_register_files(ring, files, 100); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + + fds = open_files(10, 0, 1); + ret = io_uring_register_files_update(ring, 90, fds, 10); + if (ret != 10) { + fprintf(stderr, "%s: update ret=%d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } + + close_files(files, 100, 0); + if (fds) + close_files(fds, 10, 1); + return 0; +err: + close_files(files, 100, 0); + if (fds) + close_files(fds, 10, 1); + return 1; +} + +static int test_removals(struct io_uring *ring) +{ + int *files, *fds = NULL; + int ret, i; + + files = open_files(100, 0, 0); + ret = io_uring_register_files(ring, files, 100); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + + fds = t_calloc(10, sizeof(int)); + for (i = 0; i < 10; i++) + fds[i] = -1; + + ret = io_uring_register_files_update(ring, 50, fds, 10); + if (ret != 10) { + fprintf(stderr, "%s: update ret=%d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } + + close_files(files, 100, 0); + if (fds) + free(fds); + return 0; +err: + close_files(files, 100, 0); + if (fds) + free(fds); + return 1; +} + +static int test_additions(struct io_uring *ring) +{ + int *files, *fds = NULL; + int ret; + + files = open_files(100, 100, 0); + ret = io_uring_register_files(ring, files, 200); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + + fds = open_files(2, 0, 1); + ret = io_uring_register_files_update(ring, 100, fds, 2); + if (ret != 2) { + fprintf(stderr, "%s: update ret=%d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } + + close_files(files, 100, 0); + if (fds) + close_files(fds, 2, 1); + return 0; +err: + close_files(files, 100, 0); + if (fds) + close_files(fds, 2, 1); + return 1; +} + +static int test_sparse(struct io_uring *ring) +{ + int *files; + int ret; + + files = open_files(100, 100, 0); + ret = io_uring_register_files(ring, files, 200); + if (ret) { + if (ret == -EBADF) { + fprintf(stdout, "Sparse files not supported, skipping\n"); + no_update = 1; + goto done; + } + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } +done: + close_files(files, 100, 0); + return 0; +err: + close_files(files, 100, 0); + return 1; +} + +static int test_basic_many(struct io_uring *ring) +{ + int *files; + int ret; + + files = open_files(768, 0, 0); + ret = io_uring_register_files(ring, files, 768); + if (ret) { + fprintf(stderr, "%s: register %d\n", __FUNCTION__, ret); + goto err; + } + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister %d\n", __FUNCTION__, ret); + goto err; + } + close_files(files, 768, 0); + return 0; +err: + close_files(files, 768, 0); + return 1; +} + +static int test_basic(struct io_uring *ring, int fail) +{ + int *files; + int ret; + int nr_files = fail ? 10 : 100; + + files = open_files(nr_files, 0, 0); + ret = io_uring_register_files(ring, files, 100); + if (ret) { + if (fail) { + if (ret == -EBADF || ret == -EFAULT) + return 0; + } + fprintf(stderr, "%s: register %d\n", __FUNCTION__, ret); + goto err; + } + if (fail) { + fprintf(stderr, "Registration succeeded, but expected fail\n"); + goto err; + } + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister %d\n", __FUNCTION__, ret); + goto err; + } + close_files(files, nr_files, 0); + return 0; +err: + close_files(files, nr_files, 0); + return 1; +} + +/* + * Register 0 files, but reserve space for 10. Then add one file. + */ +static int test_zero(struct io_uring *ring) +{ + int *files, *fds = NULL; + int ret; + + files = open_files(0, 10, 0); + ret = io_uring_register_files(ring, files, 10); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + + fds = open_files(1, 0, 1); + ret = io_uring_register_files_update(ring, 0, fds, 1); + if (ret != 1) { + fprintf(stderr, "%s: update ret=%d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } + + if (fds) + close_files(fds, 1, 1); + free(files); + return 0; +err: + if (fds) + close_files(fds, 1, 1); + free(files); + return 1; +} + +static int test_fixed_read_write(struct io_uring *ring, int index) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct iovec iov[2]; + int ret; + + iov[0].iov_base = t_malloc(4096); + iov[0].iov_len = 4096; + memset(iov[0].iov_base, 0x5a, 4096); + + iov[1].iov_base = t_malloc(4096); + iov[1].iov_len = 4096; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: failed to get sqe\n", __FUNCTION__); + return 1; + } + io_uring_prep_writev(sqe, index, &iov[0], 1, 0); + sqe->flags |= IOSQE_FIXED_FILE; + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: io_uring_wait_cqe=%d\n", __FUNCTION__, ret); + return 1; + } + if (cqe->res != 4096) { + fprintf(stderr, "%s: write cqe->res=%d\n", __FUNCTION__, cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: failed to get sqe\n", __FUNCTION__); + return 1; + } + io_uring_prep_readv(sqe, index, &iov[1], 1, 0); + sqe->flags |= IOSQE_FIXED_FILE; + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: io_uring_wait_cqe=%d\n", __FUNCTION__, ret); + return 1; + } + if (cqe->res != 4096) { + fprintf(stderr, "%s: read cqe->res=%d\n", __FUNCTION__, cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + + if (memcmp(iov[1].iov_base, iov[0].iov_base, 4096)) { + fprintf(stderr, "%s: data mismatch\n", __FUNCTION__); + return 1; + } + + free(iov[0].iov_base); + free(iov[1].iov_base); + return 0; +} + +static void adjust_nfiles(int want_files) +{ + struct rlimit rlim; + + if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) + return; + if (rlim.rlim_cur >= want_files) + return; + rlim.rlim_cur = want_files; + setrlimit(RLIMIT_NOFILE, &rlim); +} + +/* + * Register 8K of sparse files, update one at a random spot, then do some + * file IO to verify it works. + */ +static int test_huge(struct io_uring *ring) +{ + int *files; + int ret; + + adjust_nfiles(16384); + + files = open_files(0, 8192, 0); + ret = io_uring_register_files(ring, files, 8192); + if (ret) { + /* huge sets not supported */ + if (ret == -EMFILE) { + fprintf(stdout, "%s: No huge file set support, skipping\n", __FUNCTION__); + goto out; + } + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + + files[7193] = open(".reg.7193", O_RDWR | O_CREAT, 0644); + if (files[7193] < 0) { + fprintf(stderr, "%s: open=%d\n", __FUNCTION__, errno); + goto err; + } + + ret = io_uring_register_files_update(ring, 7193, &files[7193], 1); + if (ret != 1) { + fprintf(stderr, "%s: update ret=%d\n", __FUNCTION__, ret); + goto err; + } + + if (test_fixed_read_write(ring, 7193)) + goto err; + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } + + if (files[7193] != -1) { + close(files[7193]); + unlink(".reg.7193"); + } +out: + free(files); + return 0; +err: + if (files[7193] != -1) { + close(files[7193]); + unlink(".reg.7193"); + } + free(files); + return 1; +} + +static int test_skip(struct io_uring *ring) +{ + int *files; + int ret; + + files = open_files(100, 0, 0); + ret = io_uring_register_files(ring, files, 100); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + goto err; + } + + files[90] = IORING_REGISTER_FILES_SKIP; + ret = io_uring_register_files_update(ring, 90, &files[90], 1); + if (ret != 1) { + if (ret == -EBADF) { + fprintf(stdout, "Skipping files not supported\n"); + goto done; + } + fprintf(stderr, "%s: update ret=%d\n", __FUNCTION__, ret); + goto err; + } + + /* verify can still use file index 90 */ + if (test_fixed_read_write(ring, 90)) + goto err; + + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "%s: unregister ret=%d\n", __FUNCTION__, ret); + goto err; + } + +done: + close_files(files, 100, 0); + return 0; +err: + close_files(files, 100, 0); + return 1; +} + +static int test_sparse_updates(void) +{ + struct io_uring ring; + int ret, i, *fds, newfd; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "queue_init: %d\n", ret); + return ret; + } + + fds = t_malloc(256 * sizeof(int)); + for (i = 0; i < 256; i++) + fds[i] = -1; + + ret = io_uring_register_files(&ring, fds, 256); + if (ret) { + fprintf(stderr, "file_register: %d\n", ret); + return ret; + } + + newfd = 1; + for (i = 0; i < 256; i++) { + ret = io_uring_register_files_update(&ring, i, &newfd, 1); + if (ret != 1) { + fprintf(stderr, "file_update: %d\n", ret); + return ret; + } + } + io_uring_unregister_files(&ring); + + for (i = 0; i < 256; i++) + fds[i] = 1; + + ret = io_uring_register_files(&ring, fds, 256); + if (ret) { + fprintf(stderr, "file_register: %d\n", ret); + return ret; + } + + newfd = -1; + for (i = 0; i < 256; i++) { + ret = io_uring_register_files_update(&ring, i, &newfd, 1); + if (ret != 1) { + fprintf(stderr, "file_update: %d\n", ret); + return ret; + } + } + io_uring_unregister_files(&ring); + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_fixed_removal_ordering(void) +{ + char buffer[128]; + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + int ret, fd, i, fds[2]; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret < 0) { + fprintf(stderr, "failed to init io_uring: %s\n", strerror(-ret)); + return ret; + } + if (pipe(fds)) { + perror("pipe"); + return -1; + } + ret = io_uring_register_files(&ring, fds, 2); + if (ret) { + fprintf(stderr, "file_register: %d\n", ret); + return ret; + } + /* ring should have fds referenced, can close them */ + close(fds[0]); + close(fds[1]); + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + return 1; + } + /* outwait file recycling delay */ + ts.tv_sec = 3; + ts.tv_nsec = 0; + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->flags |= IOSQE_IO_LINK | IOSQE_IO_HARDLINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + printf("get sqe failed\n"); + return -1; + } + io_uring_prep_write(sqe, 1, buffer, sizeof(buffer), 0); + sqe->flags |= IOSQE_FIXED_FILE; + sqe->user_data = 2; + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "%s: got %d, wanted 2\n", __FUNCTION__, ret); + return -1; + } + + /* remove unused pipe end */ + fd = -1; + ret = io_uring_register_files_update(&ring, 0, &fd, 1); + if (ret != 1) { + fprintf(stderr, "update off=0 failed\n"); + return -1; + } + + /* remove used pipe end */ + fd = -1; + ret = io_uring_register_files_update(&ring, 1, &fd, 1); + if (ret != 1) { + fprintf(stderr, "update off=1 failed\n"); + return -1; + } + + for (i = 0; i < 2; ++i) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: io_uring_wait_cqe=%d\n", __FUNCTION__, ret); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + return 0; +} + +/* mix files requiring SCM-accounting and not in a single register */ +static int test_mixed_af_unix(void) +{ + struct io_uring ring; + int i, ret, fds[2]; + int reg_fds[32]; + int sp[2]; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret < 0) { + fprintf(stderr, "failed to init io_uring: %s\n", strerror(-ret)); + return ret; + } + if (pipe(fds)) { + perror("pipe"); + return -1; + } + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, sp) != 0) { + perror("Failed to create Unix-domain socket pair\n"); + return 1; + } + + for (i = 0; i < 16; i++) { + reg_fds[i * 2] = fds[0]; + reg_fds[i * 2 + 1] = sp[0]; + } + + ret = io_uring_register_files(&ring, reg_fds, 32); + if (ret) { + fprintf(stderr, "file_register: %d\n", ret); + return ret; + } + + close(fds[0]); + close(fds[1]); + close(sp[0]); + close(sp[1]); + io_uring_queue_exit(&ring); + return 0; +} + +static int test_partial_register_fail(void) +{ + char buffer[128]; + struct io_uring ring; + int ret, fds[2]; + int reg_fds[5]; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret < 0) { + fprintf(stderr, "failed to init io_uring: %s\n", strerror(-ret)); + return ret; + } + if (pipe(fds)) { + perror("pipe"); + return -1; + } + + /* + * Expect register to fail as it doesn't support io_uring fds, shouldn't + * leave any fds referenced afterwards. + */ + reg_fds[0] = fds[0]; + reg_fds[1] = fds[1]; + reg_fds[2] = -1; + reg_fds[3] = ring.ring_fd; + reg_fds[4] = -1; + ret = io_uring_register_files(&ring, reg_fds, 5); + if (!ret) { + fprintf(stderr, "file_register unexpectedly succeeded\n"); + return 1; + } + + /* ring should have fds referenced, can close them */ + close(fds[1]); + + /* confirm that fds[1] is actually close and to ref'ed by io_uring */ + ret = read(fds[0], buffer, 10); + if (ret < 0) + perror("read"); + close(fds[0]); + io_uring_queue_exit(&ring); + return 0; +} + +static int file_update_alloc(struct io_uring *ring, int *fd) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_files_update(sqe, fd, 1, IORING_FILE_INDEX_ALLOC); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return -1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: io_uring_wait_cqe=%d\n", __FUNCTION__, ret); + return -1; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +} + +static int test_out_of_range_file_ranges(struct io_uring *ring) +{ + int ret; + + ret = io_uring_register_file_alloc_range(ring, 8, 3); + if (ret != -EINVAL) { + fprintf(stderr, "overlapping range %i\n", ret); + return 1; + } + + ret = io_uring_register_file_alloc_range(ring, 10, 1); + if (ret != -EINVAL) { + fprintf(stderr, "out of range index %i\n", ret); + return 1; + } + + ret = io_uring_register_file_alloc_range(ring, 7, ~1U); + if (ret != -EOVERFLOW) { + fprintf(stderr, "overflow %i\n", ret); + return 1; + } + + return 0; +} + +static int test_overallocating_file_range(struct io_uring *ring, int fds[2]) +{ + int roff = 7, rlen = 2; + int ret, i, fd; + + ret = io_uring_register_file_alloc_range(ring, roff, rlen); + if (ret) { + fprintf(stderr, "io_uring_register_file_alloc_range %i\n", ret); + return 1; + } + + for (i = 0; i < rlen; i++) { + fd = fds[0]; + ret = file_update_alloc(ring, &fd); + if (ret != 1) { + fprintf(stderr, "file_update_alloc\n"); + return 1; + } + + if (fd < roff || fd >= roff + rlen) { + fprintf(stderr, "invalid off result %i\n", fd); + return 1; + } + } + + fd = fds[0]; + ret = file_update_alloc(ring, &fd); + if (ret != -ENFILE) { + fprintf(stderr, "overallocated %i, off %i\n", ret, fd); + return 1; + } + + return 0; +} + +static int test_zero_range_alloc(struct io_uring *ring, int fds[2]) +{ + int ret, fd; + + ret = io_uring_register_file_alloc_range(ring, 7, 0); + if (ret) { + fprintf(stderr, "io_uring_register_file_alloc_range failed %i\n", ret); + return 1; + } + + fd = fds[0]; + ret = file_update_alloc(ring, &fd); + if (ret != -ENFILE) { + fprintf(stderr, "zero alloc %i\n", ret); + return 1; + } + return 0; +} + +static int test_file_alloc_ranges(void) +{ + struct io_uring ring; + int ret, pipe_fds[2]; + + if (pipe(pipe_fds)) { + fprintf(stderr, "pipes\n"); + return 1; + } + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "queue_init: %d\n", ret); + return 1; + } + + ret = io_uring_register_files_sparse(&ring, 10); + if (ret == -EINVAL) { +not_supported: + close(pipe_fds[0]); + close(pipe_fds[1]); + io_uring_queue_exit(&ring); + printf("file alloc ranges are not supported, skip\n"); + return 0; + } else if (ret) { + fprintf(stderr, "io_uring_register_files_sparse %i\n", ret); + return ret; + } + + ret = io_uring_register_file_alloc_range(&ring, 0, 1); + if (ret) { + if (ret == -EINVAL) + goto not_supported; + fprintf(stderr, "io_uring_register_file_alloc_range %i\n", ret); + return 1; + } + + ret = test_overallocating_file_range(&ring, pipe_fds); + if (ret) { + fprintf(stderr, "test_overallocating_file_range() failed\n"); + return 1; + } + + ret = test_out_of_range_file_ranges(&ring); + if (ret) { + fprintf(stderr, "test_out_of_range_file_ranges() failed\n"); + return 1; + } + + ret = test_zero_range_alloc(&ring, pipe_fds); + if (ret) { + fprintf(stderr, "test_zero_range_alloc() failed\n"); + return 1; + } + + close(pipe_fds[0]); + close(pipe_fds[1]); + io_uring_queue_exit(&ring); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return T_EXIT_FAIL; + } + + ret = test_basic(&ring, 0); + if (ret) { + fprintf(stderr, "test_basic failed\n"); + return T_EXIT_FAIL; + } + + ret = test_basic(&ring, 1); + if (ret) { + fprintf(stderr, "test_basic failed\n"); + return T_EXIT_FAIL; + } + + ret = test_basic_many(&ring); + if (ret) { + fprintf(stderr, "test_basic_many failed\n"); + return T_EXIT_FAIL; + } + + ret = test_sparse(&ring); + if (ret) { + fprintf(stderr, "test_sparse failed\n"); + return T_EXIT_FAIL; + } + + if (no_update) + return T_EXIT_SKIP; + + ret = test_additions(&ring); + if (ret) { + fprintf(stderr, "test_additions failed\n"); + return T_EXIT_FAIL; + } + + ret = test_removals(&ring); + if (ret) { + fprintf(stderr, "test_removals failed\n"); + return T_EXIT_FAIL; + } + + ret = test_replace(&ring); + if (ret) { + fprintf(stderr, "test_replace failed\n"); + return T_EXIT_FAIL; + } + + ret = test_replace_all(&ring); + if (ret) { + fprintf(stderr, "test_replace_all failed\n"); + return T_EXIT_FAIL; + } + + ret = test_grow(&ring); + if (ret) { + fprintf(stderr, "test_grow failed\n"); + return T_EXIT_FAIL; + } + + ret = test_shrink(&ring); + if (ret) { + fprintf(stderr, "test_shrink failed\n"); + return T_EXIT_FAIL; + } + + ret = test_zero(&ring); + if (ret) { + fprintf(stderr, "test_zero failed\n"); + return T_EXIT_FAIL; + } + + ret = test_huge(&ring); + if (ret) { + fprintf(stderr, "test_huge failed\n"); + return T_EXIT_FAIL; + } + + ret = test_skip(&ring); + if (ret) { + fprintf(stderr, "test_skip failed\n"); + return T_EXIT_FAIL; + } + + ret = test_sparse_updates(); + if (ret) { + fprintf(stderr, "test_sparse_updates failed\n"); + return T_EXIT_FAIL; + } + + ret = test_fixed_removal_ordering(); + if (ret) { + fprintf(stderr, "test_fixed_removal_ordering failed\n"); + return T_EXIT_FAIL; + } + + ret = test_mixed_af_unix(); + if (ret) { + fprintf(stderr, "test_mixed_af_unix failed\n"); + return T_EXIT_FAIL; + } + + ret = test_partial_register_fail(); + if (ret) { + fprintf(stderr, "test_partial_register_fail failed\n"); + return T_EXIT_FAIL; + } + + ret = test_file_alloc_ranges(); + if (ret) { + fprintf(stderr, "test_partial_register_fail failed\n"); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/file-update.c b/contrib/libs/liburing/test/file-update.c new file mode 100644 index 0000000000..0896807ddf --- /dev/null +++ b/contrib/libs/liburing/test/file-update.c @@ -0,0 +1,232 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various file registration tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "helpers.h" +#include "liburing.h" + +static void close_files(int *files, int nr_files, int add) +{ + char fname[32]; + int i; + + for (i = 0; i < nr_files; i++) { + if (files) + close(files[i]); + if (!add) + sprintf(fname, ".reg.%d", i); + else + sprintf(fname, ".add.%d", i + add); + unlink(fname); + } + if (files) + free(files); +} + +static int *open_files(int nr_files, int extra, int add) +{ + char fname[32]; + int *files; + int i; + + files = t_calloc(nr_files + extra, sizeof(int)); + + for (i = 0; i < nr_files; i++) { + if (!add) + sprintf(fname, ".reg.%d", i); + else + sprintf(fname, ".add.%d", i + add); + files[i] = open(fname, O_RDWR | O_CREAT, 0644); + if (files[i] < 0) { + perror("open"); + free(files); + files = NULL; + break; + } + } + if (extra) { + for (i = nr_files; i < nr_files + extra; i++) + files[i] = -1; + } + + return files; +} + +static int test_update_multiring(struct io_uring *r1, struct io_uring *r2, + struct io_uring *r3, int do_unreg) +{ + int *fds, *newfds; + + fds = open_files(10, 0, 0); + newfds = open_files(10, 0, 1); + + if (io_uring_register_files(r1, fds, 10) || + io_uring_register_files(r2, fds, 10) || + io_uring_register_files(r3, fds, 10)) { + fprintf(stderr, "%s: register files failed\n", __FUNCTION__); + goto err; + } + + if (io_uring_register_files_update(r1, 0, newfds, 10) != 10 || + io_uring_register_files_update(r2, 0, newfds, 10) != 10 || + io_uring_register_files_update(r3, 0, newfds, 10) != 10) { + fprintf(stderr, "%s: update files failed\n", __FUNCTION__); + goto err; + } + + if (!do_unreg) + goto done; + + if (io_uring_unregister_files(r1) || + io_uring_unregister_files(r2) || + io_uring_unregister_files(r3)) { + fprintf(stderr, "%s: unregister files failed\n", __FUNCTION__); + goto err; + } + +done: + close_files(fds, 10, 0); + close_files(newfds, 10, 1); + return 0; +err: + close_files(fds, 10, 0); + close_files(newfds, 10, 1); + return 1; +} + +static int test_sqe_update(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int *fds, i, ret; + + fds = t_malloc(sizeof(int) * 10); + for (i = 0; i < 10; i++) + fds[i] = -1; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_files_update(sqe, fds, 10, 0); + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return 1; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + free(fds); + if (ret == -EINVAL) { + fprintf(stdout, "IORING_OP_FILES_UPDATE not supported, skipping\n"); + return T_EXIT_SKIP; + } + return ret != 10; +} + +static int test_update_no_table(void) +{ + int up_fd, fds[4] = {-1, 0, 1, 4}; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret; + + ret = t_create_ring(2, &ring, 0); + if (ret == T_SETUP_SKIP) + return T_EXIT_SKIP; + else if (ret != T_SETUP_OK) + return ret; + + ret = io_uring_register_files(&ring, fds, 4); + /* ignore other failures */ + if (ret && ret != -EBADF) { + fprintf(stderr, "Failed registering file table: %d\n", ret); + goto fail; + } + + sqe = io_uring_get_sqe(&ring); + up_fd = ring.ring_fd; + io_uring_prep_files_update(sqe, &up_fd, 1, -1); //offset = -1 + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "Failed submit: %d\n", ret); + goto fail; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "Failed wait: %d\n", ret); + goto fail; + } + ret = cqe->res; + io_uring_cqe_seen(&ring, cqe); + if (ret != -EMFILE && ret != -EINVAL && ret != -EOVERFLOW && + ret != -ENXIO) { + fprintf(stderr, "Bad cqe res: %d\n", ret); + goto fail; + } + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +fail: + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} + +int main(int argc, char *argv[]) +{ + struct io_uring r1, r2, r3; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + if (io_uring_queue_init(8, &r1, 0) || + io_uring_queue_init(8, &r2, 0) || + io_uring_queue_init(8, &r3, 0)) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + ret = test_update_multiring(&r1, &r2, &r3, 1); + if (ret) { + fprintf(stderr, "test_update_multiring w/unreg\n"); + return ret; + } + + ret = test_update_multiring(&r1, &r2, &r3, 0); + if (ret) { + fprintf(stderr, "test_update_multiring wo/unreg\n"); + return ret; + } + + ret = test_sqe_update(&r1); + if (ret) { + if (ret != T_EXIT_SKIP) + fprintf(stderr, "test_sqe_update failed\n"); + return ret; + } + + ret = test_update_no_table(); + if (ret) { + if (ret != T_EXIT_SKIP) + fprintf(stderr, "test_sqe_update failed\n"); + return ret; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/file-verify.c b/contrib/libs/liburing/test/file-verify.c new file mode 100644 index 0000000000..7950b739cc --- /dev/null +++ b/contrib/libs/liburing/test/file-verify.c @@ -0,0 +1,634 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various reads tests, verifying data + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <assert.h> +#include <string.h> +#include <sys/ioctl.h> +#include <sys/stat.h> +#include <linux/fs.h> + +#include "helpers.h" +#include "liburing.h" + +#define FSIZE 128*1024*1024 +#define CHUNK_SIZE 131072 +#define PUNCH_SIZE 32768 + +/* + * 8 because it fits within the on-stack iov, 16 because it's larger than 8 + */ +#define MIN_VECS 8 +#define MAX_VECS 16 + +/* + * Can be anything, let's just do something for a bit of parallellism + */ +#define READ_BATCH 16 + +/* + * Each offset in the file has the offset / sizeof(int) stored for every + * sizeof(int) address. + */ +static int verify_buf(void *buf, size_t size, off_t off) +{ + int i, u_in_buf = size / sizeof(unsigned int); + unsigned int *ptr; + + off /= sizeof(unsigned int); + ptr = buf; + for (i = 0; i < u_in_buf; i++) { + if (off != *ptr) { + fprintf(stderr, "Found %u, wanted %lu\n", *ptr, off); + return 1; + } + ptr++; + off++; + } + + return 0; +} + +static int test_truncate(struct io_uring *ring, const char *fname, int buffered, + int vectored, int provide_buf) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct iovec vec; + struct stat sb; + off_t punch_off, off, file_size; + void *buf = NULL; + int u_in_buf, i, ret, fd, first_pass = 1; + unsigned int *ptr; + + if (buffered) + fd = open(fname, O_RDWR); + else + fd = open(fname, O_DIRECT | O_RDWR); + if (fd < 0) { + perror("open"); + return 1; + } + + if (fstat(fd, &sb) < 0) { + perror("stat"); + close(fd); + return 1; + } + + if (S_ISREG(sb.st_mode)) { + file_size = sb.st_size; + } else if (S_ISBLK(sb.st_mode)) { + unsigned long long bytes; + + if (ioctl(fd, BLKGETSIZE64, &bytes) < 0) { + perror("ioctl"); + close(fd); + return 1; + } + file_size = bytes; + } else { + goto out; + } + + if (file_size < CHUNK_SIZE) + goto out; + + t_posix_memalign(&buf, 4096, CHUNK_SIZE); + + off = file_size - (CHUNK_SIZE / 2); + punch_off = off + CHUNK_SIZE / 4; + + u_in_buf = CHUNK_SIZE / sizeof(unsigned int); + ptr = buf; + for (i = 0; i < u_in_buf; i++) { + *ptr = i; + ptr++; + } + ret = pwrite(fd, buf, CHUNK_SIZE / 2, off); + if (ret < 0) { + perror("pwrite"); + goto err; + } else if (ret != CHUNK_SIZE / 2) + goto out; + +again: + /* + * Read in last bit of file so it's known cached, then remove half of that + * last bit so we get a short read that needs retry + */ + ret = pread(fd, buf, CHUNK_SIZE / 2, off); + if (ret < 0) { + perror("pread"); + goto err; + } else if (ret != CHUNK_SIZE / 2) + goto out; + + if (posix_fadvise(fd, punch_off, CHUNK_SIZE / 4, POSIX_FADV_DONTNEED) < 0) { + perror("posix_fadivse"); + goto err; + } + + if (provide_buf) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_provide_buffers(sqe, buf, CHUNK_SIZE, 1, 0, 0); + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "submit failed %d\n", ret); + goto err; + } + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (ret) { + fprintf(stderr, "Provide buffer failed %d\n", ret); + goto err; + } + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + if (vectored) { + assert(!provide_buf); + vec.iov_base = buf; + vec.iov_len = CHUNK_SIZE; + io_uring_prep_readv(sqe, fd, &vec, 1, off); + } else { + if (provide_buf) { + io_uring_prep_read(sqe, fd, NULL, CHUNK_SIZE, off); + sqe->flags |= IOSQE_BUFFER_SELECT; + } else { + io_uring_prep_read(sqe, fd, buf, CHUNK_SIZE, off); + } + } + memset(buf, 0, CHUNK_SIZE); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "Submit failed %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (ret != CHUNK_SIZE / 2) { + fprintf(stderr, "Unexpected truncated read %d\n", ret); + goto err; + } + + if (verify_buf(buf, CHUNK_SIZE / 2, 0)) + goto err; + + /* + * Repeat, but punch first part instead of last + */ + if (first_pass) { + punch_off = file_size - CHUNK_SIZE / 4; + first_pass = 0; + goto again; + } + +out: + free(buf); + close(fd); + return 0; +err: + free(buf); + close(fd); + return 1; +} + +enum { + PUNCH_NONE, + PUNCH_FRONT, + PUNCH_MIDDLE, + PUNCH_END, +}; + +/* + * For each chunk in file, DONTNEED a start, end, or middle segment of it. + * We enter here with the file fully cached every time, either freshly + * written or after other reads. This forces (at least) the buffered reads + * to be handled incrementally, exercising that path. + */ +static int do_punch(int fd) +{ + off_t offset = 0; + int punch_type; + + while (offset + CHUNK_SIZE <= FSIZE) { + off_t punch_off; + + punch_type = rand() % (PUNCH_END + 1); + switch (punch_type) { + default: + case PUNCH_NONE: + punch_off = -1; /* gcc... */ + break; + case PUNCH_FRONT: + punch_off = offset; + break; + case PUNCH_MIDDLE: + punch_off = offset + PUNCH_SIZE; + break; + case PUNCH_END: + punch_off = offset + CHUNK_SIZE - PUNCH_SIZE; + break; + } + + offset += CHUNK_SIZE; + if (punch_type == PUNCH_NONE) + continue; + if (posix_fadvise(fd, punch_off, PUNCH_SIZE, POSIX_FADV_DONTNEED) < 0) { + perror("posix_fadivse"); + return 1; + } + } + + return 0; +} + +static int provide_buffers(struct io_uring *ring, void **buf) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int i, ret; + + /* real use case would have one buffer chopped up, but... */ + for (i = 0; i < READ_BATCH; i++) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_provide_buffers(sqe, buf[i], CHUNK_SIZE, 1, 0, i); + } + + ret = io_uring_submit(ring); + if (ret != READ_BATCH) { + fprintf(stderr, "Submit failed %d\n", ret); + return 1; + } + + for (i = 0; i < READ_BATCH; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe %d\n", ret); + return 1; + } + if (cqe->res < 0) { + fprintf(stderr, "cqe res provide %d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +} + +static int test(struct io_uring *ring, const char *fname, int buffered, + int vectored, int small_vecs, int registered, int provide) +{ + struct iovec vecs[READ_BATCH][MAX_VECS]; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + void *buf[READ_BATCH]; + int ret, fd, flags; + int i, j, nr_vecs; + off_t off, voff; + size_t left; + + if (registered) { + assert(!provide); + assert(!vectored && !small_vecs); + } + if (provide) { + assert(!registered); + assert(!vectored && !small_vecs); + } + + flags = O_RDONLY; + if (!buffered) + flags |= O_DIRECT; + fd = open(fname, flags); + if (fd < 0) { + perror("open"); + return 1; + } + + if (do_punch(fd)) + return 1; + + if (vectored) { + if (small_vecs) + nr_vecs = MIN_VECS; + else + nr_vecs = MAX_VECS; + + for (j = 0; j < READ_BATCH; j++) { + for (i = 0; i < nr_vecs; i++) { + void *ptr; + + t_posix_memalign(&ptr, 4096, CHUNK_SIZE / nr_vecs); + vecs[j][i].iov_base = ptr; + vecs[j][i].iov_len = CHUNK_SIZE / nr_vecs; + } + } + } else { + for (j = 0; j < READ_BATCH; j++) + t_posix_memalign(&buf[j], 4096, CHUNK_SIZE); + nr_vecs = 0; + } + + if (registered) { + struct iovec v[READ_BATCH]; + + for (i = 0; i < READ_BATCH; i++) { + v[i].iov_base = buf[i]; + v[i].iov_len = CHUNK_SIZE; + } + ret = io_uring_register_buffers(ring, v, READ_BATCH); + if (ret) { + fprintf(stderr, "Error buffer reg %d\n", ret); + goto err; + } + } + + i = 0; + left = FSIZE; + off = 0; + while (left) { + int pending = 0; + + if (provide && provide_buffers(ring, buf)) + goto err; + + for (i = 0; i < READ_BATCH; i++) { + size_t this = left; + + if (this > CHUNK_SIZE) + this = CHUNK_SIZE; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + if (vectored) { + io_uring_prep_readv(sqe, fd, vecs[i], nr_vecs, off); + } else { + if (registered) { + io_uring_prep_read_fixed(sqe, fd, buf[i], this, off, i); + } else if (provide) { + io_uring_prep_read(sqe, fd, NULL, this, off); + sqe->flags |= IOSQE_BUFFER_SELECT; + } else { + io_uring_prep_read(sqe, fd, buf[i], this, off); + } + } + sqe->user_data = ((uint64_t)off << 32) | i; + off += this; + left -= this; + pending++; + if (!left) + break; + } + + ret = io_uring_submit(ring); + if (ret != pending) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < pending; i++) { + int index; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res < 0) { + fprintf(stderr, "bad read %d, read %d\n", cqe->res, i); + goto err; + } + if (cqe->res < CHUNK_SIZE) { + fprintf(stderr, "short read %d, read %d\n", cqe->res, i); + goto err; + } + if (cqe->flags & IORING_CQE_F_BUFFER) + index = cqe->flags >> 16; + else + index = cqe->user_data & 0xffffffff; + voff = cqe->user_data >> 32; + io_uring_cqe_seen(ring, cqe); + if (vectored) { + for (j = 0; j < nr_vecs; j++) { + void *buf = vecs[index][j].iov_base; + size_t len = vecs[index][j].iov_len; + + if (verify_buf(buf, len, voff)) + goto err; + voff += len; + } + } else { + if (verify_buf(buf[index], CHUNK_SIZE, voff)) + goto err; + } + } + } + + ret = 0; +done: + if (registered) + io_uring_unregister_buffers(ring); + if (vectored) { + for (j = 0; j < READ_BATCH; j++) + for (i = 0; i < nr_vecs; i++) + free(vecs[j][i].iov_base); + } else { + for (j = 0; j < READ_BATCH; j++) + free(buf[j]); + } + close(fd); + return ret; +err: + ret = 1; + goto done; +} + +static int fill_pattern(const char *fname) +{ + size_t left = FSIZE; + unsigned int val, *ptr; + void *buf; + int fd, i; + + fd = open(fname, O_WRONLY); + if (fd < 0) { + perror("open"); + return 1; + } + + val = 0; + buf = t_malloc(4096); + while (left) { + int u_in_buf = 4096 / sizeof(val); + size_t this = left; + + if (this > 4096) + this = 4096; + ptr = buf; + for (i = 0; i < u_in_buf; i++) { + *ptr = val; + val++; + ptr++; + } + if (write(fd, buf, 4096) != 4096) + return 1; + left -= 4096; + } + + fsync(fd); + close(fd); + free(buf); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + const char *fname; + char buf[32]; + int ret; + + srand(getpid()); + + if (argc > 1) { + fname = argv[1]; + } else { + sprintf(buf, ".file-verify.%d", getpid()); + fname = buf; + t_create_file(fname, FSIZE); + } + + ret = io_uring_queue_init(READ_BATCH, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + goto err; + } + + if (fill_pattern(fname)) + goto err; + + ret = test(&ring, fname, 1, 0, 0, 0, 0); + if (ret) { + fprintf(stderr, "Buffered novec test failed\n"); + goto err; + } + ret = test(&ring, fname, 1, 0, 0, 1, 0); + if (ret) { + fprintf(stderr, "Buffered novec reg test failed\n"); + goto err; + } + ret = test(&ring, fname, 1, 0, 0, 0, 1); + if (ret) { + fprintf(stderr, "Buffered novec provide test failed\n"); + goto err; + } + ret = test(&ring, fname, 1, 1, 0, 0, 0); + if (ret) { + fprintf(stderr, "Buffered vec test failed\n"); + goto err; + } + ret = test(&ring, fname, 1, 1, 1, 0, 0); + if (ret) { + fprintf(stderr, "Buffered small vec test failed\n"); + goto err; + } + + ret = test(&ring, fname, 0, 0, 0, 0, 0); + if (ret) { + fprintf(stderr, "O_DIRECT novec test failed\n"); + goto err; + } + ret = test(&ring, fname, 0, 0, 0, 1, 0); + if (ret) { + fprintf(stderr, "O_DIRECT novec reg test failed\n"); + goto err; + } + ret = test(&ring, fname, 0, 0, 0, 0, 1); + if (ret) { + fprintf(stderr, "O_DIRECT novec provide test failed\n"); + goto err; + } + ret = test(&ring, fname, 0, 1, 0, 0, 0); + if (ret) { + fprintf(stderr, "O_DIRECT vec test failed\n"); + goto err; + } + ret = test(&ring, fname, 0, 1, 1, 0, 0); + if (ret) { + fprintf(stderr, "O_DIRECT small vec test failed\n"); + goto err; + } + + ret = test_truncate(&ring, fname, 1, 0, 0); + if (ret) { + fprintf(stderr, "Buffered end truncate read failed\n"); + goto err; + } + ret = test_truncate(&ring, fname, 1, 1, 0); + if (ret) { + fprintf(stderr, "Buffered end truncate vec read failed\n"); + goto err; + } + ret = test_truncate(&ring, fname, 1, 0, 1); + if (ret) { + fprintf(stderr, "Buffered end truncate pbuf read failed\n"); + goto err; + } + + ret = test_truncate(&ring, fname, 0, 0, 0); + if (ret) { + fprintf(stderr, "O_DIRECT end truncate read failed\n"); + goto err; + } + ret = test_truncate(&ring, fname, 0, 1, 0); + if (ret) { + fprintf(stderr, "O_DIRECT end truncate vec read failed\n"); + goto err; + } + ret = test_truncate(&ring, fname, 0, 0, 1); + if (ret) { + fprintf(stderr, "O_DIRECT end truncate pbuf read failed\n"); + goto err; + } + + if (buf == fname) + unlink(fname); + return T_EXIT_PASS; +err: + if (buf == fname) + unlink(fname); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/files-exit-hang-poll.c b/contrib/libs/liburing/test/files-exit-hang-poll.c new file mode 100644 index 0000000000..2428a10283 --- /dev/null +++ b/contrib/libs/liburing/test/files-exit-hang-poll.c @@ -0,0 +1,115 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Based on a test case from Josef Grieb - test that we can exit without + * hanging if we have the task file table pinned by a request that is linked + * to another request that doesn't finish. + */ +#include <errno.h> +#include <fcntl.h> +#include <netinet/in.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <strings.h> +#include <sys/socket.h> +#include <unistd.h> +#include <poll.h> +#include "liburing.h" +#include "helpers.h" + +#define BACKLOG 512 + +static struct io_uring ring; + +static void add_poll(struct io_uring *ring, int fd) +{ + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_poll_add(sqe, fd, POLLIN); + sqe->flags |= IOSQE_IO_LINK; +} + +static void add_accept(struct io_uring *ring, int fd) +{ + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_accept(sqe, fd, 0, 0, SOCK_NONBLOCK | SOCK_CLOEXEC); +} + +static int setup_io_uring(void) +{ + int ret; + + ret = io_uring_queue_init(16, &ring, 0); + if (ret) { + fprintf(stderr, "Unable to setup io_uring: %s\n", strerror(-ret)); + return 1; + } + + return 0; +} + +static void alarm_sig(int sig) +{ + exit(0); +} + +int main(int argc, char *argv[]) +{ + struct sockaddr_in serv_addr; + struct io_uring_cqe *cqe; + int ret, sock_listen_fd; + const int val = 1; + + if (argc > 1) + return T_EXIT_SKIP; + + sock_listen_fd = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0); + if (sock_listen_fd < 0) { + perror("socket"); + return T_EXIT_FAIL; + } + + setsockopt(sock_listen_fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + + memset(&serv_addr, 0, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_addr.s_addr = INADDR_ANY; + + if (t_bind_ephemeral_port(sock_listen_fd, &serv_addr)) { + perror("bind"); + return T_EXIT_FAIL; + } + + if (listen(sock_listen_fd, BACKLOG) < 0) { + perror("Error listening on socket\n"); + return T_EXIT_FAIL; + } + + if (setup_io_uring()) + return T_EXIT_FAIL; + + add_poll(&ring, sock_listen_fd); + add_accept(&ring, sock_listen_fd); + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "submit=%d\n", ret); + return T_EXIT_FAIL; + } + + signal(SIGALRM, alarm_sig); + alarm(1); + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return T_EXIT_FAIL; + } + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/files-exit-hang-timeout.c b/contrib/libs/liburing/test/files-exit-hang-timeout.c new file mode 100644 index 0000000000..708e42cded --- /dev/null +++ b/contrib/libs/liburing/test/files-exit-hang-timeout.c @@ -0,0 +1,138 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Based on a test case from Josef Grieb - test that we can exit without + * hanging if we have the task file table pinned by a request that is linked + * to another request that doesn't finish. + */ +#include <errno.h> +#include <fcntl.h> +#include <netinet/in.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <strings.h> +#include <sys/socket.h> +#include <unistd.h> +#include <poll.h> +#include "liburing.h" +#include "helpers.h" + +#define BACKLOG 512 + +#define PORT 9100 + +struct io_uring ring; + +struct __kernel_timespec ts = { + .tv_sec = 300, + .tv_nsec = 0, +}; + +static void add_timeout(struct io_uring *ring, int fd) +{ + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_timeout(sqe, &ts, 100, 0); + sqe->flags |= IOSQE_IO_LINK; +} + +static void add_accept(struct io_uring *ring, int fd) +{ + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_accept(sqe, fd, 0, 0, SOCK_NONBLOCK | SOCK_CLOEXEC); + sqe->flags |= IOSQE_IO_LINK; +} + +static int setup_io_uring(void) +{ + int ret; + + ret = io_uring_queue_init(16, &ring, 0); + if (ret) { + fprintf(stderr, "Unable to setup io_uring: %s\n", strerror(-ret)); + return 1; + } + + return 0; +} + +static void alarm_sig(int sig) +{ + exit(0); +} + +int main(int argc, char *argv[]) +{ + struct sockaddr_in serv_addr; + struct io_uring_cqe *cqe; + int ret, sock_listen_fd; + const int val = 1; + int i; + + if (argc > 1) + return T_EXIT_SKIP; + + sock_listen_fd = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0); + if (sock_listen_fd < 0) { + perror("socket"); + return T_EXIT_FAIL; + } + + setsockopt(sock_listen_fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + + memset(&serv_addr, 0, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_addr.s_addr = INADDR_ANY; + + for (i = 0; i < 100; i++) { + serv_addr.sin_port = htons(PORT + i); + + ret = bind(sock_listen_fd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)); + if (!ret) + break; + if (errno != EADDRINUSE) { + fprintf(stderr, "bind: %s\n", strerror(errno)); + return T_EXIT_FAIL; + } + if (i == 99) { + printf("Gave up on finding a port, skipping\n"); + goto skip; + } + } + + if (listen(sock_listen_fd, BACKLOG) < 0) { + perror("Error listening on socket\n"); + return T_EXIT_FAIL; + } + + if (setup_io_uring()) + return T_EXIT_FAIL; + + add_timeout(&ring, sock_listen_fd); + add_accept(&ring, sock_listen_fd); + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "submit=%d\n", ret); + return T_EXIT_FAIL; + } + + signal(SIGALRM, alarm_sig); + alarm(1); + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return T_EXIT_FAIL; + } + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +skip: + io_uring_queue_exit(&ring); + return T_EXIT_SKIP; +} diff --git a/contrib/libs/liburing/test/fixed-buf-iter.c b/contrib/libs/liburing/test/fixed-buf-iter.c new file mode 100644 index 0000000000..1cdb327166 --- /dev/null +++ b/contrib/libs/liburing/test/fixed-buf-iter.c @@ -0,0 +1,116 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test fixed buffers with non-iterators. + * + * Taken from: https://github.com/axboe/liburing/issues/549 + */ +#include <stdio.h> +#include <string.h> +#include <fcntl.h> +#include <stdlib.h> + +#include "liburing.h" +#include "helpers.h" + +#define BUF_SIZE 4096 +#define BUFFERS 1 +#define IN_FD "/dev/urandom" +#define OUT_FD "/dev/zero" + +static int test(struct io_uring *ring) +{ + struct iovec iov[BUFFERS]; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, fd_in, fd_out, i; + + fd_in = open(IN_FD, O_RDONLY, 0644); + if (fd_in < 0) { + perror("open in"); + return 1; + } + + fd_out = open(OUT_FD, O_RDWR, 0644); + if (fd_out < 0) { + perror("open out"); + return 1; + } + + for (i = 0; i < BUFFERS; i++) { + iov[i].iov_base = malloc(BUF_SIZE); + iov[i].iov_len = BUF_SIZE; + memset(iov[i].iov_base, 0, BUF_SIZE); + } + + ret = io_uring_register_buffers(ring, iov, BUFFERS); + if (ret) { + fprintf(stderr, "Error registering buffers: %s", strerror(-ret)); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "Could not get SQE.\n"); + return 1; + } + + io_uring_prep_read_fixed(sqe, fd_in, iov[0].iov_base, BUF_SIZE, 0, 0); + io_uring_submit(ring); + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret)); + return 1; + } + + if (cqe->res < 0) { + fprintf(stderr, "Error in async operation: %s\n", strerror(-cqe->res)); + return 1; + } + io_uring_cqe_seen(ring, cqe); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "Could not get SQE.\n"); + return 1; + } + io_uring_prep_write_fixed(sqe, fd_out, iov[0].iov_base, BUF_SIZE, 0, 0); + io_uring_submit(ring); + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret)); + return 1; + } + if (cqe->res < 0) { + fprintf(stderr, "Error in async operation: %s\n", strerror(-cqe->res)); + return 1; + } + io_uring_cqe_seen(ring, cqe); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = t_create_ring(8, &ring, 0); + if (ret == T_SETUP_SKIP) + return T_EXIT_SKIP; + else if (ret < 0) + return T_EXIT_FAIL; + + ret = test(&ring); + if (ret) { + fprintf(stderr, "Test failed\n"); + return T_EXIT_FAIL; + } + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/fixed-link.c b/contrib/libs/liburing/test/fixed-link.c new file mode 100644 index 0000000000..3e9eb8c4f1 --- /dev/null +++ b/contrib/libs/liburing/test/fixed-link.c @@ -0,0 +1,91 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> + +#include "helpers.h" +#include "liburing.h" + +#define IOVECS_LEN 2 + +int main(int argc, char *argv[]) +{ + struct iovec iovecs[IOVECS_LEN]; + struct io_uring ring; + int i, fd, ret; + + if (argc > 1) + return T_EXIT_SKIP; + + fd = open("/dev/zero", O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Failed to open /dev/zero\n"); + return T_EXIT_FAIL; + } + + if (io_uring_queue_init(32, &ring, 0) < 0) { + fprintf(stderr, "Failed to init io_uring\n"); + close(fd); + return T_EXIT_FAIL; + } + + for (i = 0; i < IOVECS_LEN; ++i) { + iovecs[i].iov_base = t_malloc(64); + iovecs[i].iov_len = 64; + }; + + ret = io_uring_register_buffers(&ring, iovecs, IOVECS_LEN); + if (ret) { + fprintf(stderr, "Failed to register buffers\n"); + return T_EXIT_FAIL; + } + + for (i = 0; i < IOVECS_LEN; ++i) { + struct io_uring_sqe *sqe = io_uring_get_sqe(&ring); + const char *str = "#include <errno.h>"; + + iovecs[i].iov_len = strlen(str); + io_uring_prep_read_fixed(sqe, fd, iovecs[i].iov_base, strlen(str), 0, i); + if (i == 0) + io_uring_sqe_set_flags(sqe, IOSQE_IO_LINK); + io_uring_sqe_set_data(sqe, (void *)str); + } + + ret = io_uring_submit_and_wait(&ring, IOVECS_LEN); + if (ret < 0) { + fprintf(stderr, "Failed to submit IO\n"); + return T_EXIT_FAIL; + } else if (ret < 2) { + fprintf(stderr, "Submitted %d, wanted %d\n", ret, IOVECS_LEN); + return T_EXIT_FAIL; + } + + for (i = 0; i < IOVECS_LEN; i++) { + struct io_uring_cqe *cqe; + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return T_EXIT_FAIL; + } + if (cqe->res != iovecs[i].iov_len) { + fprintf(stderr, "read: wanted %ld, got %d\n", + (long) iovecs[i].iov_len, cqe->res); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(&ring, cqe); + } + + close(fd); + io_uring_queue_exit(&ring); + + for (i = 0; i < IOVECS_LEN; ++i) + free(iovecs[i].iov_base); + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/fixed-reuse.c b/contrib/libs/liburing/test/fixed-reuse.c new file mode 100644 index 0000000000..5383f48af1 --- /dev/null +++ b/contrib/libs/liburing/test/fixed-reuse.c @@ -0,0 +1,161 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: link <open file><read from file><close file> with an existing + * file present in the opened slot, verifying that we get the new file + * rather than the old one. + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +#define MAX_FILES 8 +#define FNAME1 ".slot.reuse.1" +#define FNAME2 ".slot.reuse.2" +#define PAT1 0xaa +#define PAT2 0x55 +#define BSIZE 4096 + +static int test(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + char buf[BSIZE]; + int ret, i; + + /* open FNAME1 in slot 0 */ + sqe = io_uring_get_sqe(ring); + io_uring_prep_openat_direct(sqe, AT_FDCWD, FNAME1, O_RDONLY, 0, 0); + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res != 0) { + fprintf(stderr, "open res %d\n", ret); + goto err; + } + io_uring_cqe_seen(ring, cqe); + + /* + * Now open FNAME2 in that same slot, verifying we get data from + * FNAME2 and not FNAME1. + */ + sqe = io_uring_get_sqe(ring); + io_uring_prep_openat_direct(sqe, AT_FDCWD, FNAME2, O_RDONLY, 0, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 2; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_read(sqe, 0, buf, sizeof(buf), 0); + sqe->flags |= IOSQE_FIXED_FILE; + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 3; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_close_direct(sqe, 0); + sqe->user_data = 4; + + ret = io_uring_submit(ring); + if (ret != 3) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 2: + if (cqe->res) { + fprintf(stderr, "bad open %d\n", cqe->res); + goto err; + } + break; + case 3: + if (cqe->res != sizeof(buf)) { + fprintf(stderr, "bad read %d\n", cqe->res); + goto err; + } + break; + case 4: + if (cqe->res) { + fprintf(stderr, "bad close %d\n", cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + for (i = 0; i < sizeof(buf); i++) { + if (buf[i] == PAT2) + continue; + fprintf(stderr, "Bad pattern %x at %d\n", buf[i], i); + goto err; + } + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct io_uring_params p = { }; + int ret, files[MAX_FILES]; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init_params(8, &ring, &p); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + if (!(p.features & IORING_FEAT_CQE_SKIP)) + return T_EXIT_SKIP; + + memset(files, -1, sizeof(files)); + ret = io_uring_register_files(&ring, files, ARRAY_SIZE(files)); + if (ret) { + fprintf(stderr, "Failed registering files\n"); + return T_EXIT_FAIL; + } + + t_create_file_pattern(FNAME1, 4096, PAT1); + t_create_file_pattern(FNAME2, 4096, PAT2); + + ret = test(&ring); + if (ret) { + fprintf(stderr, "test failed\n"); + goto err; + } + + unlink(FNAME1); + unlink(FNAME2); + return T_EXIT_PASS; +err: + unlink(FNAME1); + unlink(FNAME2); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/fpos.c b/contrib/libs/liburing/test/fpos.c new file mode 100644 index 0000000000..239a5b2959 --- /dev/null +++ b/contrib/libs/liburing/test/fpos.c @@ -0,0 +1,256 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring fpos handling + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <assert.h> + +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE 5000 +#define QUEUE_SIZE 2048 + +static void create_file(const char *file, size_t size) +{ + ssize_t ret; + char *buf; + size_t idx; + int fd; + + buf = t_malloc(size); + for (idx = 0; idx < size; ++idx) { + /* write 0 or 1 */ + buf[idx] = (unsigned char)(idx & 0x01); + } + + fd = open(file, O_WRONLY | O_CREAT, 0644); + assert(fd >= 0); + + ret = write(fd, buf, size); + fsync(fd); + close(fd); + free(buf); + assert(ret == size); +} + +static int test_read(struct io_uring *ring, bool async, int blocksize) +{ + int ret, fd, i; + bool done = false; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + loff_t current, expected = 0; + int count_ok; + int count_0 = 0, count_1 = 0; + unsigned char buff[QUEUE_SIZE * blocksize]; + unsigned char reordered[QUEUE_SIZE * blocksize]; + + memset(buff, 0, QUEUE_SIZE * blocksize); + memset(reordered, 0, QUEUE_SIZE * blocksize); + + create_file(".test_fpos_read", FILE_SIZE); + fd = open(".test_fpos_read", O_RDONLY); + unlink(".test_fpos_read"); + assert(fd >= 0); + + while (!done) { + for (i = 0; i < QUEUE_SIZE; ++i) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "no sqe\n"); + return -1; + } + io_uring_prep_read(sqe, fd, + buff + i * blocksize, + blocksize, -1); + sqe->user_data = i; + if (async) + sqe->flags |= IOSQE_ASYNC; + if (i != QUEUE_SIZE - 1) + sqe->flags |= IOSQE_IO_LINK; + } + ret = io_uring_submit_and_wait(ring, QUEUE_SIZE); + if (ret != QUEUE_SIZE) { + fprintf(stderr, "submit failed: %d\n", ret); + return 1; + } + count_ok = 0; + for (i = 0; i < QUEUE_SIZE; ++i) { + int res; + + ret = io_uring_peek_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "peek failed: %d\n", ret); + return ret; + } + assert(cqe->user_data < QUEUE_SIZE); + memcpy(reordered + count_ok, + buff + cqe->user_data * blocksize, blocksize); + res = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (res == 0) { + done = true; + } else if (res == -ECANCELED) { + /* cancelled, probably ok */ + } else if (res < 0 || res > blocksize) { + fprintf(stderr, "bad read: %d\n", res); + return -1; + } else { + expected += res; + count_ok += res; + } + } + ret = 0; + for (i = 0; i < count_ok; i++) { + if (reordered[i] == 1) { + count_1++; + } else if (reordered[i] == 0) { + count_0++; + } else { + fprintf(stderr, "odd read %d\n", + (int)reordered[i]); + ret = -1; + break; + } + } + if (labs(count_1 - count_0) > 1) { + fprintf(stderr, "inconsistent reads, got 0s:%d 1s:%d\n", + count_0, count_1); + ret = -1; + } + current = lseek(fd, 0, SEEK_CUR); + if (current != expected) { + fprintf(stderr, "f_pos incorrect, expected %ld have %ld\n", + (long) expected, (long) current); + ret = -1; + } + if (ret) + return ret; + } + return 0; +} + + +static int test_write(struct io_uring *ring, bool async, int blocksize) +{ + int ret, fd, i; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + bool fail = false; + loff_t current; + char data[blocksize+1]; + char readbuff[QUEUE_SIZE*blocksize+1]; + + fd = open(".test_fpos_write", O_RDWR | O_CREAT, 0644); + unlink(".test_fpos_write"); + assert(fd >= 0); + + for (i = 0; i < blocksize; i++) + data[i] = 'A' + i; + + data[blocksize] = '\0'; + + for (i = 0; i < QUEUE_SIZE; ++i) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "no sqe\n"); + return -1; + } + io_uring_prep_write(sqe, fd, data + (i % blocksize), 1, -1); + sqe->user_data = 1; + if (async) + sqe->flags |= IOSQE_ASYNC; + if (i != QUEUE_SIZE - 1) + sqe->flags |= IOSQE_IO_LINK; + } + ret = io_uring_submit_and_wait(ring, QUEUE_SIZE); + if (ret != QUEUE_SIZE) { + fprintf(stderr, "submit failed: %d\n", ret); + return 1; + } + for (i = 0; i < QUEUE_SIZE; ++i) { + int res; + + ret = io_uring_peek_cqe(ring, &cqe); + res = cqe->res; + if (ret) { + fprintf(stderr, "peek failed: %d\n", ret); + return ret; + } + io_uring_cqe_seen(ring, cqe); + if (!fail && res != 1) { + fprintf(stderr, "bad result %d\n", res); + fail = true; + } + } + current = lseek(fd, 0, SEEK_CUR); + if (current != QUEUE_SIZE) { + fprintf(stderr, "f_pos incorrect, expected %ld have %d\n", + (long) current, QUEUE_SIZE); + fail = true; + } + current = lseek(fd, 0, SEEK_SET); + if (current != 0) { + perror("seek to start"); + return -1; + } + ret = read(fd, readbuff, QUEUE_SIZE); + if (ret != QUEUE_SIZE) { + fprintf(stderr, "did not write enough: %d\n", ret); + return -1; + } + i = 0; + while (i < QUEUE_SIZE - blocksize) { + if (strncmp(readbuff + i, data, blocksize)) { + char bad[QUEUE_SIZE+1]; + + memcpy(bad, readbuff + i, blocksize); + bad[blocksize] = '\0'; + fprintf(stderr, "unexpected data %s\n", bad); + fail = true; + } + i += blocksize; + } + + return fail ? -1 : 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(QUEUE_SIZE, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return T_EXIT_FAIL; + } + + for (int test = 0; test < 8; test++) { + int async = test & 0x01; + int write = test & 0x02; + int blocksize = test & 0x04 ? 1 : 7; + + ret = write + ? test_write(&ring, !!async, blocksize) + : test_read(&ring, !!async, blocksize); + if (ret) { + fprintf(stderr, "failed %s async=%d blocksize=%d\n", + write ? "write" : "read", + async, blocksize); + return -1; + } + } + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/fsync.c b/contrib/libs/liburing/test/fsync.c new file mode 100644 index 0000000000..22cb705e48 --- /dev/null +++ b/contrib/libs/liburing/test/fsync.c @@ -0,0 +1,225 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring fsync handling + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "helpers.h" +#include "liburing.h" + +static int test_single_fsync(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + char buf[32]; + int fd, ret; + + sprintf(buf, "./XXXXXX"); + fd = mkstemp(buf); + if (fd < 0) { + perror("open"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_fsync(sqe, fd, 0); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + unlink(buf); + return 0; +err: + unlink(buf); + return 1; +} + +static int test_barrier_fsync(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct iovec iovecs[4]; + int i, fd, ret; + off_t off; + + fd = open("fsync-testfile", O_WRONLY | O_CREAT, 0644); + if (fd < 0) { + perror("open"); + return 1; + } + unlink("fsync-testfile"); + + for (i = 0; i < ARRAY_SIZE(iovecs); i++) { + iovecs[i].iov_base = t_malloc(4096); + iovecs[i].iov_len = 4096; + } + + off = 0; + for (i = 0; i < 4; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_writev(sqe, fd, &iovecs[i], 1, off); + sqe->user_data = 0; + off += 4096; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_fsync(sqe, fd, IORING_FSYNC_DATASYNC); + sqe->user_data = 1; + io_uring_sqe_set_flags(sqe, IOSQE_IO_DRAIN); + + ret = io_uring_submit(ring); + if (ret < 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } else if (ret < 5) { + fprintf(stderr, "Submitted only %d\n", ret); + goto err; + } + + for (i = 0; i < 5; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + /* kernel doesn't support IOSQE_IO_DRAIN */ + if (cqe->res == -EINVAL) + break; + if (i <= 3) { + if (cqe->user_data) { + fprintf(stderr, "Got fsync early?\n"); + goto err; + } + } else { + if (!cqe->user_data) { + fprintf(stderr, "Got write late?\n"); + goto err; + } + } + io_uring_cqe_seen(ring, cqe); + } + + + ret = 0; + goto out; +err: + ret = 1; +out: + for (i = 0; i < ARRAY_SIZE(iovecs); i++) + free(iovecs[i].iov_base); + return ret; +} + +#define FILE_SIZE 1024 + +static int test_sync_file_range(struct io_uring *ring) +{ + int ret, fd, save_errno; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + + t_create_file(".sync_file_range", FILE_SIZE); + + fd = open(".sync_file_range", O_RDWR); + save_errno = errno; + unlink(".sync_file_range"); + errno = save_errno; + if (fd < 0) { + perror("file open"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + return 1; + } + io_uring_prep_sync_file_range(sqe, fd, 0, 0, 0); + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "submit failed: %d\n", ret); + return 1; + } + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe failed: %d\n", ret); + return 1; + } + if (cqe->res) { + fprintf(stderr, "sfr failed: %d\n", cqe->res); + return 1; + } + + io_uring_cqe_seen(ring, cqe); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return T_EXIT_FAIL; + + } + + ret = test_single_fsync(&ring); + if (ret) { + fprintf(stderr, "test_single_fsync failed\n"); + return ret; + } + + ret = test_barrier_fsync(&ring); + if (ret) { + fprintf(stderr, "test_barrier_fsync failed\n"); + return ret; + } + + ret = test_sync_file_range(&ring); + if (ret) { + fprintf(stderr, "test_sync_file_range failed\n"); + return ret; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/hardlink.c b/contrib/libs/liburing/test/hardlink.c new file mode 100644 index 0000000000..1d85fe0046 --- /dev/null +++ b/contrib/libs/liburing/test/hardlink.c @@ -0,0 +1,141 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring linkat handling + */ +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" + + +static int do_linkat(struct io_uring *ring, const char *oldname, const char *newname) +{ + int ret; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + io_uring_prep_linkat(sqe, AT_FDCWD, oldname, AT_FDCWD, newname, 0); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqes(ring, &cqe, 1, 0, 0); + if (ret) { + fprintf(stderr, "wait_cqe failed: %d\n", ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +err: + return 1; +} + +int files_linked_ok(const char* fn1, const char *fn2) +{ + struct stat s1, s2; + + if (stat(fn1, &s1)) { + fprintf(stderr, "stat(%s): %s\n", fn1, strerror(errno)); + return 0; + } + if (stat(fn2, &s2)) { + fprintf(stderr, "stat(%s): %s\n", fn2, strerror(errno)); + return 0; + } + if (s1.st_dev != s2.st_dev || s1.st_ino != s2.st_ino) { + fprintf(stderr, "linked files have different device / inode numbers\n"); + return 0; + } + if (s1.st_nlink != 2 || s2.st_nlink != 2) { + fprintf(stderr, "linked files have unexpected links count\n"); + return 0; + } + return 1; +} + +int main(int argc, char *argv[]) +{ + static const char target[] = "io_uring-linkat-test-target"; + static const char linkname[] = "io_uring-linkat-test-link"; + int ret; + struct io_uring ring; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return ret; + } + + ret = open(target, O_CREAT | O_RDWR | O_EXCL, 0600); + if (ret < 0) { + perror("open"); + goto err; + } + if (write(ret, "linktest", 8) != 8) { + close(ret); + goto err1; + } + close(ret); + + ret = do_linkat(&ring, target, linkname); + if (ret < 0) { + if (ret == -EBADF || ret == -EINVAL) { + fprintf(stdout, "linkat not supported, skipping\n"); + goto skip; + } + fprintf(stderr, "linkat: %s\n", strerror(-ret)); + goto err1; + } else if (ret) { + goto err1; + } + + if (!files_linked_ok(linkname, target)) + goto err2; + + ret = do_linkat(&ring, target, linkname); + if (ret != -EEXIST) { + fprintf(stderr, "test_linkat linkname already exists failed: %d\n", ret); + goto err2; + } + + ret = do_linkat(&ring, target, "surely/this/does/not/exist"); + if (ret != -ENOENT) { + fprintf(stderr, "test_linkat no parent failed: %d\n", ret); + goto err2; + } + + unlinkat(AT_FDCWD, linkname, 0); + unlinkat(AT_FDCWD, target, 0); + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +skip: + unlinkat(AT_FDCWD, linkname, 0); + unlinkat(AT_FDCWD, target, 0); + io_uring_queue_exit(&ring); + return T_EXIT_SKIP; +err2: + unlinkat(AT_FDCWD, linkname, 0); +err1: + unlinkat(AT_FDCWD, target, 0); +err: + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/helpers.c b/contrib/libs/liburing/test/helpers.c new file mode 100644 index 0000000000..a29c7b5966 --- /dev/null +++ b/contrib/libs/liburing/test/helpers.c @@ -0,0 +1,269 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: Helpers for tests. + */ +#include <stdlib.h> +#include <assert.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/types.h> + +#include <arpa/inet.h> +#include <netinet/ip.h> +#include <netinet/tcp.h> + +#include "helpers.h" +#include "liburing.h" + +/* + * Helper for allocating memory in tests. + */ +void *t_malloc(size_t size) +{ + void *ret; + ret = malloc(size); + assert(ret); + return ret; +} + +/* + * Helper for binding socket to an ephemeral port. + * The port number to be bound is returned in @addr->sin_port. + */ +int t_bind_ephemeral_port(int fd, struct sockaddr_in *addr) +{ + socklen_t addrlen; + + addr->sin_port = 0; + if (bind(fd, (struct sockaddr *)addr, sizeof(*addr))) + return -errno; + + addrlen = sizeof(*addr); + assert(!getsockname(fd, (struct sockaddr *)addr, &addrlen)); + assert(addr->sin_port != 0); + return 0; +} + +/* + * Helper for allocating size bytes aligned on a boundary. + */ +void t_posix_memalign(void **memptr, size_t alignment, size_t size) +{ + int ret; + ret = posix_memalign(memptr, alignment, size); + assert(!ret); +} + +/* + * Helper for allocating space for an array of nmemb elements + * with size bytes for each element. + */ +void *t_calloc(size_t nmemb, size_t size) +{ + void *ret; + ret = calloc(nmemb, size); + assert(ret); + return ret; +} + +/* + * Helper for creating file and write @size byte buf with 0xaa value in the file. + */ +static void __t_create_file(const char *file, size_t size, char pattern) +{ + ssize_t ret; + char *buf; + int fd; + + buf = t_malloc(size); + memset(buf, pattern, size); + + fd = open(file, O_WRONLY | O_CREAT, 0644); + assert(fd >= 0); + + ret = write(fd, buf, size); + fsync(fd); + close(fd); + free(buf); + assert(ret == size); +} + +void t_create_file(const char *file, size_t size) +{ + __t_create_file(file, size, 0xaa); +} + +void t_create_file_pattern(const char *file, size_t size, char pattern) +{ + __t_create_file(file, size, pattern); +} + +/* + * Helper for creating @buf_num number of iovec + * with @buf_size bytes buffer of each iovec. + */ +struct iovec *t_create_buffers(size_t buf_num, size_t buf_size) +{ + struct iovec *vecs; + int i; + + vecs = t_malloc(buf_num * sizeof(struct iovec)); + for (i = 0; i < buf_num; i++) { + t_posix_memalign(&vecs[i].iov_base, buf_size, buf_size); + vecs[i].iov_len = buf_size; + } + return vecs; +} + +/* + * Helper for setting up an io_uring instance, skipping if the given user isn't + * allowed to. + */ +enum t_setup_ret t_create_ring_params(int depth, struct io_uring *ring, + struct io_uring_params *p) +{ + int ret; + + ret = io_uring_queue_init_params(depth, ring, p); + if (!ret) + return T_SETUP_OK; + if ((p->flags & IORING_SETUP_SQPOLL) && ret == -EPERM && geteuid()) { + fprintf(stdout, "SQPOLL skipped for regular user\n"); + return T_SETUP_SKIP; + } + + fprintf(stderr, "queue_init: %s\n", strerror(-ret)); + return ret; +} + +enum t_setup_ret t_create_ring(int depth, struct io_uring *ring, + unsigned int flags) +{ + struct io_uring_params p = { }; + + p.flags = flags; + return t_create_ring_params(depth, ring, &p); +} + +enum t_setup_ret t_register_buffers(struct io_uring *ring, + const struct iovec *iovecs, + unsigned nr_iovecs) +{ + int ret; + + ret = io_uring_register_buffers(ring, iovecs, nr_iovecs); + if (!ret) + return T_SETUP_OK; + + if ((ret == -EPERM || ret == -ENOMEM) && geteuid()) { + fprintf(stdout, "too large non-root buffer registration, skip\n"); + return T_SETUP_SKIP; + } + + fprintf(stderr, "buffer register failed: %s\n", strerror(-ret)); + return ret; +} + +int t_create_socket_pair(int fd[2], bool stream) +{ + int ret; + int type = stream ? SOCK_STREAM : SOCK_DGRAM; + int val; + struct sockaddr_in serv_addr; + struct sockaddr *paddr; + size_t paddrlen; + + type |= SOCK_CLOEXEC; + fd[0] = socket(AF_INET, type, 0); + if (fd[0] < 0) + return errno; + fd[1] = socket(AF_INET, type, 0); + if (fd[1] < 0) { + ret = errno; + close(fd[0]); + return ret; + } + + val = 1; + if (setsockopt(fd[0], SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val))) + goto errno_cleanup; + + memset(&serv_addr, 0, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_port = 0; + inet_pton(AF_INET, "127.0.0.1", &serv_addr.sin_addr); + + paddr = (struct sockaddr *)&serv_addr; + paddrlen = sizeof(serv_addr); + + if (bind(fd[0], paddr, paddrlen)) { + fprintf(stderr, "bind failed\n"); + goto errno_cleanup; + } + + if (stream && listen(fd[0], 16)) { + fprintf(stderr, "listen failed\n"); + goto errno_cleanup; + } + + if (getsockname(fd[0], (struct sockaddr *)&serv_addr, + (socklen_t *)&paddrlen)) { + fprintf(stderr, "getsockname failed\n"); + goto errno_cleanup; + } + inet_pton(AF_INET, "127.0.0.1", &serv_addr.sin_addr); + + if (connect(fd[1], (struct sockaddr *)&serv_addr, paddrlen)) { + fprintf(stderr, "connect failed\n"); + goto errno_cleanup; + } + + if (!stream) { + /* connect the other udp side */ + if (getsockname(fd[1], (struct sockaddr *)&serv_addr, + (socklen_t *)&paddrlen)) { + fprintf(stderr, "getsockname failed\n"); + goto errno_cleanup; + } + inet_pton(AF_INET, "127.0.0.1", &serv_addr.sin_addr); + + if (connect(fd[0], (struct sockaddr *)&serv_addr, paddrlen)) { + fprintf(stderr, "connect failed\n"); + goto errno_cleanup; + } + return 0; + } + + /* for stream case we must accept and cleanup the listen socket */ + + ret = accept(fd[0], NULL, NULL); + if (ret < 0) + goto errno_cleanup; + + close(fd[0]); + fd[0] = ret; + + return 0; + +errno_cleanup: + ret = errno; + close(fd[0]); + close(fd[1]); + return ret; +} + +bool t_probe_defer_taskrun(void) +{ + struct io_uring ring; + int ret; + + ret = io_uring_queue_init(1, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN); + if (ret < 0) + return false; + io_uring_queue_exit(&ring); + return true; +} diff --git a/contrib/libs/liburing/test/helpers.h b/contrib/libs/liburing/test/helpers.h new file mode 100644 index 0000000000..4375a9e465 --- /dev/null +++ b/contrib/libs/liburing/test/helpers.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Description: Helpers for tests. + */ +#ifndef LIBURING_HELPERS_H +#define LIBURING_HELPERS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "liburing.h" +#include <arpa/inet.h> + +enum t_setup_ret { + T_SETUP_OK = 0, + T_SETUP_SKIP, +}; + +enum t_test_result { + T_EXIT_PASS = 0, + T_EXIT_FAIL = 1, + T_EXIT_SKIP = 77, +}; + +/* + * Helper for binding socket to an ephemeral port. + * The port number to be bound is returned in @addr->sin_port. + */ +int t_bind_ephemeral_port(int fd, struct sockaddr_in *addr); + + +/* + * Helper for allocating memory in tests. + */ +void *t_malloc(size_t size); + + +/* + * Helper for allocating size bytes aligned on a boundary. + */ +void t_posix_memalign(void **memptr, size_t alignment, size_t size); + + +/* + * Helper for allocating space for an array of nmemb elements + * with size bytes for each element. + */ +void *t_calloc(size_t nmemb, size_t size); + + +/* + * Helper for creating file and write @size byte buf with 0xaa value in the file. + */ +void t_create_file(const char *file, size_t size); + +/* + * Helper for creating file and write @size byte buf with @pattern value in + * the file. + */ +void t_create_file_pattern(const char *file, size_t size, char pattern); + +/* + * Helper for creating @buf_num number of iovec + * with @buf_size bytes buffer of each iovec. + */ +struct iovec *t_create_buffers(size_t buf_num, size_t buf_size); + +/* + * Helper for creating connected socket pairs + */ +int t_create_socket_pair(int fd[2], bool stream); + +/* + * Helper for setting up a ring and checking for user privs + */ +enum t_setup_ret t_create_ring_params(int depth, struct io_uring *ring, + struct io_uring_params *p); +enum t_setup_ret t_create_ring(int depth, struct io_uring *ring, + unsigned int flags); + +enum t_setup_ret t_register_buffers(struct io_uring *ring, + const struct iovec *iovecs, + unsigned nr_iovecs); + +bool t_probe_defer_taskrun(void); + +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/libs/liburing/test/io-cancel.c b/contrib/libs/liburing/test/io-cancel.c new file mode 100644 index 0000000000..59932ff9a4 --- /dev/null +++ b/contrib/libs/liburing/test/io-cancel.c @@ -0,0 +1,556 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: Basic IO cancel test + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/time.h> +#include <sys/wait.h> +#include <poll.h> + +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE (128 * 1024) +#define BS 4096 +#define BUFFERS (FILE_SIZE / BS) + +static struct iovec *vecs; + +static unsigned long long utime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000000; + return sec + usec; +} + +static unsigned long long utime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return utime_since(tv, &end); +} + +static int start_io(struct io_uring *ring, int fd, int do_write) +{ + struct io_uring_sqe *sqe; + int i, ret; + + for (i = 0; i < BUFFERS; i++) { + off_t offset; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + offset = BS * (rand() % BUFFERS); + if (do_write) { + io_uring_prep_writev(sqe, fd, &vecs[i], 1, offset); + } else { + io_uring_prep_readv(sqe, fd, &vecs[i], 1, offset); + } + sqe->user_data = i + 1; + } + + ret = io_uring_submit(ring); + if (ret != BUFFERS) { + fprintf(stderr, "submit got %d, wanted %d\n", ret, BUFFERS); + goto err; + } + + return 0; +err: + return 1; +} + +static int wait_io(struct io_uring *ring, unsigned nr_io, int do_partial) +{ + struct io_uring_cqe *cqe; + int i, ret; + + for (i = 0; i < nr_io; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + goto err; + } + if (do_partial && cqe->user_data) { + if (!(cqe->user_data & 1)) { + if (cqe->res != BS) { + fprintf(stderr, "IO %d wasn't cancelled but got error %d\n", (unsigned) cqe->user_data, cqe->res); + goto err; + } + } + } + io_uring_cqe_seen(ring, cqe); + } + return 0; +err: + return 1; + +} + +static int do_io(struct io_uring *ring, int fd, int do_write) +{ + if (start_io(ring, fd, do_write)) + return 1; + if (wait_io(ring, BUFFERS, 0)) + return 1; + return 0; +} + +static int start_cancel(struct io_uring *ring, int do_partial, int async_cancel) +{ + struct io_uring_sqe *sqe; + int i, ret, submitted = 0; + + for (i = 0; i < BUFFERS; i++) { + if (do_partial && (i & 1)) + continue; + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + io_uring_prep_cancel64(sqe, i + 1, 0); + if (async_cancel) + sqe->flags |= IOSQE_ASYNC; + sqe->user_data = 0; + submitted++; + } + + ret = io_uring_submit(ring); + if (ret != submitted) { + fprintf(stderr, "submit got %d, wanted %d\n", ret, submitted); + goto err; + } + return 0; +err: + return 1; +} + +/* + * Test cancels. If 'do_partial' is set, then we only attempt to cancel half of + * the submitted IO. This is done to verify that cancelling one piece of IO doesn't + * impact others. + */ +static int test_io_cancel(const char *file, int do_write, int do_partial, + int async_cancel) +{ + struct io_uring ring; + struct timeval start_tv; + unsigned long usecs; + unsigned to_wait; + int fd, ret; + + fd = open(file, O_RDWR | O_DIRECT); + if (fd < 0) { + perror("file open"); + goto err; + } + + ret = io_uring_queue_init(4 * BUFFERS, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + goto err; + } + + if (do_io(&ring, fd, do_write)) + goto err; + gettimeofday(&start_tv, NULL); + if (do_io(&ring, fd, do_write)) + goto err; + usecs = utime_since_now(&start_tv); + + if (start_io(&ring, fd, do_write)) + goto err; + /* sleep for 1/3 of the total time, to allow some to start/complete */ + usleep(usecs / 3); + if (start_cancel(&ring, do_partial, async_cancel)) + goto err; + to_wait = BUFFERS; + if (do_partial) + to_wait += BUFFERS / 2; + else + to_wait += BUFFERS; + if (wait_io(&ring, to_wait, do_partial)) + goto err; + + io_uring_queue_exit(&ring); + close(fd); + return 0; +err: + if (fd != -1) + close(fd); + return 1; +} + +static int test_dont_cancel_another_ring(void) +{ + struct io_uring ring1, ring2; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + char buffer[128]; + int ret, fds[2]; + struct __kernel_timespec ts = { .tv_sec = 0, .tv_nsec = 100000000, }; + + ret = io_uring_queue_init(8, &ring1, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + ret = io_uring_queue_init(8, &ring2, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(&ring1); + if (!sqe) { + fprintf(stderr, "%s: failed to get sqe\n", __FUNCTION__); + return 1; + } + io_uring_prep_read(sqe, fds[0], buffer, 10, 0); + sqe->flags |= IOSQE_ASYNC; + sqe->user_data = 1; + + ret = io_uring_submit(&ring1); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + + /* make sure it doesn't cancel requests of the other ctx */ + sqe = io_uring_get_sqe(&ring2); + if (!sqe) { + fprintf(stderr, "%s: failed to get sqe\n", __FUNCTION__); + return 1; + } + io_uring_prep_cancel64(sqe, 1, 0); + sqe->user_data = 2; + + ret = io_uring_submit(&ring2); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + + ret = io_uring_wait_cqe(&ring2, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + if (cqe->user_data != 2 || cqe->res != -ENOENT) { + fprintf(stderr, "error: cqe %i: res=%i, but expected -ENOENT\n", + (int)cqe->user_data, (int)cqe->res); + return 1; + } + io_uring_cqe_seen(&ring2, cqe); + + ret = io_uring_wait_cqe_timeout(&ring1, &cqe, &ts); + if (ret != -ETIME) { + fprintf(stderr, "read got cancelled or wait failed\n"); + return 1; + } + io_uring_cqe_seen(&ring1, cqe); + + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring1); + io_uring_queue_exit(&ring2); + return 0; +} + +static int test_cancel_req_across_fork(void) +{ + struct io_uring ring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + char buffer[128]; + int ret, i, fds[2]; + pid_t p; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + if (pipe(fds)) { + perror("pipe"); + return 1; + } + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "%s: failed to get sqe\n", __FUNCTION__); + return 1; + } + io_uring_prep_read(sqe, fds[0], buffer, 10, 0); + sqe->flags |= IOSQE_ASYNC; + sqe->user_data = 1; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + + p = fork(); + if (p == -1) { + fprintf(stderr, "fork() failed\n"); + return 1; + } + + if (p == 0) { + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "%s: failed to get sqe\n", __FUNCTION__); + return 1; + } + io_uring_prep_cancel64(sqe, 1, 0); + sqe->user_data = 2; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + + for (i = 0; i < 2; ++i) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + switch (cqe->user_data) { + case 1: + if (cqe->res != -EINTR && + cqe->res != -ECANCELED) { + fprintf(stderr, "%i %i\n", (int)cqe->user_data, cqe->res); + exit(1); + } + break; + case 2: + if (cqe->res != -EALREADY && cqe->res) { + fprintf(stderr, "%i %i\n", (int)cqe->user_data, cqe->res); + exit(1); + } + break; + default: + fprintf(stderr, "%i %i\n", (int)cqe->user_data, cqe->res); + exit(1); + } + + io_uring_cqe_seen(&ring, cqe); + } + exit(0); + } else { + int wstatus; + pid_t childpid; + + do { + childpid = waitpid(p, &wstatus, 0); + } while (childpid == (pid_t)-1 && errno == EINTR); + + if (childpid == (pid_t)-1) { + perror("waitpid()"); + return 1; + } + if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus)) { + fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus)); + return 1; + } + } + + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring); + return 0; +} + +static int test_cancel_inflight_exit(void) +{ + struct __kernel_timespec ts = { .tv_sec = 1, .tv_nsec = 0, }; + struct io_uring ring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + pid_t p; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + p = fork(); + if (p == -1) { + fprintf(stderr, "fork() failed\n"); + return 1; + } + + if (p == 0) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, ring.ring_fd, POLLIN); + sqe->user_data = 1; + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 2; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 3; + + ret = io_uring_submit(&ring); + if (ret != 3) { + fprintf(stderr, "io_uring_submit() failed %s, ret %i\n", __FUNCTION__, ret); + exit(1); + } + exit(0); + } else { + int wstatus; + + if (waitpid(p, &wstatus, 0) == (pid_t)-1) { + perror("waitpid()"); + return 1; + } + if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus)) { + fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus)); + return 1; + } + } + + for (i = 0; i < 3; ++i) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + if ((cqe->user_data == 1 && cqe->res != -ECANCELED) || + (cqe->user_data == 2 && cqe->res != -ECANCELED) || + (cqe->user_data == 3 && cqe->res != -ETIME)) { + fprintf(stderr, "%i %i\n", (int)cqe->user_data, cqe->res); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_sqpoll_cancel_iowq_requests(void) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + int ret, fds[2]; + char buffer[16]; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SQPOLL); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + if (pipe(fds)) { + perror("pipe"); + return 1; + } + /* pin both pipe ends via io-wq */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, fds[0], buffer, 10, 0); + sqe->flags |= IOSQE_ASYNC | IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_write(sqe, fds[1], buffer, 10, 0); + sqe->flags |= IOSQE_ASYNC; + sqe->user_data = 2; + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + + /* wait for sqpoll to kick in and submit before exit */ + sleep(1); + io_uring_queue_exit(&ring); + + /* close the write end, so if ring is cancelled properly read() fails*/ + close(fds[1]); + ret = read(fds[0], buffer, 10); + close(fds[0]); + return 0; +} + +int main(int argc, char *argv[]) +{ + const char *fname = ".io-cancel-test"; + int i, ret; + + if (argc > 1) + return T_EXIT_SKIP; + + if (test_dont_cancel_another_ring()) { + fprintf(stderr, "test_dont_cancel_another_ring() failed\n"); + return T_EXIT_FAIL; + } + + if (test_cancel_req_across_fork()) { + fprintf(stderr, "test_cancel_req_across_fork() failed\n"); + return T_EXIT_FAIL; + } + + if (test_cancel_inflight_exit()) { + fprintf(stderr, "test_cancel_inflight_exit() failed\n"); + return T_EXIT_FAIL; + } + + if (test_sqpoll_cancel_iowq_requests()) { + fprintf(stderr, "test_sqpoll_cancel_iowq_requests() failed\n"); + return T_EXIT_FAIL; + } + + t_create_file(fname, FILE_SIZE); + + vecs = t_create_buffers(BUFFERS, BS); + + for (i = 0; i < 8; i++) { + int write = (i & 1) != 0; + int partial = (i & 2) != 0; + int async = (i & 4) != 0; + + ret = test_io_cancel(fname, write, partial, async); + if (ret) { + fprintf(stderr, "test_io_cancel %d %d %d failed\n", + write, partial, async); + goto err; + } + } + + unlink(fname); + return T_EXIT_PASS; +err: + unlink(fname); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/io_uring_enter.c b/contrib/libs/liburing/test/io_uring_enter.c new file mode 100644 index 0000000000..7f87f0cf55 --- /dev/null +++ b/contrib/libs/liburing/test/io_uring_enter.c @@ -0,0 +1,262 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * io_uring_enter.c + * + * Description: Unit tests for the io_uring_enter system call. + * + * Copyright 2019, Red Hat, Inc. + * Author: Jeff Moyer <jmoyer@redhat.com> + */ +#include <stdio.h> +#include <fcntl.h> +#include <string.h> +#include <stdlib.h> +#include <unistd.h> +#include <errno.h> +#include <sys/sysinfo.h> +#include <poll.h> +#include <assert.h> +#include <sys/uio.h> +#include <sys/mman.h> +#include <linux/mman.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <limits.h> +#include <sys/time.h> + +#include "helpers.h" +#include "liburing.h" +#include "liburing/barrier.h" +#include "../src/syscall.h" + +#define IORING_MAX_ENTRIES 4096 +#define IORING_MAX_ENTRIES_FALLBACK 128 + +static int expect_fail(int fd, unsigned int to_submit, + unsigned int min_complete, unsigned int flags, + sigset_t *sig, int error) +{ + int ret; + + ret = io_uring_enter(fd, to_submit, min_complete, flags, sig); + if (ret >= 0) { + fprintf(stderr, "expected %s, but call succeeded\n", strerror(-error)); + return 1; + } + + if (ret != error) { + fprintf(stderr, "expected %d, got %d\n", error, ret); + return 1; + } + + return 0; +} + +static int try_io_uring_enter(int fd, unsigned int to_submit, + unsigned int min_complete, unsigned int flags, + sigset_t *sig, int expect) +{ + int ret; + + if (expect < 0) + return expect_fail(fd, to_submit, min_complete, flags, sig, + expect); + + ret = io_uring_enter(fd, to_submit, min_complete, flags, sig); + if (ret != expect) { + fprintf(stderr, "Expected %d, got %d\n", expect, ret); + return 1; + } + + return 0; +} + +/* + * prep a read I/O. index is treated like a block number. + */ +static int setup_file(char *template, off_t len) +{ + int fd, ret; + char buf[4096]; + + fd = mkstemp(template); + if (fd < 0) { + perror("mkstemp"); + exit(1); + } + ret = ftruncate(fd, len); + if (ret < 0) { + perror("ftruncate"); + exit(1); + } + + ret = read(fd, buf, 4096); + if (ret != 4096) { + fprintf(stderr, "read returned %d, expected 4096\n", ret); + exit(1); + } + + return fd; +} + +static void io_prep_read(struct io_uring_sqe *sqe, int fd, off_t offset, + size_t len) +{ + struct iovec *iov; + + iov = t_malloc(sizeof(*iov)); + assert(iov); + + iov->iov_base = t_malloc(len); + assert(iov->iov_base); + iov->iov_len = len; + + io_uring_prep_readv(sqe, fd, iov, 1, offset); + io_uring_sqe_set_data(sqe, iov); // free on completion +} + +static void reap_events(struct io_uring *ring, unsigned nr) +{ + int ret; + unsigned left = nr; + struct io_uring_cqe *cqe; + struct iovec *iov; + struct timeval start, now, elapsed; + + gettimeofday(&start, NULL); + while (left) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "io_uring_wait_cqe returned %d\n", ret); + exit(1); + } + if (cqe->res != 4096) + fprintf(stderr, "cqe->res: %d, expected 4096\n", cqe->res); + iov = io_uring_cqe_get_data(cqe); + free(iov->iov_base); + free(iov); + left--; + io_uring_cqe_seen(ring, cqe); + + gettimeofday(&now, NULL); + timersub(&now, &start, &elapsed); + if (elapsed.tv_sec > 10) { + fprintf(stderr, "Timed out waiting for I/Os to complete.\n"); + fprintf(stderr, "%u expected, %u completed\n", nr, left); + break; + } + } +} + +static void submit_io(struct io_uring *ring, unsigned nr) +{ + int fd, ret; + off_t file_len; + unsigned i; + static char template[32] = "/tmp/io_uring_enter-test.XXXXXX"; + struct io_uring_sqe *sqe; + + file_len = nr * 4096; + fd = setup_file(template, file_len); + for (i = 0; i < nr; i++) { + /* allocate an sqe */ + sqe = io_uring_get_sqe(ring); + /* fill it in */ + io_prep_read(sqe, fd, i * 4096, 4096); + } + + /* submit the I/Os */ + ret = io_uring_submit(ring); + unlink(template); + if (ret < 0) { + perror("io_uring_enter"); + exit(1); + } +} + +int main(int argc, char **argv) +{ + int ret; + unsigned int status = 0; + struct io_uring ring; + struct io_uring_sq *sq = &ring.sq; + unsigned ktail, mask, index; + unsigned sq_entries; + unsigned completed, dropped; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(IORING_MAX_ENTRIES, &ring, 0); + if (ret == -ENOMEM) + ret = io_uring_queue_init(IORING_MAX_ENTRIES_FALLBACK, &ring, 0); + if (ret < 0) { + perror("io_uring_queue_init"); + exit(T_EXIT_FAIL); + } + mask = sq->ring_mask; + + /* invalid flags */ + status |= try_io_uring_enter(ring.ring_fd, 1, 0, ~0U, NULL, -EINVAL); + + /* invalid fd, EBADF */ + status |= try_io_uring_enter(-1, 0, 0, 0, NULL, -EBADF); + + /* valid, non-ring fd, EOPNOTSUPP */ + status |= try_io_uring_enter(0, 0, 0, 0, NULL, -EOPNOTSUPP); + + /* to_submit: 0, flags: 0; should get back 0. */ + status |= try_io_uring_enter(ring.ring_fd, 0, 0, 0, NULL, 0); + + /* fill the sq ring */ + sq_entries = ring.sq.ring_entries; + submit_io(&ring, sq_entries); + ret = io_uring_enter(ring.ring_fd, 0, sq_entries, + IORING_ENTER_GETEVENTS, NULL); + if (ret < 0) { + fprintf(stderr, "io_uring_enter: %s\n", strerror(-ret)); + status = 1; + } else { + /* + * This is a non-IOPOLL ring, which means that io_uring_enter + * should not return until min_complete events are available + * in the completion queue. + */ + completed = *ring.cq.ktail - *ring.cq.khead; + if (completed != sq_entries) { + fprintf(stderr, "Submitted %u I/Os, but only got %u completions\n", + sq_entries, completed); + status = 1; + } + reap_events(&ring, sq_entries); + } + + /* + * Add an invalid index to the submission queue. This should + * result in the dropped counter increasing. + */ + index = sq->ring_entries + 1; // invalid index + dropped = *sq->kdropped; + ktail = *sq->ktail; + sq->array[ktail & mask] = index; + ++ktail; + /* + * Ensure that the kernel sees the SQE update before it sees the tail + * update. + */ + io_uring_smp_store_release(sq->ktail, ktail); + + ret = io_uring_enter(ring.ring_fd, 1, 0, 0, NULL); + /* now check to see if our sqe was dropped */ + if (*sq->kdropped == dropped) { + fprintf(stderr, "dropped counter did not increase\n"); + status = 1; + } + + if (!status) + return T_EXIT_PASS; + + fprintf(stderr, "FAIL\n"); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/io_uring_passthrough.c b/contrib/libs/liburing/test/io_uring_passthrough.c new file mode 100644 index 0000000000..156347456e --- /dev/null +++ b/contrib/libs/liburing/test/io_uring_passthrough.c @@ -0,0 +1,452 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: basic read/write tests for io_uring passthrough commands + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> + +#include "helpers.h" +#include "liburing.h" +#include "../src/syscall.h" +#include "nvme.h" + +#define FILE_SIZE (256 * 1024) +#define BS 8192 +#define BUFFERS (FILE_SIZE / BS) + +static struct iovec *vecs; + +/* + * Each offset in the file has the ((test_case / 2) * FILE_SIZE) + * + (offset / sizeof(int)) stored for every + * sizeof(int) address. + */ +static int verify_buf(int tc, void *buf, off_t off) +{ + int i, u_in_buf = BS / sizeof(unsigned int); + unsigned int *ptr; + + off /= sizeof(unsigned int); + off += (tc / 2) * FILE_SIZE; + ptr = buf; + for (i = 0; i < u_in_buf; i++) { + if (off != *ptr) { + fprintf(stderr, "Found %u, wanted %lu\n", *ptr, off); + return 1; + } + ptr++; + off++; + } + + return 0; +} + +static int fill_pattern(int tc) +{ + unsigned int val, *ptr; + int i, j; + int u_in_buf = BS / sizeof(val); + + val = (tc / 2) * FILE_SIZE; + for (i = 0; i < BUFFERS; i++) { + ptr = vecs[i].iov_base; + for (j = 0; j < u_in_buf; j++) { + *ptr = val; + val++; + ptr++; + } + } + + return 0; +} + +static int __test_io(const char *file, struct io_uring *ring, int tc, int read, + int sqthread, int fixed, int nonvec) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct nvme_uring_cmd *cmd; + int open_flags; + int do_fixed; + int i, ret, fd = -1; + off_t offset; + __u64 slba; + __u32 nlb; + + if (read) + open_flags = O_RDONLY; + else + open_flags = O_WRONLY; + + if (fixed) { + ret = t_register_buffers(ring, vecs, BUFFERS); + if (ret == T_SETUP_SKIP) + return 0; + if (ret != T_SETUP_OK) { + fprintf(stderr, "buffer reg failed: %d\n", ret); + goto err; + } + } + + fd = open(file, open_flags); + if (fd < 0) { + perror("file open"); + goto err; + } + + if (sqthread) { + ret = io_uring_register_files(ring, &fd, 1); + if (ret) { + fprintf(stderr, "file reg failed: %d\n", ret); + goto err; + } + } + + if (!read) + fill_pattern(tc); + + offset = 0; + for (i = 0; i < BUFFERS; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + if (read) { + int use_fd = fd; + + do_fixed = fixed; + + if (sqthread) + use_fd = 0; + if (fixed && (i & 1)) + do_fixed = 0; + if (do_fixed) { + io_uring_prep_read_fixed(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, + offset, i); + sqe->cmd_op = NVME_URING_CMD_IO; + } else if (nonvec) { + io_uring_prep_read(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, offset); + sqe->cmd_op = NVME_URING_CMD_IO; + } else { + io_uring_prep_readv(sqe, use_fd, &vecs[i], 1, + offset); + sqe->cmd_op = NVME_URING_CMD_IO_VEC; + } + } else { + int use_fd = fd; + + do_fixed = fixed; + + if (sqthread) + use_fd = 0; + if (fixed && (i & 1)) + do_fixed = 0; + if (do_fixed) { + io_uring_prep_write_fixed(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, + offset, i); + sqe->cmd_op = NVME_URING_CMD_IO; + } else if (nonvec) { + io_uring_prep_write(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, offset); + sqe->cmd_op = NVME_URING_CMD_IO; + } else { + io_uring_prep_writev(sqe, use_fd, &vecs[i], 1, + offset); + sqe->cmd_op = NVME_URING_CMD_IO_VEC; + } + } + sqe->opcode = IORING_OP_URING_CMD; + sqe->user_data = ((uint64_t)offset << 32) | i; + if (sqthread) + sqe->flags |= IOSQE_FIXED_FILE; + + cmd = (struct nvme_uring_cmd *)sqe->cmd; + memset(cmd, 0, sizeof(struct nvme_uring_cmd)); + + cmd->opcode = read ? nvme_cmd_read : nvme_cmd_write; + + slba = offset >> lba_shift; + nlb = (BS >> lba_shift) - 1; + + /* cdw10 and cdw11 represent starting lba */ + cmd->cdw10 = slba & 0xffffffff; + cmd->cdw11 = slba >> 32; + /* cdw12 represent number of lba's for read/write */ + cmd->cdw12 = nlb; + if (do_fixed || nonvec) { + cmd->addr = (__u64)(uintptr_t)vecs[i].iov_base; + cmd->data_len = vecs[i].iov_len; + } else { + cmd->addr = (__u64)(uintptr_t)&vecs[i]; + cmd->data_len = 1; + } + cmd->nsid = nsid; + + offset += BS; + } + + ret = io_uring_submit(ring); + if (ret != BUFFERS) { + fprintf(stderr, "submit got %d, wanted %d\n", ret, BUFFERS); + goto err; + } + + for (i = 0; i < BUFFERS; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + goto err; + } + if (cqe->res != 0) { + fprintf(stderr, "cqe res %d, wanted 0\n", cqe->res); + goto err; + } + io_uring_cqe_seen(ring, cqe); + if (read) { + int index = cqe->user_data & 0xffffffff; + void *buf = vecs[index].iov_base; + off_t voff = cqe->user_data >> 32; + + if (verify_buf(tc, buf, voff)) + goto err; + } + } + + if (fixed) { + ret = io_uring_unregister_buffers(ring); + if (ret) { + fprintf(stderr, "buffer unreg failed: %d\n", ret); + goto err; + } + } + if (sqthread) { + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "file unreg failed: %d\n", ret); + goto err; + } + } + + close(fd); + return 0; +err: + if (fd != -1) + close(fd); + return 1; +} + +static int test_io(const char *file, int tc, int read, int sqthread, + int fixed, int nonvec) +{ + struct io_uring ring; + int ret, ring_flags = 0; + + ring_flags |= IORING_SETUP_SQE128; + ring_flags |= IORING_SETUP_CQE32; + + if (sqthread) + ring_flags |= IORING_SETUP_SQPOLL; + + ret = t_create_ring(64, &ring, ring_flags); + if (ret == T_SETUP_SKIP) + return 0; + if (ret != T_SETUP_OK) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + ret = __test_io(file, &ring, tc, read, sqthread, fixed, nonvec); + io_uring_queue_exit(&ring); + + return ret; +} + +extern unsigned __io_uring_flush_sq(struct io_uring *ring); + +/* + * Send a passthrough command that nvme will fail during submission. + * This comes handy for testing error handling. + */ +static int test_invalid_passthru_submit(const char *file) +{ + struct io_uring ring; + int fd, ret, ring_flags, open_flags; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct nvme_uring_cmd *cmd; + + ring_flags = IORING_SETUP_IOPOLL | IORING_SETUP_SQE128; + ring_flags |= IORING_SETUP_CQE32; + + ret = t_create_ring(1, &ring, ring_flags); + if (ret != T_SETUP_OK) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + open_flags = O_RDONLY; + fd = open(file, open_flags); + if (fd < 0) { + perror("file open"); + goto err; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, fd, vecs[0].iov_base, vecs[0].iov_len, 0); + sqe->cmd_op = NVME_URING_CMD_IO; + sqe->opcode = IORING_OP_URING_CMD; + sqe->user_data = 1; + cmd = (struct nvme_uring_cmd *)sqe->cmd; + memset(cmd, 0, sizeof(struct nvme_uring_cmd)); + cmd->opcode = nvme_cmd_read; + cmd->addr = (__u64)(uintptr_t)&vecs[0].iov_base; + cmd->data_len = vecs[0].iov_len; + /* populate wrong nsid to force failure */ + cmd->nsid = nsid + 1; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit got %d, wanted %d\n", ret, 1); + goto err; + } + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + goto err; + } + if (cqe->res == 0) { + fprintf(stderr, "cqe res %d, wanted failure\n", cqe->res); + goto err; + } + io_uring_cqe_seen(&ring, cqe); + close(fd); + io_uring_queue_exit(&ring); + return 0; +err: + if (fd != -1) + close(fd); + io_uring_queue_exit(&ring); + return 1; +} + +/* + * if we are polling io_uring_submit needs to always enter the + * kernel to fetch events + */ +static int test_io_uring_submit_enters(const char *file) +{ + struct io_uring ring; + int fd, i, ret, ring_flags, open_flags; + unsigned head; + struct io_uring_cqe *cqe; + + ring_flags = IORING_SETUP_IOPOLL; + ring_flags |= IORING_SETUP_SQE128; + ring_flags |= IORING_SETUP_CQE32; + + ret = io_uring_queue_init(64, &ring, ring_flags); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + open_flags = O_WRONLY; + fd = open(file, open_flags); + if (fd < 0) { + perror("file open"); + goto err; + } + + for (i = 0; i < BUFFERS; i++) { + struct io_uring_sqe *sqe; + off_t offset = BS * (rand() % BUFFERS); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, fd, &vecs[i], 1, offset); + sqe->user_data = 1; + } + + /* submit manually to avoid adding IORING_ENTER_GETEVENTS */ + ret = __sys_io_uring_enter(ring.ring_fd, __io_uring_flush_sq(&ring), 0, + 0, NULL); + if (ret < 0) + goto err; + + for (i = 0; i < 500; i++) { + ret = io_uring_submit(&ring); + if (ret != 0) { + fprintf(stderr, "still had %d sqes to submit\n", ret); + goto err; + } + + io_uring_for_each_cqe(&ring, head, cqe) { + if (cqe->res == -EOPNOTSUPP) + fprintf(stdout, "Device doesn't support polled IO\n"); + goto ok; + } + usleep(10000); + } +err: + ret = 1; + if (fd != -1) + close(fd); + +ok: + io_uring_queue_exit(&ring); + return ret; +} + +int main(int argc, char *argv[]) +{ + int i, ret; + char *fname; + + if (argc < 2) + return T_EXIT_SKIP; + + fname = argv[1]; + ret = nvme_get_info(fname); + + if (ret) + return T_EXIT_SKIP; + + vecs = t_create_buffers(BUFFERS, BS); + + for (i = 0; i < 16; i++) { + int read = (i & 1) != 0; + int sqthread = (i & 2) != 0; + int fixed = (i & 4) != 0; + int nonvec = (i & 8) != 0; + + ret = test_io(fname, i, read, sqthread, fixed, nonvec); + if (ret) { + fprintf(stderr, "test_io failed %d/%d/%d/%d\n", + read, sqthread, fixed, nonvec); + goto err; + } + } + + ret = test_io_uring_submit_enters(fname); + if (ret) { + fprintf(stderr, "test_io_uring_submit_enters failed\n"); + goto err; + } + + ret = test_invalid_passthru_submit(fname); + if (ret) { + fprintf(stderr, "test_invalid_passthru_submit failed\n"); + goto err; + } + + return T_EXIT_PASS; +err: + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/io_uring_register.c b/contrib/libs/liburing/test/io_uring_register.c new file mode 100644 index 0000000000..b484ef4c4f --- /dev/null +++ b/contrib/libs/liburing/test/io_uring_register.c @@ -0,0 +1,507 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * io_uring_register.c + * + * Description: Unit tests for the io_uring_register system call. + * + * Copyright 2019, Red Hat, Inc. + * Author: Jeff Moyer <jmoyer@redhat.com> + */ +#include <stdio.h> +#include <fcntl.h> +#include <string.h> +#include <stdlib.h> +#include <unistd.h> +#include <errno.h> +#include <sys/sysinfo.h> +#include <poll.h> +#include <assert.h> +#include <sys/uio.h> +#include <sys/mman.h> +#include <linux/mman.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <limits.h> + +#include "helpers.h" +#include "liburing.h" +#include "../src/syscall.h" + +static int pagesize; +static rlim_t mlock_limit; +static int devnull; + +static int expect_fail(int fd, unsigned int opcode, void *arg, + unsigned int nr_args, int error) +{ + int ret; + + ret = io_uring_register(fd, opcode, arg, nr_args); + if (ret >= 0) { + int ret2 = 0; + + fprintf(stderr, "expected %s, but call succeeded\n", strerror(error)); + if (opcode == IORING_REGISTER_BUFFERS) { + ret2 = io_uring_register(fd, IORING_UNREGISTER_BUFFERS, + 0, 0); + } else if (opcode == IORING_REGISTER_FILES) { + ret2 = io_uring_register(fd, IORING_UNREGISTER_FILES, 0, + 0); + } + if (ret2) { + fprintf(stderr, "internal error: failed to unregister\n"); + exit(1); + } + return 1; + } + + if (ret != error) { + fprintf(stderr, "expected %d, got %d\n", error, ret); + return 1; + } + return 0; +} + +static int new_io_uring(int entries, struct io_uring_params *p) +{ + int fd; + + fd = io_uring_setup(entries, p); + if (fd < 0) { + perror("io_uring_setup"); + exit(1); + } + return fd; +} + +#define MAXFDS (UINT_MAX * sizeof(int)) + +static void *map_filebacked(size_t size) +{ + int fd, ret; + void *addr; + char template[32] = "io_uring_register-test-XXXXXXXX"; + + fd = mkstemp(template); + if (fd < 0) { + perror("mkstemp"); + return NULL; + } + unlink(template); + + ret = ftruncate(fd, size); + if (ret < 0) { + perror("ftruncate"); + close(fd); + return NULL; + } + + addr = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + perror("mmap"); + close(fd); + return NULL; + } + + close(fd); + return addr; +} + +/* + * NOTE: this is now limited by SCM_MAX_FD (253). Keep the code for now, + * but probably should augment it to test 253 and 254, specifically. + */ +static int test_max_fds(int uring_fd) +{ + int status = 1; + int ret; + void *fd_as; /* file descriptor address space */ + int fdtable_fd; /* fd for the file that will be mapped over and over */ + int io_fd; /* the valid fd for I/O -- /dev/null */ + int *fds; /* used to map the file into the address space */ + char template[32] = "io_uring_register-test-XXXXXXXX"; + unsigned long long i, nr_maps, nr_fds; + + /* + * First, mmap anonymous the full size. That will guarantee the + * mapping will fit in the memory area selected by mmap. Then, + * over-write that mapping using a file-backed mapping, 128MiB at + * a time using MAP_FIXED. + */ + fd_as = mmap(NULL, UINT_MAX * sizeof(int), PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (fd_as == MAP_FAILED) { + if (errno == ENOMEM) + return 0; + perror("mmap fd_as"); + exit(1); + } + + fdtable_fd = mkstemp(template); + if (fdtable_fd < 0) { + perror("mkstemp"); + exit(1); + } + unlink(template); + ret = ftruncate(fdtable_fd, 128*1024*1024); + if (ret < 0) { + perror("ftruncate"); + exit(1); + } + + io_fd = open("/dev/null", O_RDWR); + if (io_fd < 0) { + perror("open /dev/null"); + exit(1); + } + fds = mmap(fd_as, 128*1024*1024, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_FIXED, fdtable_fd, 0); + if (fds == MAP_FAILED) { + perror("mmap fdtable"); + exit(1); + } + + /* fill the fd table */ + nr_fds = 128*1024*1024 / sizeof(int); + for (i = 0; i < nr_fds; i++) + fds[i] = io_fd; + + /* map the file through the rest of the address space */ + nr_maps = (UINT_MAX * sizeof(int)) / (128*1024*1024); + for (i = 0; i < nr_maps; i++) { + fds = &fds[nr_fds]; /* advance fds by 128MiB */ + fds = mmap(fds, 128*1024*1024, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_FIXED, fdtable_fd, 0); + if (fds == MAP_FAILED) { + fprintf(stderr, "mmap failed at offset %lu\n", + (unsigned long)((char *)fd_as - (char *)fds)); + exit(1); + } + } + + /* Now fd_as points to the file descriptor array. */ + /* + * We may not be able to map all of these files. Let's back off + * until success. + */ + nr_fds = UINT_MAX; + while (nr_fds) { + ret = io_uring_register(uring_fd, IORING_REGISTER_FILES, fd_as, + nr_fds); + if (ret != 0) { + nr_fds /= 2; + continue; + } + status = 0; + ret = io_uring_register(uring_fd, IORING_UNREGISTER_FILES, 0, 0); + if (ret < 0) { + ret = errno; + errno = ret; + perror("io_uring_register UNREGISTER_FILES"); + exit(1); + } + break; + } + + close(io_fd); + close(fdtable_fd); + ret = munmap(fd_as, UINT_MAX * sizeof(int)); + if (ret != 0) { + fprintf(stderr, "munmap(%zu) failed\n", UINT_MAX * sizeof(int)); + exit(1); + } + + return status; +} + +static int test_memlock_exceeded(int fd) +{ + int ret; + void *buf; + struct iovec iov; + + /* if limit is larger than 2gb, just skip this test */ + if (mlock_limit >= 2 * 1024 * 1024 * 1024ULL) + return 0; + + iov.iov_len = mlock_limit * 2; + buf = t_malloc(iov.iov_len); + iov.iov_base = buf; + + while (iov.iov_len) { + ret = io_uring_register(fd, IORING_REGISTER_BUFFERS, &iov, 1); + if (ret < 0) { + if (errno == ENOMEM) { + iov.iov_len /= 2; + continue; + } + if (errno == EFAULT) { + free(buf); + return 0; + } + fprintf(stderr, "expected success or EFAULT, got %d\n", errno); + free(buf); + return 1; + } + ret = io_uring_register(fd, IORING_UNREGISTER_BUFFERS, NULL, 0); + if (ret != 0) { + fprintf(stderr, "error: unregister failed with %d\n", errno); + free(buf); + return 1; + } + break; + } + if (!iov.iov_len) + printf("Unable to register buffers. Check memlock rlimit.\n"); + + free(buf); + return 0; +} + +static int test_iovec_nr(int fd) +{ + int i, ret, status = 0; + unsigned int nr = 1000000; + struct iovec *iovs; + void *buf; + + iovs = malloc(nr * sizeof(struct iovec)); + if (!iovs) { + fprintf(stdout, "can't allocate iovecs, skip\n"); + return 0; + } + buf = t_malloc(pagesize); + + for (i = 0; i < nr; i++) { + iovs[i].iov_base = buf; + iovs[i].iov_len = pagesize; + } + + status |= expect_fail(fd, IORING_REGISTER_BUFFERS, iovs, nr, -EINVAL); + + /* reduce to UIO_MAXIOV */ + nr = UIO_MAXIOV; + ret = io_uring_register(fd, IORING_REGISTER_BUFFERS, iovs, nr); + if (ret && (errno == ENOMEM || errno == EPERM) && geteuid()) { + fprintf(stderr, "can't register large iovec for regular users, skip\n"); + } else if (ret != 0) { + fprintf(stderr, "expected success, got %d\n", errno); + status = 1; + } else { + io_uring_register(fd, IORING_UNREGISTER_BUFFERS, 0, 0); + } + free(buf); + free(iovs); + return status; +} + +/* + * io_uring limit is 1G. iov_len limit is ~OUL, I think + */ +static int test_iovec_size(int fd) +{ + unsigned int status = 0; + int ret; + struct iovec iov; + void *buf; + + /* NULL pointer for base */ + iov.iov_base = 0; + iov.iov_len = 4096; + status |= expect_fail(fd, IORING_REGISTER_BUFFERS, &iov, 1, -EFAULT); + + /* valid base, 0 length */ + iov.iov_base = &buf; + iov.iov_len = 0; + status |= expect_fail(fd, IORING_REGISTER_BUFFERS, &iov, 1, -EFAULT); + + /* valid base, length exceeds size */ + /* this requires an unampped page directly after buf */ + buf = mmap(NULL, 2 * pagesize, PROT_READ|PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + assert(buf != MAP_FAILED); + ret = munmap(buf + pagesize, pagesize); + assert(ret == 0); + iov.iov_base = buf; + iov.iov_len = 2 * pagesize; + status |= expect_fail(fd, IORING_REGISTER_BUFFERS, &iov, 1, -EFAULT); + munmap(buf, pagesize); + + /* huge page */ + buf = mmap(NULL, 2*1024*1024, PROT_READ|PROT_WRITE, + MAP_PRIVATE | MAP_HUGETLB | MAP_HUGE_2MB | MAP_ANONYMOUS, + -1, 0); + if (buf == MAP_FAILED) { + printf("Unable to map a huge page. Try increasing " + "/proc/sys/vm/nr_hugepages by at least 1.\n"); + printf("Skipping the hugepage test\n"); + } else { + /* + * This should succeed, so long as RLIMIT_MEMLOCK is + * not exceeded + */ + iov.iov_base = buf; + iov.iov_len = 2*1024*1024; + ret = io_uring_register(fd, IORING_REGISTER_BUFFERS, &iov, 1); + if (ret < 0) { + if (ret == -ENOMEM) + printf("Unable to test registering of a huge " + "page. Try increasing the " + "RLIMIT_MEMLOCK resource limit by at " + "least 2MB."); + else { + fprintf(stderr, "expected success, got %d\n", ret); + status = 1; + } + } else { + ret = io_uring_register(fd, IORING_UNREGISTER_BUFFERS, + 0, 0); + if (ret < 0) { + fprintf(stderr, "io_uring_unregister: %s\n", + strerror(-ret)); + status = 1; + } + } + } + ret = munmap(iov.iov_base, iov.iov_len); + assert(ret == 0); + + /* file-backed buffers -- not supported */ + buf = map_filebacked(2*1024*1024); + if (!buf) + status = 1; + iov.iov_base = buf; + iov.iov_len = 2*1024*1024; + status |= expect_fail(fd, IORING_REGISTER_BUFFERS, &iov, 1, -EOPNOTSUPP); + munmap(buf, 2*1024*1024); + + /* bump up against the soft limit and make sure we get EFAULT + * or whatever we're supposed to get. NOTE: this requires + * running the test as non-root. */ + if (getuid() != 0) + status |= test_memlock_exceeded(fd); + + return status; +} + +static int ioring_poll(struct io_uring *ring, int fd, int fixed) +{ + int ret; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + + sqe = io_uring_get_sqe(ring); + memset(sqe, 0, sizeof(*sqe)); + sqe->opcode = IORING_OP_POLL_ADD; + if (fixed) + sqe->flags = IOSQE_FIXED_FILE; + sqe->fd = fd; + sqe->poll_events = POLLIN|POLLOUT; + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "failed to submit poll sqe: %d.\n", ret); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "io_uring_wait_cqe failed with %d\n", ret); + return 1; + } + ret = 0; + if (cqe->res != POLLOUT) { + fprintf(stderr, "io_uring_wait_cqe: expected 0x%.8x, got 0x%.8x\n", + POLLOUT, cqe->res); + ret = 1; + } + + io_uring_cqe_seen(ring, cqe); + return ret; +} + +static int test_poll_ringfd(void) +{ + int status = 0; + int ret; + int fd; + struct io_uring ring; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + perror("io_uring_queue_init"); + return 1; + } + fd = ring.ring_fd; + + /* try polling the ring fd */ + status = ioring_poll(&ring, fd, 0); + + /* + * now register the ring fd, and try the poll again. This should + * fail, because the kernel does not allow registering of the + * ring_fd. + */ + status |= expect_fail(fd, IORING_REGISTER_FILES, &fd, 1, -EBADF); + + /* tear down queue */ + io_uring_queue_exit(&ring); + + return status; +} + +int main(int argc, char **argv) +{ + int fd, ret; + unsigned int status = 0; + struct io_uring_params p; + struct rlimit rlim; + + if (argc > 1) + return T_EXIT_SKIP; + + /* setup globals */ + pagesize = getpagesize(); + ret = getrlimit(RLIMIT_MEMLOCK, &rlim); + if (ret < 0) { + perror("getrlimit"); + return T_EXIT_PASS; + } + mlock_limit = rlim.rlim_cur; + devnull = open("/dev/null", O_RDWR); + if (devnull < 0) { + perror("open /dev/null"); + exit(T_EXIT_FAIL); + } + + /* invalid fd */ + status |= expect_fail(-1, 0, NULL, 0, -EBADF); + /* valid fd that is not an io_uring fd */ + status |= expect_fail(devnull, 0, NULL, 0, -EOPNOTSUPP); + + /* invalid opcode */ + memset(&p, 0, sizeof(p)); + fd = new_io_uring(1, &p); + ret = expect_fail(fd, ~0U, NULL, 0, -EINVAL); + if (ret) { + /* if this succeeds, tear down the io_uring instance + * and start clean for the next test. */ + close(fd); + fd = new_io_uring(1, &p); + } + + /* IORING_REGISTER_BUFFERS */ + status |= test_iovec_size(fd); + status |= test_iovec_nr(fd); + /* IORING_REGISTER_FILES */ + status |= test_max_fds(fd); + close(fd); + /* uring poll on the uring fd */ + status |= test_poll_ringfd(); + + if (status) + fprintf(stderr, "FAIL\n"); + + return status; +} diff --git a/contrib/libs/liburing/test/io_uring_setup.c b/contrib/libs/liburing/test/io_uring_setup.c new file mode 100644 index 0000000000..3d5a6c4bca --- /dev/null +++ b/contrib/libs/liburing/test/io_uring_setup.c @@ -0,0 +1,188 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * io_uring_setup.c + * + * Description: Unit tests for the io_uring_setup system call. + * + * Copyright 2019, Red Hat, Inc. + * Author: Jeff Moyer <jmoyer@redhat.com> + */ +#include <stdio.h> +#include <fcntl.h> +#include <string.h> +#include <stdlib.h> +#include <unistd.h> +#include <errno.h> +#include <sys/sysinfo.h> +#include "liburing.h" +#include "helpers.h" + +#include "../syscall.h" + +char *features_string(struct io_uring_params *p) +{ + static char flagstr[64]; + + if (!p || !p->features) + return "none"; + + if (p->features & ~IORING_FEAT_SINGLE_MMAP) { + snprintf(flagstr, 64, "0x%.8x", p->features); + return flagstr; + } + + if (p->features & IORING_FEAT_SINGLE_MMAP) + strncat(flagstr, "IORING_FEAT_SINGLE_MMAP", 64 - strlen(flagstr)); + + return flagstr; +} + +/* + * Attempt the call with the given args. Return 0 when expect matches + * the return value of the system call, 1 otherwise. + */ +char * +flags_string(struct io_uring_params *p) +{ + static char flagstr[64]; + int add_pipe = 0; + + memset(flagstr, 0, sizeof(flagstr)); + + if (!p || p->flags == 0) + return "none"; + + /* + * If unsupported flags are present, just print the bitmask. + */ + if (p->flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL | + IORING_SETUP_SQ_AFF)) { + snprintf(flagstr, 64, "0x%.8x", p->flags); + return flagstr; + } + + if (p->flags & IORING_SETUP_IOPOLL) { + strncat(flagstr, "IORING_SETUP_IOPOLL", 64 - strlen(flagstr)); + add_pipe = 1; + } + if (p->flags & IORING_SETUP_SQPOLL) { + if (add_pipe) + strncat(flagstr, "|", 64 - strlen(flagstr)); + else + add_pipe = 1; + strncat(flagstr, "IORING_SETUP_SQPOLL", 64 - strlen(flagstr)); + } + if (p->flags & IORING_SETUP_SQ_AFF) { + if (add_pipe) + strncat(flagstr, "|", 64 - strlen(flagstr)); + strncat(flagstr, "IORING_SETUP_SQ_AFF", 64 - strlen(flagstr)); + } + + return flagstr; +} + +char * +dump_resv(struct io_uring_params *p) +{ + static char resvstr[4096]; + + if (!p) + return ""; + + sprintf(resvstr, "0x%.8x 0x%.8x 0x%.8x", p->resv[0], + p->resv[1], p->resv[2]); + + return resvstr; +} + +/* bogus: setup returns a valid fd on success... expect can't predict the + fd we'll get, so this really only takes 1 parameter: error */ +int +try_io_uring_setup(unsigned entries, struct io_uring_params *p, int expect) +{ + int ret; + + ret = io_uring_setup(entries, p); + if (ret != expect) { + fprintf(stderr, "expected %d, got %d\n", expect, ret); + /* if we got a valid uring, close it */ + if (ret > 0) + close(ret); + return 1; + } + + if (expect < 0 && expect != ret) { + if (ret == -EPERM && geteuid() != 0) { + printf("Needs root, not flagging as an error\n"); + return 0; + } + fprintf(stderr, "expected errno %d, got %d\n", expect, ret); + return 1; + } + + return 0; +} + +int +main(int argc, char **argv) +{ + int fd; + unsigned int status = 0; + struct io_uring_params p; + + if (argc > 1) + return T_EXIT_SKIP; + + memset(&p, 0, sizeof(p)); + status |= try_io_uring_setup(0, &p, -EINVAL); + status |= try_io_uring_setup(1, NULL, -EFAULT); + + /* resv array is non-zero */ + memset(&p, 0, sizeof(p)); + p.resv[0] = p.resv[1] = p.resv[2] = 1; + status |= try_io_uring_setup(1, &p, -EINVAL); + + /* invalid flags */ + memset(&p, 0, sizeof(p)); + p.flags = ~0U; + status |= try_io_uring_setup(1, &p, -EINVAL); + + /* IORING_SETUP_SQ_AFF set but not IORING_SETUP_SQPOLL */ + memset(&p, 0, sizeof(p)); + p.flags = IORING_SETUP_SQ_AFF; + status |= try_io_uring_setup(1, &p, -EINVAL); + + /* attempt to bind to invalid cpu */ + memset(&p, 0, sizeof(p)); + p.flags = IORING_SETUP_SQPOLL | IORING_SETUP_SQ_AFF; + p.sq_thread_cpu = get_nprocs_conf(); + status |= try_io_uring_setup(1, &p, -EINVAL); + + /* I think we can limit a process to a set of cpus. I assume + * we shouldn't be able to setup a kernel thread outside of that. + * try to do that. (task->cpus_allowed) */ + + /* read/write on io_uring_fd */ + memset(&p, 0, sizeof(p)); + fd = io_uring_setup(1, &p); + if (fd < 0) { + fprintf(stderr, "io_uring_setup failed with %d, expected success\n", + -fd); + status = 1; + } else { + char buf[4096]; + int ret; + ret = read(fd, buf, 4096); + if (ret >= 0) { + fprintf(stderr, "read from io_uring fd succeeded. expected fail\n"); + status = 1; + } + } + + if (!status) + return T_EXIT_PASS; + + fprintf(stderr, "FAIL\n"); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/iopoll-leak.c b/contrib/libs/liburing/test/iopoll-leak.c new file mode 100644 index 0000000000..01b98fb64a --- /dev/null +++ b/contrib/libs/liburing/test/iopoll-leak.c @@ -0,0 +1,86 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test a mem leak with IOPOLL + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/wait.h> +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE (128 * 1024) +#define BS 4096 +#define BUFFERS (FILE_SIZE / BS) + +static int do_iopoll(const char *fname) +{ + struct io_uring_sqe *sqe; + struct io_uring ring; + struct iovec *iov; + int fd; + + fd = open(fname, O_RDONLY | O_DIRECT); + if (fd < 0) { + perror("open"); + return T_EXIT_SKIP; + } + + iov = t_create_buffers(1, 4096); + + t_create_ring(2, &ring, IORING_SETUP_IOPOLL); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, fd, iov->iov_base, iov->iov_len, 0); + io_uring_submit(&ring); + + close(fd); + return T_EXIT_PASS; +} + +static int test(const char *fname) +{ + if (fork()) { + int stat; + + wait(&stat); + return WEXITSTATUS(stat); + } else { + int ret; + + ret = do_iopoll(fname); + exit(ret); + } +} + +int main(int argc, char *argv[]) +{ + char buf[256]; + char *fname; + int i, ret; + + if (argc > 1) { + fname = argv[1]; + } else { + srand((unsigned)time(NULL)); + snprintf(buf, sizeof(buf), ".iopoll-leak-%u-%u", + (unsigned)rand(), (unsigned)getpid()); + fname = buf; + t_create_file(fname, FILE_SIZE); + } + + for (i = 0; i < 16; i++) { + ret = test(fname); + if (ret == T_EXIT_SKIP || ret == T_EXIT_FAIL) + break; + } + + if (fname != argv[1]) + unlink(fname); + return ret; +} diff --git a/contrib/libs/liburing/test/iopoll.c b/contrib/libs/liburing/test/iopoll.c new file mode 100644 index 0000000000..dfb73265a8 --- /dev/null +++ b/contrib/libs/liburing/test/iopoll.c @@ -0,0 +1,380 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: basic read/write tests with polled IO + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <poll.h> +#include <sys/eventfd.h> +#include <sys/resource.h> +#include "helpers.h" +#include "liburing.h" +#include "../src/syscall.h" + +#define FILE_SIZE (128 * 1024) +#define BS 4096 +#define BUFFERS (FILE_SIZE / BS) + +static struct iovec *vecs; +static int no_buf_select; +static int no_iopoll; + +static int provide_buffers(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, i; + + for (i = 0; i < BUFFERS; i++) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_provide_buffers(sqe, vecs[i].iov_base, + vecs[i].iov_len, 1, 1, i); + } + + ret = io_uring_submit(ring); + if (ret != BUFFERS) { + fprintf(stderr, "submit: %d\n", ret); + return 1; + } + + for (i = 0; i < BUFFERS; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (cqe->res < 0) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +} + +static int __test_io(const char *file, struct io_uring *ring, int write, int sqthread, + int fixed, int buf_select) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int open_flags; + int i, fd = -1, ret; + off_t offset; + + if (buf_select) { + write = 0; + fixed = 0; + } + if (buf_select && provide_buffers(ring)) + return 1; + + if (write) + open_flags = O_WRONLY; + else + open_flags = O_RDONLY; + open_flags |= O_DIRECT; + + if (fixed) { + ret = t_register_buffers(ring, vecs, BUFFERS); + if (ret == T_SETUP_SKIP) + return 0; + if (ret != T_SETUP_OK) { + fprintf(stderr, "buffer reg failed: %d\n", ret); + goto err; + } + } + fd = open(file, open_flags); + if (fd < 0) { + perror("file open"); + goto err; + } + if (sqthread) { + ret = io_uring_register_files(ring, &fd, 1); + if (ret) { + fprintf(stderr, "file reg failed: %d\n", ret); + goto err; + } + } + + offset = 0; + for (i = 0; i < BUFFERS; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + offset = BS * (rand() % BUFFERS); + if (write) { + int do_fixed = fixed; + int use_fd = fd; + + if (sqthread) + use_fd = 0; + if (fixed && (i & 1)) + do_fixed = 0; + if (do_fixed) { + io_uring_prep_write_fixed(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, + offset, i); + } else { + io_uring_prep_writev(sqe, use_fd, &vecs[i], 1, + offset); + } + } else { + int do_fixed = fixed; + int use_fd = fd; + + if (sqthread) + use_fd = 0; + if (fixed && (i & 1)) + do_fixed = 0; + if (do_fixed) { + io_uring_prep_read_fixed(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, + offset, i); + } else { + io_uring_prep_readv(sqe, use_fd, &vecs[i], 1, + offset); + } + + } + if (sqthread) + sqe->flags |= IOSQE_FIXED_FILE; + if (buf_select) { + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = buf_select; + sqe->user_data = i; + } + } + + ret = io_uring_submit(ring); + if (ret != BUFFERS) { + ret = io_uring_peek_cqe(ring, &cqe); + if (!ret && cqe->res == -EOPNOTSUPP) { + no_iopoll = 1; + io_uring_cqe_seen(ring, cqe); + goto out; + } + fprintf(stderr, "submit got %d, wanted %d\n", ret, BUFFERS); + goto err; + } + + for (i = 0; i < BUFFERS; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + goto err; + } else if (cqe->res == -EOPNOTSUPP) { + fprintf(stdout, "File/device/fs doesn't support polled IO\n"); + no_iopoll = 1; + goto out; + } else if (cqe->res != BS) { + fprintf(stderr, "cqe res %d, wanted %d\n", cqe->res, BS); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + if (fixed) { + ret = io_uring_unregister_buffers(ring); + if (ret) { + fprintf(stderr, "buffer unreg failed: %d\n", ret); + goto err; + } + } + if (sqthread) { + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "file unreg failed: %d\n", ret); + goto err; + } + } + +out: + close(fd); + return 0; +err: + if (fd != -1) + close(fd); + return 1; +} + +extern unsigned __io_uring_flush_sq(struct io_uring *ring); + +/* + * if we are polling io_uring_submit needs to always enter the + * kernel to fetch events + */ +static int test_io_uring_submit_enters(const char *file) +{ + struct io_uring ring; + int fd, i, ret, ring_flags, open_flags; + unsigned head; + struct io_uring_cqe *cqe; + + if (no_iopoll) + return 0; + + ring_flags = IORING_SETUP_IOPOLL; + ret = io_uring_queue_init(64, &ring, ring_flags); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + open_flags = O_WRONLY | O_DIRECT; + fd = open(file, open_flags); + if (fd < 0) { + perror("file open"); + goto err; + } + + for (i = 0; i < BUFFERS; i++) { + struct io_uring_sqe *sqe; + off_t offset = BS * (rand() % BUFFERS); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, fd, &vecs[i], 1, offset); + sqe->user_data = 1; + } + + /* submit manually to avoid adding IORING_ENTER_GETEVENTS */ + ret = __sys_io_uring_enter(ring.ring_fd, __io_uring_flush_sq(&ring), 0, + 0, NULL); + if (ret < 0) + goto err; + + for (i = 0; i < 500; i++) { + ret = io_uring_submit(&ring); + if (ret != 0) { + fprintf(stderr, "still had %d sqes to submit, this is unexpected", ret); + goto err; + } + + io_uring_for_each_cqe(&ring, head, cqe) { + /* runs after test_io so should not have happened */ + if (cqe->res == -EOPNOTSUPP) { + fprintf(stdout, "File/device/fs doesn't support polled IO\n"); + goto err; + } + goto ok; + } + usleep(10000); + } +err: + ret = 1; + if (fd != -1) + close(fd); + +ok: + io_uring_queue_exit(&ring); + return ret; +} + +static int test_io(const char *file, int write, int sqthread, int fixed, + int buf_select, int defer) +{ + struct io_uring ring; + int ret, ring_flags = IORING_SETUP_IOPOLL; + + if (no_iopoll) + return 0; + + if (defer) + ring_flags |= IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN; + + ret = t_create_ring(64, &ring, ring_flags); + if (ret == T_SETUP_SKIP) + return 0; + if (ret != T_SETUP_OK) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + ret = __test_io(file, &ring, write, sqthread, fixed, buf_select); + io_uring_queue_exit(&ring); + return ret; +} + +static int probe_buf_select(void) +{ + struct io_uring_probe *p; + struct io_uring ring; + int ret; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + p = io_uring_get_probe_ring(&ring); + if (!p || !io_uring_opcode_supported(p, IORING_OP_PROVIDE_BUFFERS)) { + no_buf_select = 1; + fprintf(stdout, "Buffer select not supported, skipping\n"); + return 0; + } + io_uring_free_probe(p); + return 0; +} + +int main(int argc, char *argv[]) +{ + int i, ret, nr; + char buf[256]; + char *fname; + + if (probe_buf_select()) + return T_EXIT_FAIL; + + if (argc > 1) { + fname = argv[1]; + } else { + srand((unsigned)time(NULL)); + snprintf(buf, sizeof(buf), ".basic-rw-%u-%u", + (unsigned)rand(), (unsigned)getpid()); + fname = buf; + t_create_file(fname, FILE_SIZE); + } + + vecs = t_create_buffers(BUFFERS, BS); + + nr = 32; + if (no_buf_select) + nr = 8; + else if (!t_probe_defer_taskrun()) + nr = 16; + for (i = 0; i < nr; i++) { + int write = (i & 1) != 0; + int sqthread = (i & 2) != 0; + int fixed = (i & 4) != 0; + int buf_select = (i & 8) != 0; + int defer = (i & 16) != 0; + + ret = test_io(fname, write, sqthread, fixed, buf_select, defer); + if (ret) { + fprintf(stderr, "test_io failed %d/%d/%d/%d/%d\n", + write, sqthread, fixed, buf_select, defer); + goto err; + } + if (no_iopoll) + break; + } + + ret = test_io_uring_submit_enters(fname); + if (ret) { + fprintf(stderr, "test_io_uring_submit_enters failed\n"); + goto err; + } + + if (fname != argv[1]) + unlink(fname); + return T_EXIT_PASS; +err: + if (fname != argv[1]) + unlink(fname); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/lfs-openat-write.c b/contrib/libs/liburing/test/lfs-openat-write.c new file mode 100644 index 0000000000..8e3c404de4 --- /dev/null +++ b/contrib/libs/liburing/test/lfs-openat-write.c @@ -0,0 +1,122 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ + +#define _LARGEFILE_SOURCE +#define _FILE_OFFSET_BITS 64 + +#include <liburing.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/resource.h> +#include <unistd.h> + +#include "helpers.h" + +static const int RSIZE = 2; +static const int OPEN_FLAGS = O_RDWR | O_CREAT; +static const mode_t OPEN_MODE = S_IRUSR | S_IWUSR; + +#define DIE(...) do {\ + fprintf(stderr, __VA_ARGS__);\ + abort();\ + } while(0); + +static int do_write(struct io_uring *ring, int fd, off_t offset) +{ + char buf[] = "some test write buf"; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int res, ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "failed to get sqe\n"); + return 1; + } + io_uring_prep_write(sqe, fd, buf, sizeof(buf), offset); + + ret = io_uring_submit(ring); + if (ret < 0) { + fprintf(stderr, "failed to submit write: %s\n", strerror(-ret)); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait_cqe failed: %s\n", strerror(-ret)); + return 1; + } + + res = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (res < 0) { + fprintf(stderr, "write failed: %s\n", strerror(-res)); + return 1; + } + + return 0; +} + +static int test_open_write(struct io_uring *ring, int dfd, const char *fn) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, fd = -1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "failed to get sqe\n"); + return 1; + } + io_uring_prep_openat(sqe, dfd, fn, OPEN_FLAGS, OPEN_MODE); + + ret = io_uring_submit(ring); + if (ret < 0) { + fprintf(stderr, "failed to submit openat: %s\n", strerror(-ret)); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait_cqe failed: %s\n", strerror(-ret)); + return 1; + } + + fd = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (fd < 0) { + fprintf(stderr, "openat failed: %s\n", strerror(-fd)); + return 1; + } + + return do_write(ring, fd, 1ULL << 32); +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int dfd, ret; + + if (argc > 1) + return T_EXIT_SKIP; + + dfd = open("/tmp", O_RDONLY | O_DIRECTORY); + if (dfd < 0) + DIE("open /tmp: %s\n", strerror(errno)); + + ret = io_uring_queue_init(RSIZE, &ring, 0); + if (ret < 0) + DIE("failed to init io_uring: %s\n", strerror(-ret)); + + ret = test_open_write(&ring, dfd, "io_uring_openat_write_test1"); + + io_uring_queue_exit(&ring); + close(dfd); + unlink("/tmp/io_uring_openat_write_test1"); + return ret; +} diff --git a/contrib/libs/liburing/test/lfs-openat.c b/contrib/libs/liburing/test/lfs-openat.c new file mode 100644 index 0000000000..1d93df7e4c --- /dev/null +++ b/contrib/libs/liburing/test/lfs-openat.c @@ -0,0 +1,276 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ + +#define _LARGEFILE_SOURCE +#define _FILE_OFFSET_BITS 64 + +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/resource.h> +#include <unistd.h> + +#include "liburing.h" + +#define DIE(...) do {\ + fprintf(stderr, __VA_ARGS__);\ + abort();\ + } while(0); + +static const int RSIZE = 2; +static const int OPEN_FLAGS = O_RDWR | O_CREAT; +static const mode_t OPEN_MODE = S_IRUSR | S_IWUSR; + +static int open_io_uring(struct io_uring *ring, int dfd, const char *fn) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, fd; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "failed to get sqe\n"); + return 1; + } + io_uring_prep_openat(sqe, dfd, fn, OPEN_FLAGS, OPEN_MODE); + + ret = io_uring_submit(ring); + if (ret < 0) { + fprintf(stderr, "failed to submit openat: %s\n", strerror(-ret)); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + fd = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (ret < 0) { + fprintf(stderr, "wait_cqe failed: %s\n", strerror(-ret)); + return 1; + } else if (fd < 0) { + fprintf(stderr, "io_uring openat failed: %s\n", strerror(-fd)); + return 1; + } + + close(fd); + return 0; +} + +static int prepare_file(int dfd, const char* fn) +{ + const char buf[] = "foo"; + int fd, res; + + fd = openat(dfd, fn, OPEN_FLAGS, OPEN_MODE); + if (fd < 0) { + fprintf(stderr, "prepare/open: %s\n", strerror(errno)); + return -1; + } + + res = pwrite(fd, buf, sizeof(buf), 1ull << 32); + if (res < 0) + fprintf(stderr, "prepare/pwrite: %s\n", strerror(errno)); + + close(fd); + return res < 0 ? res : 0; +} + +static int test_linked_files(int dfd, const char *fn, bool async) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + char buffer[128]; + struct iovec iov = {.iov_base = buffer, .iov_len = sizeof(buffer), }; + int ret, fd; + int fds[2]; + + ret = io_uring_queue_init(10, &ring, 0); + if (ret < 0) + DIE("failed to init io_uring: %s\n", strerror(-ret)); + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + printf("get sqe failed\n"); + return -1; + } + io_uring_prep_readv(sqe, fds[0], &iov, 1, 0); + sqe->flags |= IOSQE_IO_LINK; + if (async) + sqe->flags |= IOSQE_ASYNC; + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "failed to get sqe\n"); + return 1; + } + io_uring_prep_openat(sqe, dfd, fn, OPEN_FLAGS, OPEN_MODE); + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "failed to submit openat: %s\n", strerror(-ret)); + return 1; + } + + fd = dup(ring.ring_fd); + if (fd < 0) { + fprintf(stderr, "dup() failed: %s\n", strerror(-fd)); + return 1; + } + + /* io_uring->flush() */ + close(fd); + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_drained_files(int dfd, const char *fn, bool linked, bool prepend) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + char buffer[128]; + struct iovec iov = {.iov_base = buffer, .iov_len = sizeof(buffer), }; + int ret, fd, fds[2], to_cancel = 0; + + ret = io_uring_queue_init(10, &ring, 0); + if (ret < 0) + DIE("failed to init io_uring: %s\n", strerror(-ret)); + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + printf("get sqe failed\n"); + return -1; + } + io_uring_prep_readv(sqe, fds[0], &iov, 1, 0); + sqe->user_data = 0; + + if (prepend) { + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "failed to get sqe\n"); + return 1; + } + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_DRAIN; + to_cancel++; + sqe->user_data = to_cancel; + } + + if (linked) { + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "failed to get sqe\n"); + return 1; + } + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_DRAIN | IOSQE_IO_LINK; + to_cancel++; + sqe->user_data = to_cancel; + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "failed to get sqe\n"); + return 1; + } + io_uring_prep_openat(sqe, dfd, fn, OPEN_FLAGS, OPEN_MODE); + sqe->flags |= IOSQE_IO_DRAIN; + to_cancel++; + sqe->user_data = to_cancel; + + + ret = io_uring_submit(&ring); + if (ret != 1 + to_cancel) { + fprintf(stderr, "failed to submit openat: %s\n", strerror(-ret)); + return 1; + } + + fd = dup(ring.ring_fd); + if (fd < 0) { + fprintf(stderr, "dup() failed: %s\n", strerror(-fd)); + return 1; + } + + /* + * close(), which triggers ->flush(), and io_uring_queue_exit() + * should successfully return and not hang. + */ + close(fd); + io_uring_queue_exit(&ring); + return 0; +} + +int main(int argc, char *argv[]) +{ + const char *fn = "io_uring_openat_test"; + struct io_uring ring; + int ret, dfd; + + if (argc > 1) + return 0; + + dfd = open("/tmp", O_PATH); + if (dfd < 0) + DIE("open /tmp: %s\n", strerror(errno)); + + ret = io_uring_queue_init(RSIZE, &ring, 0); + if (ret < 0) + DIE("failed to init io_uring: %s\n", strerror(-ret)); + + if (prepare_file(dfd, fn)) + return 1; + + ret = open_io_uring(&ring, dfd, fn); + if (ret) { + fprintf(stderr, "open_io_uring() failed\n"); + goto out; + } + + ret = test_linked_files(dfd, fn, false); + if (ret) { + fprintf(stderr, "test_linked_files() !async failed\n"); + goto out; + } + + ret = test_linked_files(dfd, fn, true); + if (ret) { + fprintf(stderr, "test_linked_files() async failed\n"); + goto out; + } + + ret = test_drained_files(dfd, fn, false, false); + if (ret) { + fprintf(stderr, "test_drained_files() failed\n"); + goto out; + } + + ret = test_drained_files(dfd, fn, false, true); + if (ret) { + fprintf(stderr, "test_drained_files() middle failed\n"); + goto out; + } + + ret = test_drained_files(dfd, fn, true, false); + if (ret) { + fprintf(stderr, "test_drained_files() linked failed\n"); + goto out; + } +out: + io_uring_queue_exit(&ring); + close(dfd); + unlink("/tmp/io_uring_openat_test"); + return ret; +} diff --git a/contrib/libs/liburing/test/link-timeout.c b/contrib/libs/liburing/test/link-timeout.c new file mode 100644 index 0000000000..c59543c53f --- /dev/null +++ b/contrib/libs/liburing/test/link-timeout.c @@ -0,0 +1,1109 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various linked timeout cases + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <poll.h> + +#include "liburing.h" +#include "helpers.h" + +static int test_fail_lone_link_timeouts(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_link_timeout(sqe, &ts, 0); + ts.tv_sec = 1; + ts.tv_nsec = 0; + sqe->user_data = 1; + sqe->flags |= IOSQE_IO_LINK; + + ret = io_uring_submit(ring); + if (ret != 1) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + + if (cqe->user_data != 1) { + fprintf(stderr, "invalid user data %d\n", cqe->res); + goto err; + } + if (cqe->res != -EINVAL) { + fprintf(stderr, "got %d, wanted -EINVAL\n", cqe->res); + goto err; + } + io_uring_cqe_seen(ring, cqe); + + return 0; +err: + return 1; +} + +static int test_fail_two_link_timeouts(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i, nr_wait; + + ts.tv_sec = 1; + ts.tv_nsec = 0; + + /* + * sqe_1: write destined to fail + * use buf=NULL, to do that during the issuing stage + */ + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_writev(sqe, 0, NULL, 1, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + + /* sqe_2: valid linked timeout */ + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 2; + sqe->flags |= IOSQE_IO_LINK; + + + /* sqe_3: invalid linked timeout */ + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 3; + + /* sqe_4: invalid linked timeout */ + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 4; + + ret = io_uring_submit(ring); + if (ret < 3) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + nr_wait = ret; + + for (i = 0; i < nr_wait; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + + switch (cqe->user_data) { + case 1: + if (cqe->res != -EFAULT && cqe->res != -ECANCELED) { + fprintf(stderr, "write got %d, wanted -EFAULT " + "or -ECANCELED\n", cqe->res); + goto err; + } + break; + case 2: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Link timeout got %d, wanted -ECACNCELED\n", cqe->res); + goto err; + } + break; + case 3: + /* fall through */ + case 4: + if (cqe->res != -ECANCELED && cqe->res != -EINVAL) { + fprintf(stderr, "Invalid link timeout got %d" + ", wanted -ECACNCELED || -EINVAL\n", cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test linked timeout with timeout (timeoutception) + */ +static int test_single_link_timeout_ception(struct io_uring *ring) +{ + struct __kernel_timespec ts1, ts2; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + ts1.tv_sec = 1; + ts1.tv_nsec = 0; + io_uring_prep_timeout(sqe, &ts1, -1U, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + ts2.tv_sec = 2; + ts2.tv_nsec = 0; + io_uring_prep_link_timeout(sqe, &ts2, 0); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret != 2) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1: + /* newer kernels allow timeout links */ + if (cqe->res != -EINVAL && cqe->res != -ETIME) { + fprintf(stderr, "Timeout got %d, wanted " + "-EINVAL or -ETIME\n", cqe->res); + goto err; + } + break; + case 2: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Link timeout got %d, wanted -ECANCELED\n", cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test linked timeout with NOP + */ +static int test_single_link_timeout_nop(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + ts.tv_sec = 1; + ts.tv_nsec = 0; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret != 2) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1: + if (cqe->res) { + fprintf(stderr, "NOP got %d, wanted 0\n", cqe->res); + goto err; + } + break; + case 2: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Link timeout got %d, wanted -ECACNCELED\n", cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test read that will not complete, with a linked timeout behind it that + * has errors in the SQE + */ +static int test_single_link_timeout_error(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fds[2], ret, i; + struct iovec iov; + char buffer[128]; + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + iov.iov_base = buffer; + iov.iov_len = sizeof(buffer); + io_uring_prep_readv(sqe, fds[0], &iov, 1, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + ts.tv_sec = 1; + ts.tv_nsec = 0; + io_uring_prep_link_timeout(sqe, &ts, 0); + /* set invalid field, it'll get failed */ + sqe->ioprio = 89; + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret != 2) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Read got %d, wanted -ECANCELED\n", + cqe->res); + goto err; + } + break; + case 2: + if (cqe->res != -EINVAL) { + fprintf(stderr, "Link timeout got %d, wanted -EINVAL\n", cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test read that will complete, with a linked timeout behind it + */ +static int test_single_link_no_timeout(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fds[2], ret, i; + struct iovec iov; + char buffer[128]; + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + iov.iov_base = buffer; + iov.iov_len = sizeof(buffer); + io_uring_prep_readv(sqe, fds[0], &iov, 1, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + ts.tv_sec = 1; + ts.tv_nsec = 0; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 2; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + iov.iov_base = buffer; + iov.iov_len = sizeof(buffer); + io_uring_prep_writev(sqe, fds[1], &iov, 1, 0); + sqe->user_data = 3; + + ret = io_uring_submit(ring); + if (ret != 3) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1: + case 3: + if (cqe->res != sizeof(buffer)) { + fprintf(stderr, "R/W got %d, wanted %d\n", cqe->res, + (int) sizeof(buffer)); + goto err; + } + break; + case 2: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Link timeout %d, wanted -ECANCELED\n", + cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test read that will not complete, with a linked timeout behind it + */ +static int test_single_link_timeout(struct io_uring *ring, unsigned nsec) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fds[2], ret, i; + struct iovec iov; + char buffer[128]; + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + iov.iov_base = buffer; + iov.iov_len = sizeof(buffer); + io_uring_prep_readv(sqe, fds[0], &iov, 1, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + ts.tv_sec = 0; + ts.tv_nsec = nsec; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret != 2) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1: + if (cqe->res != -EINTR && cqe->res != -ECANCELED) { + fprintf(stderr, "Read got %d\n", cqe->res); + goto err; + } + break; + case 2: + if (cqe->res != -EALREADY && cqe->res != -ETIME && + cqe->res != 0) { + fprintf(stderr, "Link timeout got %d\n", cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + close(fds[0]); + close(fds[1]); + return 0; +err: + return 1; +} + +static int test_timeout_link_chain1(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fds[2], ret, i; + struct iovec iov; + char buffer[128]; + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + iov.iov_base = buffer; + iov.iov_len = sizeof(buffer); + io_uring_prep_readv(sqe, fds[0], &iov, 1, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + ts.tv_sec = 0; + ts.tv_nsec = 1000000; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 2; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->user_data = 3; + + ret = io_uring_submit(ring); + if (ret != 3) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1: + if (cqe->res != -EINTR && cqe->res != -ECANCELED) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + case 2: + /* FASTPOLL kernels can cancel successfully */ + if (cqe->res != -EALREADY && cqe->res != -ETIME) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + case 3: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + } + + io_uring_cqe_seen(ring, cqe); + } + + close(fds[0]); + close(fds[1]); + return 0; +err: + return 1; +} + +static int test_timeout_link_chain2(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fds[2], ret, i; + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_poll_add(sqe, fds[0], POLLIN); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + ts.tv_sec = 0; + ts.tv_nsec = 1000000; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 2; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 3; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->user_data = 4; + + ret = io_uring_submit(ring); + if (ret != 4) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 4; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + /* poll cancel really should return -ECANCEL... */ + case 1: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + case 2: + if (cqe->res != -ETIME) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + case 3: + case 4: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + close(fds[0]); + close(fds[1]); + return 0; +err: + return 1; +} + +static int test_timeout_link_chain3(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fds[2], ret, i; + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_poll_add(sqe, fds[0], POLLIN); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + ts.tv_sec = 0; + ts.tv_nsec = 1000000; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 2; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 3; + + /* POLL -> TIMEOUT -> NOP */ + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_poll_add(sqe, fds[0], POLLIN); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 4; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + ts.tv_sec = 0; + ts.tv_nsec = 1000000; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 5; + + /* poll on pipe + timeout */ + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->user_data = 6; + + /* nop */ + + ret = io_uring_submit(ring); + if (ret != 6) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 6; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 2: + if (cqe->res != -ETIME) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + case 1: + case 3: + case 4: + case 5: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + case 6: + if (cqe->res) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + close(fds[0]); + close(fds[1]); + return 0; +err: + return 1; +} + +static int test_timeout_link_chain4(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fds[2], ret, i; + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_poll_add(sqe, fds[0], POLLIN); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 2; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + ts.tv_sec = 0; + ts.tv_nsec = 1000000; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 3; + + ret = io_uring_submit(ring); + if (ret != 3) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + /* poll cancel really should return -ECANCEL... */ + case 1: + if (cqe->res) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + case 2: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + case 3: + if (cqe->res != -ETIME) { + fprintf(stderr, "Req %" PRIu64 " got %d\n", (uint64_t) cqe->user_data, + cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + close(fds[0]); + close(fds[1]); + return 0; +err: + return 1; +} + +static int test_timeout_link_chain5(struct io_uring *ring) +{ + struct __kernel_timespec ts1, ts2; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + ts1.tv_sec = 1; + ts1.tv_nsec = 0; + io_uring_prep_link_timeout(sqe, &ts1, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 2; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + ts2.tv_sec = 2; + ts2.tv_nsec = 0; + io_uring_prep_link_timeout(sqe, &ts2, 0); + sqe->user_data = 3; + + ret = io_uring_submit(ring); + if (ret != 3) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1: + case 2: + if (cqe->res && cqe->res != -ECANCELED) { + fprintf(stderr, "Request got %d, wanted -EINVAL " + "or -ECANCELED\n", + cqe->res); + goto err; + } + break; + case 3: + if (cqe->res != -ECANCELED && cqe->res != -EINVAL) { + fprintf(stderr, "Link timeout got %d, wanted -ECANCELED\n", cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return T_EXIT_FAIL; + } + + ret = test_timeout_link_chain1(&ring); + if (ret) { + printf("test_single_link_chain1 failed\n"); + return ret; + } + + ret = test_timeout_link_chain2(&ring); + if (ret) { + printf("test_single_link_chain2 failed\n"); + return ret; + } + + ret = test_timeout_link_chain3(&ring); + if (ret) { + printf("test_single_link_chain3 failed\n"); + return ret; + } + + ret = test_timeout_link_chain4(&ring); + if (ret) { + printf("test_single_link_chain4 failed\n"); + return ret; + } + + ret = test_timeout_link_chain5(&ring); + if (ret) { + printf("test_single_link_chain5 failed\n"); + return ret; + } + + ret = test_single_link_timeout(&ring, 10); + if (ret) { + printf("test_single_link_timeout 10 failed\n"); + return ret; + } + + ret = test_single_link_timeout(&ring, 100000ULL); + if (ret) { + printf("test_single_link_timeout 100000 failed\n"); + return ret; + } + + ret = test_single_link_timeout(&ring, 500000000ULL); + if (ret) { + printf("test_single_link_timeout 500000000 failed\n"); + return ret; + } + + ret = test_single_link_no_timeout(&ring); + if (ret) { + printf("test_single_link_no_timeout failed\n"); + return ret; + } + + ret = test_single_link_timeout_error(&ring); + if (ret) { + printf("test_single_link_timeout_error failed\n"); + return ret; + } + + ret = test_single_link_timeout_nop(&ring); + if (ret) { + printf("test_single_link_timeout_nop failed\n"); + return ret; + } + + ret = test_single_link_timeout_ception(&ring); + if (ret) { + printf("test_single_link_timeout_ception failed\n"); + return ret; + } + + ret = test_fail_lone_link_timeouts(&ring); + if (ret) { + printf("test_fail_lone_link_timeouts failed\n"); + return ret; + } + + ret = test_fail_two_link_timeouts(&ring); + if (ret) { + printf("test_fail_two_link_timeouts failed\n"); + return ret; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/link.c b/contrib/libs/liburing/test/link.c new file mode 100644 index 0000000000..6f839394c3 --- /dev/null +++ b/contrib/libs/liburing/test/link.c @@ -0,0 +1,498 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various linked sqe tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +static int no_hardlink; + +/* + * Timer with single nop + */ +static int test_single_hardlink(struct io_uring *ring) +{ + struct __kernel_timespec ts; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + ts.tv_sec = 0; + ts.tv_nsec = 10000000ULL; + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->flags |= IOSQE_IO_LINK | IOSQE_IO_HARDLINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (!cqe) { + fprintf(stderr, "failed to get cqe\n"); + goto err; + } + if (no_hardlink) + goto next; + if (cqe->user_data == 1 && cqe->res == -EINVAL) { + fprintf(stdout, "Hard links not supported, skipping\n"); + no_hardlink = 1; + goto next; + } + if (cqe->user_data == 1 && cqe->res != -ETIME) { + fprintf(stderr, "timeout failed with %d\n", cqe->res); + goto err; + } + if (cqe->user_data == 2 && cqe->res) { + fprintf(stderr, "nop failed with %d\n", cqe->res); + goto err; + } +next: + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Timer -> timer -> nop + */ +static int test_double_hardlink(struct io_uring *ring) +{ + struct __kernel_timespec ts1, ts2; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + if (no_hardlink) + return 0; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + ts1.tv_sec = 0; + ts1.tv_nsec = 10000000ULL; + io_uring_prep_timeout(sqe, &ts1, 0, 0); + sqe->flags |= IOSQE_IO_LINK | IOSQE_IO_HARDLINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + ts2.tv_sec = 0; + ts2.tv_nsec = 15000000ULL; + io_uring_prep_timeout(sqe, &ts2, 0, 0); + sqe->flags |= IOSQE_IO_LINK | IOSQE_IO_HARDLINK; + sqe->user_data = 2; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->user_data = 3; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (!cqe) { + fprintf(stderr, "failed to get cqe\n"); + goto err; + } + if (cqe->user_data == 1 && cqe->res != -ETIME) { + fprintf(stderr, "timeout failed with %d\n", cqe->res); + goto err; + } + if (cqe->user_data == 2 && cqe->res != -ETIME) { + fprintf(stderr, "timeout failed with %d\n", cqe->res); + goto err; + } + if (cqe->user_data == 3 && cqe->res) { + fprintf(stderr, "nop failed with %d\n", cqe->res); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; + +} + +/* + * Test failing head of chain, and dependent getting -ECANCELED + */ +static int test_single_link_fail(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_remove_buffers(sqe, 10, 1); + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_peek_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + if (!cqe) { + printf("failed to get cqe\n"); + goto err; + } + if (i == 0 && cqe->res != -ENOENT) { + printf("sqe0 failed with %d, wanted -ENOENT\n", cqe->res); + goto err; + } + if (i == 1 && cqe->res != -ECANCELED) { + printf("sqe1 failed with %d, wanted -ECANCELED\n", cqe->res); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test two independent chains + */ +static int test_double_chain(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 4; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test multiple dependents + */ +static int test_double_link(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test single dependency + */ +static int test_single_link(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + + ret = io_uring_submit(ring); + if (ret <= 0) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +static int test_early_fail_and_wait(void) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + int ret, invalid_fd = 42; + struct iovec iov = { .iov_base = NULL, .iov_len = 0 }; + + /* create a new ring as it leaves it dirty */ + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return 1; + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_readv(sqe, invalid_fd, &iov, 1, 0); + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + printf("get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + + ret = io_uring_submit_and_wait(&ring, 2); + if (ret <= 0 && ret != -EAGAIN) { + printf("sqe submit failed: %d\n", ret); + goto err; + } + + io_uring_queue_exit(&ring); + return 0; +err: + io_uring_queue_exit(&ring); + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring, poll_ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return T_EXIT_FAIL; + + } + + ret = io_uring_queue_init(8, &poll_ring, IORING_SETUP_IOPOLL); + if (ret) { + printf("poll_ring setup failed\n"); + return T_EXIT_FAIL; + } + + ret = test_single_link(&ring); + if (ret) { + printf("test_single_link failed\n"); + return ret; + } + + ret = test_double_link(&ring); + if (ret) { + printf("test_double_link failed\n"); + return ret; + } + + ret = test_double_chain(&ring); + if (ret) { + printf("test_double_chain failed\n"); + return ret; + } + + ret = test_single_link_fail(&poll_ring); + if (ret) { + printf("test_single_link_fail failed\n"); + return ret; + } + + ret = test_single_hardlink(&ring); + if (ret) { + fprintf(stderr, "test_single_hardlink\n"); + return ret; + } + + ret = test_double_hardlink(&ring); + if (ret) { + fprintf(stderr, "test_double_hardlink\n"); + return ret; + } + + ret = test_early_fail_and_wait(); + if (ret) { + fprintf(stderr, "test_early_fail_and_wait\n"); + return ret; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/link_drain.c b/contrib/libs/liburing/test/link_drain.c new file mode 100644 index 0000000000..ec8021f807 --- /dev/null +++ b/contrib/libs/liburing/test/link_drain.c @@ -0,0 +1,230 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring link io with drain io + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "helpers.h" +#include "liburing.h" + +static int test_link_drain_one(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe[5]; + struct iovec iovecs; + int i, fd, ret; + off_t off = 0; + char data[5] = {0}; + char expect[5] = {0, 1, 2, 3, 4}; + + fd = open("testfile", O_WRONLY | O_CREAT, 0644); + if (fd < 0) { + perror("open"); + return 1; + } + + iovecs.iov_base = t_malloc(4096); + iovecs.iov_len = 4096; + + for (i = 0; i < 5; i++) { + sqe[i] = io_uring_get_sqe(ring); + if (!sqe[i]) { + printf("get sqe failed\n"); + goto err; + } + } + + /* normal heavy io */ + io_uring_prep_writev(sqe[0], fd, &iovecs, 1, off); + sqe[0]->user_data = 0; + + /* link io */ + io_uring_prep_nop(sqe[1]); + sqe[1]->flags |= IOSQE_IO_LINK; + sqe[1]->user_data = 1; + + /* link drain io */ + io_uring_prep_nop(sqe[2]); + sqe[2]->flags |= (IOSQE_IO_LINK | IOSQE_IO_DRAIN); + sqe[2]->user_data = 2; + + /* link io */ + io_uring_prep_nop(sqe[3]); + sqe[3]->user_data = 3; + + /* normal nop io */ + io_uring_prep_nop(sqe[4]); + sqe[4]->user_data = 4; + + ret = io_uring_submit(ring); + if (ret < 0) { + printf("sqe submit failed\n"); + goto err; + } else if (ret < 5) { + printf("Submitted only %d\n", ret); + goto err; + } + + for (i = 0; i < 5; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("child: wait completion %d\n", ret); + goto err; + } + + data[i] = cqe->user_data; + io_uring_cqe_seen(ring, cqe); + } + + if (memcmp(data, expect, 5) != 0) + goto err; + + free(iovecs.iov_base); + close(fd); + unlink("testfile"); + return 0; +err: + free(iovecs.iov_base); + close(fd); + unlink("testfile"); + return 1; +} + +int test_link_drain_multi(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe[9]; + struct iovec iovecs; + int i, fd, ret; + off_t off = 0; + char data[9] = {0}; + char expect[9] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + + fd = open("testfile", O_WRONLY | O_CREAT, 0644); + if (fd < 0) { + perror("open"); + return 1; + } + unlink("testfile"); + + iovecs.iov_base = t_malloc(4096); + iovecs.iov_len = 4096; + + for (i = 0; i < 9; i++) { + sqe[i] = io_uring_get_sqe(ring); + if (!sqe[i]) { + printf("get sqe failed\n"); + goto err; + } + } + + /* normal heavy io */ + io_uring_prep_writev(sqe[0], fd, &iovecs, 1, off); + sqe[0]->user_data = 0; + + /* link1 io head */ + io_uring_prep_nop(sqe[1]); + sqe[1]->flags |= IOSQE_IO_LINK; + sqe[1]->user_data = 1; + + /* link1 drain io */ + io_uring_prep_nop(sqe[2]); + sqe[2]->flags |= (IOSQE_IO_LINK | IOSQE_IO_DRAIN); + sqe[2]->user_data = 2; + + /* link1 io end*/ + io_uring_prep_nop(sqe[3]); + sqe[3]->user_data = 3; + + /* link2 io head */ + io_uring_prep_nop(sqe[4]); + sqe[4]->flags |= IOSQE_IO_LINK; + sqe[4]->user_data = 4; + + /* link2 io */ + io_uring_prep_nop(sqe[5]); + sqe[5]->flags |= IOSQE_IO_LINK; + sqe[5]->user_data = 5; + + /* link2 drain io */ + io_uring_prep_writev(sqe[6], fd, &iovecs, 1, off); + sqe[6]->flags |= (IOSQE_IO_LINK | IOSQE_IO_DRAIN); + sqe[6]->user_data = 6; + + /* link2 io end */ + io_uring_prep_nop(sqe[7]); + sqe[7]->user_data = 7; + + /* normal io */ + io_uring_prep_nop(sqe[8]); + sqe[8]->user_data = 8; + + ret = io_uring_submit(ring); + if (ret < 0) { + printf("sqe submit failed\n"); + goto err; + } else if (ret < 9) { + printf("Submitted only %d\n", ret); + goto err; + } + + for (i = 0; i < 9; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("child: wait completion %d\n", ret); + goto err; + } + + data[i] = cqe->user_data; + io_uring_cqe_seen(ring, cqe); + } + + if (memcmp(data, expect, 9) != 0) + goto err; + + free(iovecs.iov_base); + close(fd); + return 0; +err: + free(iovecs.iov_base); + close(fd); + return 1; + +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int i, ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(100, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return 1; + } + + for (i = 0; i < 1000; i++) { + ret = test_link_drain_one(&ring); + if (ret) { + fprintf(stderr, "test_link_drain_one failed\n"); + break; + } + ret = test_link_drain_multi(&ring); + if (ret) { + fprintf(stderr, "test_link_drain_multi failed\n"); + break; + } + } + + return ret; +} diff --git a/contrib/libs/liburing/test/madvise.c b/contrib/libs/liburing/test/madvise.c new file mode 100644 index 0000000000..bd44a9741c --- /dev/null +++ b/contrib/libs/liburing/test/madvise.c @@ -0,0 +1,196 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: basic madvise test + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/time.h> +#include <sys/mman.h> + +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE (128 * 1024) + +#define LOOPS 100 +#define MIN_LOOPS 10 + +static unsigned long long utime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000000; + return sec + usec; +} + +static unsigned long long utime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return utime_since(tv, &end); +} + +static int do_madvise(struct io_uring *ring, void *addr, off_t len, int advice) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "failed to get sqe\n"); + return 1; + } + + io_uring_prep_madvise(sqe, addr, len, advice); + sqe->user_data = advice; + ret = io_uring_submit_and_wait(ring, 1); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return 1; + } + + ret = cqe->res; + if (ret == -EINVAL || ret == -EBADF) { + fprintf(stdout, "Madvise not supported, skipping\n"); + unlink(".madvise.tmp"); + exit(0); + } else if (ret) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + } + io_uring_cqe_seen(ring, cqe); + return ret; +} + +static long do_copy(int fd, char *buf, void *ptr) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + memcpy(buf, ptr, FILE_SIZE); + return utime_since_now(&tv); +} + +static int test_madvise(struct io_uring *ring, const char *filename) +{ + unsigned long cached_read, uncached_read, cached_read2; + int fd, ret; + char *buf; + void *ptr; + + fd = open(filename, O_RDONLY); + if (fd < 0) { + perror("open"); + return 1; + } + + buf = t_malloc(FILE_SIZE); + + ptr = mmap(NULL, FILE_SIZE, PROT_READ, MAP_PRIVATE, fd, 0); + if (ptr == MAP_FAILED) { + perror("mmap"); + return 1; + } + + cached_read = do_copy(fd, buf, ptr); + if (cached_read == -1) + return 1; + + cached_read = do_copy(fd, buf, ptr); + if (cached_read == -1) + return 1; + + ret = do_madvise(ring, ptr, FILE_SIZE, MADV_DONTNEED); + if (ret) + return 1; + + uncached_read = do_copy(fd, buf, ptr); + if (uncached_read == -1) + return 1; + + ret = do_madvise(ring, ptr, FILE_SIZE, MADV_DONTNEED); + if (ret) + return 1; + + ret = do_madvise(ring, ptr, FILE_SIZE, MADV_WILLNEED); + if (ret) + return 1; + + msync(ptr, FILE_SIZE, MS_SYNC); + + cached_read2 = do_copy(fd, buf, ptr); + if (cached_read2 == -1) + return 1; + + if (cached_read < uncached_read && + cached_read2 < uncached_read) + return 0; + + return 2; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret, i, good, bad; + char *fname; + + if (argc > 1) { + fname = argv[1]; + } else { + fname = ".madvise.tmp"; + t_create_file(fname, FILE_SIZE); + } + + if (io_uring_queue_init(8, &ring, 0)) { + fprintf(stderr, "ring creation failed\n"); + goto err; + } + + good = bad = 0; + for (i = 0; i < LOOPS; i++) { + ret = test_madvise(&ring, fname); + if (ret == 1) { + fprintf(stderr, "test_madvise failed\n"); + goto err; + } else if (!ret) + good++; + else if (ret == 2) + bad++; + if (i >= MIN_LOOPS && !bad) + break; + } + + /* too hard to reliably test, just ignore */ + if (0 && bad > good) + fprintf(stderr, "Suspicious timings (%u > %u)\n", bad, good); + if (fname != argv[1]) + unlink(fname); + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +err: + if (fname != argv[1]) + unlink(fname); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/mkdir.c b/contrib/libs/liburing/test/mkdir.c new file mode 100644 index 0000000000..630672cc93 --- /dev/null +++ b/contrib/libs/liburing/test/mkdir.c @@ -0,0 +1,113 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring mkdirat handling + */ +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" + +static int do_mkdirat(struct io_uring *ring, const char *fn) +{ + int ret; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + io_uring_prep_mkdirat(sqe, AT_FDCWD, fn, 0700); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqes(ring, &cqe, 1, 0, 0); + if (ret) { + fprintf(stderr, "wait_cqe failed: %d\n", ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +err: + return 1; +} + +static int stat_file(const char *fn) +{ + struct stat sb; + + if (!stat(fn, &sb)) + return 0; + + return errno; +} + +int main(int argc, char *argv[]) +{ + static const char fn[] = "io_uring-mkdirat-test"; + int ret; + struct io_uring ring; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return ret; + } + + ret = do_mkdirat(&ring, fn); + if (ret < 0) { + if (ret == -EBADF || ret == -EINVAL) { + fprintf(stdout, "mkdirat not supported, skipping\n"); + goto skip; + } + fprintf(stderr, "mkdirat: %s\n", strerror(-ret)); + goto err; + } else if (ret) { + goto err; + } + + if (stat_file(fn)) { + perror("stat"); + goto err; + } + + ret = do_mkdirat(&ring, fn); + if (ret != -EEXIST) { + fprintf(stderr, "do_mkdirat already exists failed: %d\n", ret); + goto err1; + } + + ret = do_mkdirat(&ring, "surely/this/wont/exist"); + if (ret != -ENOENT) { + fprintf(stderr, "do_mkdirat no parent failed: %d\n", ret); + goto err1; + } + + unlinkat(AT_FDCWD, fn, AT_REMOVEDIR); + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +skip: + unlinkat(AT_FDCWD, fn, AT_REMOVEDIR); + io_uring_queue_exit(&ring); + return T_EXIT_SKIP; +err1: + unlinkat(AT_FDCWD, fn, AT_REMOVEDIR); +err: + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/msg-ring.c b/contrib/libs/liburing/test/msg-ring.c new file mode 100644 index 0000000000..079c1609d0 --- /dev/null +++ b/contrib/libs/liburing/test/msg-ring.c @@ -0,0 +1,261 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test ring messaging command + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <pthread.h> + +#include "liburing.h" +#include "helpers.h" + +static int no_msg; + +static int test_own(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_msg_ring(sqe, ring->ring_fd, 0x10, 0x1234, 0); + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1: + if (cqe->res == -EINVAL || cqe->res == -EOPNOTSUPP) { + no_msg = 1; + return 0; + } + if (cqe->res != 0) { + fprintf(stderr, "cqe res %d\n", cqe->res); + return -1; + } + break; + case 0x1234: + if (cqe->res != 0x10) { + fprintf(stderr, "invalid len %x\n", cqe->res); + return -1; + } + break; + default: + fprintf(stderr, "Invalid user_data\n"); + return -1; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +static void *wait_cqe_fn(void *data) +{ + struct io_uring *ring = data; + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe %d\n", ret); + goto err; + } + + if (cqe->user_data != 0x5aa5) { + fprintf(stderr, "user_data %llx\n", (long long) cqe->user_data); + goto err; + } + if (cqe->res != 0x20) { + fprintf(stderr, "len %x\n", cqe->res); + goto err; + } + + return NULL; +err: + return (void *) (unsigned long) 1; +} + +static int test_remote(struct io_uring *ring, struct io_uring *target) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_msg_ring(sqe, target->ring_fd, 0x20, 0x5aa5, 0); + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res != 0) { + fprintf(stderr, "cqe res %d\n", cqe->res); + return -1; + } + if (cqe->user_data != 1) { + fprintf(stderr, "user_data %llx\n", (long long) cqe->user_data); + return -1; + } + + io_uring_cqe_seen(ring, cqe); + return 0; +err: + return 1; +} + +static int test_invalid(struct io_uring *ring, bool fixed) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, fd = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return 1; + } + + if (fixed) { + ret = io_uring_register_files(ring, &fd, 1); + if (ret) { + fprintf(stderr, "file register %d\n", ret); + return 1; + } + io_uring_prep_msg_ring(sqe, 0, 0, 0x8989, 0); + sqe->flags |= IOSQE_FIXED_FILE; + } else { + io_uring_prep_msg_ring(sqe, 1, 0, 0x8989, 0); + } + + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res != -EBADFD) { + fprintf(stderr, "cqe res %d\n", cqe->res); + return -1; + } + + io_uring_cqe_seen(ring, cqe); + if (fixed) + io_uring_unregister_files(ring); + return 0; +err: + if (fixed) + io_uring_unregister_files(ring); + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring, ring2, pring; + pthread_t thread; + void *tret; + int ret, i; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + ret = io_uring_queue_init(8, &ring2, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + ret = io_uring_queue_init(8, &pring, IORING_SETUP_IOPOLL); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = test_own(&ring); + if (ret) { + fprintf(stderr, "test_own failed\n"); + return ret; + } + if (no_msg) { + fprintf(stdout, "Skipped\n"); + return T_EXIT_SKIP; + } + ret = test_own(&pring); + if (ret) { + fprintf(stderr, "test_own iopoll failed\n"); + return ret; + } + + ret = test_invalid(&ring, 0); + if (ret) { + fprintf(stderr, "test_invalid failed\n"); + return ret; + } + + for (i = 0; i < 2; i++) { + ret = test_invalid(&ring, 1); + if (ret) { + fprintf(stderr, "test_invalid fixed failed\n"); + return ret; + } + } + + pthread_create(&thread, NULL, wait_cqe_fn, &ring2); + + ret = test_remote(&ring, &ring2); + if (ret) { + fprintf(stderr, "test_remote failed\n"); + return ret; + } + + pthread_join(thread, &tret); + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/multicqes_drain.c b/contrib/libs/liburing/test/multicqes_drain.c new file mode 100644 index 0000000000..99e5fe1247 --- /dev/null +++ b/contrib/libs/liburing/test/multicqes_drain.c @@ -0,0 +1,427 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: generic tests for io_uring drain io + * + * The main idea is to randomly generate different type of sqe to + * challenge the drain logic. There are some restrictions for the + * generated sqes, details in io_uring maillist: + * https://lore.kernel.org/io-uring/39a49b4c-27c2-1035-b250-51daeccaab9b@linux.alibaba.com/ + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <poll.h> + +#include "liburing.h" +#include "helpers.h" + +enum { + multi, + single, + nop, + cancel, + op_last, +}; + +struct sqe_info { + __u8 op; + unsigned flags; +}; + +#define max_entry 50 + +/* + * sqe_flags: combination of sqe flags + * multi_sqes: record the user_data/index of all the multishot sqes + * cnt: how many entries there are in multi_sqes + * we can leverage multi_sqes array for cancellation: we randomly pick + * up an entry in multi_sqes when form a cancellation sqe. + * multi_cap: limitation of number of multishot sqes + */ +const unsigned sqe_flags[4] = {0, IOSQE_IO_LINK, IOSQE_IO_DRAIN, + IOSQE_IO_LINK | IOSQE_IO_DRAIN}; +int multi_sqes[max_entry], cnt = 0; +int multi_cap = max_entry / 5; + +int write_pipe(int pipe, char *str) +{ + int ret; + do { + errno = 0; + ret = write(pipe, str, 3); + } while (ret == -1 && errno == EINTR); + return ret; +} + +void read_pipe(int pipe) +{ + char str[4] = {0}; + int ret; + + ret = read(pipe, &str, 3); + if (ret < 0) + perror("read"); +} + +int trigger_event(int p[]) +{ + int ret; + if ((ret = write_pipe(p[1], "foo")) != 3) { + fprintf(stderr, "bad write return %d\n", ret); + return 1; + } + read_pipe(p[0]); + return 0; +} + +void io_uring_sqe_prep(int op, struct io_uring_sqe *sqe, unsigned sqe_flags, int arg) +{ + switch (op) { + case multi: + io_uring_prep_poll_add(sqe, arg, POLLIN); + sqe->len |= IORING_POLL_ADD_MULTI; + break; + case single: + io_uring_prep_poll_add(sqe, arg, POLLIN); + break; + case nop: + io_uring_prep_nop(sqe); + break; + case cancel: + io_uring_prep_poll_remove(sqe, arg); + break; + } + sqe->flags = sqe_flags; +} + +__u8 generate_flags(int sqe_op) +{ + __u8 flags = 0; + /* + * drain sqe must be put after multishot sqes cancelled + */ + do { + flags = sqe_flags[rand() % 4]; + } while ((flags & IOSQE_IO_DRAIN) && cnt); + + /* + * cancel req cannot have drain or link flag + */ + if (sqe_op == cancel) { + flags &= ~(IOSQE_IO_DRAIN | IOSQE_IO_LINK); + } + /* + * avoid below case: + * sqe0(multishot, link)->sqe1(nop, link)->sqe2(nop)->sqe3(cancel_sqe0) + * sqe3 may execute before sqe0 so that sqe0 isn't cancelled + */ + if (sqe_op == multi) + flags &= ~IOSQE_IO_LINK; + + return flags; + +} + +/* + * function to generate opcode of a sqe + * several restrictions here: + * - cancel all the previous multishot sqes as soon as possible when + * we reach high watermark. + * - ensure there is some multishot sqe when generating a cancel sqe + * - ensure a cancel/multshot sqe is not in a linkchain + * - ensure number of multishot sqes doesn't exceed multi_cap + * - don't generate multishot sqes after high watermark + */ +int generate_opcode(int i, int pre_flags) +{ + int sqe_op; + int high_watermark = max_entry - max_entry / 5; + bool retry0 = false, retry1 = false, retry2 = false; + + if ((i >= high_watermark) && cnt) { + sqe_op = cancel; + } else { + do { + sqe_op = rand() % op_last; + retry0 = (sqe_op == cancel) && (!cnt || (pre_flags & IOSQE_IO_LINK)); + retry1 = (sqe_op == multi) && ((multi_cap - 1 < 0) || i >= high_watermark); + retry2 = (sqe_op == multi) && (pre_flags & IOSQE_IO_LINK); + } while (retry0 || retry1 || retry2); + } + + if (sqe_op == multi) + multi_cap--; + return sqe_op; +} + +static inline void add_multishot_sqe(int index) +{ + multi_sqes[cnt++] = index; +} + +int remove_multishot_sqe() +{ + int ret; + + int rem_index = rand() % cnt; + ret = multi_sqes[rem_index]; + multi_sqes[rem_index] = multi_sqes[cnt - 1]; + cnt--; + + return ret; +} + +static int test_generic_drain(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe[max_entry]; + struct sqe_info si[max_entry]; + int cqe_data[max_entry << 1], cqe_res[max_entry << 1]; + int i, j, ret, arg = 0; + int pipes[max_entry][2]; + int pre_flags = 0; + + for (i = 0; i < max_entry; i++) { + if (pipe(pipes[i]) != 0) { + perror("pipe"); + return 1; + } + } + + srand((unsigned)time(NULL)); + for (i = 0; i < max_entry; i++) { + sqe[i] = io_uring_get_sqe(ring); + if (!sqe[i]) { + printf("get sqe failed\n"); + goto err; + } + + int sqe_op = generate_opcode(i, pre_flags); + __u8 flags = generate_flags(sqe_op); + + if (sqe_op == cancel) + arg = remove_multishot_sqe(); + if (sqe_op == multi || sqe_op == single) + arg = pipes[i][0]; + io_uring_sqe_prep(sqe_op, sqe[i], flags, arg); + sqe[i]->user_data = i; + si[i].op = sqe_op; + si[i].flags = flags; + pre_flags = flags; + if (sqe_op == multi) + add_multishot_sqe(i); + } + + ret = io_uring_submit(ring); + if (ret < 0) { + printf("sqe submit failed\n"); + goto err; + } else if (ret < max_entry) { + printf("Submitted only %d\n", ret); + goto err; + } + + sleep(1); + // TODO: randomize event triggerring order + for (i = 0; i < max_entry; i++) { + if (si[i].op != multi && si[i].op != single) + continue; + + if (trigger_event(pipes[i])) + goto err; + + io_uring_get_events(ring); + } + sleep(1); + i = 0; + while (!io_uring_peek_cqe(ring, &cqe)) { + cqe_data[i] = cqe->user_data; + cqe_res[i++] = cqe->res; + io_uring_cqe_seen(ring, cqe); + } + + /* + * compl_bits is a bit map to record completions. + * eg. sqe[0], sqe[1], sqe[2] fully completed + * then compl_bits is 000...00111b + * + */ + unsigned long long compl_bits = 0; + for (j = 0; j < i; j++) { + int index = cqe_data[j]; + if ((si[index].flags & IOSQE_IO_DRAIN) && index) { + if ((~compl_bits) & ((1ULL << index) - 1)) { + printf("drain failed\n"); + goto err; + } + } + /* + * for multishot sqes, record them only when it is cancelled + */ + if ((si[index].op != multi) || (cqe_res[j] == -ECANCELED)) + compl_bits |= (1ULL << index); + } + + return 0; +err: + return 1; +} + +static int test_simple_drain(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe[2]; + int i, ret; + int pipe1[2], pipe2[2]; + + if (pipe(pipe1) != 0 || pipe(pipe2) != 0) { + perror("pipe"); + return 1; + } + + for (i = 0; i < 2; i++) { + sqe[i] = io_uring_get_sqe(ring); + if (!sqe[i]) { + printf("get sqe failed\n"); + goto err; + } + } + + io_uring_prep_poll_multishot(sqe[0], pipe1[0], POLLIN); + sqe[0]->user_data = 0; + + io_uring_prep_poll_add(sqe[1], pipe2[0], POLLIN); + sqe[1]->user_data = 1; + + /* This test relies on multishot poll to trigger events continually. + * however with IORING_SETUP_DEFER_TASKRUN this will only happen when + * triggered with a get_events. Hence we sprinkle get_events whenever + * there might be work to process in order to get the same result + */ + ret = io_uring_submit_and_get_events(ring); + if (ret < 0) { + printf("sqe submit failed\n"); + goto err; + } else if (ret < 2) { + printf("Submitted only %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + if (trigger_event(pipe1)) + goto err; + io_uring_get_events(ring); + } + if (trigger_event(pipe2)) + goto err; + io_uring_get_events(ring); + + for (i = 0; i < 2; i++) { + sqe[i] = io_uring_get_sqe(ring); + if (!sqe[i]) { + printf("get sqe failed\n"); + goto err; + } + } + + io_uring_prep_poll_remove(sqe[0], 0); + sqe[0]->user_data = 2; + + io_uring_prep_nop(sqe[1]); + sqe[1]->flags |= IOSQE_IO_DRAIN; + sqe[1]->user_data = 3; + + ret = io_uring_submit(ring); + if (ret < 0) { + printf("sqe submit failed\n"); + goto err; + } else if (ret < 2) { + printf("Submitted only %d\n", ret); + goto err; + } + + for (i = 0; i < 6; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + printf("wait completion %d\n", ret); + goto err; + } + if ((i == 5) && (cqe->user_data != 3)) + goto err; + io_uring_cqe_seen(ring, cqe); + } + + close(pipe1[0]); + close(pipe1[1]); + close(pipe2[0]); + close(pipe2[1]); + return 0; +err: + return 1; +} + +static int test(bool defer_taskrun) +{ + struct io_uring ring; + int i, ret; + unsigned int flags = 0; + + if (defer_taskrun) + flags = IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN; + + ret = io_uring_queue_init(1024, &ring, flags); + if (ret) { + printf("ring setup failed\n"); + return T_EXIT_FAIL; + } + + for (i = 0; i < 5; i++) { + ret = test_simple_drain(&ring); + if (ret) { + fprintf(stderr, "test_simple_drain failed\n"); + return T_EXIT_FAIL; + } + } + + for (i = 0; i < 5; i++) { + ret = test_generic_drain(&ring); + if (ret) { + fprintf(stderr, "test_generic_drain failed\n"); + return T_EXIT_FAIL; + } + } + + io_uring_queue_exit(&ring); + + return T_EXIT_PASS; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = test(false); + if (ret != T_EXIT_PASS) { + fprintf(stderr, "%s: test(false) failed\n", argv[0]); + return ret; + } + + if (t_probe_defer_taskrun()) { + ret = test(true); + if (ret != T_EXIT_PASS) { + fprintf(stderr, "%s: test(true) failed\n", argv[0]); + return ret; + } + } + + return ret; +} diff --git a/contrib/libs/liburing/test/nolibc.c b/contrib/libs/liburing/test/nolibc.c new file mode 100644 index 0000000000..fd07fa13cd --- /dev/null +++ b/contrib/libs/liburing/test/nolibc.c @@ -0,0 +1,61 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test liburing nolibc functionality. + * + * Currently, supported architectures are: + * 1) x86 + * 2) x86-64 + * 3) aarch64 + * + */ +#include "helpers.h" + +#if !defined(__x86_64__) && !defined(__i386__) && !defined(__aarch64__) + +/* + * This arch doesn't support nolibc. + */ +int main(void) +{ + return T_EXIT_SKIP; +} + +#else /* #if !defined(__x86_64__) && !defined(__i386__) && !defined(__aarch64__) */ + +#ifndef CONFIG_NOLIBC +#define CONFIG_NOLIBC +#endif + +#include <stdio.h> +#include <unistd.h> +#include "../src/lib.h" + +static int test_get_page_size(void) +{ + long a, b; + + a = sysconf(_SC_PAGESIZE); + b = get_page_size(); + if (a != b) { + fprintf(stderr, "get_page_size() fails, %ld != %ld", a, b); + return -1; + } + return 0; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = test_get_page_size(); + if (ret) + return T_EXIT_FAIL; + + return T_EXIT_PASS; +} + +#endif /* #if !defined(__x86_64__) && !defined(__i386__) && !defined(__aarch64__) */ diff --git a/contrib/libs/liburing/test/nop-all-sizes.c b/contrib/libs/liburing/test/nop-all-sizes.c new file mode 100644 index 0000000000..f7bc55b910 --- /dev/null +++ b/contrib/libs/liburing/test/nop-all-sizes.c @@ -0,0 +1,100 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: exercise full filling of SQ and CQ ring + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" + +#define MAX_ENTRIES 32768 + +static int fill_nops(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + int filled = 0; + + do { + sqe = io_uring_get_sqe(ring); + if (!sqe) + break; + + io_uring_prep_nop(sqe); + filled++; + } while (1); + + return filled; +} + +static int test_nops(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int ret, nr, total = 0, i; + + nr = fill_nops(ring); + + ret = io_uring_submit(ring); + if (ret != nr) { + fprintf(stderr, "submit %d, wanted %d\n", ret, nr); + goto err; + } + total += ret; + + nr = fill_nops(ring); + + ret = io_uring_submit(ring); + if (ret != nr) { + fprintf(stderr, "submit %d, wanted %d\n", ret, nr); + goto err; + } + total += ret; + + for (i = 0; i < total; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + } + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret, depth; + + if (argc > 1) + return 0; + + depth = 1; + while (depth <= MAX_ENTRIES) { + ret = io_uring_queue_init(depth, &ring, 0); + if (ret) { + if (ret == -ENOMEM) + break; + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + ret = test_nops(&ring); + if (ret) { + fprintf(stderr, "test_single_nop failed\n"); + return ret; + } + depth <<= 1; + io_uring_queue_exit(&ring); + } + + return 0; +} diff --git a/contrib/libs/liburing/test/nop.c b/contrib/libs/liburing/test/nop.c new file mode 100644 index 0000000000..a701b3d8ed --- /dev/null +++ b/contrib/libs/liburing/test/nop.c @@ -0,0 +1,178 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various nop tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "test.h" + +static int seq; + +static int test_single_nop(struct io_uring *ring, unsigned req_flags) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + bool cqe32 = (ring->flags & IORING_SETUP_CQE32); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + sqe->user_data = ++seq; + sqe->flags |= req_flags; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (!cqe->user_data) { + fprintf(stderr, "Unexpected 0 user_data\n"); + goto err; + } + if (cqe32) { + if (cqe->big_cqe[0] != 0) { + fprintf(stderr, "Unexpected extra1\n"); + goto err; + + } + if (cqe->big_cqe[1] != 0) { + fprintf(stderr, "Unexpected extra2\n"); + goto err; + } + } + io_uring_cqe_seen(ring, cqe); + return 0; +err: + return 1; +} + +static int test_barrier_nop(struct io_uring *ring, unsigned req_flags) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + bool cqe32 = (ring->flags & IORING_SETUP_CQE32); + + for (i = 0; i < 8; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_nop(sqe); + if (i == 4) + sqe->flags = IOSQE_IO_DRAIN; + sqe->user_data = ++seq; + sqe->flags |= req_flags; + } + + ret = io_uring_submit(ring); + if (ret < 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } else if (ret < 8) { + fprintf(stderr, "Submitted only %d\n", ret); + goto err; + } + + for (i = 0; i < 8; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (!cqe->user_data) { + fprintf(stderr, "Unexpected 0 user_data\n"); + goto err; + } + if (cqe32) { + if (cqe->big_cqe[0] != 0) { + fprintf(stderr, "Unexpected extra1\n"); + goto err; + } + if (cqe->big_cqe[1] != 0) { + fprintf(stderr, "Unexpected extra2\n"); + goto err; + } + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +static int test_ring(unsigned flags) +{ + struct io_uring ring; + struct io_uring_params p = { }; + int ret, i; + + p.flags = flags; + ret = io_uring_queue_init_params(8, &ring, &p); + if (ret) { + if (ret == -EINVAL) + return 0; + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + for (i = 0; i < 1000; i++) { + unsigned req_flags = (i & 1) ? IOSQE_ASYNC : 0; + + ret = test_single_nop(&ring, req_flags); + if (ret) { + fprintf(stderr, "test_single_nop failed\n"); + goto err; + } + + ret = test_barrier_nop(&ring, req_flags); + if (ret) { + fprintf(stderr, "test_barrier_nop failed\n"); + goto err; + } + } +err: + io_uring_queue_exit(&ring); + return ret; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + FOR_ALL_TEST_CONFIGS { + ret = test_ring(IORING_GET_TEST_CONFIG_FLAGS()); + if (ret) { + fprintf(stderr, "Normal ring test failed: %s\n", + IORING_GET_TEST_CONFIG_DESCRIPTION()); + return ret; + } + } + + return 0; +} diff --git a/contrib/libs/liburing/test/nvme.h b/contrib/libs/liburing/test/nvme.h new file mode 100644 index 0000000000..14dc338b85 --- /dev/null +++ b/contrib/libs/liburing/test/nvme.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Description: Helpers for NVMe uring passthrough commands + */ +#ifndef LIBURING_NVME_H +#define LIBURING_NVME_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include <sys/ioctl.h> +#include <linux/nvme_ioctl.h> + +/* + * If the uapi headers installed on the system lacks nvme uring command + * support, use the local version to prevent compilation issues. + */ +#ifndef CONFIG_HAVE_NVME_URING +struct nvme_uring_cmd { + __u8 opcode; + __u8 flags; + __u16 rsvd1; + __u32 nsid; + __u32 cdw2; + __u32 cdw3; + __u64 metadata; + __u64 addr; + __u32 metadata_len; + __u32 data_len; + __u32 cdw10; + __u32 cdw11; + __u32 cdw12; + __u32 cdw13; + __u32 cdw14; + __u32 cdw15; + __u32 timeout_ms; + __u32 rsvd2; +}; + +#define NVME_URING_CMD_IO _IOWR('N', 0x80, struct nvme_uring_cmd) +#define NVME_URING_CMD_IO_VEC _IOWR('N', 0x81, struct nvme_uring_cmd) +#endif /* CONFIG_HAVE_NVME_URING */ + +#define NVME_DEFAULT_IOCTL_TIMEOUT 0 +#define NVME_IDENTIFY_DATA_SIZE 4096 +#define NVME_IDENTIFY_CSI_SHIFT 24 +#define NVME_IDENTIFY_CNS_NS 0 +#define NVME_CSI_NVM 0 + +enum nvme_admin_opcode { + nvme_admin_identify = 0x06, +}; + +enum nvme_io_opcode { + nvme_cmd_write = 0x01, + nvme_cmd_read = 0x02, +}; + +int nsid; +__u32 lba_shift; + +struct nvme_lbaf { + __le16 ms; + __u8 ds; + __u8 rp; +}; + +struct nvme_id_ns { + __le64 nsze; + __le64 ncap; + __le64 nuse; + __u8 nsfeat; + __u8 nlbaf; + __u8 flbas; + __u8 mc; + __u8 dpc; + __u8 dps; + __u8 nmic; + __u8 rescap; + __u8 fpi; + __u8 dlfeat; + __le16 nawun; + __le16 nawupf; + __le16 nacwu; + __le16 nabsn; + __le16 nabo; + __le16 nabspf; + __le16 noiob; + __u8 nvmcap[16]; + __le16 npwg; + __le16 npwa; + __le16 npdg; + __le16 npda; + __le16 nows; + __le16 mssrl; + __le32 mcl; + __u8 msrc; + __u8 rsvd81[11]; + __le32 anagrpid; + __u8 rsvd96[3]; + __u8 nsattr; + __le16 nvmsetid; + __le16 endgid; + __u8 nguid[16]; + __u8 eui64[8]; + struct nvme_lbaf lbaf[16]; + __u8 rsvd192[192]; + __u8 vs[3712]; +}; + +static inline int ilog2(uint32_t i) +{ + int log = -1; + + while (i) { + i >>= 1; + log++; + } + return log; +} + +int nvme_get_info(const char *file) +{ + struct nvme_id_ns ns; + int fd, err; + __u32 lba_size; + + fd = open(file, O_RDONLY); + if (fd < 0) { + perror("file open"); + return -errno; + } + + nsid = ioctl(fd, NVME_IOCTL_ID); + if (nsid < 0) { + close(fd); + return -errno; + } + + struct nvme_passthru_cmd cmd = { + .opcode = nvme_admin_identify, + .nsid = nsid, + .addr = (__u64)(uintptr_t)&ns, + .data_len = NVME_IDENTIFY_DATA_SIZE, + .cdw10 = NVME_IDENTIFY_CNS_NS, + .cdw11 = NVME_CSI_NVM << NVME_IDENTIFY_CSI_SHIFT, + .timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT, + }; + + err = ioctl(fd, NVME_IOCTL_ADMIN_CMD, &cmd); + if (err) { + close(fd); + return err; + } + + lba_size = 1 << ns.lbaf[(ns.flbas & 0x0f)].ds; + lba_shift = ilog2(lba_size); + + close(fd); + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/libs/liburing/test/open-close.c b/contrib/libs/liburing/test/open-close.c new file mode 100644 index 0000000000..6b236e463a --- /dev/null +++ b/contrib/libs/liburing/test/open-close.c @@ -0,0 +1,262 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various openat(2) tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <assert.h> + +#include "helpers.h" +#include "liburing.h" + +static int submit_wait(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return 1; + } + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + return 1; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +} + +static inline int try_close(struct io_uring *ring, int fd, int slot) +{ + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_close(sqe, fd); + __io_uring_set_target_fixed_file(sqe, slot); + return submit_wait(ring); +} + +static int test_close_fixed(void) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + int ret, fds[2]; + char buf[1]; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return -1; + } + if (pipe(fds)) { + perror("pipe"); + return -1; + } + + ret = try_close(&ring, 0, 0); + if (ret == -EINVAL) { + fprintf(stderr, "close for fixed files is not supported\n"); + return 0; + } else if (ret != -ENXIO) { + fprintf(stderr, "no table failed %i\n", ret); + return -1; + } + + ret = try_close(&ring, 1, 0); + if (ret != -EINVAL) { + fprintf(stderr, "set fd failed %i\n", ret); + return -1; + } + + ret = io_uring_register_files(&ring, fds, 2); + if (ret) { + fprintf(stderr, "file_register: %d\n", ret); + return ret; + } + + ret = try_close(&ring, 0, 2); + if (ret != -EINVAL) { + fprintf(stderr, "out of table failed %i\n", ret); + return -1; + } + + ret = try_close(&ring, 0, 0); + if (ret != 0) { + fprintf(stderr, "close failed %i\n", ret); + return -1; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, 0, buf, sizeof(buf), 0); + sqe->flags |= IOSQE_FIXED_FILE; + ret = submit_wait(&ring); + if (ret != -EBADF) { + fprintf(stderr, "read failed %i\n", ret); + return -1; + } + + ret = try_close(&ring, 0, 1); + if (ret != 0) { + fprintf(stderr, "close 2 failed %i\n", ret); + return -1; + } + + ret = try_close(&ring, 0, 0); + if (ret != -EBADF) { + fprintf(stderr, "empty slot failed %i\n", ret); + return -1; + } + + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring); + return 0; +} + +static int test_close(struct io_uring *ring, int fd, int is_ring_fd) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_close(sqe, fd); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + if (!(is_ring_fd && ret == -EBADF)) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + return ret; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +err: + return -1; +} + +static int test_openat(struct io_uring *ring, const char *path, int dfd) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_openat(sqe, dfd, path, O_RDONLY, 0); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +err: + return -1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + const char *path, *path_rel; + int ret, do_unlink; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + if (argc > 1) { + path = "/tmp/.open.close"; + path_rel = argv[1]; + do_unlink = 0; + } else { + path = "/tmp/.open.close"; + path_rel = ".open.close"; + do_unlink = 1; + } + + t_create_file(path, 4096); + + if (do_unlink) + t_create_file(path_rel, 4096); + + ret = test_openat(&ring, path, -1); + if (ret < 0) { + if (ret == -EINVAL) { + fprintf(stdout, "Open not supported, skipping\n"); + goto done; + } + fprintf(stderr, "test_openat absolute failed: %d\n", ret); + goto err; + } + + ret = test_openat(&ring, path_rel, AT_FDCWD); + if (ret < 0) { + fprintf(stderr, "test_openat relative failed: %d\n", ret); + goto err; + } + + ret = test_close(&ring, ret, 0); + if (ret) { + fprintf(stderr, "test_close normal failed\n"); + goto err; + } + + ret = test_close(&ring, ring.ring_fd, 1); + if (ret != -EBADF) { + fprintf(stderr, "test_close ring_fd failed\n"); + goto err; + } + + ret = test_close_fixed(); + if (ret) { + fprintf(stderr, "test_close_fixed failed\n"); + goto err; + } + +done: + unlink(path); + if (do_unlink) + unlink(path_rel); + return 0; +err: + unlink(path); + if (do_unlink) + unlink(path_rel); + return 1; +} diff --git a/contrib/libs/liburing/test/open-direct-link.c b/contrib/libs/liburing/test/open-direct-link.c new file mode 100644 index 0000000000..ba3b8ea368 --- /dev/null +++ b/contrib/libs/liburing/test/open-direct-link.c @@ -0,0 +1,189 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: link <open file><read from file><close file> + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +#define MAX_FILES 8 +#define FNAME ".link.direct" + +static int test(struct io_uring *ring, int skip_success, int drain, int async) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + char buf[4096]; + int ret, i; + + /* drain and cqe skip are mutually exclusive */ + if (skip_success && drain) + return 1; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_openat_direct(sqe, AT_FDCWD, FNAME, O_RDONLY, 0, 0); + if (!drain) + sqe->flags |= IOSQE_IO_LINK; + if (skip_success) + sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; + if (async) + sqe->flags |= IOSQE_ASYNC; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_read(sqe, 0, buf, sizeof(buf), 0); + sqe->flags |= IOSQE_FIXED_FILE; + if (drain) + sqe->flags |= IOSQE_IO_DRAIN; + else + sqe->flags |= IOSQE_IO_LINK; + if (async) + sqe->flags |= IOSQE_ASYNC; + sqe->user_data = 2; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_close_direct(sqe, 0); + sqe->user_data = 3; + if (skip_success) + sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; + if (drain) + sqe->flags |= IOSQE_IO_DRAIN; + if (async) + sqe->flags |= IOSQE_ASYNC; + + ret = io_uring_submit(ring); + if (ret != 3) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + if (skip_success) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->user_data != 2) { + fprintf(stderr, "Unexpected cqe %lu/%d\n", + (unsigned long) cqe->user_data, + cqe->res); + goto err; + } + if (cqe->res != sizeof(buf)) { + fprintf(stderr, "bad read %d\n", cqe->res); + goto err; + } + io_uring_cqe_seen(ring, cqe); + return 0; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1: + if (cqe->res) { + fprintf(stderr, "bad open %d\n", cqe->res); + goto err; + } + break; + case 2: + if (cqe->res != sizeof(buf)) { + fprintf(stderr, "bad read %d\n", cqe->res); + goto err; + } + break; + case 3: + if (cqe->res) { + fprintf(stderr, "bad close %d\n", cqe->res); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct io_uring_params p = { }; + int ret, files[MAX_FILES]; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init_params(8, &ring, &p); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + if (!(p.features & IORING_FEAT_CQE_SKIP)) + return 0; + + memset(files, -1, sizeof(files)); + ret = io_uring_register_files(&ring, files, ARRAY_SIZE(files)); + if (ret) { + fprintf(stderr, "Failed registering files\n"); + return 1; + } + + t_create_file(FNAME, 4096); + + ret = test(&ring, 0, 0, 0); + if (ret) { + fprintf(stderr, "test 0 0 0 failed\n"); + goto err; + } + + ret = test(&ring, 0, 1, 0); + if (ret) { + fprintf(stderr, "test 0 1 0 failed\n"); + goto err; + } + + ret = test(&ring, 0, 0, 1); + if (ret) { + fprintf(stderr, "test 0 0 1 failed\n"); + goto err; + } + + ret = test(&ring, 0, 1, 1); + if (ret) { + fprintf(stderr, "test 0 1 1 failed\n"); + goto err; + } + + ret = test(&ring, 1, 0, 0); + if (ret) { + fprintf(stderr, "test 1 0 0 failed\n"); + goto err; + } + + ret = test(&ring, 1, 0, 1); + if (ret) { + fprintf(stderr, "test 1 0 1 failed\n"); + goto err; + } + + unlink(FNAME); + return 0; +err: + unlink(FNAME); + return 1; +} diff --git a/contrib/libs/liburing/test/open-direct-pick.c b/contrib/libs/liburing/test/open-direct-pick.c new file mode 100644 index 0000000000..0d04d8127e --- /dev/null +++ b/contrib/libs/liburing/test/open-direct-pick.c @@ -0,0 +1,181 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various openat(2) tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <limits.h> + +#include "helpers.h" +#include "liburing.h" + +#define FDS 800 + +static int no_direct_pick; + +static int submit_wait(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return 1; + } + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + return 1; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +} + +static inline int try_close(struct io_uring *ring, int slot) +{ + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_close_direct(sqe, slot); + return submit_wait(ring); +} + +static int do_opens(struct io_uring *ring, const char *path, int nr, + int expect_enfile) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int i, ret; + + for (i = 0; i < nr; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_openat_direct(sqe, -1, path, O_RDONLY, 0, 0); + sqe->file_index = UINT_MAX; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + } + + for (i = 0; i < nr; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + ret = cqe->res; + if (ret < 0) { + if (!expect_enfile || ret != -ENFILE) { + printf("open=%d, %d\n", cqe->res, i); + goto err; + } + if (!i && ret == -EINVAL) { + no_direct_pick = 1; + return 0; + } + } + io_uring_cqe_seen(ring, cqe); + } + return 0; +err: + return 1; +} + +static int test_openat(struct io_uring *ring, const char *path) +{ + int ret, i; + + /* open all */ + ret = do_opens(ring, path, FDS, 0); + if (ret) + goto err; + if (no_direct_pick) + return 0; + + /* now close 100 randomly */ + for (i = 0; i < 100; i++) { + do { + int slot = rand() % FDS; + ret = try_close(ring, slot); + if (ret == -EBADF) + continue; + break; + } while (1); + } + + /* opening 100 should work, we closed 100 */ + ret = do_opens(ring, path, 100, 0); + if (ret) + goto err; + + /* we should be full now, expect -ENFILE */ + ret = do_opens(ring, path, 1, 1); + if (ret) + goto err; + + return ret; +err: + fprintf(stderr,"%s: err=%d\n", __FUNCTION__, ret); + return -1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + const char *path; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + ret = io_uring_register_files_sparse(&ring, FDS); + if (ret ) { + if (ret != -EINVAL) { + fprintf(stderr, "Sparse file registration failed\n"); + return 1; + } + /* skip, kernel doesn't support sparse file array */ + return 0; + } + + path = "/tmp/.open.direct.pick"; + t_create_file(path, 4096); + + ret = test_openat(&ring, path); + if (ret < 0) { + if (ret == -EINVAL) { + fprintf(stdout, "Open not supported, skipping\n"); + goto done; + } + fprintf(stderr, "test_openat absolute failed: %d\n", ret); + goto err; + } + +done: + unlink(path); + return 0; +err: + unlink(path); + return 1; +} diff --git a/contrib/libs/liburing/test/openat2.c b/contrib/libs/liburing/test/openat2.c new file mode 100644 index 0000000000..a3ad70e143 --- /dev/null +++ b/contrib/libs/liburing/test/openat2.c @@ -0,0 +1,309 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various openat(2) tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/uio.h> + +#include "helpers.h" +#include "liburing.h" + +static int test_openat2(struct io_uring *ring, const char *path, int dfd, + bool direct, int fixed_index) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct open_how how; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return -1; + } + memset(&how, 0, sizeof(how)); + how.flags = O_RDWR; + + if (!direct) + io_uring_prep_openat2(sqe, dfd, path, &how); + else + io_uring_prep_openat2_direct(sqe, dfd, path, &how, fixed_index); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return -1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + return -1; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + + if (direct && ret > 0) { + close(ret); + return -EINVAL; + } + return ret; +} + +static int test_open_fixed(const char *path, int dfd) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring ring; + const char pattern = 0xac; + char buffer[] = { 0, 0 }; + int i, ret, fd = -1; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return -1; + } + ret = io_uring_register_files(&ring, &fd, 1); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + return -1; + } + + ret = test_openat2(&ring, path, dfd, true, 0); + if (ret == -EINVAL) { + printf("fixed open isn't supported\n"); + return 1; + } else if (ret) { + fprintf(stderr, "direct open failed %d\n", ret); + return -1; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_write(sqe, 0, &pattern, 1, 0); + sqe->user_data = 1; + sqe->flags |= IOSQE_FIXED_FILE | IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, 0, buffer, 1, 0); + sqe->user_data = 2; + sqe->flags |= IOSQE_FIXED_FILE; + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "%s: got %d, wanted 2\n", __FUNCTION__, ret); + return -1; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + return -1; + } + if (cqe->res != 1) { + fprintf(stderr, "unexpectetd ret %d\n", cqe->res); + return -1; + } + io_uring_cqe_seen(&ring, cqe); + } + if (memcmp(&pattern, buffer, 1) != 0) { + fprintf(stderr, "buf validation failed\n"); + return -1; + } + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_open_fixed_fail(const char *path, int dfd) +{ + struct io_uring ring; + int ret, fd = -1; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return -1; + } + + ret = test_openat2(&ring, path, dfd, true, 0); + if (ret != -ENXIO) { + fprintf(stderr, "install into not existing table, %i\n", ret); + return 1; + } + + ret = io_uring_register_files(&ring, &fd, 1); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + return -1; + } + + ret = test_openat2(&ring, path, dfd, true, 1); + if (ret != -EINVAL) { + fprintf(stderr, "install out of bounds, %i\n", ret); + return -1; + } + + ret = test_openat2(&ring, path, dfd, true, (1u << 16)); + if (ret != -EINVAL) { + fprintf(stderr, "install out of bounds or u16 overflow, %i\n", ret); + return -1; + } + + ret = test_openat2(&ring, path, dfd, true, (1u << 16) + 1); + if (ret != -EINVAL) { + fprintf(stderr, "install out of bounds or u16 overflow, %i\n", ret); + return -1; + } + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_direct_reinstall(const char *path, int dfd) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + char buf[1] = { 0xfa }; + struct io_uring ring; + int ret, pipe_fds[2]; + ssize_t ret2; + + if (pipe2(pipe_fds, O_NONBLOCK)) { + fprintf(stderr, "pipe() failed\n"); + return -1; + } + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return -1; + } + ret = io_uring_register_files(&ring, pipe_fds, 2); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + return -1; + } + + /* reinstall into the second slot */ + ret = test_openat2(&ring, path, dfd, true, 1); + if (ret != 0) { + fprintf(stderr, "reinstall failed, %i\n", ret); + return -1; + } + + /* verify it's reinstalled, first write into the slot... */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_write(sqe, 1, buf, sizeof(buf), 0); + sqe->flags |= IOSQE_FIXED_FILE; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return -1; + } + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + return ret; + } + ret = cqe->res; + io_uring_cqe_seen(&ring, cqe); + if (ret != 1) { + fprintf(stderr, "invalid write %i\n", ret); + return -1; + } + + /* ... and make sure nothing has been written to the pipe */ + ret2 = read(pipe_fds[0], buf, 1); + if (ret2 != 0 && !(ret2 < 0 && errno == EAGAIN)) { + fprintf(stderr, "invalid pipe read, %d %d\n", errno, (int)ret2); + return -1; + } + + close(pipe_fds[0]); + close(pipe_fds[1]); + io_uring_queue_exit(&ring); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + const char *path, *path_rel; + int ret, do_unlink; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + if (argc > 1) { + path = "/tmp/.open.at2"; + path_rel = argv[1]; + do_unlink = 0; + } else { + path = "/tmp/.open.at2"; + path_rel = ".open.at2"; + do_unlink = 1; + } + + t_create_file(path, 4096); + + if (do_unlink) + t_create_file(path_rel, 4096); + + ret = test_openat2(&ring, path, -1, false, 0); + if (ret < 0) { + if (ret == -EINVAL) { + fprintf(stdout, "openat2 not supported, skipping\n"); + goto done; + } + fprintf(stderr, "test_openat2 absolute failed: %d\n", ret); + goto err; + } + + ret = test_openat2(&ring, path_rel, AT_FDCWD, false, 0); + if (ret < 0) { + fprintf(stderr, "test_openat2 relative failed: %d\n", ret); + goto err; + } + + ret = test_open_fixed(path, -1); + if (ret > 0) + goto done; + if (ret) { + fprintf(stderr, "test_open_fixed failed\n"); + goto err; + } + ret = test_open_fixed_fail(path, -1); + if (ret) { + fprintf(stderr, "test_open_fixed_fail failed\n"); + goto err; + } + + ret = test_direct_reinstall(path, -1); + if (ret) { + fprintf(stderr, "test_direct_reinstall failed\n"); + goto err; + } + +done: + unlink(path); + if (do_unlink) + unlink(path_rel); + return 0; +err: + unlink(path); + if (do_unlink) + unlink(path_rel); + return 1; +} diff --git a/contrib/libs/liburing/test/personality.c b/contrib/libs/liburing/test/personality.c new file mode 100644 index 0000000000..577e356833 --- /dev/null +++ b/contrib/libs/liburing/test/personality.c @@ -0,0 +1,205 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test if personalities work + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" + +#define FNAME "/tmp/.tmp.access" +#define USE_UID 1000 + +static int no_personality; + +static int open_file(struct io_uring *ring, int cred_id, int with_link) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i, to_submit = 1; + + if (with_link) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_nop(sqe); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + to_submit++; + } + + sqe = io_uring_get_sqe(ring); + io_uring_prep_openat(sqe, -1, FNAME, O_RDONLY, 0); + sqe->user_data = 2; + + if (cred_id != -1) + sqe->personality = cred_id; + + ret = io_uring_submit(ring); + if (ret != to_submit) { + fprintf(stderr, "submit got: %d\n", ret); + goto err; + } + + for (i = 0; i < to_submit; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + } +err: + return ret; +} + +static int test_personality(struct io_uring *ring) +{ + int ret, cred_id; + + ret = io_uring_register_personality(ring); + if (ret < 0) { + if (ret == -EINVAL) { + fprintf(stdout, "Personalities not supported, skipping\n"); + no_personality = 1; + goto out; + } + fprintf(stderr, "register_personality: %d\n", ret); + goto err; + } + cred_id = ret; + + /* create file only owner can open */ + ret = open(FNAME, O_RDONLY | O_CREAT, 0600); + if (ret < 0) { + perror("open"); + goto err; + } + close(ret); + + /* verify we can open it */ + ret = open_file(ring, -1, 0); + if (ret < 0) { + fprintf(stderr, "current open got: %d\n", ret); + goto err; + } + + if (seteuid(USE_UID) < 0) { + fprintf(stdout, "Can't switch to UID %u, skipping\n", USE_UID); + goto out; + } + + /* verify we can't open it with current credentials */ + ret = open_file(ring, -1, 0); + if (ret != -EACCES) { + fprintf(stderr, "open got: %d\n", ret); + goto err; + } + + /* verify we can open with registered credentials */ + ret = open_file(ring, cred_id, 0); + if (ret < 0) { + fprintf(stderr, "credential open: %d\n", ret); + goto err; + } + close(ret); + + /* verify we can open with registered credentials and as a link */ + ret = open_file(ring, cred_id, 1); + if (ret < 0) { + fprintf(stderr, "credential open: %d\n", ret); + goto err; + } + + if (seteuid(0)) + perror("seteuid"); + + ret = io_uring_unregister_personality(ring, cred_id); + if (ret) { + fprintf(stderr, "register_personality: %d\n", ret); + goto err; + } + +out: + unlink(FNAME); + return 0; +err: + unlink(FNAME); + return 1; +} + +static int test_invalid_personality(struct io_uring *ring) +{ + int ret; + + ret = open_file(ring, 2, 0); + if (ret != -EINVAL) { + fprintf(stderr, "invalid personality got: %d\n", ret); + goto err; + } + return 0; +err: + return 1; +} + +static int test_invalid_unregister(struct io_uring *ring) +{ + int ret; + + ret = io_uring_unregister_personality(ring, 2); + if (ret != -EINVAL) { + fprintf(stderr, "invalid personality unregister got: %d\n", ret); + goto err; + } + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return 0; + + if (geteuid()) { + fprintf(stderr, "Not root, skipping\n"); + return 0; + } + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + ret = test_personality(&ring); + if (ret) { + fprintf(stderr, "test_personality failed\n"); + return ret; + } + if (no_personality) + return 0; + + ret = test_invalid_personality(&ring); + if (ret) { + fprintf(stderr, "test_invalid_personality failed\n"); + return ret; + } + + ret = test_invalid_unregister(&ring); + if (ret) { + fprintf(stderr, "test_invalid_unregister failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/pipe-eof.c b/contrib/libs/liburing/test/pipe-eof.c new file mode 100644 index 0000000000..a917ae00a7 --- /dev/null +++ b/contrib/libs/liburing/test/pipe-eof.c @@ -0,0 +1,84 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ + +/* + * Test that closed pipe reads returns 0, instead of waiting for more + * data. + */ +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <pthread.h> +#include <string.h> +#include "liburing.h" + +#define BUFSIZE 512 + +struct data { + char *str; + int fds[2]; +}; + +static void *t(void *data) +{ + struct data *d = data; + int ret; + + strcpy(d->str, "This is a test string"); + ret = write(d->fds[1], d->str, strlen(d->str)); + close(d->fds[1]); + if (ret < 0) + perror("write"); + + return NULL; +} + +int main(int argc, char *argv[]) +{ + static char buf[BUFSIZE]; + struct io_uring ring; + pthread_t thread; + struct data d; + int ret; + + if (pipe(d.fds) < 0) { + perror("pipe"); + return 1; + } + d.str = buf; + + io_uring_queue_init(8, &ring, 0); + + pthread_create(&thread, NULL, t, &d); + + while (1) { + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, d.fds[0], buf, BUFSIZE, 0); + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return 1; + } + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return 1; + } + + if (cqe->res < 0) { + fprintf(stderr, "Read error: %s\n", strerror(-cqe->res)); + return 1; + } + if (cqe->res == 0) + break; + io_uring_cqe_seen(&ring, cqe); + } + + pthread_join(thread, NULL); + io_uring_queue_exit(&ring); + return 0; +} diff --git a/contrib/libs/liburing/test/pipe-reuse.c b/contrib/libs/liburing/test/pipe-reuse.c new file mode 100644 index 0000000000..7005e59781 --- /dev/null +++ b/contrib/libs/liburing/test/pipe-reuse.c @@ -0,0 +1,106 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Check split up read is handled correctly + */ +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <pthread.h> +#include <string.h> +#include "liburing.h" + +#define BUFSIZE 16384 +#define BUFFERS 16 + +int main(int argc, char *argv[]) +{ + char buf[BUFSIZE], wbuf[BUFSIZE]; + struct iovec iov[BUFFERS]; + struct io_uring_params p = { }; + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, i, fds[2]; + void *ptr; + + if (pipe(fds) < 0) { + perror("pipe"); + return 1; + } + + ptr = buf; + for (i = 0; i < BUFFERS; i++) { + unsigned bsize = BUFSIZE / BUFFERS; + + iov[i].iov_base = ptr; + iov[i].iov_len = bsize; + ptr += bsize; + } + + ret = io_uring_queue_init_params(8, &ring, &p); + if (ret) { + fprintf(stderr, "queue_init: %d\n", ret); + return 1; + } + if (!(p.features & IORING_FEAT_SUBMIT_STABLE)) { + fprintf(stdout, "FEAT_SUBMIT_STABLE not there, skipping\n"); + return 0; + } + + ptr = wbuf; + memset(ptr, 0x11, sizeof(wbuf) / 2); + ptr += sizeof(wbuf) / 2; + memset(ptr, 0x22, sizeof(wbuf) / 2); + + ret = write(fds[1], wbuf, sizeof(wbuf) / 2); + if (ret != sizeof(wbuf) / 2) { + fprintf(stderr, "Bad write\n"); + ret = 1; + goto err; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, fds[0], iov, BUFFERS, 0); + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return 1; + } + + for (i = 0; i < BUFFERS; i++) { + iov[i].iov_base = NULL; + iov[i].iov_len = 1000000; + } + + ret = write(fds[1], ptr, sizeof(wbuf) / 2); + if (ret != sizeof(wbuf) / 2) { + fprintf(stderr, "Bad write\n"); + ret = 1; + goto err; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return 1; + } + + if (cqe->res < 0) { + fprintf(stderr, "Read error: %s\n", strerror(-cqe->res)); + return 1; + } else if (cqe->res != sizeof(wbuf)) { + /* ignore short read, not a failure */ + goto err; + } + io_uring_cqe_seen(&ring, cqe); + + ret = memcmp(wbuf, buf, sizeof(wbuf)); + if (ret) + fprintf(stderr, "Read data mismatch\n"); + +err: + io_uring_queue_exit(&ring); + return ret; +} diff --git a/contrib/libs/liburing/test/poll-cancel-all.c b/contrib/libs/liburing/test/poll-cancel-all.c new file mode 100644 index 0000000000..83c48abeea --- /dev/null +++ b/contrib/libs/liburing/test/poll-cancel-all.c @@ -0,0 +1,473 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: Test IORING_ASYNC_CANCEL_{ALL,FD} + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <poll.h> + +#include "liburing.h" + +static int no_cancel_flags; + +static int test1(struct io_uring *ring, int *fd) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, i; + + for (i = 0; i < 8; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return 1; + } + + io_uring_prep_poll_add(sqe, fd[0], POLLIN); + sqe->user_data = i + 1; + } + + ret = io_uring_submit(ring); + if (ret < 8) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return 1; + } + + /* + * Mark CANCEL_ALL to cancel all matching the key, and use + * CANCEL_FD to cancel requests matching the specified fd. + * This should cancel all the pending poll requests on the pipe + * input. + */ + io_uring_prep_cancel(sqe, 0, IORING_ASYNC_CANCEL_ALL); + sqe->cancel_flags |= IORING_ASYNC_CANCEL_FD; + sqe->fd = fd[0]; + sqe->user_data = 100; + + ret = io_uring_submit(ring); + if (ret < 1) { + fprintf(stderr, "child: sqe submit failed: %d\n", ret); + return 1; + } + + for (i = 0; i < 9; i++) { + if (no_cancel_flags) + break; + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait=%d\n", ret); + return 1; + } + switch (cqe->user_data) { + case 100: + if (cqe->res == -EINVAL) { + no_cancel_flags = 1; + break; + } + if (cqe->res != 8) { + fprintf(stderr, "canceled %d\n", cqe->res); + return 1; + } + break; + case 1 ... 8: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "poll res %d\n", cqe->res); + return 1; + } + break; + default: + fprintf(stderr, "invalid user_data %lu\n", + (unsigned long) cqe->user_data); + return 1; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +} + +static int test2(struct io_uring *ring, int *fd) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, i, fd2[2]; + + if (pipe(fd2) < 0) { + perror("pipe"); + return 1; + } + + for (i = 0; i < 8; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + if (!(i & 1)) + io_uring_prep_poll_add(sqe, fd[0], POLLIN); + else + io_uring_prep_poll_add(sqe, fd2[0], POLLIN); + sqe->user_data = i & 1; + } + + ret = io_uring_submit(ring); + if (ret < 8) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + /* + * Mark CANCEL_ALL to cancel all matching the key, and use + * CANCEL_FD to cancel requests matching the specified fd. + * This should cancel all the pending poll requests on the pipe + * input. + */ + io_uring_prep_cancel(sqe, 0, IORING_ASYNC_CANCEL_ALL); + sqe->cancel_flags |= IORING_ASYNC_CANCEL_FD; + sqe->fd = fd[0]; + sqe->user_data = 100; + + ret = io_uring_submit(ring); + if (ret < 1) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 5; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait=%d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 100: + if (cqe->res != 4) { + fprintf(stderr, "canceled %d\n", cqe->res); + goto err; + } + break; + case 0: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "poll res %d\n", cqe->res); + goto err; + } + break; + default: + fprintf(stderr, "invalid user_data %lu\n", + (unsigned long) cqe->user_data); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + usleep(1000); + + /* + * Should not have any pending CQEs now + */ + ret = io_uring_peek_cqe(ring, &cqe); + if (!ret) { + fprintf(stderr, "Unexpected extra cancel cqe\n"); + goto err; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + /* + * Mark CANCEL_ALL to cancel all matching the key, and use + * CANCEL_FD to cancel requests matching the specified fd. + * This should cancel all the pending poll requests on the pipe + * input. + */ + io_uring_prep_cancel(sqe, 0, IORING_ASYNC_CANCEL_ALL); + sqe->cancel_flags |= IORING_ASYNC_CANCEL_FD; + sqe->fd = fd2[0]; + sqe->user_data = 100; + + ret = io_uring_submit(ring); + if (ret < 1) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 5; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait=%d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 100: + if (cqe->res != 4) { + fprintf(stderr, "canceled %d\n", cqe->res); + goto err; + } + break; + case 1: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "poll res %d\n", cqe->res); + goto err; + } + break; + default: + fprintf(stderr, "invalid user_data %lu\n", + (unsigned long) cqe->user_data); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + close(fd2[0]); + close(fd2[1]); + return 0; +err: + close(fd2[0]); + close(fd2[1]); + return 1; +} + +static int test3(struct io_uring *ring, int *fd) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, i, fd2[2]; + + if (pipe(fd2) < 0) { + perror("pipe"); + return 1; + } + + for (i = 0; i < 8; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + if (!(i & 1)) { + io_uring_prep_poll_add(sqe, fd[0], POLLIN); + sqe->flags |= IOSQE_ASYNC; + } else + io_uring_prep_poll_add(sqe, fd2[0], POLLIN); + sqe->user_data = i & 1; + } + + ret = io_uring_submit(ring); + if (ret < 8) { + fprintf(stderr, "child: sqe submit failed: %d\n", ret); + goto err; + } + + usleep(10000); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + /* + * Mark CANCEL_ALL to cancel all matching the key, and use + * CANCEL_FD to cancel requests matching the specified fd. + * This should cancel all the pending poll requests on the pipe + * input. + */ + io_uring_prep_cancel(sqe, 0, IORING_ASYNC_CANCEL_ALL); + sqe->cancel_flags |= IORING_ASYNC_CANCEL_ANY; + sqe->fd = 0; + sqe->user_data = 100; + + ret = io_uring_submit(ring); + if (ret < 1) { + fprintf(stderr, "child: sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 9; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait=%d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 100: + if (cqe->res != 8) { + fprintf(stderr, "canceled %d\n", cqe->res); + goto err; + } + break; + case 0: + case 1: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "poll res %d\n", cqe->res); + goto err; + } + break; + default: + fprintf(stderr, "invalid user_data %lu\n", + (unsigned long) cqe->user_data); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + close(fd2[0]); + close(fd2[1]); + return 0; +err: + close(fd2[0]); + close(fd2[1]); + return 1; +} + +static int test4(struct io_uring *ring, int *fd) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + char buffer[32]; + int ret, i; + + for (i = 0; i < 8; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_read(sqe, fd[0], &buffer, sizeof(buffer), 0); + sqe->flags |= IOSQE_ASYNC; + sqe->user_data = i + 1; + } + + ret = io_uring_submit(ring); + if (ret < 8) { + fprintf(stderr, "child: sqe submit failed: %d\n", ret); + goto err; + } + + usleep(10000); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + /* + * Mark CANCEL_ALL to cancel all matching the key, and use + * CANCEL_FD to cancel requests matching the specified fd. + * This should cancel all the pending poll requests on the pipe + * input. + */ + io_uring_prep_cancel(sqe, 0, IORING_ASYNC_CANCEL_ALL); + sqe->cancel_flags |= IORING_ASYNC_CANCEL_ANY; + sqe->fd = 0; + sqe->user_data = 100; + + ret = io_uring_submit(ring); + if (ret < 1) { + fprintf(stderr, "child: sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 9; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait=%d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 100: + if (cqe->res != 8) { + fprintf(stderr, "canceled %d\n", cqe->res); + goto err; + } + break; + case 1 ... 8: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "poll res %d\n", cqe->res); + goto err; + } + break; + default: + fprintf(stderr, "invalid user_data %lu\n", + (unsigned long) cqe->user_data); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret, fd[2]; + + if (argc > 1) + return 0; + + if (pipe(fd) < 0) { + perror("pipe"); + return 1; + } + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + ret = test1(&ring, fd); + if (ret) { + fprintf(stderr, "test1 failed\n"); + return ret; + } + if (no_cancel_flags) + return 0; + + ret = test2(&ring, fd); + if (ret) { + fprintf(stderr, "test2 failed\n"); + return ret; + } + + ret = test3(&ring, fd); + if (ret) { + fprintf(stderr, "test3 failed\n"); + return ret; + } + + ret = test4(&ring, fd); + if (ret) { + fprintf(stderr, "test4 failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/poll-cancel-ton.c b/contrib/libs/liburing/test/poll-cancel-ton.c new file mode 100644 index 0000000000..5c77ce2b81 --- /dev/null +++ b/contrib/libs/liburing/test/poll-cancel-ton.c @@ -0,0 +1,136 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test massive amounts of poll with cancel + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <inttypes.h> +#include <poll.h> +#include <sys/wait.h> +#include <signal.h> + +#include "liburing.h" + +#define POLL_COUNT 30000 + +static void *sqe_index[POLL_COUNT]; + +static int reap_events(struct io_uring *ring, unsigned nr_events, int nowait) +{ + struct io_uring_cqe *cqe; + int i, ret = 0; + + for (i = 0; i < nr_events; i++) { + if (!i && !nowait) + ret = io_uring_wait_cqe(ring, &cqe); + else + ret = io_uring_peek_cqe(ring, &cqe); + if (ret) { + if (ret != -EAGAIN) + fprintf(stderr, "cqe peek failed: %d\n", ret); + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return i ? i : ret; +} + +static int del_polls(struct io_uring *ring, int fd, int nr) +{ + int batch, i, ret; + struct io_uring_sqe *sqe; + + while (nr) { + batch = 1024; + if (batch > nr) + batch = nr; + + for (i = 0; i < batch; i++) { + void *data; + + sqe = io_uring_get_sqe(ring); + data = sqe_index[lrand48() % nr]; + io_uring_prep_poll_remove(sqe, (__u64)(uintptr_t)data); + } + + ret = io_uring_submit(ring); + if (ret != batch) { + fprintf(stderr, "%s: failed submit, %d\n", __FUNCTION__, ret); + return 1; + } + nr -= batch; + ret = reap_events(ring, 2 * batch, 0); + } + return 0; +} + +static int add_polls(struct io_uring *ring, int fd, int nr) +{ + int batch, i, count, ret; + struct io_uring_sqe *sqe; + + count = 0; + while (nr) { + batch = 1024; + if (batch > nr) + batch = nr; + + for (i = 0; i < batch; i++) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_poll_add(sqe, fd, POLLIN); + sqe_index[count++] = sqe; + sqe->user_data = (unsigned long) sqe; + } + + ret = io_uring_submit(ring); + if (ret != batch) { + fprintf(stderr, "%s: failed submit, %d\n", __FUNCTION__, ret); + return 1; + } + nr -= batch; + reap_events(ring, batch, 1); + } + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct io_uring_params p = { }; + int pipe1[2]; + int ret; + + if (argc > 1) + return 0; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return 1; + } + + p.flags = IORING_SETUP_CQSIZE; + p.cq_entries = 16384; + ret = io_uring_queue_init_params(1024, &ring, &p); + if (ret) { + if (ret == -EINVAL) { + fprintf(stdout, "No CQSIZE, trying without\n"); + ret = io_uring_queue_init(1024, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + } + } + + add_polls(&ring, pipe1[0], 30000); + del_polls(&ring, pipe1[0], 30000); + + io_uring_queue_exit(&ring); + return 0; +} diff --git a/contrib/libs/liburing/test/poll-cancel.c b/contrib/libs/liburing/test/poll-cancel.c new file mode 100644 index 0000000000..2d953b5676 --- /dev/null +++ b/contrib/libs/liburing/test/poll-cancel.c @@ -0,0 +1,229 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring poll cancel handling + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <inttypes.h> +#include <poll.h> +#include <sys/wait.h> +#include <signal.h> + +#include "liburing.h" + +struct poll_data { + unsigned is_poll; + unsigned is_cancel; +}; + +static void sig_alrm(int sig) +{ + fprintf(stderr, "Timed out!\n"); + exit(1); +} + +static int test_poll_cancel(void) +{ + struct io_uring ring; + int pipe1[2]; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct poll_data *pd, pds[2]; + struct sigaction act; + int ret; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return 1; + } + + ret = io_uring_queue_init(2, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + memset(&act, 0, sizeof(act)); + act.sa_handler = sig_alrm; + act.sa_flags = SA_RESTART; + sigaction(SIGALRM, &act, NULL); + alarm(1); + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return 1; + } + + io_uring_prep_poll_add(sqe, pipe1[0], POLLIN); + + pds[0].is_poll = 1; + pds[0].is_cancel = 0; + io_uring_sqe_set_data(sqe, &pds[0]); + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed\n"); + return 1; + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return 1; + } + + pds[1].is_poll = 0; + pds[1].is_cancel = 1; + io_uring_prep_poll_remove(sqe, (__u64)(uintptr_t)&pds[0]); + io_uring_sqe_set_data(sqe, &pds[1]); + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return 1; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait cqe failed: %d\n", ret); + return 1; + } + + pd = io_uring_cqe_get_data(cqe); + if (pd->is_poll && cqe->res != -ECANCELED) { + fprintf(stderr ,"sqe (add=%d/remove=%d) failed with %ld\n", + pd->is_poll, pd->is_cancel, + (long) cqe->res); + return 1; + } else if (pd->is_cancel && cqe->res) { + fprintf(stderr, "sqe (add=%d/remove=%d) failed with %ld\n", + pd->is_poll, pd->is_cancel, + (long) cqe->res); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait_cqe: %d\n", ret); + return 1; + } + + pd = io_uring_cqe_get_data(cqe); + if (pd->is_poll && cqe->res != -ECANCELED) { + fprintf(stderr, "sqe (add=%d/remove=%d) failed with %ld\n", + pd->is_poll, pd->is_cancel, + (long) cqe->res); + return 1; + } else if (pd->is_cancel && cqe->res) { + fprintf(stderr, "sqe (add=%d/remove=%d) failed with %ld\n", + pd->is_poll, pd->is_cancel, + (long) cqe->res); + return 1; + } + + close(pipe1[0]); + close(pipe1[1]); + io_uring_cqe_seen(&ring, cqe); + io_uring_queue_exit(&ring); + return 0; +} + + +static int __test_poll_cancel_with_timeouts(void) +{ + struct __kernel_timespec ts = { .tv_sec = 10, }; + struct io_uring ring, ring2; + struct io_uring_sqe *sqe; + int ret, off_nr = 1000; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + ret = io_uring_queue_init(1, &ring2, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + /* test timeout-offset triggering path during cancellation */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_timeout(sqe, &ts, off_nr, 0); + + /* poll ring2 to trigger cancellation on exit() */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, ring2.ring_fd, POLLIN); + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_link_timeout(sqe, &ts, 0); + + ret = io_uring_submit(&ring); + if (ret != 3) { + fprintf(stderr, "sqe submit failed\n"); + return 1; + } + + /* just drop all rings/etc. intact, exit() will clean them up */ + return 0; +} + +static int test_poll_cancel_with_timeouts(void) +{ + int ret; + pid_t p; + + p = fork(); + if (p == -1) { + fprintf(stderr, "fork() failed\n"); + return 1; + } + + if (p == 0) { + ret = __test_poll_cancel_with_timeouts(); + exit(ret); + } else { + int wstatus; + + if (waitpid(p, &wstatus, 0) == (pid_t)-1) { + perror("waitpid()"); + return 1; + } + if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus)) { + fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus)); + return 1; + } + } + return 0; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test_poll_cancel(); + if (ret) { + fprintf(stderr, "test_poll_cancel failed\n"); + return -1; + } + + ret = test_poll_cancel_with_timeouts(); + if (ret) { + fprintf(stderr, "test_poll_cancel_with_timeouts failed\n"); + return -1; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/poll-link.c b/contrib/libs/liburing/test/poll-link.c new file mode 100644 index 0000000000..27346c65ae --- /dev/null +++ b/contrib/libs/liburing/test/poll-link.c @@ -0,0 +1,222 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <assert.h> +#include <pthread.h> +#include <sys/socket.h> +#include <netinet/tcp.h> +#include <netinet/in.h> +#include <poll.h> +#include <arpa/inet.h> + +#include "helpers.h" +#include "liburing.h" + +pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + +static int recv_thread_ready = 0; +static int recv_thread_done = 0; + +static void signal_var(int *var) +{ + pthread_mutex_lock(&mutex); + *var = 1; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&mutex); +} + +static void wait_for_var(int *var) +{ + pthread_mutex_lock(&mutex); + + while (!*var) + pthread_cond_wait(&cond, &mutex); + + pthread_mutex_unlock(&mutex); +} + +struct data { + unsigned expected[2]; + unsigned is_mask[2]; + unsigned long timeout; + unsigned short port; + unsigned int addr; + int stop; +}; + +static void *send_thread(void *arg) +{ + struct sockaddr_in addr; + struct data *data = arg; + int s0; + + wait_for_var(&recv_thread_ready); + + s0 = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + assert(s0 != -1); + + addr.sin_family = AF_INET; + addr.sin_port = data->port; + addr.sin_addr.s_addr = data->addr; + + if (connect(s0, (struct sockaddr*)&addr, sizeof(addr)) != -1) + wait_for_var(&recv_thread_done); + + close(s0); + return 0; +} + +void *recv_thread(void *arg) +{ + struct sockaddr_in addr = { }; + struct data *data = arg; + struct io_uring_sqe *sqe; + struct io_uring ring; + int i, ret; + + ret = io_uring_queue_init(8, &ring, 0); + assert(ret == 0); + + int s0 = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + assert(s0 != -1); + + int32_t val = 1; + ret = setsockopt(s0, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + assert(ret != -1); + ret = setsockopt(s0, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + assert(ret != -1); + + addr.sin_family = AF_INET; + data->addr = inet_addr("127.0.0.1"); + addr.sin_addr.s_addr = data->addr; + + if (t_bind_ephemeral_port(s0, &addr)) { + perror("bind"); + data->stop = 1; + signal_var(&recv_thread_ready); + goto err; + } + data->port = addr.sin_port; + + ret = listen(s0, 128); + assert(ret != -1); + + signal_var(&recv_thread_ready); + + sqe = io_uring_get_sqe(&ring); + assert(sqe != NULL); + + io_uring_prep_poll_add(sqe, s0, POLLIN | POLLHUP | POLLERR); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring); + assert(sqe != NULL); + + struct __kernel_timespec ts; + ts.tv_sec = data->timeout / 1000000000; + ts.tv_nsec = data->timeout % 1000000000; + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 2; + + ret = io_uring_submit(&ring); + assert(ret == 2); + + for (i = 0; i < 2; i++) { + struct io_uring_cqe *cqe; + int idx; + + if (io_uring_wait_cqe(&ring, &cqe)) { + fprintf(stderr, "wait cqe failed\n"); + goto err; + } + idx = cqe->user_data - 1; + if (data->is_mask[idx] && !(data->expected[idx] & cqe->res)) { + fprintf(stderr, "cqe %" PRIu64 " got %x, wanted mask %x\n", + (uint64_t) cqe->user_data, cqe->res, + data->expected[idx]); + goto err; + } else if (!data->is_mask[idx] && cqe->res != data->expected[idx]) { + fprintf(stderr, "cqe %" PRIu64 " got %d, wanted %d\n", + (uint64_t) cqe->user_data, cqe->res, + data->expected[idx]); + goto err; + } + io_uring_cqe_seen(&ring, cqe); + } + + signal_var(&recv_thread_done); + close(s0); + io_uring_queue_exit(&ring); + return NULL; +err: + signal_var(&recv_thread_done); + close(s0); + io_uring_queue_exit(&ring); + return (void *) 1; +} + +static int test_poll_timeout(int do_connect, unsigned long timeout) +{ + pthread_t t1, t2; + struct data d; + void *tret; + int ret = 0; + + recv_thread_ready = 0; + recv_thread_done = 0; + + memset(&d, 0, sizeof(d)); + d.timeout = timeout; + if (!do_connect) { + d.expected[0] = -ECANCELED; + d.expected[1] = -ETIME; + } else { + d.expected[0] = POLLIN; + d.is_mask[0] = 1; + d.expected[1] = -ECANCELED; + } + + pthread_create(&t1, NULL, recv_thread, &d); + + if (do_connect) + pthread_create(&t2, NULL, send_thread, &d); + + pthread_join(t1, &tret); + if (tret) + ret++; + + if (do_connect) { + pthread_join(t2, &tret); + if (tret) + ret++; + } + + return ret; +} + +int main(int argc, char *argv[]) +{ + if (argc > 1) + return 0; + + srand(getpid()); + + if (test_poll_timeout(0, 200000000)) { + fprintf(stderr, "poll timeout 0 failed\n"); + return 1; + } + + if (test_poll_timeout(1, 1000000000)) { + fprintf(stderr, "poll timeout 1 failed\n"); + return 1; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/poll-many.c b/contrib/libs/liburing/test/poll-many.c new file mode 100644 index 0000000000..8f6a89efdd --- /dev/null +++ b/contrib/libs/liburing/test/poll-many.c @@ -0,0 +1,209 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test many files being polled for + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <signal.h> +#include <poll.h> +#include <sys/resource.h> +#include <fcntl.h> + +#include "liburing.h" + +#define NFILES 5000 +#define BATCH 500 +#define NLOOPS 1000 + +#define RING_SIZE 512 + +struct p { + int fd[2]; + int triggered; +}; + +static struct p p[NFILES]; + +static int arm_poll(struct io_uring *ring, int off) +{ + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "failed getting sqe\n"); + return 1; + } + + io_uring_prep_poll_add(sqe, p[off].fd[0], POLLIN); + sqe->user_data = off; + return 0; +} + +static int reap_polls(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int i, ret, off; + char c; + + for (i = 0; i < BATCH; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe %d\n", ret); + return ret; + } + off = cqe->user_data; + p[off].triggered = 0; + ret = read(p[off].fd[0], &c, 1); + if (ret != 1) { + fprintf(stderr, "read got %d/%d\n", ret, errno); + break; + } + if (arm_poll(ring, off)) + break; + io_uring_cqe_seen(ring, cqe); + } + + if (i != BATCH) { + fprintf(stderr, "gave up at %d\n", i); + return 1; + } + + ret = io_uring_submit(ring); + if (ret != BATCH) { + fprintf(stderr, "submitted %d, %d\n", ret, BATCH); + return 1; + } + + return 0; +} + +static int trigger_polls(void) +{ + char c = 89; + int i, ret; + + for (i = 0; i < BATCH; i++) { + int off; + + do { + off = rand() % NFILES; + if (!p[off].triggered) + break; + } while (1); + + p[off].triggered = 1; + ret = write(p[off].fd[1], &c, 1); + if (ret != 1) { + fprintf(stderr, "write got %d/%d\n", ret, errno); + return 1; + } + } + + return 0; +} + +static int arm_polls(struct io_uring *ring) +{ + int ret, to_arm = NFILES, i, off; + + off = 0; + while (to_arm) { + int this_arm; + + this_arm = to_arm; + if (this_arm > RING_SIZE) + this_arm = RING_SIZE; + + for (i = 0; i < this_arm; i++) { + if (arm_poll(ring, off)) { + fprintf(stderr, "arm failed at %d\n", off); + return 1; + } + off++; + } + + ret = io_uring_submit(ring); + if (ret != this_arm) { + fprintf(stderr, "submitted %d, %d\n", ret, this_arm); + return 1; + } + to_arm -= this_arm; + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct io_uring_params params = { }; + struct rlimit rlim; + int i, ret; + + if (argc > 1) + return 0; + + if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) { + perror("getrlimit"); + goto err_noring; + } + + if (rlim.rlim_cur < (2 * NFILES + 5)) { + rlim.rlim_cur = (2 * NFILES + 5); + rlim.rlim_max = rlim.rlim_cur; + if (setrlimit(RLIMIT_NOFILE, &rlim) < 0) { + if (errno == EPERM) + goto err_nofail; + perror("setrlimit"); + goto err_noring; + } + } + + for (i = 0; i < NFILES; i++) { + if (pipe(p[i].fd) < 0) { + perror("pipe"); + goto err_noring; + } + } + + params.flags = IORING_SETUP_CQSIZE; + params.cq_entries = 4096; + ret = io_uring_queue_init_params(RING_SIZE, &ring, ¶ms); + if (ret) { + if (ret == -EINVAL) { + fprintf(stdout, "No CQSIZE, trying without\n"); + ret = io_uring_queue_init(RING_SIZE, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + } + } + + if (arm_polls(&ring)) + goto err; + + for (i = 0; i < NLOOPS; i++) { + trigger_polls(); + ret = reap_polls(&ring); + if (ret) + goto err; + } + + io_uring_queue_exit(&ring); + return 0; +err: + io_uring_queue_exit(&ring); +err_noring: + fprintf(stderr, "poll-many failed\n"); + return 1; +err_nofail: + fprintf(stderr, "poll-many: not enough files available (and not root), " + "skipped\n"); + return 0; +} diff --git a/contrib/libs/liburing/test/poll-mshot-overflow.c b/contrib/libs/liburing/test/poll-mshot-overflow.c new file mode 100644 index 0000000000..b31633d507 --- /dev/null +++ b/contrib/libs/liburing/test/poll-mshot-overflow.c @@ -0,0 +1,163 @@ +#include "../config-host.h" +// SPDX-License-Identifier: MIT + +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <signal.h> +#include <poll.h> +#include <sys/wait.h> + +#include "liburing.h" +#include "helpers.h" + +int check_final_cqe(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int count = 0; + bool signalled_no_more = false; + + while (!io_uring_peek_cqe(ring, &cqe)) { + if (cqe->user_data == 1) { + count++; + if (signalled_no_more) { + fprintf(stderr, "signalled no more!\n"); + return T_EXIT_FAIL; + } + if (!(cqe->flags & IORING_CQE_F_MORE)) + signalled_no_more = true; + } else if (cqe->user_data != 3) { + fprintf(stderr, "%d: got unexpected %d\n", count, (int)cqe->user_data); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(ring, cqe); + } + + if (!count) { + fprintf(stderr, "no cqe\n"); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} + +static int test(bool defer_taskrun) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring ring; + int pipe1[2]; + int ret, i; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return T_EXIT_FAIL; + } + + struct io_uring_params params = { + /* cheat using SINGLE_ISSUER existence to know if this behaviour + * is updated + */ + .flags = IORING_SETUP_CQSIZE | IORING_SETUP_SINGLE_ISSUER, + .cq_entries = 2 + }; + + if (defer_taskrun) + params.flags |= IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN; + + ret = io_uring_queue_init_params(2, &ring, ¶ms); + if (ret) + return T_EXIT_SKIP; + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return T_EXIT_FAIL; + } + io_uring_prep_poll_multishot(sqe, pipe1[0], POLLIN); + io_uring_sqe_set_data64(sqe, 1); + + if (io_uring_cq_ready(&ring)) { + fprintf(stderr, "unexpected cqe\n"); + return T_EXIT_FAIL; + } + + for (i = 0; i < 2; i++) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + io_uring_sqe_set_data64(sqe, 2); + io_uring_submit(&ring); + } + + do { + errno = 0; + ret = write(pipe1[1], "foo", 3); + } while (ret == -1 && errno == EINTR); + + if (ret <= 0) { + fprintf(stderr, "write failed: %d\n", errno); + return T_EXIT_FAIL; + } + + /* should have 2 cqe + 1 overflow now, so take out two cqes */ + for (i = 0; i < 2; i++) { + if (io_uring_peek_cqe(&ring, &cqe)) { + fprintf(stderr, "unexpectedly no cqe\n"); + return T_EXIT_FAIL; + } + if (cqe->user_data != 2) { + fprintf(stderr, "unexpected user_data\n"); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(&ring, cqe); + } + + /* make sure everything is processed */ + io_uring_get_events(&ring); + + /* now remove the poll */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_remove(sqe, 1); + io_uring_sqe_set_data64(sqe, 3); + ret = io_uring_submit(&ring); + + if (ret != 1) { + fprintf(stderr, "bad poll remove\n"); + return T_EXIT_FAIL; + } + + ret = check_final_cqe(&ring); + + close(pipe1[0]); + close(pipe1[1]); + io_uring_queue_exit(&ring); + + return ret; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = test(false); + if (ret != T_EXIT_PASS) { + fprintf(stderr, "%s: test(false) failed\n", argv[0]); + return ret; + } + + if (t_probe_defer_taskrun()) { + ret = test(true); + if (ret != T_EXIT_PASS) { + fprintf(stderr, "%s: test(true) failed\n", argv[0]); + return ret; + } + } + + return ret; +} diff --git a/contrib/libs/liburing/test/poll-mshot-update.c b/contrib/libs/liburing/test/poll-mshot-update.c new file mode 100644 index 0000000000..c3b687260c --- /dev/null +++ b/contrib/libs/liburing/test/poll-mshot-update.c @@ -0,0 +1,324 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test many files being polled for and updated + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <signal.h> +#include <poll.h> +#include <sys/resource.h> +#include <fcntl.h> +#include <pthread.h> + +#include "liburing.h" + +#define NFILES 5000 +#define BATCH 500 +#define NLOOPS 1000 + +#define RING_SIZE 512 + +struct p { + int fd[2]; + int triggered; +}; + +static struct p p[NFILES]; + +static int has_poll_update(void) +{ + struct io_uring ring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + bool has_update = false; + int ret; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) + return -1; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_update(sqe, 0, 0, POLLIN, IORING_TIMEOUT_UPDATE); + + ret = io_uring_submit(&ring); + if (ret != 1) + return -1; + + ret = io_uring_wait_cqe(&ring, &cqe); + if (!ret) { + if (cqe->res == -ENOENT) + has_update = true; + else if (cqe->res != -EINVAL) + return -1; + io_uring_cqe_seen(&ring, cqe); + } + io_uring_queue_exit(&ring); + return has_update; +} + +static int arm_poll(struct io_uring *ring, int off) +{ + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "failed getting sqe\n"); + return 1; + } + + io_uring_prep_poll_multishot(sqe, p[off].fd[0], POLLIN); + sqe->user_data = off; + return 0; +} + +static int submit_arm_poll(struct io_uring *ring, int off) +{ + int ret; + + ret = arm_poll(ring, off); + if (ret) + return ret; + + ret = io_uring_submit(ring); + if (ret < 0) + return ret; + return ret == 1 ? 0 : -1; +} + +static int reap_polls(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int i, ret, off; + char c; + + for (i = 0; i < BATCH; i++) { + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + /* update event */ + io_uring_prep_poll_update(sqe, i, 0, POLLIN, + IORING_POLL_UPDATE_EVENTS); + sqe->user_data = 0x12345678; + } + + ret = io_uring_submit(ring); + if (ret != BATCH) { + fprintf(stderr, "submitted %d, %d\n", ret, BATCH); + return 1; + } + + for (i = 0; i < 2 * BATCH; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe %d\n", ret); + return ret; + } + off = cqe->user_data; + if (off == 0x12345678) + goto seen; + if (!(cqe->flags & IORING_CQE_F_MORE)) { + /* need to re-arm poll */ + ret = submit_arm_poll(ring, off); + if (ret) + break; + if (cqe->res <= 0) { + /* retry this one */ + i--; + goto seen; + } + } + + ret = read(p[off].fd[0], &c, 1); + if (ret != 1) { + if (ret == -1 && errno == EAGAIN) + goto seen; + fprintf(stderr, "read got %d/%d\n", ret, errno); + break; + } +seen: + io_uring_cqe_seen(ring, cqe); + } + + if (i != 2 * BATCH) { + fprintf(stderr, "gave up at %d\n", i); + return 1; + } + + return 0; +} + +static int trigger_polls(void) +{ + char c = 89; + int i, ret; + + for (i = 0; i < BATCH; i++) { + int off; + + do { + off = rand() % NFILES; + if (!p[off].triggered) + break; + } while (1); + + p[off].triggered = 1; + ret = write(p[off].fd[1], &c, 1); + if (ret != 1) { + fprintf(stderr, "write got %d/%d\n", ret, errno); + return 1; + } + } + + return 0; +} + +static void *trigger_polls_fn(void *data) +{ + trigger_polls(); + return NULL; +} + +static int arm_polls(struct io_uring *ring) +{ + int ret, to_arm = NFILES, i, off; + + off = 0; + while (to_arm) { + int this_arm; + + this_arm = to_arm; + if (this_arm > RING_SIZE) + this_arm = RING_SIZE; + + for (i = 0; i < this_arm; i++) { + if (arm_poll(ring, off)) { + fprintf(stderr, "arm failed at %d\n", off); + return 1; + } + off++; + } + + ret = io_uring_submit(ring); + if (ret != this_arm) { + fprintf(stderr, "submitted %d, %d\n", ret, this_arm); + return 1; + } + to_arm -= this_arm; + } + + return 0; +} + +static int run(int cqe) +{ + struct io_uring ring; + struct io_uring_params params = { }; + pthread_t thread; + int i, j, ret; + + for (i = 0; i < NFILES; i++) { + if (pipe(p[i].fd) < 0) { + perror("pipe"); + return 1; + } + fcntl(p[i].fd[0], F_SETFL, O_NONBLOCK); + } + + params.flags = IORING_SETUP_CQSIZE; + params.cq_entries = cqe; + ret = io_uring_queue_init_params(RING_SIZE, &ring, ¶ms); + if (ret) { + if (ret == -EINVAL) { + fprintf(stdout, "No CQSIZE, trying without\n"); + ret = io_uring_queue_init(RING_SIZE, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + } + } + + if (arm_polls(&ring)) + goto err; + + for (i = 0; i < NLOOPS; i++) { + pthread_create(&thread, NULL, trigger_polls_fn, NULL); + ret = reap_polls(&ring); + if (ret) + goto err; + pthread_join(thread, NULL); + + for (j = 0; j < NFILES; j++) + p[j].triggered = 0; + } + + io_uring_queue_exit(&ring); + for (i = 0; i < NFILES; i++) { + close(p[i].fd[0]); + close(p[i].fd[1]); + } + return 0; +err: + io_uring_queue_exit(&ring); + return 1; +} + +int main(int argc, char *argv[]) +{ + struct rlimit rlim; + int ret; + + if (argc > 1) + return 0; + + ret = has_poll_update(); + if (ret < 0) { + fprintf(stderr, "poll update check failed %i\n", ret); + return -1; + } else if (!ret) { + fprintf(stderr, "no poll update, skip\n"); + return 0; + } + + if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) { + perror("getrlimit"); + goto err; + } + + if (rlim.rlim_cur < (2 * NFILES + 5)) { + rlim.rlim_cur = (2 * NFILES + 5); + rlim.rlim_max = rlim.rlim_cur; + if (setrlimit(RLIMIT_NOFILE, &rlim) < 0) { + if (errno == EPERM) + goto err_nofail; + perror("setrlimit"); + goto err; + } + } + + ret = run(1024); + if (ret) { + fprintf(stderr, "run(1024) failed\n"); + goto err; + } + + ret = run(8192); + if (ret) { + fprintf(stderr, "run(8192) failed\n"); + goto err; + } + + return 0; +err: + fprintf(stderr, "poll-many failed\n"); + return 1; +err_nofail: + fprintf(stderr, "poll-many: not enough files available (and not root), " + "skipped\n"); + return 0; +} diff --git a/contrib/libs/liburing/test/poll-ring.c b/contrib/libs/liburing/test/poll-ring.c new file mode 100644 index 0000000000..0c58046179 --- /dev/null +++ b/contrib/libs/liburing/test/poll-ring.c @@ -0,0 +1,49 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: Test poll against ring itself. A buggy kernel will end up + * having io_wq_* workers pending, as the circular reference + * will prevent full exit. + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <poll.h> + +#include "liburing.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_sqe *sqe; + struct io_uring ring; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "child: ring setup failed: %d\n", ret); + return 1; + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return 1; + } + + io_uring_prep_poll_add(sqe, ring.ring_fd, POLLIN); + io_uring_sqe_set_data(sqe, sqe); + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "child: sqe submit failed: %d\n", ret); + return 1; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/poll-v-poll.c b/contrib/libs/liburing/test/poll-v-poll.c new file mode 100644 index 0000000000..207f7bd5da --- /dev/null +++ b/contrib/libs/liburing/test/poll-v-poll.c @@ -0,0 +1,354 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring poll handling + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <signal.h> +#include <fcntl.h> +#include <poll.h> +#include <sys/wait.h> +#include <sys/select.h> +#include <pthread.h> +#include <sys/epoll.h> + +#include "liburing.h" + +struct thread_data { + struct io_uring *ring; + int fd; + int events; + const char *test; + int out[2]; +}; + +static void *epoll_wait_fn(void *data) +{ + struct thread_data *td = data; + struct epoll_event ev; + + if (epoll_wait(td->fd, &ev, 1, -1) < 0) { + perror("epoll_wait"); + goto err; + } + + return NULL; +err: + return (void *) 1; +} + +static void *iou_poll(void *data) +{ + struct thread_data *td = data; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(td->ring); + io_uring_prep_poll_add(sqe, td->fd, td->events); + + ret = io_uring_submit(td->ring); + if (ret != 1) { + fprintf(stderr, "submit got %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(td->ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe: %d\n", ret); + goto err; + } + + td->out[0] = cqe->res & 0x3f; + io_uring_cqe_seen(td->ring, cqe); + return NULL; +err: + return (void *) 1; +} + +static void *poll_pipe(void *data) +{ + struct thread_data *td = data; + struct pollfd pfd; + int ret; + + pfd.fd = td->fd; + pfd.events = td->events; + + ret = poll(&pfd, 1, -1); + if (ret < 0) + perror("poll"); + + td->out[1] = pfd.revents; + return NULL; +} + +static int do_pipe_pollin_test(struct io_uring *ring) +{ + struct thread_data td; + pthread_t threads[2]; + int ret, pipe1[2]; + char buf; + + if (pipe(pipe1) < 0) { + perror("pipe"); + return 1; + } + + td.ring = ring; + td.fd = pipe1[0]; + td.events = POLLIN; + td.test = __FUNCTION__; + + pthread_create(&threads[1], NULL, iou_poll, &td); + pthread_create(&threads[0], NULL, poll_pipe, &td); + usleep(100000); + + buf = 0x89; + ret = write(pipe1[1], &buf, sizeof(buf)); + if (ret != sizeof(buf)) { + fprintf(stderr, "write failed: %d\n", ret); + return 1; + } + + pthread_join(threads[0], NULL); + pthread_join(threads[1], NULL); + + if (td.out[0] != td.out[1]) { + fprintf(stderr, "%s: res %x/%x differ\n", __FUNCTION__, + td.out[0], td.out[1]); + return 1; + } + return 0; +} + +static int do_pipe_pollout_test(struct io_uring *ring) +{ + struct thread_data td; + pthread_t threads[2]; + int ret, pipe1[2]; + char buf; + + if (pipe(pipe1) < 0) { + perror("pipe"); + return 1; + } + + td.ring = ring; + td.fd = pipe1[1]; + td.events = POLLOUT; + td.test = __FUNCTION__; + + pthread_create(&threads[0], NULL, poll_pipe, &td); + pthread_create(&threads[1], NULL, iou_poll, &td); + usleep(100000); + + buf = 0x89; + ret = write(pipe1[1], &buf, sizeof(buf)); + if (ret != sizeof(buf)) { + fprintf(stderr, "write failed: %d\n", ret); + return 1; + } + + pthread_join(threads[0], NULL); + pthread_join(threads[1], NULL); + + if (td.out[0] != td.out[1]) { + fprintf(stderr, "%s: res %x/%x differ\n", __FUNCTION__, + td.out[0], td.out[1]); + return 1; + } + + return 0; +} + +static int do_fd_test(struct io_uring *ring, const char *fname, int events) +{ + struct thread_data td; + pthread_t threads[2]; + int fd; + + fd = open(fname, O_RDONLY); + if (fd < 0) { + perror("open"); + return 1; + } + + td.ring = ring; + td.fd = fd; + td.events = events; + td.test = __FUNCTION__; + + pthread_create(&threads[0], NULL, poll_pipe, &td); + pthread_create(&threads[1], NULL, iou_poll, &td); + + pthread_join(threads[0], NULL); + pthread_join(threads[1], NULL); + + if (td.out[0] != td.out[1]) { + fprintf(stderr, "%s: res %x/%x differ\n", __FUNCTION__, + td.out[0], td.out[1]); + return 1; + } + + return 0; +} + +static int iou_epoll_ctl(struct io_uring *ring, int epfd, int fd, + struct epoll_event *ev) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "Failed to get sqe\n"); + return 1; + } + + io_uring_prep_epoll_ctl(sqe, epfd, fd, EPOLL_CTL_ADD, ev); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe: %d\n", ret); + return 1; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +} + +static int do_test_epoll(struct io_uring *ring, int iou_epoll_add) +{ + struct epoll_event ev; + struct thread_data td; + pthread_t threads[2]; + int ret, pipe1[2]; + char buf; + int fd; + + fd = epoll_create1(0); + if (fd < 0) { + perror("epoll_create"); + return 1; + } + + if (pipe(pipe1) < 0) { + perror("pipe"); + return 1; + } + + ev.events = EPOLLIN; + ev.data.fd = pipe1[0]; + + if (!iou_epoll_add) { + if (epoll_ctl(fd, EPOLL_CTL_ADD, pipe1[0], &ev) < 0) { + perror("epoll_ctrl"); + return 1; + } + } else { + ret = iou_epoll_ctl(ring, fd, pipe1[0], &ev); + if (ret == -EINVAL) { + fprintf(stdout, "epoll not supported, skipping\n"); + return 0; + } else if (ret < 0) { + return 1; + } + } + + td.ring = ring; + td.fd = fd; + td.events = POLLIN; + td.test = __FUNCTION__; + + pthread_create(&threads[0], NULL, iou_poll, &td); + pthread_create(&threads[1], NULL, epoll_wait_fn, &td); + usleep(100000); + + buf = 0x89; + ret = write(pipe1[1], &buf, sizeof(buf)); + if (ret != sizeof(buf)) { + fprintf(stderr, "write failed: %d\n", ret); + return 1; + } + + pthread_join(threads[0], NULL); + pthread_join(threads[1], NULL); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + const char *fname; + int ret; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + ret = do_pipe_pollin_test(&ring); + if (ret) { + fprintf(stderr, "pipe pollin test failed\n"); + return ret; + } + + ret = do_pipe_pollout_test(&ring); + if (ret) { + fprintf(stderr, "pipe pollout test failed\n"); + return ret; + } + + ret = do_test_epoll(&ring, 0); + if (ret) { + fprintf(stderr, "epoll test 0 failed\n"); + return ret; + } + + ret = do_test_epoll(&ring, 1); + if (ret) { + fprintf(stderr, "epoll test 1 failed\n"); + return ret; + } + + if (argc > 1) + fname = argv[1]; + else + fname = argv[0]; + + ret = do_fd_test(&ring, fname, POLLIN); + if (ret) { + fprintf(stderr, "fd test IN failed\n"); + return ret; + } + + ret = do_fd_test(&ring, fname, POLLOUT); + if (ret) { + fprintf(stderr, "fd test OUT failed\n"); + return ret; + } + + ret = do_fd_test(&ring, fname, POLLOUT | POLLIN); + if (ret) { + fprintf(stderr, "fd test IN|OUT failed\n"); + return ret; + } + + return 0; + +} diff --git a/contrib/libs/liburing/test/poll.c b/contrib/libs/liburing/test/poll.c new file mode 100644 index 0000000000..cfba077af6 --- /dev/null +++ b/contrib/libs/liburing/test/poll.c @@ -0,0 +1,110 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring poll handling + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <signal.h> +#include <poll.h> +#include <sys/wait.h> + +#include "liburing.h" + +static void sig_alrm(int sig) +{ + fprintf(stderr, "Timed out!\n"); + exit(1); +} + +int main(int argc, char *argv[]) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring ring; + int pipe1[2]; + pid_t p; + int ret; + + if (argc > 1) + return 0; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return 1; + } + + p = fork(); + switch (p) { + case -1: + perror("fork"); + exit(2); + case 0: { + struct sigaction act; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "child: ring setup failed: %d\n", ret); + return 1; + } + + memset(&act, 0, sizeof(act)); + act.sa_handler = sig_alrm; + act.sa_flags = SA_RESTART; + sigaction(SIGALRM, &act, NULL); + alarm(1); + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return 1; + } + + io_uring_prep_poll_add(sqe, pipe1[0], POLLIN); + io_uring_sqe_set_data(sqe, sqe); + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "child: sqe submit failed: %d\n", ret); + return 1; + } + + do { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "child: wait completion %d\n", ret); + break; + } + io_uring_cqe_seen(&ring, cqe); + } while (ret != 0); + + if (ret < 0) + return 1; + if (cqe->user_data != (unsigned long) sqe) { + fprintf(stderr, "child: cqe doesn't match sqe\n"); + return 1; + } + if ((cqe->res & POLLIN) != POLLIN) { + fprintf(stderr, "child: bad return value %ld\n", + (long) cqe->res); + return 1; + } + exit(0); + } + default: + do { + errno = 0; + ret = write(pipe1[1], "foo", 3); + } while (ret == -1 && errno == EINTR); + + if (ret != 3) { + fprintf(stderr, "parent: bad write return %d\n", ret); + return 1; + } + return 0; + } +} diff --git a/contrib/libs/liburing/test/pollfree.c b/contrib/libs/liburing/test/pollfree.c new file mode 100644 index 0000000000..4ed61e3091 --- /dev/null +++ b/contrib/libs/liburing/test/pollfree.c @@ -0,0 +1,427 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// https://syzkaller.appspot.com/bug?id=5f5a44abb4cba056fe24255c4fcb7e7bbe13de7a +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <dirent.h> +#include <endian.h> +#include <errno.h> +#include <fcntl.h> +#include <pthread.h> +#include <signal.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/prctl.h> +#include <sys/stat.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <time.h> +#include <unistd.h> + +#include <linux/futex.h> + +#ifdef __NR_futex + +static void sleep_ms(uint64_t ms) +{ + usleep(ms * 1000); +} + +static uint64_t current_time_ms(void) +{ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts)) + exit(1); + return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000; +} + +static void thread_start(void* (*fn)(void*), void* arg) +{ + pthread_t th; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 128 << 10); + int i = 0; + for (; i < 100; i++) { + if (pthread_create(&th, &attr, fn, arg) == 0) { + pthread_attr_destroy(&attr); + return; + } + if (errno == EAGAIN) { + usleep(50); + continue; + } + break; + } + exit(1); +} + +typedef struct { + int state; +} event_t; + +static void event_init(event_t* ev) +{ + ev->state = 0; +} + +static void event_reset(event_t* ev) +{ + ev->state = 0; +} + +static void event_set(event_t* ev) +{ + if (ev->state) + exit(1); + __atomic_store_n(&ev->state, 1, __ATOMIC_RELEASE); + syscall(__NR_futex, &ev->state, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1000000); +} + +static void event_wait(event_t* ev) +{ + while (!__atomic_load_n(&ev->state, __ATOMIC_ACQUIRE)) + syscall(__NR_futex, &ev->state, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, 0, 0); +} + +static int event_isset(event_t* ev) +{ + return __atomic_load_n(&ev->state, __ATOMIC_ACQUIRE); +} + +static int event_timedwait(event_t* ev, uint64_t timeout) +{ + uint64_t start = current_time_ms(); + uint64_t now = start; + for (;;) { + uint64_t remain = timeout - (now - start); + struct timespec ts; + ts.tv_sec = remain / 1000; + ts.tv_nsec = (remain % 1000) * 1000 * 1000; + syscall(__NR_futex, &ev->state, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, 0, &ts); + if (__atomic_load_n(&ev->state, __ATOMIC_ACQUIRE)) + return 1; + now = current_time_ms(); + if (now - start > timeout) + return 0; + } +} + +#define SIZEOF_IO_URING_SQE 64 +#define SIZEOF_IO_URING_CQE 16 +#define SQ_HEAD_OFFSET 0 +#define SQ_TAIL_OFFSET 64 +#define SQ_RING_MASK_OFFSET 256 +#define SQ_RING_ENTRIES_OFFSET 264 +#define SQ_FLAGS_OFFSET 276 +#define SQ_DROPPED_OFFSET 272 +#define CQ_HEAD_OFFSET 128 +#define CQ_TAIL_OFFSET 192 +#define CQ_RING_MASK_OFFSET 260 +#define CQ_RING_ENTRIES_OFFSET 268 +#define CQ_RING_OVERFLOW_OFFSET 284 +#define CQ_FLAGS_OFFSET 280 +#define CQ_CQES_OFFSET 320 + +struct io_sqring_offsets { + uint32_t head; + uint32_t tail; + uint32_t ring_mask; + uint32_t ring_entries; + uint32_t flags; + uint32_t dropped; + uint32_t array; + uint32_t resv1; + uint64_t resv2; +}; + +struct io_cqring_offsets { + uint32_t head; + uint32_t tail; + uint32_t ring_mask; + uint32_t ring_entries; + uint32_t overflow; + uint32_t cqes; + uint64_t resv[2]; +}; + +struct io_uring_params { + uint32_t sq_entries; + uint32_t cq_entries; + uint32_t flags; + uint32_t sq_thread_cpu; + uint32_t sq_thread_idle; + uint32_t features; + uint32_t resv[4]; + struct io_sqring_offsets sq_off; + struct io_cqring_offsets cq_off; +}; + +#define IORING_OFF_SQ_RING 0 +#define IORING_OFF_SQES 0x10000000ULL + +#define sys_io_uring_setup 425 +static long syz_io_uring_setup(volatile long a0, volatile long a1, + volatile long a2, volatile long a3, + volatile long a4, volatile long a5) +{ + uint32_t entries = (uint32_t)a0; + struct io_uring_params* setup_params = (struct io_uring_params*)a1; + void* vma1 = (void*)a2; + void* vma2 = (void*)a3; + void** ring_ptr_out = (void**)a4; + void** sqes_ptr_out = (void**)a5; + uint32_t fd_io_uring = syscall(sys_io_uring_setup, entries, setup_params); + uint32_t sq_ring_sz = + setup_params->sq_off.array + setup_params->sq_entries * sizeof(uint32_t); + uint32_t cq_ring_sz = setup_params->cq_off.cqes + + setup_params->cq_entries * SIZEOF_IO_URING_CQE; + uint32_t ring_sz = sq_ring_sz > cq_ring_sz ? sq_ring_sz : cq_ring_sz; + *ring_ptr_out = mmap(vma1, ring_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, + IORING_OFF_SQ_RING); + uint32_t sqes_sz = setup_params->sq_entries * SIZEOF_IO_URING_SQE; + *sqes_ptr_out = + mmap(vma2, sqes_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, IORING_OFF_SQES); + return fd_io_uring; +} + +static long syz_io_uring_submit(volatile long a0, volatile long a1, + volatile long a2, volatile long a3) +{ + char* ring_ptr = (char*)a0; + char* sqes_ptr = (char*)a1; + char* sqe = (char*)a2; + uint32_t sqes_index = (uint32_t)a3; + uint32_t sq_ring_entries = *(uint32_t*)(ring_ptr + SQ_RING_ENTRIES_OFFSET); + uint32_t cq_ring_entries = *(uint32_t*)(ring_ptr + CQ_RING_ENTRIES_OFFSET); + uint32_t sq_array_off = + (CQ_CQES_OFFSET + cq_ring_entries * SIZEOF_IO_URING_CQE + 63) & ~63; + if (sq_ring_entries) + sqes_index %= sq_ring_entries; + char* sqe_dest = sqes_ptr + sqes_index * SIZEOF_IO_URING_SQE; + memcpy(sqe_dest, sqe, SIZEOF_IO_URING_SQE); + uint32_t sq_ring_mask = *(uint32_t*)(ring_ptr + SQ_RING_MASK_OFFSET); + uint32_t* sq_tail_ptr = (uint32_t*)(ring_ptr + SQ_TAIL_OFFSET); + uint32_t sq_tail = *sq_tail_ptr & sq_ring_mask; + uint32_t sq_tail_next = *sq_tail_ptr + 1; + uint32_t* sq_array = (uint32_t*)(ring_ptr + sq_array_off); + *(sq_array + sq_tail) = sqes_index; + __atomic_store_n(sq_tail_ptr, sq_tail_next, __ATOMIC_RELEASE); + return 0; +} + +static void kill_and_wait(int pid, int* status) +{ + kill(-pid, SIGKILL); + kill(pid, SIGKILL); + for (int i = 0; i < 100; i++) { + if (waitpid(-1, status, WNOHANG | __WALL) == pid) + return; + usleep(1000); + } + DIR* dir = opendir("/sys/fs/fuse/connections"); + if (dir) { + for (;;) { + struct dirent* ent = readdir(dir); + if (!ent) + break; + if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) + continue; + char abort[300]; + snprintf(abort, sizeof(abort), "/sys/fs/fuse/connections/%s/abort", + ent->d_name); + int fd = open(abort, O_WRONLY); + if (fd == -1) { + continue; + } + if (write(fd, abort, 1) < 0) { + } + close(fd); + } + closedir(dir); + } else { + } + while (waitpid(-1, status, __WALL) != pid) { + } +} + +static void setup_test() +{ + prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); + setpgrp(); +} + +struct thread_t { + int created, call; + event_t ready, done; +}; + +static struct thread_t threads[16]; +static void execute_call(int call); +static int running; + +static void* thr(void* arg) +{ + struct thread_t* th = (struct thread_t*)arg; + for (;;) { + event_wait(&th->ready); + event_reset(&th->ready); + execute_call(th->call); + __atomic_fetch_sub(&running, 1, __ATOMIC_RELAXED); + event_set(&th->done); + } + return 0; +} + +static void execute_one(void) +{ + int i, call, thread; + for (call = 0; call < 4; call++) { + for (thread = 0; thread < (int)(sizeof(threads) / sizeof(threads[0])); + thread++) { + struct thread_t* th = &threads[thread]; + if (!th->created) { + th->created = 1; + event_init(&th->ready); + event_init(&th->done); + event_set(&th->done); + thread_start(thr, th); + } + if (!event_isset(&th->done)) + continue; + event_reset(&th->done); + th->call = call; + __atomic_fetch_add(&running, 1, __ATOMIC_RELAXED); + event_set(&th->ready); + event_timedwait(&th->done, 50); + break; + } + } + for (i = 0; i < 100 && __atomic_load_n(&running, __ATOMIC_RELAXED); i++) + sleep_ms(1); +} + +static void execute_one(void); + +#define WAIT_FLAGS __WALL + +static void loop(void) +{ + int iter = 0; + for (; iter < 5000; iter++) { + int pid = fork(); + if (pid < 0) + exit(1); + if (pid == 0) { + setup_test(); + execute_one(); + exit(0); + } + int status = 0; + uint64_t start = current_time_ms(); + for (;;) { + if (waitpid(-1, &status, WNOHANG | WAIT_FLAGS) == pid) + break; + sleep_ms(1); + if (current_time_ms() - start < 5000) + continue; + kill_and_wait(pid, &status); + break; + } + } +} + +#ifndef __NR_io_uring_enter +#define __NR_io_uring_enter 426 +#endif + +uint64_t r[4] = {0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0x0}; + +void execute_call(int call) +{ + intptr_t res = 0; + switch (call) { + case 0: + *(uint64_t*)0x200000c0 = 0; + res = syscall(__NR_signalfd4, -1, 0x200000c0ul, 8ul, 0ul); + if (res != -1) + r[0] = res; + break; + case 1: + *(uint32_t*)0x20000a84 = 0; + *(uint32_t*)0x20000a88 = 0; + *(uint32_t*)0x20000a8c = 0; + *(uint32_t*)0x20000a90 = 0; + *(uint32_t*)0x20000a98 = -1; + memset((void*)0x20000a9c, 0, 12); + res = -1; + res = syz_io_uring_setup(0x87, 0x20000a80, 0x206d6000, 0x206d7000, + 0x20000000, 0x20000040); + if (res != -1) { + r[1] = res; + r[2] = *(uint64_t*)0x20000000; + r[3] = *(uint64_t*)0x20000040; + } + break; + case 2: + *(uint8_t*)0x20002240 = 6; + *(uint8_t*)0x20002241 = 0; + *(uint16_t*)0x20002242 = 0; + *(uint32_t*)0x20002244 = r[0]; + *(uint64_t*)0x20002248 = 0; + *(uint64_t*)0x20002250 = 0; + *(uint32_t*)0x20002258 = 0; + *(uint16_t*)0x2000225c = 0; + *(uint16_t*)0x2000225e = 0; + *(uint64_t*)0x20002260 = 0; + *(uint16_t*)0x20002268 = 0; + *(uint16_t*)0x2000226a = 0; + memset((void*)0x2000226c, 0, 20); + syz_io_uring_submit(r[2], r[3], 0x20002240, 0); + break; + case 3: + syscall(__NR_io_uring_enter, r[1], 0x1523a, 0, 0ul, 0ul, 0xaul); + break; + } +} + +int main(int argc, char *argv[]) +{ + void *ret; + +#if !defined(__i386) && !defined(__x86_64__) + return 0; +#endif + + if (argc > 1) + return 0; + + ret = mmap((void *)0x1ffff000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); + if (ret == MAP_FAILED) + return 0; + ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul); + if (ret == MAP_FAILED) + return 0; + ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); + if (ret == MAP_FAILED) + return 0; + loop(); + return 0; +} + +#else /* __NR_futex */ + +int main(int argc, char *argv[]) +{ + return 0; +} + +#endif /* __NR_futex */ diff --git a/contrib/libs/liburing/test/probe.c b/contrib/libs/liburing/test/probe.c new file mode 100644 index 0000000000..5a40e289e4 --- /dev/null +++ b/contrib/libs/liburing/test/probe.c @@ -0,0 +1,136 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test IORING_REGISTER_PROBE + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "helpers.h" +#include "liburing.h" + +static int no_probe; + +static int verify_probe(struct io_uring_probe *p, int full) +{ + if (!full && p->ops_len) { + fprintf(stderr, "Got ops_len=%u\n", p->ops_len); + return 1; + } + if (!p->last_op) { + fprintf(stderr, "Got last_op=%u\n", p->last_op); + return 1; + } + if (!full) + return 0; + /* check a few ops that must be supported */ + if (!(p->ops[IORING_OP_NOP].flags & IO_URING_OP_SUPPORTED)) { + fprintf(stderr, "NOP not supported!?\n"); + return 1; + } + if (!(p->ops[IORING_OP_READV].flags & IO_URING_OP_SUPPORTED)) { + fprintf(stderr, "READV not supported!?\n"); + return 1; + } + if (!(p->ops[IORING_OP_WRITE].flags & IO_URING_OP_SUPPORTED)) { + fprintf(stderr, "WRITE not supported!?\n"); + return 1; + } + + return 0; +} + +static int test_probe_helper(struct io_uring *ring) +{ + int ret; + struct io_uring_probe *p; + + p = io_uring_get_probe_ring(ring); + if (!p) { + fprintf(stderr, "Failed getting probe data\n"); + return 1; + } + + ret = verify_probe(p, 1); + io_uring_free_probe(p); + return ret; +} + +static int test_probe(struct io_uring *ring) +{ + struct io_uring_probe *p; + size_t len; + int ret; + + len = sizeof(*p) + 256 * sizeof(struct io_uring_probe_op); + p = t_calloc(1, len); + ret = io_uring_register_probe(ring, p, 0); + if (ret == -EINVAL) { + fprintf(stdout, "Probe not supported, skipping\n"); + no_probe = 1; + goto out; + } else if (ret) { + fprintf(stdout, "Probe returned %d\n", ret); + goto err; + } + + if (verify_probe(p, 0)) + goto err; + + /* now grab for all entries */ + memset(p, 0, len); + ret = io_uring_register_probe(ring, p, 256); + if (ret == -EINVAL) { + fprintf(stdout, "Probe not supported, skipping\n"); + goto err; + } else if (ret) { + fprintf(stdout, "Probe returned %d\n", ret); + goto err; + } + + if (verify_probe(p, 1)) + goto err; + +out: + free(p); + return 0; +err: + free(p); + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + ret = test_probe(&ring); + if (ret) { + fprintf(stderr, "test_probe failed\n"); + return ret; + } + if (no_probe) + return 0; + + ret = test_probe_helper(&ring); + if (ret) { + fprintf(stderr, "test_probe failed\n"); + return ret; + } + + + return 0; +} diff --git a/contrib/libs/liburing/test/read-before-exit.c b/contrib/libs/liburing/test/read-before-exit.c new file mode 100644 index 0000000000..6510a09263 --- /dev/null +++ b/contrib/libs/liburing/test/read-before-exit.c @@ -0,0 +1,113 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test if issuing IO from thread and immediately exiting will + * proceed correctly. + * + * Original test case from: https://github.com/axboe/liburing/issues/582 + */ +#include <unistd.h> +#include <pthread.h> +#include <sys/timerfd.h> +#include <string.h> +#include <stdio.h> + +#include "liburing.h" +#include "helpers.h" + +struct data { + struct io_uring *ring; + int timer_fd1; + int timer_fd2; + uint64_t buf1; + uint64_t buf2; +}; + +void *submit(void *data) +{ + struct io_uring_sqe *sqe; + struct data *d = data; + int ret; + + sqe = io_uring_get_sqe(d->ring); + io_uring_prep_read(sqe, d->timer_fd1, &d->buf1, sizeof(d->buf1), 0); + + sqe = io_uring_get_sqe(d->ring); + io_uring_prep_read(sqe, d->timer_fd2, &d->buf2, sizeof(d->buf2), 0); + + ret = io_uring_submit(d->ring); + if (ret != 2) + return (void *) (uintptr_t) 1; + + /* Exit suddenly. */ + return NULL; +} + +static int test(int flags) +{ + struct io_uring_params params = { .flags = flags, }; + struct io_uring ring; + struct data d = { .ring = &ring, }; + pthread_t thread; + void *res; + int ret; + + ret = t_create_ring_params(8, &ring, ¶ms); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret != T_SETUP_OK) + return 1; + + d.timer_fd1 = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC); + if (d.timer_fd1 < 0) { + perror("timerfd_create"); + return 1; + } + d.timer_fd2 = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC); + if (d.timer_fd2 < 0) { + perror("timerfd_create"); + return 1; + } + + pthread_create(&thread, NULL, submit, &d); + pthread_join(thread, &res); + + /** Wait for completions and do stuff ... **/ + + io_uring_queue_exit(&ring); + + close(d.timer_fd1); + close(d.timer_fd2); + return !!res; +} + +int main(int argc, char *argv[]) +{ + int ret, i; + + for (i = 0; i < 1000; i++) { + ret = test(0); + if (ret) { + fprintf(stderr, "Test failed\n"); + return ret; + } + } + + for (i = 0; i < 1000; i++) { + ret = test(IORING_SETUP_IOPOLL); + if (ret) { + fprintf(stderr, "Test IOPOLL failed\n"); + return ret; + } + } + + for (i = 0; i < 100; i++) { + ret = test(IORING_SETUP_SQPOLL); + if (ret) { + fprintf(stderr, "Test SQPOLL failed\n"); + return ret; + } + } + + return 0; +} diff --git a/contrib/libs/liburing/test/read-write.c b/contrib/libs/liburing/test/read-write.c new file mode 100644 index 0000000000..953b0afc53 --- /dev/null +++ b/contrib/libs/liburing/test/read-write.c @@ -0,0 +1,959 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: basic read/write tests with buffered, O_DIRECT, and SQPOLL + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <poll.h> +#include <sys/eventfd.h> +#include <sys/resource.h> + +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE (256 * 1024) +#define BS 8192 +#define BUFFERS (FILE_SIZE / BS) + +static struct iovec *vecs; +static int no_read; +static int no_buf_select; +static int warned; + +static int create_nonaligned_buffers(void) +{ + int i; + + vecs = t_malloc(BUFFERS * sizeof(struct iovec)); + for (i = 0; i < BUFFERS; i++) { + char *p = t_malloc(3 * BS); + + if (!p) + return 1; + vecs[i].iov_base = p + (rand() % BS); + vecs[i].iov_len = 1 + (rand() % BS); + } + + return 0; +} + +static int __test_io(const char *file, struct io_uring *ring, int write, + int buffered, int sqthread, int fixed, int nonvec, + int buf_select, int seq, int exp_len) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int open_flags; + int i, fd = -1, ret; + off_t offset; + +#ifdef VERBOSE + fprintf(stdout, "%s: start %d/%d/%d/%d/%d: ", __FUNCTION__, write, + buffered, sqthread, + fixed, nonvec); +#endif + if (write) + open_flags = O_WRONLY; + else + open_flags = O_RDONLY; + if (!buffered) + open_flags |= O_DIRECT; + + if (fixed) { + ret = t_register_buffers(ring, vecs, BUFFERS); + if (ret == T_SETUP_SKIP) + return 0; + if (ret != T_SETUP_OK) { + fprintf(stderr, "buffer reg failed: %d\n", ret); + goto err; + } + } + + fd = open(file, open_flags); + if (fd < 0) { + perror("file open"); + goto err; + } + + if (sqthread) { + ret = io_uring_register_files(ring, &fd, 1); + if (ret) { + fprintf(stderr, "file reg failed: %d\n", ret); + goto err; + } + } + + offset = 0; + for (i = 0; i < BUFFERS; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + if (!seq) + offset = BS * (rand() % BUFFERS); + if (write) { + int do_fixed = fixed; + int use_fd = fd; + + if (sqthread) + use_fd = 0; + if (fixed && (i & 1)) + do_fixed = 0; + if (do_fixed) { + io_uring_prep_write_fixed(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, + offset, i); + } else if (nonvec) { + io_uring_prep_write(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, offset); + } else { + io_uring_prep_writev(sqe, use_fd, &vecs[i], 1, + offset); + } + } else { + int do_fixed = fixed; + int use_fd = fd; + + if (sqthread) + use_fd = 0; + if (fixed && (i & 1)) + do_fixed = 0; + if (do_fixed) { + io_uring_prep_read_fixed(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, + offset, i); + } else if (nonvec) { + io_uring_prep_read(sqe, use_fd, vecs[i].iov_base, + vecs[i].iov_len, offset); + } else { + io_uring_prep_readv(sqe, use_fd, &vecs[i], 1, + offset); + } + + } + sqe->user_data = i; + if (sqthread) + sqe->flags |= IOSQE_FIXED_FILE; + if (buf_select) { + if (nonvec) + sqe->addr = 0; + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = buf_select; + } + if (seq) + offset += BS; + } + + ret = io_uring_submit(ring); + if (ret != BUFFERS) { + fprintf(stderr, "submit got %d, wanted %d\n", ret, BUFFERS); + goto err; + } + + for (i = 0; i < BUFFERS; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + goto err; + } + if (cqe->res == -EINVAL && nonvec) { + if (!warned) { + fprintf(stdout, "Non-vectored IO not " + "supported, skipping\n"); + warned = 1; + no_read = 1; + } + } else if (exp_len == -1) { + int iov_len = vecs[cqe->user_data].iov_len; + + if (cqe->res != iov_len) { + fprintf(stderr, "cqe res %d, wanted %d\n", + cqe->res, iov_len); + goto err; + } + } else if (cqe->res != exp_len) { + fprintf(stderr, "cqe res %d, wanted %d\n", cqe->res, exp_len); + goto err; + } + if (buf_select && exp_len == BS) { + int bid = cqe->flags >> 16; + unsigned char *ptr = vecs[bid].iov_base; + int j; + + for (j = 0; j < BS; j++) { + if (ptr[j] == cqe->user_data) + continue; + + fprintf(stderr, "Data mismatch! bid=%d, " + "wanted=%d, got=%d\n", bid, + (int)cqe->user_data, ptr[j]); + return 1; + } + } + io_uring_cqe_seen(ring, cqe); + } + + if (fixed) { + ret = io_uring_unregister_buffers(ring); + if (ret) { + fprintf(stderr, "buffer unreg failed: %d\n", ret); + goto err; + } + } + if (sqthread) { + ret = io_uring_unregister_files(ring); + if (ret) { + fprintf(stderr, "file unreg failed: %d\n", ret); + goto err; + } + } + + close(fd); +#ifdef VERBOSE + fprintf(stdout, "PASS\n"); +#endif + return 0; +err: +#ifdef VERBOSE + fprintf(stderr, "FAILED\n"); +#endif + if (fd != -1) + close(fd); + return 1; +} +static int test_io(const char *file, int write, int buffered, int sqthread, + int fixed, int nonvec, int exp_len) +{ + struct io_uring ring; + int ret, ring_flags = 0; + + if (sqthread) + ring_flags = IORING_SETUP_SQPOLL; + + ret = t_create_ring(64, &ring, ring_flags); + if (ret == T_SETUP_SKIP) + return 0; + if (ret != T_SETUP_OK) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + ret = __test_io(file, &ring, write, buffered, sqthread, fixed, nonvec, + 0, 0, exp_len); + io_uring_queue_exit(&ring); + return ret; +} + +static int read_poll_link(const char *file) +{ + struct __kernel_timespec ts; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int i, fd, ret, fds[2]; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) + return ret; + + fd = open(file, O_WRONLY); + if (fd < 0) { + perror("open"); + return 1; + } + + if (pipe(fds)) { + perror("pipe"); + return 1; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, fd, &vecs[0], 1, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, fds[0], POLLIN); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 2; + + ts.tv_sec = 1; + ts.tv_nsec = 0; + sqe = io_uring_get_sqe(&ring); + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->user_data = 3; + + ret = io_uring_submit(&ring); + if (ret != 3) { + fprintf(stderr, "submitted %d\n", ret); + return 1; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + } + + return 0; +} + +static int has_nonvec_read(void) +{ + struct io_uring_probe *p; + struct io_uring ring; + int ret; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + exit(ret); + } + + p = t_calloc(1, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); + ret = io_uring_register_probe(&ring, p, 256); + /* if we don't have PROBE_REGISTER, we don't have OP_READ/WRITE */ + if (ret == -EINVAL) { +out: + io_uring_queue_exit(&ring); + return 0; + } else if (ret) { + fprintf(stderr, "register_probe: %d\n", ret); + goto out; + } + + if (p->ops_len <= IORING_OP_READ) + goto out; + if (!(p->ops[IORING_OP_READ].flags & IO_URING_OP_SUPPORTED)) + goto out; + io_uring_queue_exit(&ring); + return 1; +} + +static int test_eventfd_read(void) +{ + struct io_uring ring; + int fd, ret; + eventfd_t event; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + + if (no_read) + return 0; + ret = io_uring_queue_init(8, &ring, 0); + if (ret) + return ret; + + fd = eventfd(1, 0); + if (fd < 0) { + perror("eventfd"); + return 1; + } + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, fd, &event, sizeof(eventfd_t), 0); + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submitted %d\n", ret); + return 1; + } + eventfd_write(fd, 1); + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + if (cqe->res == -EINVAL) { + fprintf(stdout, "eventfd IO not supported, skipping\n"); + } else if (cqe->res != sizeof(eventfd_t)) { + fprintf(stderr, "cqe res %d, wanted %d\n", cqe->res, + (int) sizeof(eventfd_t)); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + return 0; +} + +static int test_buf_select_short(const char *filename, int nonvec) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret, i, exp_len; + + if (no_buf_select) + return 0; + + ret = io_uring_queue_init(64, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + exp_len = 0; + for (i = 0; i < BUFFERS; i++) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_provide_buffers(sqe, vecs[i].iov_base, + vecs[i].iov_len / 2, 1, 1, i); + if (!exp_len) + exp_len = vecs[i].iov_len / 2; + } + + ret = io_uring_submit(&ring); + if (ret != BUFFERS) { + fprintf(stderr, "submit: %d\n", ret); + return -1; + } + + for (i = 0; i < BUFFERS; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (cqe->res < 0) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + } + + ret = __test_io(filename, &ring, 0, 0, 0, 0, nonvec, 1, 1, exp_len); + + io_uring_queue_exit(&ring); + return ret; +} + +static int provide_buffers_iovec(struct io_uring *ring, int bgid) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int i, ret; + + for (i = 0; i < BUFFERS; i++) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_provide_buffers(sqe, vecs[i].iov_base, + vecs[i].iov_len, 1, bgid, i); + } + + ret = io_uring_submit(ring); + if (ret != BUFFERS) { + fprintf(stderr, "submit: %d\n", ret); + return -1; + } + + for (i = 0; i < BUFFERS; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + if (cqe->res < 0) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +} + +static int test_buf_select_pipe(void) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret, i; + int fds[2]; + + if (no_buf_select) + return 0; + + ret = io_uring_queue_init(64, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + ret = provide_buffers_iovec(&ring, 0); + if (ret) { + fprintf(stderr, "provide buffers failed: %d\n", ret); + return 1; + } + + ret = pipe(fds); + if (ret) { + fprintf(stderr, "pipe failed: %d\n", ret); + return 1; + } + + for (i = 0; i < 5; i++) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, fds[0], NULL, 1 /* max read 1 per go */, -1); + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = 0; + } + io_uring_submit(&ring); + + ret = write(fds[1], "01234", 5); + if (ret != 5) { + fprintf(stderr, "pipe write failed %d\n", ret); + return 1; + } + + for (i = 0; i < 5; i++) { + const char *buff; + + if (io_uring_wait_cqe(&ring, &cqe)) { + fprintf(stderr, "bad wait %d\n", i); + return 1; + } + if (cqe->res != 1) { + fprintf(stderr, "expected read %d\n", cqe->res); + return 1; + } + if (!(cqe->flags & IORING_CQE_F_BUFFER)) { + fprintf(stderr, "no buffer %d\n", cqe->res); + return 1; + } + buff = vecs[cqe->flags >> 16].iov_base; + if (*buff != '0' + i) { + fprintf(stderr, "%d: expected %c, got %c\n", i, '0' + i, *buff); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + } + + + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring); + return 0; +} + +static int test_buf_select(const char *filename, int nonvec) +{ + struct io_uring_probe *p; + struct io_uring ring; + int ret, i; + + ret = io_uring_queue_init(64, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + p = io_uring_get_probe_ring(&ring); + if (!p || !io_uring_opcode_supported(p, IORING_OP_PROVIDE_BUFFERS)) { + no_buf_select = 1; + fprintf(stdout, "Buffer select not supported, skipping\n"); + return 0; + } + io_uring_free_probe(p); + + /* + * Write out data with known pattern + */ + for (i = 0; i < BUFFERS; i++) + memset(vecs[i].iov_base, i, vecs[i].iov_len); + + ret = __test_io(filename, &ring, 1, 0, 0, 0, 0, 0, 1, BS); + if (ret) { + fprintf(stderr, "failed writing data\n"); + return 1; + } + + for (i = 0; i < BUFFERS; i++) + memset(vecs[i].iov_base, 0x55, vecs[i].iov_len); + + ret = provide_buffers_iovec(&ring, 1); + if (ret) + return ret; + + ret = __test_io(filename, &ring, 0, 0, 0, 0, nonvec, 1, 1, BS); + io_uring_queue_exit(&ring); + return ret; +} + +static int test_rem_buf(int batch, int sqe_flags) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int left, ret, nr = 0; + int bgid = 1; + + if (no_buf_select) + return 0; + + ret = io_uring_queue_init(64, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + ret = provide_buffers_iovec(&ring, bgid); + if (ret) + return ret; + + left = BUFFERS; + while (left) { + int to_rem = (left < batch) ? left : batch; + + left -= to_rem; + sqe = io_uring_get_sqe(&ring); + io_uring_prep_remove_buffers(sqe, to_rem, bgid); + sqe->user_data = to_rem; + sqe->flags |= sqe_flags; + ++nr; + } + + ret = io_uring_submit(&ring); + if (ret != nr) { + fprintf(stderr, "submit: %d\n", ret); + return -1; + } + + for (; nr > 0; nr--) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + if (cqe->res != cqe->user_data) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + return ret; +} + +static int test_io_link(const char *file) +{ + const int nr_links = 100; + const int link_len = 100; + const int nr_sqes = nr_links * link_len; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int i, j, fd, ret; + + fd = open(file, O_WRONLY); + if (fd < 0) { + perror("file open"); + goto err; + } + + ret = io_uring_queue_init(nr_sqes, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + goto err; + } + + for (i = 0; i < nr_links; ++i) { + for (j = 0; j < link_len; ++j) { + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + io_uring_prep_writev(sqe, fd, &vecs[0], 1, 0); + sqe->flags |= IOSQE_ASYNC; + if (j != link_len - 1) + sqe->flags |= IOSQE_IO_LINK; + } + } + + ret = io_uring_submit(&ring); + if (ret != nr_sqes) { + ret = io_uring_peek_cqe(&ring, &cqe); + if (!ret && cqe->res == -EINVAL) { + fprintf(stdout, "IOSQE_ASYNC not supported, skipped\n"); + goto out; + } + fprintf(stderr, "submit got %d, wanted %d\n", ret, nr_sqes); + goto err; + } + + for (i = 0; i < nr_sqes; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + goto err; + } + if (cqe->res == -EINVAL) { + if (!warned) { + fprintf(stdout, "Non-vectored IO not " + "supported, skipping\n"); + warned = 1; + no_read = 1; + } + } else if (cqe->res != BS) { + fprintf(stderr, "cqe res %d, wanted %d\n", cqe->res, BS); + goto err; + } + io_uring_cqe_seen(&ring, cqe); + } + +out: + io_uring_queue_exit(&ring); + close(fd); + return 0; +err: + if (fd != -1) + close(fd); + return 1; +} + +static int test_write_efbig(void) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + struct rlimit rlim, old_rlim; + int i, fd, ret; + loff_t off; + + if (geteuid()) { + fprintf(stdout, "Not root, skipping %s\n", __FUNCTION__); + return 0; + } + + if (getrlimit(RLIMIT_FSIZE, &old_rlim) < 0) { + perror("getrlimit"); + return 1; + } + rlim = old_rlim; + rlim.rlim_cur = 128 * 1024; + rlim.rlim_max = 128 * 1024; + if (setrlimit(RLIMIT_FSIZE, &rlim) < 0) { + perror("setrlimit"); + return 1; + } + + fd = open(".efbig", O_WRONLY | O_CREAT, 0644); + if (fd < 0) { + perror("file open"); + goto err; + } + unlink(".efbig"); + + ret = io_uring_queue_init(32, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + goto err; + } + + off = 0; + for (i = 0; i < 32; i++) { + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + io_uring_prep_writev(sqe, fd, &vecs[i], 1, off); + io_uring_sqe_set_data64(sqe, i); + off += BS; + } + + ret = io_uring_submit(&ring); + if (ret != 32) { + fprintf(stderr, "submit got %d, wanted %d\n", ret, 32); + goto err; + } + + for (i = 0; i < 32; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + goto err; + } + if (cqe->user_data < 16) { + if (cqe->res != BS) { + fprintf(stderr, "bad write: %d\n", cqe->res); + goto err; + } + } else { + if (cqe->res != -EFBIG) { + fprintf(stderr, "Expected -EFBIG: %d\n", cqe->res); + goto err; + } + } + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + close(fd); + unlink(".efbig"); + + if (setrlimit(RLIMIT_FSIZE, &old_rlim) < 0) { + perror("setrlimit"); + return 1; + } + return 0; +err: + if (fd != -1) + close(fd); + return 1; +} + +int main(int argc, char *argv[]) +{ + int i, ret, nr; + char buf[256]; + char *fname; + + if (argc > 1) { + fname = argv[1]; + } else { + srand((unsigned)time(NULL)); + snprintf(buf, sizeof(buf), ".basic-rw-%u-%u", + (unsigned)rand(), (unsigned)getpid()); + fname = buf; + t_create_file(fname, FILE_SIZE); + } + + signal(SIGXFSZ, SIG_IGN); + + vecs = t_create_buffers(BUFFERS, BS); + + /* if we don't have nonvec read, skip testing that */ + nr = has_nonvec_read() ? 32 : 16; + + for (i = 0; i < nr; i++) { + int write = (i & 1) != 0; + int buffered = (i & 2) != 0; + int sqthread = (i & 4) != 0; + int fixed = (i & 8) != 0; + int nonvec = (i & 16) != 0; + + ret = test_io(fname, write, buffered, sqthread, fixed, nonvec, + BS); + if (ret) { + fprintf(stderr, "test_io failed %d/%d/%d/%d/%d\n", + write, buffered, sqthread, fixed, nonvec); + goto err; + } + } + + ret = test_buf_select(fname, 1); + if (ret) { + fprintf(stderr, "test_buf_select nonvec failed\n"); + goto err; + } + + ret = test_buf_select(fname, 0); + if (ret) { + fprintf(stderr, "test_buf_select vec failed\n"); + goto err; + } + + ret = test_buf_select_short(fname, 1); + if (ret) { + fprintf(stderr, "test_buf_select_short nonvec failed\n"); + goto err; + } + + ret = test_buf_select_short(fname, 0); + if (ret) { + fprintf(stderr, "test_buf_select_short vec failed\n"); + goto err; + } + + ret = test_buf_select_pipe(); + if (ret) { + fprintf(stderr, "test_buf_select_pipe failed\n"); + goto err; + } + + ret = test_eventfd_read(); + if (ret) { + fprintf(stderr, "test_eventfd_read failed\n"); + goto err; + } + + ret = read_poll_link(fname); + if (ret) { + fprintf(stderr, "read_poll_link failed\n"); + goto err; + } + + ret = test_io_link(fname); + if (ret) { + fprintf(stderr, "test_io_link failed\n"); + goto err; + } + + ret = test_write_efbig(); + if (ret) { + fprintf(stderr, "test_write_efbig failed\n"); + goto err; + } + + ret = test_rem_buf(1, 0); + if (ret) { + fprintf(stderr, "test_rem_buf by 1 failed\n"); + goto err; + } + + ret = test_rem_buf(10, 0); + if (ret) { + fprintf(stderr, "test_rem_buf by 10 failed\n"); + goto err; + } + + ret = test_rem_buf(2, IOSQE_IO_LINK); + if (ret) { + fprintf(stderr, "test_rem_buf link failed\n"); + goto err; + } + + ret = test_rem_buf(2, IOSQE_ASYNC); + if (ret) { + fprintf(stderr, "test_rem_buf async failed\n"); + goto err; + } + + srand((unsigned)time(NULL)); + if (create_nonaligned_buffers()) { + fprintf(stderr, "file creation failed\n"); + goto err; + } + + /* test fixed bufs with non-aligned len/offset */ + for (i = 0; i < nr; i++) { + int write = (i & 1) != 0; + int buffered = (i & 2) != 0; + int sqthread = (i & 4) != 0; + int fixed = (i & 8) != 0; + int nonvec = (i & 16) != 0; + + /* direct IO requires alignment, skip it */ + if (!buffered || !fixed || nonvec) + continue; + + ret = test_io(fname, write, buffered, sqthread, fixed, nonvec, + -1); + if (ret) { + fprintf(stderr, "test_io failed %d/%d/%d/%d/%d\n", + write, buffered, sqthread, fixed, nonvec); + goto err; + } + } + + if (fname != argv[1]) + unlink(fname); + return 0; +err: + if (fname != argv[1]) + unlink(fname); + return 1; +} diff --git a/contrib/libs/liburing/test/recv-msgall-stream.c b/contrib/libs/liburing/test/recv-msgall-stream.c new file mode 100644 index 0000000000..355b1f5cfe --- /dev/null +++ b/contrib/libs/liburing/test/recv-msgall-stream.c @@ -0,0 +1,399 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test MSG_WAITALL for recv/recvmsg and include normal sync versions just + * for comparison. + */ +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <arpa/inet.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <pthread.h> + +#include "liburing.h" +#include "helpers.h" + +#define MAX_MSG 128 + +struct recv_data { + pthread_mutex_t mutex; + int use_recvmsg; + int use_sync; + __be16 port; +}; + +static int get_conn_sock(struct recv_data *rd, int *sockout) +{ + struct sockaddr_in saddr; + int sockfd, ret, val; + + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_addr.s_addr = htonl(INADDR_ANY); + + sockfd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + if (sockfd < 0) { + perror("socket"); + goto err; + } + + val = 1; + setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + setsockopt(sockfd, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + + if (t_bind_ephemeral_port(sockfd, &saddr)) { + perror("bind"); + goto err; + } + rd->port = saddr.sin_port; + + ret = listen(sockfd, 16); + if (ret < 0) { + perror("listen"); + goto err; + } + + pthread_mutex_unlock(&rd->mutex); + + ret = accept(sockfd, NULL, NULL); + if (ret < 0) { + perror("accept"); + return -1; + } + + *sockout = sockfd; + return ret; +err: + pthread_mutex_unlock(&rd->mutex); + return -1; +} + +static int recv_prep(struct io_uring *ring, struct iovec *iov, int *sock, + struct recv_data *rd) +{ + struct io_uring_sqe *sqe; + struct msghdr msg = { }; + int sockfd, sockout = -1, ret; + + sockfd = get_conn_sock(rd, &sockout); + if (sockfd < 0) + goto err; + + sqe = io_uring_get_sqe(ring); + if (!rd->use_recvmsg) { + io_uring_prep_recv(sqe, sockfd, iov->iov_base, iov->iov_len, + MSG_WAITALL); + } else { + msg.msg_namelen = sizeof(struct sockaddr_in); + msg.msg_iov = iov; + msg.msg_iovlen = 1; + io_uring_prep_recvmsg(sqe, sockfd, &msg, MSG_WAITALL); + } + + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + *sock = sockfd; + return 0; +err: + if (sockout != -1) { + shutdown(sockout, SHUT_RDWR); + close(sockout); + } + if (sockfd != -1) { + shutdown(sockfd, SHUT_RDWR); + close(sockfd); + } + return 1; +} + +static int do_recv(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stdout, "wait_cqe: %d\n", ret); + goto err; + } + if (cqe->res == -EINVAL) { + fprintf(stdout, "recv not supported, skipping\n"); + return 0; + } + if (cqe->res < 0) { + fprintf(stderr, "failed cqe: %d\n", cqe->res); + goto err; + } + if (cqe->res != MAX_MSG * sizeof(int)) { + fprintf(stderr, "got wrong length: %d\n", cqe->res); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + return 0; +err: + return 1; +} + +static int recv_sync(struct recv_data *rd) +{ + int buf[MAX_MSG]; + struct iovec iov = { + .iov_base = buf, + .iov_len = sizeof(buf), + }; + int i, ret, sockfd, sockout = -1; + + sockfd = get_conn_sock(rd, &sockout); + + if (rd->use_recvmsg) { + struct msghdr msg = { }; + + msg.msg_namelen = sizeof(struct sockaddr_in); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + ret = recvmsg(sockfd, &msg, MSG_WAITALL); + } else { + ret = recv(sockfd, buf, sizeof(buf), MSG_WAITALL); + } + + if (ret < 0) { + perror("receive"); + goto err; + } + + if (ret != sizeof(buf)) { + ret = -1; + goto err; + } + + for (i = 0; i < MAX_MSG; i++) { + if (buf[i] != i) + goto err; + } + ret = 0; +err: + shutdown(sockout, SHUT_RDWR); + shutdown(sockfd, SHUT_RDWR); + close(sockout); + close(sockfd); + return ret; +} + +static int recv_uring(struct recv_data *rd) +{ + int buf[MAX_MSG]; + struct iovec iov = { + .iov_base = buf, + .iov_len = sizeof(buf), + }; + struct io_uring_params p = { }; + struct io_uring ring; + int ret, sock = -1, sockout = -1; + + ret = t_create_ring_params(1, &ring, &p); + if (ret == T_SETUP_SKIP) { + pthread_mutex_unlock(&rd->mutex); + ret = 0; + goto err; + } else if (ret < 0) { + pthread_mutex_unlock(&rd->mutex); + goto err; + } + + sock = recv_prep(&ring, &iov, &sockout, rd); + if (ret) { + fprintf(stderr, "recv_prep failed: %d\n", ret); + goto err; + } + ret = do_recv(&ring); + if (!ret) { + int i; + + for (i = 0; i < MAX_MSG; i++) { + if (buf[i] != i) { + fprintf(stderr, "found %d at %d\n", buf[i], i); + ret = 1; + break; + } + } + } + + shutdown(sockout, SHUT_RDWR); + shutdown(sock, SHUT_RDWR); + close(sock); + close(sockout); + io_uring_queue_exit(&ring); +err: + if (sock != -1) { + shutdown(sock, SHUT_RDWR); + close(sock); + } + if (sockout != -1) { + shutdown(sockout, SHUT_RDWR); + close(sockout); + } + return ret; +} + +static void *recv_fn(void *data) +{ + struct recv_data *rd = data; + + if (rd->use_sync) + return (void *) (uintptr_t) recv_sync(rd); + + return (void *) (uintptr_t) recv_uring(rd); +} + +static int do_send(struct recv_data *rd) +{ + struct sockaddr_in saddr; + struct io_uring ring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int sockfd, ret, i; + struct iovec iov; + int *buf; + + ret = io_uring_queue_init(2, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return 1; + } + + buf = malloc(MAX_MSG * sizeof(int)); + for (i = 0; i < MAX_MSG; i++) + buf[i] = i; + + sockfd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + if (sockfd < 0) { + perror("socket"); + return 1; + } + + pthread_mutex_lock(&rd->mutex); + assert(rd->port != 0); + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_port = rd->port; + inet_pton(AF_INET, "127.0.0.1", &saddr.sin_addr); + + ret = connect(sockfd, (struct sockaddr *)&saddr, sizeof(saddr)); + if (ret < 0) { + perror("connect"); + return 1; + } + + iov.iov_base = buf; + iov.iov_len = MAX_MSG * sizeof(int) / 2; + for (i = 0; i < 2; i++) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_send(sqe, sockfd, iov.iov_base, iov.iov_len, 0); + sqe->user_data = 1; + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + usleep(10000); + iov.iov_base += iov.iov_len; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (cqe->res == -EINVAL) { + fprintf(stdout, "send not supported, skipping\n"); + close(sockfd); + return 0; + } + if (cqe->res != iov.iov_len) { + fprintf(stderr, "failed cqe: %d\n", cqe->res); + goto err; + } + io_uring_cqe_seen(&ring, cqe); + } + + shutdown(sockfd, SHUT_RDWR); + close(sockfd); + return 0; +err: + shutdown(sockfd, SHUT_RDWR); + close(sockfd); + return 1; +} + +static int test(int use_recvmsg, int use_sync) +{ + pthread_mutexattr_t attr; + pthread_t recv_thread; + struct recv_data rd; + int ret; + void *retval; + + pthread_mutexattr_init(&attr); + pthread_mutexattr_setpshared(&attr, 1); + pthread_mutex_init(&rd.mutex, &attr); + pthread_mutex_lock(&rd.mutex); + rd.use_recvmsg = use_recvmsg; + rd.use_sync = use_sync; + rd.port = 0; + + ret = pthread_create(&recv_thread, NULL, recv_fn, &rd); + if (ret) { + fprintf(stderr, "Thread create failed: %d\n", ret); + pthread_mutex_unlock(&rd.mutex); + return 1; + } + + do_send(&rd); + pthread_join(recv_thread, &retval); + return (intptr_t)retval; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test(0, 0); + if (ret) { + fprintf(stderr, "test recv failed\n"); + return ret; + } + + ret = test(1, 0); + if (ret) { + fprintf(stderr, "test recvmsg failed\n"); + return ret; + } + + ret = test(0, 1); + if (ret) { + fprintf(stderr, "test sync recv failed\n"); + return ret; + } + + ret = test(1, 1); + if (ret) { + fprintf(stderr, "test sync recvmsg failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/recv-msgall.c b/contrib/libs/liburing/test/recv-msgall.c new file mode 100644 index 0000000000..89b12b7268 --- /dev/null +++ b/contrib/libs/liburing/test/recv-msgall.c @@ -0,0 +1,266 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test MSG_WAITALL with datagram sockets, with a send splice into two. + */ +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <arpa/inet.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <pthread.h> + +#include "liburing.h" +#include "helpers.h" + +#define MAX_MSG 128 +#define HOST "127.0.0.1" +static __be16 bind_port; + +static int recv_prep(struct io_uring *ring, struct iovec *iov, int *sock, + int use_recvmsg) +{ + struct sockaddr_in saddr; + struct io_uring_sqe *sqe; + int sockfd, ret, val; + struct msghdr msg = { }; + + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_addr.s_addr = htonl(INADDR_ANY); + + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (sockfd < 0) { + perror("socket"); + return 1; + } + + val = 1; + setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + + if (t_bind_ephemeral_port(sockfd, &saddr)) { + perror("bind"); + goto err; + } + bind_port = saddr.sin_port; + + sqe = io_uring_get_sqe(ring); + if (!use_recvmsg) { + io_uring_prep_recv(sqe, sockfd, iov->iov_base, iov->iov_len, + MSG_WAITALL); + } else { + msg.msg_namelen = sizeof(struct sockaddr_in); + msg.msg_iov = iov; + msg.msg_iovlen = 1; + io_uring_prep_recvmsg(sqe, sockfd, &msg, MSG_WAITALL); + } + + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + *sock = sockfd; + return 0; +err: + close(sockfd); + return 1; +} + +static int do_recv(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stdout, "wait_cqe: %d\n", ret); + goto err; + } + if (cqe->res == -EINVAL) { + fprintf(stdout, "recv not supported, skipping\n"); + return 0; + } + if (cqe->res < 0) { + fprintf(stderr, "failed cqe: %d\n", cqe->res); + goto err; + } + if (cqe->res != MAX_MSG * sizeof(int) / 2) { + fprintf(stderr, "got wrong length: %d\n", cqe->res); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + return 0; +err: + return 1; +} + +struct recv_data { + pthread_mutex_t mutex; + int use_recvmsg; +}; + +static void *recv_fn(void *data) +{ + struct recv_data *rd = data; + int buf[MAX_MSG]; + struct iovec iov = { + .iov_base = buf, + .iov_len = sizeof(buf), + }; + struct io_uring_params p = { }; + struct io_uring ring; + int ret, sock; + + ret = t_create_ring_params(1, &ring, &p); + if (ret == T_SETUP_SKIP) { + pthread_mutex_unlock(&rd->mutex); + ret = 0; + goto err; + } else if (ret < 0) { + pthread_mutex_unlock(&rd->mutex); + goto err; + } + + ret = recv_prep(&ring, &iov, &sock, rd->use_recvmsg); + if (ret) { + fprintf(stderr, "recv_prep failed: %d\n", ret); + goto err; + } + pthread_mutex_unlock(&rd->mutex); + ret = do_recv(&ring); + close(sock); + io_uring_queue_exit(&ring); +err: + return (void *)(intptr_t)ret; +} + +static int do_send(void) +{ + struct sockaddr_in saddr; + struct io_uring ring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int sockfd, ret, i; + struct iovec iov; + int *buf; + + ret = io_uring_queue_init(2, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return 1; + } + + buf = malloc(MAX_MSG * sizeof(int)); + for (i = 0; i < MAX_MSG; i++) + buf[i] = i; + + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_port = bind_port; + inet_pton(AF_INET, HOST, &saddr.sin_addr); + + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (sockfd < 0) { + perror("socket"); + return 1; + } + + ret = connect(sockfd, (struct sockaddr *)&saddr, sizeof(saddr)); + if (ret < 0) { + perror("connect"); + return 1; + } + + iov.iov_base = buf; + iov.iov_len = MAX_MSG * sizeof(int) / 2; + for (i = 0; i < 2; i++) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_send(sqe, sockfd, iov.iov_base, iov.iov_len, 0); + sqe->user_data = 1; + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + usleep(10000); + iov.iov_base += iov.iov_len; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (cqe->res == -EINVAL) { + fprintf(stdout, "send not supported, skipping\n"); + close(sockfd); + return 0; + } + if (cqe->res != iov.iov_len) { + fprintf(stderr, "failed cqe: %d\n", cqe->res); + goto err; + } + io_uring_cqe_seen(&ring, cqe); + } + + close(sockfd); + return 0; +err: + close(sockfd); + return 1; +} + +static int test(int use_recvmsg) +{ + pthread_mutexattr_t attr; + pthread_t recv_thread; + struct recv_data rd; + int ret; + void *retval; + + pthread_mutexattr_init(&attr); + pthread_mutexattr_setpshared(&attr, 1); + pthread_mutex_init(&rd.mutex, &attr); + pthread_mutex_lock(&rd.mutex); + rd.use_recvmsg = use_recvmsg; + + ret = pthread_create(&recv_thread, NULL, recv_fn, &rd); + if (ret) { + fprintf(stderr, "Thread create failed: %d\n", ret); + pthread_mutex_unlock(&rd.mutex); + return 1; + } + + pthread_mutex_lock(&rd.mutex); + do_send(); + pthread_join(recv_thread, &retval); + return (intptr_t)retval; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test(0); + if (ret) { + fprintf(stderr, "test recv failed\n"); + return ret; + } + + ret = test(1); + if (ret) { + fprintf(stderr, "test recvmsg failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/recv-multishot.c b/contrib/libs/liburing/test/recv-multishot.c new file mode 100644 index 0000000000..67a53567b7 --- /dev/null +++ b/contrib/libs/liburing/test/recv-multishot.c @@ -0,0 +1,506 @@ +#include "../config-host.h" +// SPDX-License-Identifier: MIT + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <arpa/inet.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <pthread.h> + +#include "liburing.h" +#include "helpers.h" + +#define ENORECVMULTISHOT 9999 + +enum early_error_t { + ERROR_NONE = 0, + ERROR_NOT_ENOUGH_BUFFERS, + ERROR_EARLY_CLOSE_SENDER, + ERROR_EARLY_CLOSE_RECEIVER, + ERROR_EARLY_OVERFLOW, + ERROR_EARLY_LAST +}; + +struct args { + bool stream; + bool wait_each; + bool recvmsg; + enum early_error_t early_error; + bool defer; +}; + +static int check_sockaddr(struct sockaddr_in *in) +{ + struct in_addr expected; + + inet_pton(AF_INET, "127.0.0.1", &expected); + if (in->sin_family != AF_INET) { + fprintf(stderr, "bad family %d\n", (int)htons(in->sin_family)); + return -1; + } + if (memcmp(&expected, &in->sin_addr, sizeof(in->sin_addr))) { + char buff[256]; + const char *addr = inet_ntop(AF_INET, &in->sin_addr, buff, sizeof(buff)); + + fprintf(stderr, "unexpected address %s\n", addr ? addr : "INVALID"); + return -1; + } + return 0; +} + +static int test(struct args *args) +{ + int const N = 8; + int const N_BUFFS = N * 64; + int const N_CQE_OVERFLOW = 4; + int const min_cqes = 2; + int const NAME_LEN = sizeof(struct sockaddr_storage); + int const CONTROL_LEN = CMSG_ALIGN(sizeof(struct sockaddr_storage)) + + sizeof(struct cmsghdr); + struct io_uring ring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int fds[2], ret, i, j; + int total_sent_bytes = 0, total_recv_bytes = 0, total_dropped_bytes = 0; + int send_buff[256]; + int *sent_buffs[N_BUFFS]; + int *recv_buffs[N_BUFFS]; + int *at; + struct io_uring_cqe recv_cqe[N_BUFFS]; + int recv_cqes = 0; + bool early_error = false; + bool early_error_started = false; + struct __kernel_timespec timeout = { + .tv_sec = 1, + }; + struct msghdr msg; + struct io_uring_params params = { }; + int n_sqe = 32; + + memset(recv_buffs, 0, sizeof(recv_buffs)); + + if (args->defer) + params.flags |= IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN; + + if (args->early_error == ERROR_EARLY_OVERFLOW) { + params.flags |= IORING_SETUP_CQSIZE; + params.cq_entries = N_CQE_OVERFLOW; + n_sqe = N_CQE_OVERFLOW; + } + + ret = io_uring_queue_init_params(n_sqe, &ring, ¶ms); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return ret; + } + + ret = t_create_socket_pair(fds, args->stream); + if (ret) { + fprintf(stderr, "t_create_socket_pair failed: %d\n", ret); + return ret; + } + + if (!args->stream) { + bool val = true; + + /* force some cmsgs to come back to us */ + ret = setsockopt(fds[0], IPPROTO_IP, IP_RECVORIGDSTADDR, &val, + sizeof(val)); + if (ret) { + fprintf(stderr, "setsockopt failed %d\n", errno); + goto cleanup; + } + } + + for (i = 0; i < ARRAY_SIZE(send_buff); i++) + send_buff[i] = i; + + for (i = 0; i < ARRAY_SIZE(recv_buffs); i++) { + /* prepare some different sized buffers */ + int buffer_size = (i % 2 == 0 && (args->stream || args->recvmsg)) ? 1 : N; + + buffer_size *= sizeof(int); + if (args->recvmsg) { + buffer_size += + sizeof(struct io_uring_recvmsg_out) + + NAME_LEN + + CONTROL_LEN; + } + + recv_buffs[i] = malloc(buffer_size); + + if (i > 2 && args->early_error == ERROR_NOT_ENOUGH_BUFFERS) + continue; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_provide_buffers(sqe, recv_buffs[i], + buffer_size, 1, 7, i); + io_uring_sqe_set_data64(sqe, 0x999); + memset(recv_buffs[i], 0xcc, buffer_size); + if (io_uring_submit_and_wait_timeout(&ring, &cqe, 1, &timeout, NULL) < 0) { + fprintf(stderr, "provide buffers failed: %d\n", ret); + ret = -1; + goto cleanup; + } + io_uring_cqe_seen(&ring, cqe); + } + + sqe = io_uring_get_sqe(&ring); + if (args->recvmsg) { + unsigned int flags = 0; + + if (!args->stream) + flags |= MSG_TRUNC; + + memset(&msg, 0, sizeof(msg)); + msg.msg_namelen = NAME_LEN; + msg.msg_controllen = CONTROL_LEN; + io_uring_prep_recvmsg_multishot(sqe, fds[0], &msg, flags); + } else { + io_uring_prep_recv_multishot(sqe, fds[0], NULL, 0, 0); + } + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = 7; + io_uring_sqe_set_data64(sqe, 1234); + io_uring_submit(&ring); + + at = &send_buff[0]; + total_sent_bytes = 0; + for (i = 0; i < N; i++) { + int to_send = sizeof(*at) * (i+1); + + total_sent_bytes += to_send; + sent_buffs[i] = at; + if (send(fds[1], at, to_send, 0) != to_send) { + if (early_error_started) + break; + fprintf(stderr, "send failed %d\n", errno); + ret = -1; + goto cleanup; + } + + if (i == 2) { + if (args->early_error == ERROR_EARLY_CLOSE_RECEIVER) { + /* allow previous sends to complete */ + usleep(1000); + io_uring_get_events(&ring); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_recv(sqe, fds[0], NULL, 0, 0); + io_uring_prep_cancel64(sqe, 1234, 0); + io_uring_sqe_set_data64(sqe, 0x888); + sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; + io_uring_submit(&ring); + early_error_started = true; + + /* allow the cancel to complete */ + usleep(1000); + io_uring_get_events(&ring); + } + if (args->early_error == ERROR_EARLY_CLOSE_SENDER) { + early_error_started = true; + shutdown(fds[1], SHUT_RDWR); + close(fds[1]); + } + } + at += (i+1); + + if (args->wait_each) { + ret = io_uring_wait_cqes(&ring, &cqe, 1, &timeout, NULL); + if (ret) { + fprintf(stderr, "wait_each failed: %d\n", ret); + ret = -1; + goto cleanup; + } + while (io_uring_peek_cqe(&ring, &cqe) == 0) { + recv_cqe[recv_cqes++] = *cqe; + if (cqe->flags & IORING_CQE_F_MORE) { + io_uring_cqe_seen(&ring, cqe); + } else { + early_error = true; + io_uring_cqe_seen(&ring, cqe); + } + } + if (early_error) + break; + } + } + + close(fds[1]); + + /* allow sends to finish */ + usleep(1000); + + if ((args->stream && !early_error) || recv_cqes < min_cqes) { + ret = io_uring_wait_cqes(&ring, &cqe, 1, &timeout, NULL); + if (ret && ret != -ETIME) { + fprintf(stderr, "wait final failed: %d\n", ret); + ret = -1; + goto cleanup; + } + } + + while (io_uring_peek_cqe(&ring, &cqe) == 0) { + recv_cqe[recv_cqes++] = *cqe; + io_uring_cqe_seen(&ring, cqe); + } + + ret = -1; + at = &send_buff[0]; + if (recv_cqes < min_cqes) { + if (recv_cqes > 0 && recv_cqe[0].res == -EINVAL) { + return -ENORECVMULTISHOT; + } + /* some kernels apparently don't check ->ioprio, skip */ + ret = -ENORECVMULTISHOT; + goto cleanup; + } + for (i = 0; i < recv_cqes; i++) { + cqe = &recv_cqe[i]; + + bool const is_last = i == recv_cqes - 1; + + bool const should_be_last = + (cqe->res <= 0) || + (args->stream && is_last) || + (args->early_error == ERROR_EARLY_OVERFLOW && + !args->wait_each && i == N_CQE_OVERFLOW); + int *this_recv; + int orig_payload_size = cqe->res; + + + if (should_be_last) { + int used_res = cqe->res; + + if (!is_last) { + fprintf(stderr, "not last cqe had error %d\n", i); + goto cleanup; + } + + switch (args->early_error) { + case ERROR_NOT_ENOUGH_BUFFERS: + if (cqe->res != -ENOBUFS) { + fprintf(stderr, + "ERROR_NOT_ENOUGH_BUFFERS: res %d\n", cqe->res); + goto cleanup; + } + break; + case ERROR_EARLY_OVERFLOW: + if (cqe->res < 0) { + fprintf(stderr, + "ERROR_EARLY_OVERFLOW: res %d\n", cqe->res); + goto cleanup; + } + break; + case ERROR_EARLY_CLOSE_RECEIVER: + if (cqe->res != -ECANCELED) { + fprintf(stderr, + "ERROR_EARLY_CLOSE_RECEIVER: res %d\n", cqe->res); + goto cleanup; + } + break; + case ERROR_NONE: + case ERROR_EARLY_CLOSE_SENDER: + if (args->recvmsg && (cqe->flags & IORING_CQE_F_BUFFER)) { + void *buff = recv_buffs[cqe->flags >> 16]; + struct io_uring_recvmsg_out *o = + io_uring_recvmsg_validate(buff, cqe->res, &msg); + + if (!o) { + fprintf(stderr, "invalid buff\n"); + goto cleanup; + } + if (o->payloadlen != 0) { + fprintf(stderr, "expected 0 payloadlen, got %u\n", + o->payloadlen); + goto cleanup; + } + used_res = 0; + } else if (cqe->res != 0) { + fprintf(stderr, "early error: res %d\n", cqe->res); + goto cleanup; + } + break; + case ERROR_EARLY_LAST: + fprintf(stderr, "bad error_early\n"); + goto cleanup; + }; + + if (cqe->res <= 0 && cqe->flags & IORING_CQE_F_BUFFER) { + fprintf(stderr, "final BUFFER flag set\n"); + goto cleanup; + } + + if (cqe->flags & IORING_CQE_F_MORE) { + fprintf(stderr, "final MORE flag set\n"); + goto cleanup; + } + + if (used_res <= 0) + continue; + } else { + if (!(cqe->flags & IORING_CQE_F_MORE)) { + fprintf(stderr, "MORE flag not set\n"); + goto cleanup; + } + } + + if (!(cqe->flags & IORING_CQE_F_BUFFER)) { + fprintf(stderr, "BUFFER flag not set\n"); + goto cleanup; + } + + this_recv = recv_buffs[cqe->flags >> 16]; + + if (args->recvmsg) { + struct io_uring_recvmsg_out *o = io_uring_recvmsg_validate( + this_recv, cqe->res, &msg); + + if (!o) { + fprintf(stderr, "bad recvmsg\n"); + goto cleanup; + } + orig_payload_size = o->payloadlen; + + if (!args->stream) { + orig_payload_size = o->payloadlen; + + struct cmsghdr *cmsg; + + if (o->namelen < sizeof(struct sockaddr_in)) { + fprintf(stderr, "bad addr len %d", + o->namelen); + goto cleanup; + } + if (check_sockaddr((struct sockaddr_in *)io_uring_recvmsg_name(o))) + goto cleanup; + + cmsg = io_uring_recvmsg_cmsg_firsthdr(o, &msg); + if (!cmsg || + cmsg->cmsg_level != IPPROTO_IP || + cmsg->cmsg_type != IP_RECVORIGDSTADDR) { + fprintf(stderr, "bad cmsg"); + goto cleanup; + } + if (check_sockaddr((struct sockaddr_in *)CMSG_DATA(cmsg))) + goto cleanup; + cmsg = io_uring_recvmsg_cmsg_nexthdr(o, &msg, cmsg); + if (cmsg) { + fprintf(stderr, "unexpected extra cmsg\n"); + goto cleanup; + } + + } + + this_recv = (int *)io_uring_recvmsg_payload(o, &msg); + cqe->res = io_uring_recvmsg_payload_length(o, cqe->res, &msg); + if (o->payloadlen != cqe->res) { + if (!(o->flags & MSG_TRUNC)) { + fprintf(stderr, "expected truncated flag\n"); + goto cleanup; + } + total_dropped_bytes += (o->payloadlen - cqe->res); + } + } + + total_recv_bytes += cqe->res; + + if (cqe->res % 4 != 0) { + /* + * doesn't seem to happen in practice, would need some + * work to remove this requirement + */ + fprintf(stderr, "unexpectedly aligned buffer cqe->res=%d\n", cqe->res); + goto cleanup; + } + + /* + * for tcp: check buffer arrived in order + * for udp: based on size validate data based on size + */ + if (!args->stream) { + int sent_idx = orig_payload_size / sizeof(*at) - 1; + + if (sent_idx < 0 || sent_idx > N) { + fprintf(stderr, "Bad sent idx: %d\n", sent_idx); + goto cleanup; + } + at = sent_buffs[sent_idx]; + } + for (j = 0; j < cqe->res / 4; j++) { + int sent = *at++; + int recv = *this_recv++; + + if (sent != recv) { + fprintf(stderr, "recv=%d sent=%d\n", recv, sent); + goto cleanup; + } + } + } + + if (args->early_error == ERROR_NONE && + total_recv_bytes + total_dropped_bytes < total_sent_bytes) { + fprintf(stderr, + "missing recv: recv=%d dropped=%d sent=%d\n", + total_recv_bytes, total_sent_bytes, total_dropped_bytes); + goto cleanup; + } + + ret = 0; +cleanup: + for (i = 0; i < ARRAY_SIZE(recv_buffs); i++) + free(recv_buffs[i]); + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring); + + return ret; +} + +int main(int argc, char *argv[]) +{ + int ret; + int loop; + int early_error = 0; + bool has_defer; + + if (argc > 1) + return T_EXIT_SKIP; + + has_defer = t_probe_defer_taskrun(); + + for (loop = 0; loop < 16; loop++) { + struct args a = { + .stream = loop & 0x01, + .wait_each = loop & 0x2, + .recvmsg = loop & 0x04, + .defer = loop & 0x08, + }; + if (a.defer && !has_defer) + continue; + for (early_error = 0; early_error < ERROR_EARLY_LAST; early_error++) { + a.early_error = (enum early_error_t)early_error; + ret = test(&a); + if (ret) { + if (ret == -ENORECVMULTISHOT) { + if (loop == 0) + return T_EXIT_SKIP; + fprintf(stderr, + "ENORECVMULTISHOT received but loop>0\n"); + } + fprintf(stderr, + "test stream=%d wait_each=%d recvmsg=%d early_error=%d " + " defer=%d failed\n", + a.stream, a.wait_each, a.recvmsg, a.early_error, a.defer); + return T_EXIT_FAIL; + } + } + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/register-restrictions.c b/contrib/libs/liburing/test/register-restrictions.c new file mode 100644 index 0000000000..af35db4b4a --- /dev/null +++ b/contrib/libs/liburing/test/register-restrictions.c @@ -0,0 +1,634 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test restrictions + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <poll.h> +#include <sys/eventfd.h> + +#include "liburing.h" + +enum { + TEST_OK, + TEST_SKIPPED, + TEST_FAILED +}; + +static int test_restrictions_sqe_op(void) +{ + struct io_uring_restriction res[2]; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret, pipe1[2]; + + uint64_t ptr; + struct iovec vec = { + .iov_base = &ptr, + .iov_len = sizeof(ptr) + }; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return TEST_FAILED; + } + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_R_DISABLED); + if (ret) { + if (ret == -EINVAL) + return TEST_SKIPPED; + fprintf(stderr, "ring setup failed: %d\n", ret); + return TEST_FAILED; + } + + res[0].opcode = IORING_RESTRICTION_SQE_OP; + res[0].sqe_op = IORING_OP_WRITEV; + + res[1].opcode = IORING_RESTRICTION_SQE_OP; + res[1].sqe_op = IORING_OP_WRITE; + + ret = io_uring_register_restrictions(&ring, res, 2); + if (ret) { + if (ret == -EINVAL) + return TEST_SKIPPED; + + fprintf(stderr, "failed to register restrictions: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_enable_rings(&ring); + if (ret) { + fprintf(stderr, "ring enabling failed: %d\n", ret); + return TEST_FAILED; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, pipe1[1], &vec, 1, 0); + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, pipe1[0], &vec, 1, 0); + sqe->user_data = 2; + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "submit: %d\n", ret); + return TEST_FAILED; + } + + for (int i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return TEST_FAILED; + } + + switch (cqe->user_data) { + case 1: /* writev */ + if (cqe->res != sizeof(ptr)) { + fprintf(stderr, "write res: %d\n", cqe->res); + return TEST_FAILED; + } + + break; + case 2: /* readv should be denied */ + if (cqe->res != -EACCES) { + fprintf(stderr, "read res: %d\n", cqe->res); + return TEST_FAILED; + } + break; + } + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + return TEST_OK; +} + +static int test_restrictions_register_op(void) +{ + struct io_uring_restriction res[1]; + struct io_uring ring; + int ret, pipe1[2]; + + uint64_t ptr; + struct iovec vec = { + .iov_base = &ptr, + .iov_len = sizeof(ptr) + }; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return TEST_FAILED; + } + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_R_DISABLED); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return TEST_FAILED; + } + + res[0].opcode = IORING_RESTRICTION_REGISTER_OP; + res[0].register_op = IORING_REGISTER_BUFFERS; + + ret = io_uring_register_restrictions(&ring, res, 1); + if (ret) { + if (ret == -EINVAL) + return TEST_SKIPPED; + + fprintf(stderr, "failed to register restrictions: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_enable_rings(&ring); + if (ret) { + fprintf(stderr, "ring enabling failed: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_register_buffers(&ring, &vec, 1); + if (ret) { + fprintf(stderr, "io_uring_register_buffers failed: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_register_files(&ring, pipe1, 2); + if (ret != -EACCES) { + fprintf(stderr, "io_uring_register_files ret: %d\n", ret); + return TEST_FAILED; + } + + io_uring_queue_exit(&ring); + return TEST_OK; +} + +static int test_restrictions_fixed_file(void) +{ + struct io_uring_restriction res[4]; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret, pipe1[2]; + + uint64_t ptr; + struct iovec vec = { + .iov_base = &ptr, + .iov_len = sizeof(ptr) + }; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return TEST_FAILED; + } + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_R_DISABLED); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return TEST_FAILED; + } + + res[0].opcode = IORING_RESTRICTION_SQE_OP; + res[0].sqe_op = IORING_OP_WRITEV; + + res[1].opcode = IORING_RESTRICTION_SQE_OP; + res[1].sqe_op = IORING_OP_READV; + + res[2].opcode = IORING_RESTRICTION_SQE_FLAGS_REQUIRED; + res[2].sqe_flags = IOSQE_FIXED_FILE; + + res[3].opcode = IORING_RESTRICTION_REGISTER_OP; + res[3].register_op = IORING_REGISTER_FILES; + + ret = io_uring_register_restrictions(&ring, res, 4); + if (ret) { + if (ret == -EINVAL) + return TEST_SKIPPED; + + fprintf(stderr, "failed to register restrictions: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_enable_rings(&ring); + if (ret) { + fprintf(stderr, "ring enabling failed: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_register_files(&ring, pipe1, 2); + if (ret) { + fprintf(stderr, "io_uring_register_files ret: %d\n", ret); + return TEST_FAILED; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, 1, &vec, 1, 0); + io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE); + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, 0, &vec, 1, 0); + io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE); + sqe->user_data = 2; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, pipe1[1], &vec, 1, 0); + sqe->user_data = 3; + + ret = io_uring_submit(&ring); + if (ret != 3) { + fprintf(stderr, "submit: %d\n", ret); + return TEST_FAILED; + } + + for (int i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return TEST_FAILED; + } + + switch (cqe->user_data) { + case 1: /* writev */ + if (cqe->res != sizeof(ptr)) { + fprintf(stderr, "write res: %d\n", cqe->res); + return TEST_FAILED; + } + + break; + case 2: /* readv */ + if (cqe->res != sizeof(ptr)) { + fprintf(stderr, "read res: %d\n", cqe->res); + return TEST_FAILED; + } + break; + case 3: /* writev without fixed_file should be denied */ + if (cqe->res != -EACCES) { + fprintf(stderr, "write res: %d\n", cqe->res); + return TEST_FAILED; + } + break; + } + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + return TEST_OK; +} + +static int test_restrictions_flags(void) +{ + struct io_uring_restriction res[3]; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret, pipe1[2]; + + uint64_t ptr; + struct iovec vec = { + .iov_base = &ptr, + .iov_len = sizeof(ptr) + }; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return TEST_FAILED; + } + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_R_DISABLED); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return TEST_FAILED; + } + + res[0].opcode = IORING_RESTRICTION_SQE_OP; + res[0].sqe_op = IORING_OP_WRITEV; + + res[1].opcode = IORING_RESTRICTION_SQE_FLAGS_ALLOWED; + res[1].sqe_flags = IOSQE_ASYNC | IOSQE_IO_LINK; + + res[2].opcode = IORING_RESTRICTION_SQE_FLAGS_REQUIRED; + res[2].sqe_flags = IOSQE_FIXED_FILE; + + ret = io_uring_register_restrictions(&ring, res, 3); + if (ret) { + if (ret == -EINVAL) + return TEST_SKIPPED; + + fprintf(stderr, "failed to register restrictions: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_register_files(&ring, pipe1, 2); + if (ret) { + fprintf(stderr, "io_uring_register_files ret: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_enable_rings(&ring); + if (ret) { + fprintf(stderr, "ring enabling failed: %d\n", ret); + return TEST_FAILED; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, 1, &vec, 1, 0); + io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE); + sqe->user_data = 1; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, 1, &vec, 1, 0); + io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE | IOSQE_ASYNC); + sqe->user_data = 2; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, 1, &vec, 1, 0); + io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE | IOSQE_IO_LINK); + sqe->user_data = 3; + + ret = io_uring_submit(&ring); + if (ret != 3) { + fprintf(stderr, "submit: %d\n", ret); + return TEST_FAILED; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, 1, &vec, 1, 0); + io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE | IOSQE_IO_DRAIN); + sqe->user_data = 4; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return TEST_FAILED; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, pipe1[1], &vec, 1, 0); + io_uring_sqe_set_flags(sqe, IOSQE_IO_DRAIN); + sqe->user_data = 5; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return TEST_FAILED; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, pipe1[1], &vec, 1, 0); + io_uring_sqe_set_flags(sqe, IOSQE_ASYNC); + sqe->user_data = 6; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return TEST_FAILED; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, pipe1[1], &vec, 1, 0); + sqe->user_data = 7; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return TEST_FAILED; + } + + for (int i = 0; i < 7; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return TEST_FAILED; + } + + switch (cqe->user_data) { + case 1: /* writev - flags = IOSQE_FIXED_FILE */ + case 2: /* writev - flags = IOSQE_FIXED_FILE | IOSQE_ASYNC */ + case 3: /* writev - flags = IOSQE_FIXED_FILE | IOSQE_IO_LINK */ + if (cqe->res != sizeof(ptr)) { + fprintf(stderr, "write res: %d user_data %" PRIu64 "\n", + cqe->res, (uint64_t) cqe->user_data); + return TEST_FAILED; + } + + break; + case 4: /* writev - flags = IOSQE_FIXED_FILE | IOSQE_IO_DRAIN */ + case 5: /* writev - flags = IOSQE_IO_DRAIN */ + case 6: /* writev - flags = IOSQE_ASYNC */ + case 7: /* writev - flags = 0 */ + if (cqe->res != -EACCES) { + fprintf(stderr, "write res: %d user_data %" PRIu64 "\n", + cqe->res, (uint64_t) cqe->user_data); + return TEST_FAILED; + } + break; + } + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + return TEST_OK; +} + +static int test_restrictions_empty(void) +{ + struct io_uring_restriction res[0]; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret, pipe1[2]; + + uint64_t ptr; + struct iovec vec = { + .iov_base = &ptr, + .iov_len = sizeof(ptr) + }; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return TEST_FAILED; + } + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_R_DISABLED); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_register_restrictions(&ring, res, 0); + if (ret) { + if (ret == -EINVAL) + return TEST_SKIPPED; + + fprintf(stderr, "failed to register restrictions: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_enable_rings(&ring); + if (ret) { + fprintf(stderr, "ring enabling failed: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_register_buffers(&ring, &vec, 1); + if (ret != -EACCES) { + fprintf(stderr, "io_uring_register_buffers ret: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_register_files(&ring, pipe1, 2); + if (ret != -EACCES) { + fprintf(stderr, "io_uring_register_files ret: %d\n", ret); + return TEST_FAILED; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, pipe1[1], &vec, 1, 0); + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return TEST_FAILED; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait: %d\n", ret); + return TEST_FAILED; + } + + if (cqe->res != -EACCES) { + fprintf(stderr, "write res: %d\n", cqe->res); + return TEST_FAILED; + } + + io_uring_cqe_seen(&ring, cqe); + + io_uring_queue_exit(&ring); + return TEST_OK; +} + +static int test_restrictions_rings_not_disabled(void) +{ + struct io_uring_restriction res[1]; + struct io_uring ring; + int ret; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return TEST_FAILED; + } + + res[0].opcode = IORING_RESTRICTION_SQE_OP; + res[0].sqe_op = IORING_OP_WRITEV; + + ret = io_uring_register_restrictions(&ring, res, 1); + if (ret != -EBADFD) { + fprintf(stderr, "io_uring_register_restrictions ret: %d\n", + ret); + return TEST_FAILED; + } + + io_uring_queue_exit(&ring); + return TEST_OK; +} + +static int test_restrictions_rings_disabled(void) +{ + struct io_uring_sqe *sqe; + struct io_uring ring; + int ret; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_R_DISABLED); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return TEST_FAILED; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + + ret = io_uring_submit(&ring); + if (ret != -EBADFD) { + fprintf(stderr, "submit: %d\n", ret); + return TEST_FAILED; + } + + io_uring_queue_exit(&ring); + return TEST_OK; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test_restrictions_sqe_op(); + if (ret == TEST_SKIPPED) { + printf("test_restrictions_sqe_op: skipped\n"); + return 0; + } else if (ret == TEST_FAILED) { + fprintf(stderr, "test_restrictions_sqe_op failed\n"); + return ret; + } + + ret = test_restrictions_register_op(); + if (ret == TEST_SKIPPED) { + printf("test_restrictions_register_op: skipped\n"); + } else if (ret == TEST_FAILED) { + fprintf(stderr, "test_restrictions_register_op failed\n"); + return ret; + } + + ret = test_restrictions_fixed_file(); + if (ret == TEST_SKIPPED) { + printf("test_restrictions_fixed_file: skipped\n"); + } else if (ret == TEST_FAILED) { + fprintf(stderr, "test_restrictions_fixed_file failed\n"); + return ret; + } + + ret = test_restrictions_flags(); + if (ret == TEST_SKIPPED) { + printf("test_restrictions_flags: skipped\n"); + } else if (ret == TEST_FAILED) { + fprintf(stderr, "test_restrictions_flags failed\n"); + return ret; + } + + ret = test_restrictions_empty(); + if (ret == TEST_SKIPPED) { + printf("test_restrictions_empty: skipped\n"); + } else if (ret == TEST_FAILED) { + fprintf(stderr, "test_restrictions_empty failed\n"); + return ret; + } + + ret = test_restrictions_rings_not_disabled(); + if (ret == TEST_SKIPPED) { + printf("test_restrictions_rings_not_disabled: skipped\n"); + } else if (ret == TEST_FAILED) { + fprintf(stderr, "test_restrictions_rings_not_disabled failed\n"); + return ret; + } + + ret = test_restrictions_rings_disabled(); + if (ret == TEST_SKIPPED) { + printf("test_restrictions_rings_disabled: skipped\n"); + } else if (ret == TEST_FAILED) { + fprintf(stderr, "test_restrictions_rings_disabled failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/rename.c b/contrib/libs/liburing/test/rename.c new file mode 100644 index 0000000000..c72a36ea1b --- /dev/null +++ b/contrib/libs/liburing/test/rename.c @@ -0,0 +1,133 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various nop tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/stat.h> + +#include "liburing.h" + +static int test_rename(struct io_uring *ring, const char *old, const char *new) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + memset(sqe, 0, sizeof(*sqe)); + + io_uring_prep_rename(sqe, old, new); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +err: + return 1; +} + +static int stat_file(const char *buf) +{ + struct stat sb; + + if (!stat(buf, &sb)) + return 0; + + return errno; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + char src[32] = "./XXXXXX"; + char dst[32] = "./XXXXXX"; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + ret = mkstemp(src); + if (ret < 0) { + perror("mkstemp"); + return 1; + } + close(ret); + + ret = mkstemp(dst); + if (ret < 0) { + perror("mkstemp"); + return 1; + } + close(ret); + + if (stat_file(src) != 0) { + perror("stat"); + return 1; + } + if (stat_file(dst) != 0) { + perror("stat"); + return 1; + } + + ret = test_rename(&ring, src, dst); + if (ret < 0) { + if (ret == -EBADF || ret == -EINVAL) { + fprintf(stdout, "Rename not supported, skipping\n"); + goto out; + } + fprintf(stderr, "rename: %s\n", strerror(-ret)); + goto err; + } else if (ret) + goto err; + + if (stat_file(src) != ENOENT) { + fprintf(stderr, "stat got %s\n", strerror(ret)); + return 1; + } + + if (stat_file(dst) != 0) { + perror("stat"); + return 1; + } + + ret = test_rename(&ring, "/x/y/1/2", "/2/1/y/x"); + if (ret != -ENOENT) { + fprintf(stderr, "test_rename invalid failed: %d\n", ret); + return ret; + } +out: + unlink(dst); + return 0; +err: + unlink(src); + unlink(dst); + return 1; +} diff --git a/contrib/libs/liburing/test/ring-leak.c b/contrib/libs/liburing/test/ring-leak.c new file mode 100644 index 0000000000..26338a48f1 --- /dev/null +++ b/contrib/libs/liburing/test/ring-leak.c @@ -0,0 +1,271 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Based on description from Al Viro - this demonstrates a leak of the + * io_uring instance, by sending the io_uring fd over a UNIX socket. + * + * See: + * + * https://lore.kernel.org/linux-block/20190129192702.3605-1-axboe@kernel.dk/T/#m6c87fc64e4d063786af6ec6fadce3ac1e95d3184 + * + */ +#include <stdio.h> +#include <stdlib.h> +#include <stddef.h> +#include <signal.h> +#include <inttypes.h> +#include <sys/types.h> +#include <sys/syscall.h> +#include <sys/socket.h> +#include <sys/wait.h> +#include <fcntl.h> +#include <unistd.h> +#include <string.h> +#include <linux/fs.h> + +#include "liburing.h" +#include "../src/syscall.h" + +static int __io_uring_register_files(int ring_fd, int fd1, int fd2) +{ + __s32 fds[2] = { fd1, fd2 }; + + return __sys_io_uring_register(ring_fd, IORING_REGISTER_FILES, fds, 2); +} + +static int get_ring_fd(void) +{ + struct io_uring_params p; + int fd; + + memset(&p, 0, sizeof(p)); + + fd = __sys_io_uring_setup(2, &p); + if (fd < 0) { + perror("io_uring_setup"); + return -1; + } + + return fd; +} + +static void send_fd(int socket, int fd) +{ + char buf[CMSG_SPACE(sizeof(fd))]; + struct cmsghdr *cmsg; + struct msghdr msg; + + memset(buf, 0, sizeof(buf)); + memset(&msg, 0, sizeof(msg)); + + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(fd)); + + memmove(CMSG_DATA(cmsg), &fd, sizeof(fd)); + + msg.msg_controllen = CMSG_SPACE(sizeof(fd)); + + if (sendmsg(socket, &msg, 0) < 0) + perror("sendmsg"); +} + +static int test_iowq_request_cancel(void) +{ + char buffer[128]; + struct io_uring ring; + struct io_uring_sqe *sqe; + int ret, fds[2]; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret < 0) { + fprintf(stderr, "failed to init io_uring: %s\n", strerror(-ret)); + return ret; + } + if (pipe(fds)) { + perror("pipe"); + return -1; + } + ret = io_uring_register_files(&ring, fds, 2); + if (ret) { + fprintf(stderr, "file_register: %d\n", ret); + return ret; + } + close(fds[1]); + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "%s: failed to get sqe\n", __FUNCTION__); + return 1; + } + /* potentially sitting in internal polling */ + io_uring_prep_read(sqe, 0, buffer, 10, 0); + sqe->flags |= IOSQE_FIXED_FILE; + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "%s: failed to get sqe\n", __FUNCTION__); + return 1; + } + /* staying in io-wq */ + io_uring_prep_read(sqe, 0, buffer, 10, 0); + sqe->flags |= IOSQE_FIXED_FILE | IOSQE_ASYNC; + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + + /* should unregister files and close the write fd */ + io_uring_queue_exit(&ring); + + /* + * We're trying to wait for the ring to "really" exit, that will be + * done async. For that rely on the registered write end to be closed + * after ring quiesce, so failing read from the other pipe end. + */ + ret = read(fds[0], buffer, 10); + if (ret < 0) + perror("read"); + close(fds[0]); + return 0; +} + +static void trigger_unix_gc(void) +{ + int fd; + + fd = socket(AF_UNIX, SOCK_DGRAM, 0); + if (fd < 0) + perror("socket dgram"); + else + close(fd); +} + +static int test_scm_cycles(bool update) +{ + char buffer[128]; + struct io_uring ring; + int i, ret; + int sp[2], fds[2], reg_fds[4]; + + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, sp) != 0) { + perror("Failed to create Unix-domain socket pair\n"); + return 1; + } + ret = io_uring_queue_init(8, &ring, 0); + if (ret < 0) { + fprintf(stderr, "failed to init io_uring: %s\n", strerror(-ret)); + return ret; + } + if (pipe(fds)) { + perror("pipe"); + return -1; + } + send_fd(sp[0], ring.ring_fd); + + /* register an empty set for updates */ + if (update) { + for (i = 0; i < 4; i++) + reg_fds[i] = -1; + ret = io_uring_register_files(&ring, reg_fds, 4); + if (ret) { + fprintf(stderr, "file_register: %d\n", ret); + return ret; + } + } + + reg_fds[0] = fds[0]; + reg_fds[1] = fds[1]; + reg_fds[2] = sp[0]; + reg_fds[3] = sp[1]; + if (update) { + ret = io_uring_register_files_update(&ring, 0, reg_fds, 4); + if (ret != 4) { + fprintf(stderr, "file_register: %d\n", ret); + return ret; + } + } else { + ret = io_uring_register_files(&ring, reg_fds, 4); + if (ret) { + fprintf(stderr, "file_register: %d\n", ret); + return ret; + } + } + + close(fds[1]); + close(sp[0]); + close(sp[1]); + + /* should unregister files and close the write fd */ + io_uring_queue_exit(&ring); + + trigger_unix_gc(); + + /* + * We're trying to wait for the ring to "really" exit, that will be + * done async. For that rely on the registered write end to be closed + * after ring quiesce, so failing read from the other pipe end. + */ + ret = read(fds[0], buffer, 10); + if (ret < 0) + perror("read"); + close(fds[0]); + return 0; +} + +int main(int argc, char *argv[]) +{ + int sp[2], pid, ring_fd, ret; + int i; + + if (argc > 1) + return 0; + + ret = test_iowq_request_cancel(); + if (ret) { + fprintf(stderr, "test_iowq_request_cancel() failed\n"); + return 1; + } + + for (i = 0; i < 2; i++) { + bool update = !!(i & 1); + + ret = test_scm_cycles(update); + if (ret) { + fprintf(stderr, "test_scm_cycles() failed %i\n", + update); + return 1; + } + break; + } + + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, sp) != 0) { + perror("Failed to create Unix-domain socket pair\n"); + return 1; + } + + ring_fd = get_ring_fd(); + if (ring_fd < 0) + return 1; + + ret = __io_uring_register_files(ring_fd, sp[0], sp[1]); + if (ret < 0) { + perror("register files"); + return 1; + } + + pid = fork(); + if (pid) + send_fd(sp[0], ring_fd); + + close(ring_fd); + close(sp[0]); + close(sp[1]); + return 0; +} diff --git a/contrib/libs/liburing/test/ring-leak2.c b/contrib/libs/liburing/test/ring-leak2.c new file mode 100644 index 0000000000..b0b43413ef --- /dev/null +++ b/contrib/libs/liburing/test/ring-leak2.c @@ -0,0 +1,250 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: Test two ring deadlock. A buggy kernel will end up + * having io_wq_* workers pending, as the circular reference + * will prevent full exit. + * + * Based on a test case from Josef <josef.grieb@gmail.com> + * + */ +#include <errno.h> +#include <fcntl.h> +#include <netinet/in.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <strings.h> +#include <poll.h> +#include <sys/socket.h> +#include <unistd.h> +#include <sys/eventfd.h> +#include <pthread.h> + +#include "liburing.h" +#include "../src/syscall.h" + +enum { + ACCEPT, + READ, + WRITE, + POLLING_IN, + POLLING_RDHUP, + CLOSE, + EVENTFD_READ, +}; + +typedef struct conn_info { + __u32 fd; + __u16 type; + __u16 bid; +} conn_info; + +static char read_eventfd_buffer[8]; + +static pthread_mutex_t lock; +static struct io_uring *client_ring; + +static int client_eventfd = -1; + +int setup_io_uring(struct io_uring *ring) +{ + struct io_uring_params p = { }; + int ret; + + ret = io_uring_queue_init_params(8, ring, &p); + if (ret) { + fprintf(stderr, "Unable to setup io_uring: %s\n", + strerror(-ret)); + return 1; + } + return 0; +} + +static void add_socket_eventfd_read(struct io_uring *ring, int fd) +{ + struct io_uring_sqe *sqe; + conn_info conn_i = { + .fd = fd, + .type = EVENTFD_READ, + }; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_read(sqe, fd, &read_eventfd_buffer, 8, 0); + io_uring_sqe_set_flags(sqe, IOSQE_ASYNC); + + memcpy(&sqe->user_data, &conn_i, sizeof(conn_i)); +} + +static void add_socket_pollin(struct io_uring *ring, int fd) +{ + struct io_uring_sqe *sqe; + conn_info conn_i = { + .fd = fd, + .type = POLLING_IN, + }; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_poll_add(sqe, fd, POLL_IN); + + memcpy(&sqe->user_data, &conn_i, sizeof(conn_i)); +} + +static void *server_thread(void *arg) +{ + struct sockaddr_in serv_addr; + int port = 0; + int sock_listen_fd, evfd; + const int val = 1; + struct io_uring ring; + + sock_listen_fd = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0); + setsockopt(sock_listen_fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + + memset(&serv_addr, 0, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_port = htons(port); + serv_addr.sin_addr.s_addr = INADDR_ANY; + + evfd = eventfd(0, EFD_CLOEXEC); + + // bind and listen + if (bind(sock_listen_fd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + perror("Error binding socket...\n"); + exit(1); + } + if (listen(sock_listen_fd, 1) < 0) { + perror("Error listening on socket...\n"); + exit(1); + } + + setup_io_uring(&ring); + add_socket_eventfd_read(&ring, evfd); + add_socket_pollin(&ring, sock_listen_fd); + + while (1) { + struct io_uring_cqe *cqe; + unsigned head; + unsigned count = 0; + + io_uring_submit_and_wait(&ring, 1); + + io_uring_for_each_cqe(&ring, head, cqe) { + struct conn_info conn_i; + + count++; + memcpy(&conn_i, &cqe->user_data, sizeof(conn_i)); + + if (conn_i.type == ACCEPT) { + int sock_conn_fd = cqe->res; + // only read when there is no error, >= 0 + if (sock_conn_fd > 0) { + add_socket_pollin(&ring, sock_listen_fd); + + pthread_mutex_lock(&lock); + io_uring_submit(client_ring); + pthread_mutex_unlock(&lock); + + } + } else if (conn_i.type == POLLING_IN) { + break; + } + } + io_uring_cq_advance(&ring, count); + } +} + +static void *client_thread(void *arg) +{ + struct io_uring ring; + int ret; + + setup_io_uring(&ring); + client_ring = ˚ + + client_eventfd = eventfd(0, EFD_CLOEXEC); + pthread_mutex_lock(&lock); + add_socket_eventfd_read(&ring, client_eventfd); + pthread_mutex_unlock(&lock); + + while (1) { + struct io_uring_cqe *cqe; + unsigned head; + unsigned count = 0; + + pthread_mutex_lock(&lock); + io_uring_submit(&ring); + pthread_mutex_unlock(&lock); + + ret = __sys_io_uring_enter(ring.ring_fd, 0, 1, IORING_ENTER_GETEVENTS, NULL); + if (ret < 0) { + perror("Error io_uring_enter...\n"); + exit(1); + } + + // go through all CQEs + io_uring_for_each_cqe(&ring, head, cqe) { + struct conn_info conn_i; + int type; + + count++; + memcpy(&conn_i, &cqe->user_data, sizeof(conn_i)); + + type = conn_i.type; + if (type == READ) { + pthread_mutex_lock(&lock); + + if (cqe->res <= 0) { + // connection closed or error + shutdown(conn_i.fd, SHUT_RDWR); + } else { + pthread_mutex_unlock(&lock); + break; + } + add_socket_pollin(&ring, conn_i.fd); + pthread_mutex_unlock(&lock); + } else if (type == WRITE) { + } else if (type == POLLING_IN) { + break; + } else if (type == POLLING_RDHUP) { + break; + } else if (type == CLOSE) { + } else if (type == EVENTFD_READ) { + add_socket_eventfd_read(&ring, client_eventfd); + } + } + + io_uring_cq_advance(&ring, count); + } +} + +static void sig_alrm(int sig) +{ + exit(0); +} + +int main(int argc, char *argv[]) +{ + pthread_t server_thread_t, client_thread_t; + struct sigaction act; + + if (argc > 1) + return 0; + + if (pthread_mutex_init(&lock, NULL) != 0) { + printf("\n mutex init failed\n"); + return 1; + } + + pthread_create(&server_thread_t, NULL, &server_thread, NULL); + pthread_create(&client_thread_t, NULL, &client_thread, NULL); + + memset(&act, 0, sizeof(act)); + act.sa_handler = sig_alrm; + act.sa_flags = SA_RESTART; + sigaction(SIGALRM, &act, NULL); + alarm(1); + + pthread_join(server_thread_t, NULL); + return 0; +} diff --git a/contrib/libs/liburing/test/ringbuf-read.c b/contrib/libs/liburing/test/ringbuf-read.c new file mode 100644 index 0000000000..2eede18213 --- /dev/null +++ b/contrib/libs/liburing/test/ringbuf-read.c @@ -0,0 +1,201 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: ring mapped provided buffers with reads + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +#define BUF_SIZE 4096 +#define NR_BUFS 64 +#define FSIZE (BUF_SIZE * NR_BUFS) + +#define BR_MASK (NR_BUFS - 1) + +static int no_buf_ring; + +static int verify_buffer(char *buf, char val) +{ + int i; + + for (i = 0; i < BUF_SIZE; i++) { + if (buf[i] != val) { + fprintf(stderr, "got %d, wanted %d\n", buf[i], val); + return 1; + } + } + + return 0; +} + +static int test(const char *filename, int dio, int async) +{ + struct io_uring_buf_reg reg = { }; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + struct io_uring_buf_ring *br; + int ret, fd, i; + char *buf; + void *ptr; + + ret = io_uring_queue_init(NR_BUFS, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + if (dio) + fd = open(filename, O_DIRECT | O_RDONLY); + else + fd = open(filename, O_RDONLY); + if (fd < 0) { + perror("open"); + return 1; + } + + posix_fadvise(fd, 0, FSIZE, POSIX_FADV_DONTNEED); + + if (posix_memalign((void **) &buf, 4096, FSIZE)) + return 1; + if (posix_memalign((void **) &br, 4096, 4096)) + return 1; + + reg.ring_addr = (unsigned long) br; + reg.ring_entries = NR_BUFS; + reg.bgid = 1; + + ret = io_uring_register_buf_ring(&ring, ®, 0); + if (ret) { + if (ret == -EINVAL) { + no_buf_ring = 1; + return 0; + } + fprintf(stderr, "Buffer ring register failed %d\n", ret); + return 1; + } + + ptr = buf; + for (i = 0; i < NR_BUFS; i++) { + io_uring_buf_ring_add(br, ptr, BUF_SIZE, i + 1, BR_MASK, i); + ptr += BUF_SIZE; + } + io_uring_buf_ring_advance(br, NR_BUFS); + + for (i = 0; i < NR_BUFS; i++) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, fd, NULL, BUF_SIZE, i * BUF_SIZE); + sqe->buf_group = 1; + sqe->flags |= IOSQE_BUFFER_SELECT; + if (async && !(i & 1)) + sqe->flags |= IOSQE_ASYNC; + sqe->user_data = i + 1; + } + + ret = io_uring_submit(&ring); + if (ret != NR_BUFS) { + fprintf(stderr, "submit: %d\n", ret); + return 1; + } + + for (i = 0; i < NR_BUFS; i++) { + int bid, ud; + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe failed %d\n", ret); + return 1; + } + if (cqe->res != BUF_SIZE) { + fprintf(stderr, "cqe res %d\n", cqe->res); + return 1; + } + if (!(cqe->flags & IORING_CQE_F_BUFFER)) { + fprintf(stderr, "no buffer selected\n"); + return 1; + } + bid = cqe->flags >> IORING_CQE_BUFFER_SHIFT; + ud = cqe->user_data; + io_uring_cqe_seen(&ring, cqe); + if (verify_buffer(buf + ((bid - 1) * BUF_SIZE), ud)) + return 1; + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + char buf[BUF_SIZE]; + char fname[80]; + int ret, fd, i, do_unlink; + + if (argc > 1) { + strcpy(fname, argv[1]); + do_unlink = 0; + } else { + sprintf(fname, ".ringbuf-read.%d", getpid()); + t_create_file(fname, FSIZE); + do_unlink = 1; + } + + fd = open(fname, O_WRONLY); + if (fd < 0) { + perror("open"); + goto err; + } + for (i = 0; i < NR_BUFS; i++) { + memset(buf, i + 1, BUF_SIZE); + ret = write(fd, buf, BUF_SIZE); + if (ret != BUF_SIZE) { + fprintf(stderr, "bad file prep write\n"); + close(fd); + goto err; + } + } + close(fd); + + ret = test(fname, 1, 0); + if (ret) { + fprintf(stderr, "dio test failed\n"); + goto err; + } + if (no_buf_ring) + goto pass; + + ret = test(fname, 0, 0); + if (ret) { + fprintf(stderr, "buffered test failed\n"); + goto err; + } + + ret = test(fname, 1, 1); + if (ret) { + fprintf(stderr, "dio async test failed\n"); + goto err; + } + + ret = test(fname, 0, 1); + if (ret) { + fprintf(stderr, "buffered async test failed\n"); + goto err; + } + +pass: + ret = T_EXIT_PASS; + goto out; +err: + ret = T_EXIT_FAIL; +out: + if (do_unlink) + unlink(fname); + return ret; +} diff --git a/contrib/libs/liburing/test/rsrc_tags.c b/contrib/libs/liburing/test/rsrc_tags.c new file mode 100644 index 0000000000..8d3fe9307b --- /dev/null +++ b/contrib/libs/liburing/test/rsrc_tags.c @@ -0,0 +1,462 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various file registration tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <assert.h> + +#include "../src/syscall.h" +#include "helpers.h" +#include "liburing.h" + +static int pipes[2]; + +enum { + TEST_IORING_RSRC_FILE = 0, + TEST_IORING_RSRC_BUFFER = 1, +}; + +static bool check_cq_empty(struct io_uring *ring) +{ + struct io_uring_cqe *cqe = NULL; + int ret; + + usleep(1000); /* doesn't happen immediately, so wait */ + ret = io_uring_peek_cqe(ring, &cqe); /* nothing should be there */ + return ret == -EAGAIN; +} + +/* + * There are io_uring_register_buffers_tags() and other wrappers, + * but they may change, so hand-code to specifically test this ABI. + */ +static int register_rsrc(struct io_uring *ring, int type, int nr, + const void *arg, const __u64 *tags) +{ + struct io_uring_rsrc_register reg; + int reg_type; + + memset(®, 0, sizeof(reg)); + reg.nr = nr; + reg.data = (__u64)(uintptr_t)arg; + reg.tags = (__u64)(uintptr_t)tags; + + reg_type = IORING_REGISTER_FILES2; + if (type != TEST_IORING_RSRC_FILE) + reg_type = IORING_REGISTER_BUFFERS2; + + return __sys_io_uring_register(ring->ring_fd, reg_type, ®, + sizeof(reg)); +} + +/* + * There are io_uring_register_buffers_update_tag() and other wrappers, + * but they may change, so hand-code to specifically test this ABI. + */ +static int update_rsrc(struct io_uring *ring, int type, int nr, int off, + const void *arg, const __u64 *tags) +{ + struct io_uring_rsrc_update2 up; + int up_type; + + memset(&up, 0, sizeof(up)); + up.offset = off; + up.data = (__u64)(uintptr_t)arg; + up.tags = (__u64)(uintptr_t)tags; + up.nr = nr; + + up_type = IORING_REGISTER_FILES_UPDATE2; + if (type != TEST_IORING_RSRC_FILE) + up_type = IORING_REGISTER_BUFFERS_UPDATE; + return __sys_io_uring_register(ring->ring_fd, up_type, &up, sizeof(up)); +} + +static bool has_rsrc_update(void) +{ + struct io_uring ring; + int ret; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "io_uring_queue_init() failed, %d\n", ret); + exit(1); + } + + ret = ring.features & IORING_FEAT_RSRC_TAGS; + io_uring_queue_exit(&ring); + return ret; +} + +static int test_tags_generic(int nr, int type, void *rsrc, int ring_flags) +{ + struct io_uring_cqe *cqe = NULL; + struct io_uring ring; + int i, ret; + __u64 *tags; + + tags = malloc(nr * sizeof(*tags)); + if (!tags) + return 1; + for (i = 0; i < nr; i++) + tags[i] = i + 1; + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return 1; + } + + ret = register_rsrc(&ring, type, nr, rsrc, tags); + if (ret) { + fprintf(stderr, "rsrc register failed %i\n", ret); + return 1; + } + + /* test that tags are set */ + tags[0] = 666; + ret = update_rsrc(&ring, type, 1, 0, rsrc, &tags[0]); + assert(ret == 1); + ret = io_uring_wait_cqe(&ring, &cqe); + assert(!ret && cqe->user_data == 1); + io_uring_cqe_seen(&ring, cqe); + + /* test that tags are updated */ + tags[0] = 0; + ret = update_rsrc(&ring, type, 1, 0, rsrc, &tags[0]); + assert(ret == 1); + ret = io_uring_wait_cqe(&ring, &cqe); + assert(!ret && cqe->user_data == 666); + io_uring_cqe_seen(&ring, cqe); + + /* test tag=0 doesn't emit CQE */ + tags[0] = 1; + ret = update_rsrc(&ring, type, 1, 0, rsrc, &tags[0]); + assert(ret == 1); + assert(check_cq_empty(&ring)); + + free(tags); + io_uring_queue_exit(&ring); + return 0; +} + +static int test_buffers_update(void) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe = NULL; + struct io_uring ring; + const int nr = 5; + int buf_idx = 1, i, ret; + int pipes[2]; + char tmp_buf[1024]; + char tmp_buf2[1024]; + struct iovec vecs[nr]; + __u64 tags[nr]; + + for (i = 0; i < nr; i++) { + vecs[i].iov_base = tmp_buf; + vecs[i].iov_len = 1024; + tags[i] = i + 1; + } + + ret = test_tags_generic(nr, TEST_IORING_RSRC_BUFFER, vecs, 0); + if (ret) + return 1; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return 1; + } + if (pipe(pipes) < 0) { + perror("pipe"); + return 1; + } + ret = register_rsrc(&ring, TEST_IORING_RSRC_BUFFER, nr, vecs, tags); + if (ret) { + fprintf(stderr, "rsrc register failed %i\n", ret); + return 1; + } + + /* test that CQE is not emitted before we're done with a buffer */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read_fixed(sqe, pipes[0], tmp_buf, 10, 0, 0); + sqe->user_data = 100; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + ret = io_uring_peek_cqe(&ring, &cqe); + assert(ret == -EAGAIN); + + vecs[buf_idx].iov_base = tmp_buf2; + ret = update_rsrc(&ring, TEST_IORING_RSRC_BUFFER, 1, buf_idx, + &vecs[buf_idx], &tags[buf_idx]); + if (ret != 1) { + fprintf(stderr, "rsrc update failed %i %i\n", ret, errno); + return 1; + } + + ret = io_uring_peek_cqe(&ring, &cqe); /* nothing should be there */ + assert(ret == -EAGAIN); + close(pipes[0]); + close(pipes[1]); + + ret = io_uring_wait_cqe(&ring, &cqe); + assert(!ret && cqe->user_data == 100); + io_uring_cqe_seen(&ring, cqe); + ret = io_uring_wait_cqe(&ring, &cqe); + assert(!ret && cqe->user_data == buf_idx + 1); + io_uring_cqe_seen(&ring, cqe); + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_buffers_empty_buffers(void) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe = NULL; + struct io_uring ring; + const int nr = 5; + int ret, i; + char tmp_buf[1024]; + struct iovec vecs[nr]; + + for (i = 0; i < nr; i++) { + vecs[i].iov_base = 0; + vecs[i].iov_len = 0; + } + vecs[0].iov_base = tmp_buf; + vecs[0].iov_len = 10; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return 1; + } + + ret = register_rsrc(&ring, TEST_IORING_RSRC_BUFFER, nr, vecs, NULL); + if (ret) { + fprintf(stderr, "rsrc register failed %i\n", ret); + return 1; + } + + /* empty to buffer */ + vecs[1].iov_base = tmp_buf; + vecs[1].iov_len = 10; + ret = update_rsrc(&ring, TEST_IORING_RSRC_BUFFER, 1, 1, &vecs[1], NULL); + if (ret != 1) { + fprintf(stderr, "rsrc update failed %i %i\n", ret, errno); + return 1; + } + + /* buffer to empty */ + vecs[0].iov_base = 0; + vecs[0].iov_len = 0; + ret = update_rsrc(&ring, TEST_IORING_RSRC_BUFFER, 1, 0, &vecs[0], NULL); + if (ret != 1) { + fprintf(stderr, "rsrc update failed %i %i\n", ret, errno); + return 1; + } + + /* zero to zero is ok */ + ret = update_rsrc(&ring, TEST_IORING_RSRC_BUFFER, 1, 2, &vecs[2], NULL); + if (ret != 1) { + fprintf(stderr, "rsrc update failed %i %i\n", ret, errno); + return 1; + } + + /* empty buf with non-zero len fails */ + vecs[3].iov_base = 0; + vecs[3].iov_len = 1; + ret = update_rsrc(&ring, TEST_IORING_RSRC_BUFFER, 1, 3, &vecs[3], NULL); + if (ret >= 0) { + fprintf(stderr, "rsrc update failed %i %i\n", ret, errno); + return 1; + } + + /* test rw on empty ubuf is failed */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read_fixed(sqe, pipes[0], tmp_buf, 10, 0, 2); + sqe->user_data = 100; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + ret = io_uring_wait_cqe(&ring, &cqe); + assert(!ret && cqe->user_data == 100); + assert(cqe->res); + io_uring_cqe_seen(&ring, cqe); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read_fixed(sqe, pipes[0], tmp_buf, 0, 0, 2); + sqe->user_data = 100; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + return 1; + } + ret = io_uring_wait_cqe(&ring, &cqe); + assert(!ret && cqe->user_data == 100); + assert(cqe->res); + io_uring_cqe_seen(&ring, cqe); + + io_uring_queue_exit(&ring); + return 0; +} + + +static int test_files(int ring_flags) +{ + struct io_uring_cqe *cqe = NULL; + struct io_uring ring; + const int nr = 50; + int off = 5, i, ret, fd; + __s32 files[nr]; + __u64 tags[nr], tag; + + for (i = 0; i < nr; ++i) { + files[i] = pipes[0]; + tags[i] = i + 1; + } + + ret = test_tags_generic(nr, TEST_IORING_RSRC_FILE, files, ring_flags); + if (ret) + return 1; + + ret = io_uring_queue_init(1, &ring, ring_flags); + if (ret) { + printf("ring setup failed\n"); + return 1; + } + ret = register_rsrc(&ring, TEST_IORING_RSRC_FILE, nr, files, tags); + if (ret) { + fprintf(stderr, "rsrc register failed %i\n", ret); + return 1; + } + + /* check update did update tag */ + fd = -1; + ret = io_uring_register_files_update(&ring, off, &fd, 1); + assert(ret == 1); + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring wait ret=%d\n", ret); + return 1; + } + if (cqe->user_data != tags[off]) { + fprintf(stderr, "data %lx != %lx\n", + (unsigned long) cqe->user_data, + (unsigned long) tags[off]); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + + /* remove removed file, shouldn't emit old tag */ + ret = io_uring_register_files_update(&ring, off, &fd, 1); + assert(ret <= 1); + assert(check_cq_empty(&ring)); + + /* non-zero tag with remove update is disallowed */ + tag = 1; + fd = -1; + ret = update_rsrc(&ring, TEST_IORING_RSRC_FILE, 1, off + 1, &fd, &tag); + assert(ret); + + io_uring_queue_exit(&ring); + return 0; +} + +static int test_notag(void) +{ + struct io_uring_cqe *cqe = NULL; + struct io_uring ring; + int i, ret, fd; + const int nr = 50; + int files[nr]; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return 1; + } + for (i = 0; i < nr; ++i) + files[i] = pipes[0]; + + ret = io_uring_register_files(&ring, files, nr); + assert(!ret); + + /* default register, update shouldn't emit CQE */ + fd = -1; + ret = io_uring_register_files_update(&ring, 0, &fd, 1); + assert(ret == 1); + assert(check_cq_empty(&ring)); + + ret = io_uring_unregister_files(&ring); + assert(!ret); + ret = io_uring_peek_cqe(&ring, &cqe); /* nothing should be there */ + assert(ret); + + io_uring_queue_exit(&ring); + return 0; +} + +int main(int argc, char *argv[]) +{ + int ring_flags[] = {0, IORING_SETUP_IOPOLL, IORING_SETUP_SQPOLL, + IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN}; + int i, ret; + + if (argc > 1) + return 0; + if (!has_rsrc_update()) { + fprintf(stderr, "doesn't support rsrc tags, skip\n"); + return 0; + } + + if (pipe(pipes) < 0) { + perror("pipe"); + return 1; + } + + ret = test_notag(); + if (ret) { + printf("test_notag failed\n"); + return ret; + } + + for (i = 0; i < sizeof(ring_flags) / sizeof(ring_flags[0]); i++) { + int flag = ring_flags[i]; + + if (flag & IORING_SETUP_DEFER_TASKRUN && !t_probe_defer_taskrun()) + continue; + + ret = test_files(flag); + if (ret) { + printf("test_tag failed, type %i\n", i); + return ret; + } + } + + ret = test_buffers_update(); + if (ret) { + printf("test_buffers_update failed\n"); + return ret; + } + + ret = test_buffers_empty_buffers(); + if (ret) { + printf("test_buffers_empty_buffers failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/rw_merge_test.c b/contrib/libs/liburing/test/rw_merge_test.c new file mode 100644 index 0000000000..e771eb7397 --- /dev/null +++ b/contrib/libs/liburing/test/rw_merge_test.c @@ -0,0 +1,99 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Regression test for incorrect async_list io_should_merge() logic + * Bug was fixed in 5.5 by (commit: 561fb04 io_uring: replace workqueue usage with io-wq") + * Affects 5.4 lts branch, at least 5.4.106 is affected. + */ +#include <stdio.h> +#include <errno.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <assert.h> +#include <fcntl.h> +#include <unistd.h> + +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret, fd, pipe1[2]; + char buf[4096]; + struct iovec vec = { + .iov_base = buf, + .iov_len = sizeof(buf) + }; + struct __kernel_timespec ts = {.tv_sec = 3, .tv_nsec = 0}; + + if (argc > 1) + return 0; + + ret = pipe(pipe1); + assert(!ret); + + fd = open("testfile", O_RDWR | O_CREAT, 0644); + assert(fd >= 0); + unlink("testfile"); + ret = ftruncate(fd, 4096); + assert(!ret); + + ret = t_create_ring(4, &ring, 0); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret < 0) + return 1; + + /* REQ1 */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, pipe1[0], &vec, 1, 0); + sqe->user_data = 1; + + /* REQ2 */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, fd, &vec, 1, 4096); + sqe->user_data = 2; + + ret = io_uring_submit(&ring); + assert(ret == 2); + + ret = io_uring_wait_cqe(&ring, &cqe); + assert(!ret); + assert(cqe->res == 0); + assert(cqe->user_data == 2); + io_uring_cqe_seen(&ring, cqe); + + /* + * REQ3 + * Prepare request adjacent to previous one, so merge logic may want to + * link it to previous request, but because of a bug in merge logic + * it may be merged with <REQ1> request + */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, fd, &vec, 1, 2048); + sqe->user_data = 3; + + ret = io_uring_submit(&ring); + assert(ret == 1); + + /* + * Read may stuck because of bug there request was be incorrectly + * merged with <REQ1> request + */ + ret = io_uring_wait_cqe_timeout(&ring, &cqe, &ts); + if (ret == -ETIME) { + printf("TEST_FAIL: readv req3 stuck\n"); + return 1; + } + assert(!ret); + + assert(cqe->res == 2048); + assert(cqe->user_data == 3); + + io_uring_cqe_seen(&ring, cqe); + io_uring_queue_exit(&ring); + return 0; +} diff --git a/contrib/libs/liburing/test/self.c b/contrib/libs/liburing/test/self.c new file mode 100644 index 0000000000..a62cc7380e --- /dev/null +++ b/contrib/libs/liburing/test/self.c @@ -0,0 +1,92 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test that pathname resolution works from async context when + * using /proc/self/ which should be the original submitting task, not the + * async worker. + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" + +static int io_openat2(struct io_uring *ring, const char *path, int dfd) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct open_how how; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + memset(&how, 0, sizeof(how)); + how.flags = O_RDONLY; + io_uring_prep_openat2(sqe, dfd, path, &how); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +err: + return -1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + char buf[64]; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + ret = io_openat2(&ring, "/proc/self/comm", -1); + if (ret < 0) { + if (ret == -EOPNOTSUPP) + return 0; + if (ret == -EINVAL) { + fprintf(stdout, "openat2 not supported, skipping\n"); + return 0; + } + fprintf(stderr, "openat2 failed: %s\n", strerror(-ret)); + return 1; + } + + memset(buf, 0, sizeof(buf)); + ret = read(ret, buf, sizeof(buf)); + if (ret < 0) { + perror("read"); + return 1; + } + + if (strncmp(buf, "self", 4)) { + fprintf(stderr, "got comm=<%s>, wanted <self>\n", buf); + return 1; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/send-zerocopy.c b/contrib/libs/liburing/test/send-zerocopy.c new file mode 100644 index 0000000000..b201e68317 --- /dev/null +++ b/contrib/libs/liburing/test/send-zerocopy.c @@ -0,0 +1,685 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <assert.h> +#include <errno.h> +#include <error.h> +#include <limits.h> +#include <fcntl.h> +#include <unistd.h> +#include <stdbool.h> +#include <string.h> + +#include <arpa/inet.h> +#include <linux/errqueue.h> +#include <linux/if_packet.h> +#include <linux/ipv6.h> +#include <linux/socket.h> +#include <linux/sockios.h> +#include <net/ethernet.h> +#include <net/if.h> +#include <netinet/ip.h> +#include <netinet/in.h> +#include <netinet/ip6.h> +#include <netinet/tcp.h> +#include <netinet/udp.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/un.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/wait.h> + +#include "liburing.h" +#include "helpers.h" + +#define MAX_MSG 128 + +#define PORT 10200 +#define HOST "127.0.0.1" +#define HOSTV6 "::1" + +#define CORK_REQS 5 +#define RX_TAG 10000 +#define BUFFER_OFFSET 41 + +#ifndef ARRAY_SIZE + #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#endif + +enum { + BUF_T_NORMAL, + BUF_T_SMALL, + BUF_T_NONALIGNED, + BUF_T_LARGE, +}; + +static char *tx_buffer, *rx_buffer; +static struct iovec buffers_iov[4]; +static bool has_sendmsg; + +static bool check_cq_empty(struct io_uring *ring) +{ + struct io_uring_cqe *cqe = NULL; + int ret; + + ret = io_uring_peek_cqe(ring, &cqe); /* nothing should be there */ + return ret == -EAGAIN; +} + +static int test_basic_send(struct io_uring *ring, int sock_tx, int sock_rx) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int msg_flags = 0; + unsigned zc_flags = 0; + int payload_size = 100; + int ret; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_send_zc(sqe, sock_tx, tx_buffer, payload_size, + msg_flags, zc_flags); + sqe->user_data = 1; + + ret = io_uring_submit(ring); + assert(ret == 1); + + ret = io_uring_wait_cqe(ring, &cqe); + assert(!ret && cqe->user_data == 1); + if (cqe->res == -EINVAL) { + assert(!(cqe->flags & IORING_CQE_F_MORE)); + return T_EXIT_SKIP; + } else if (cqe->res != payload_size) { + fprintf(stderr, "send failed %i\n", cqe->res); + return T_EXIT_FAIL; + } + + assert(cqe->flags & IORING_CQE_F_MORE); + io_uring_cqe_seen(ring, cqe); + + ret = io_uring_wait_cqe(ring, &cqe); + assert(!ret); + assert(cqe->user_data == 1); + assert(cqe->flags & IORING_CQE_F_NOTIF); + assert(!(cqe->flags & IORING_CQE_F_MORE)); + io_uring_cqe_seen(ring, cqe); + assert(check_cq_empty(ring)); + + ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC); + assert(ret == payload_size); + return T_EXIT_PASS; +} + +static int test_send_faults(struct io_uring *ring, int sock_tx, int sock_rx) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int msg_flags = 0; + unsigned zc_flags = 0; + int payload_size = 100; + int ret, i, nr_cqes = 2; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_send_zc(sqe, sock_tx, (void *)1UL, payload_size, + msg_flags, zc_flags); + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_send_zc(sqe, sock_tx, tx_buffer, payload_size, + msg_flags, zc_flags); + sqe->user_data = 2; + io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)1UL, + sizeof(struct sockaddr_in6)); + + ret = io_uring_submit(ring); + assert(ret == 2); + + for (i = 0; i < nr_cqes; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + assert(!ret); + assert(cqe->user_data <= 2); + + if (!(cqe->flags & IORING_CQE_F_NOTIF)) { + assert(cqe->res == -EFAULT); + if (cqe->flags & IORING_CQE_F_MORE) + nr_cqes++; + } + io_uring_cqe_seen(ring, cqe); + } + assert(check_cq_empty(ring)); + return T_EXIT_PASS; +} + +static int create_socketpair_ip(struct sockaddr_storage *addr, + int *sock_client, int *sock_server, + bool ipv6, bool client_connect, + bool msg_zc, bool tcp) +{ + int family, addr_size; + int ret, val; + int listen_sock = -1; + int sock; + + memset(addr, 0, sizeof(*addr)); + if (ipv6) { + struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr; + + family = AF_INET6; + saddr->sin6_family = family; + saddr->sin6_port = htons(PORT); + addr_size = sizeof(*saddr); + } else { + struct sockaddr_in *saddr = (struct sockaddr_in *)addr; + + family = AF_INET; + saddr->sin_family = family; + saddr->sin_port = htons(PORT); + saddr->sin_addr.s_addr = htonl(INADDR_ANY); + addr_size = sizeof(*saddr); + } + + /* server sock setup */ + if (tcp) { + sock = listen_sock = socket(family, SOCK_STREAM, IPPROTO_TCP); + } else { + sock = *sock_server = socket(family, SOCK_DGRAM, 0); + } + if (sock < 0) { + perror("socket"); + return 1; + } + val = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + val = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + + ret = bind(sock, (struct sockaddr *)addr, addr_size); + if (ret < 0) { + perror("bind"); + return 1; + } + if (tcp) { + ret = listen(sock, 128); + assert(ret != -1); + } + + if (ipv6) { + struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr; + + inet_pton(AF_INET6, HOSTV6, &(saddr->sin6_addr)); + } else { + struct sockaddr_in *saddr = (struct sockaddr_in *)addr; + + inet_pton(AF_INET, HOST, &saddr->sin_addr); + } + + /* client sock setup */ + if (tcp) { + *sock_client = socket(family, SOCK_STREAM, IPPROTO_TCP); + assert(client_connect); + } else { + *sock_client = socket(family, SOCK_DGRAM, 0); + } + if (*sock_client < 0) { + perror("socket"); + return 1; + } + if (client_connect) { + ret = connect(*sock_client, (struct sockaddr *)addr, addr_size); + if (ret < 0) { + perror("connect"); + return 1; + } + } + if (msg_zc) { + val = 1; + if (setsockopt(*sock_client, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val))) { + perror("setsockopt zc"); + return 1; + } + } + if (tcp) { + *sock_server = accept(listen_sock, NULL, NULL); + if (!*sock_server) { + fprintf(stderr, "can't accept\n"); + return 1; + } + close(listen_sock); + } + return 0; +} + +static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_server, + bool fixed_buf, struct sockaddr_storage *addr, + bool cork, bool mix_register, + int buf_idx, bool force_async, bool use_sendmsg) +{ + struct iovec iov[CORK_REQS]; + struct msghdr msghdr[CORK_REQS]; + const unsigned zc_flags = 0; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int nr_reqs = cork ? CORK_REQS : 1; + int i, ret, nr_cqes, addr_len = 0; + size_t send_size = buffers_iov[buf_idx].iov_len; + size_t chunk_size = send_size / nr_reqs; + size_t chunk_size_last = send_size - chunk_size * (nr_reqs - 1); + char *buf = buffers_iov[buf_idx].iov_base; + + if (addr) { + sa_family_t fam = ((struct sockaddr_in *)addr)->sin_family; + + addr_len = (fam == AF_INET) ? sizeof(struct sockaddr_in) : + sizeof(struct sockaddr_in6); + } + + memset(rx_buffer, 0, send_size); + + for (i = 0; i < nr_reqs; i++) { + bool real_fixed_buf = fixed_buf; + size_t cur_size = chunk_size; + int msg_flags = MSG_WAITALL; + + if (mix_register) + real_fixed_buf = rand() & 1; + + if (cork && i != nr_reqs - 1) + msg_flags |= MSG_MORE; + if (i == nr_reqs - 1) + cur_size = chunk_size_last; + + sqe = io_uring_get_sqe(ring); + + if (!use_sendmsg) { + io_uring_prep_send_zc(sqe, sock_client, buf + i * chunk_size, + cur_size, msg_flags, zc_flags); + if (real_fixed_buf) { + sqe->ioprio |= IORING_RECVSEND_FIXED_BUF; + sqe->buf_index = buf_idx; + } + if (addr) + io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)addr, + addr_len); + } else { + io_uring_prep_sendmsg_zc(sqe, sock_client, &msghdr[i], msg_flags); + + memset(&msghdr[i], 0, sizeof(msghdr[i])); + iov[i].iov_len = cur_size; + iov[i].iov_base = buf + i * chunk_size; + msghdr[i].msg_iov = &iov[i]; + msghdr[i].msg_iovlen = 1; + if (addr) { + msghdr[i].msg_name = addr; + msghdr[i].msg_namelen = addr_len; + } + } + sqe->user_data = i; + if (force_async) + sqe->flags |= IOSQE_ASYNC; + if (i != nr_reqs - 1) + sqe->flags |= IOSQE_IO_LINK; + } + + sqe = io_uring_get_sqe(ring); + io_uring_prep_recv(sqe, sock_server, rx_buffer, send_size, MSG_WAITALL); + sqe->user_data = RX_TAG; + + ret = io_uring_submit(ring); + if (ret != nr_reqs + 1) { + fprintf(stderr, "submit failed, got %i expected %i\n", ret, nr_reqs); + return 1; + } + + nr_cqes = 2 * nr_reqs + 1; + for (i = 0; i < nr_cqes; i++) { + int expected = chunk_size; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret); + return 1; + } + if (cqe->user_data == RX_TAG) { + if (cqe->res != send_size) { + fprintf(stderr, "rx failed %i\n", cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + continue; + } + + if (cqe->user_data >= nr_reqs) { + fprintf(stderr, "invalid user_data %lu\n", + (unsigned long)cqe->user_data); + return 1; + } + if (!(cqe->flags & IORING_CQE_F_NOTIF)) { + if (cqe->user_data == nr_reqs - 1) + expected = chunk_size_last; + if (cqe->res != expected) { + fprintf(stderr, "invalid cqe->res %d expected %d\n", + cqe->res, expected); + return 1; + } + } + if ((cqe->flags & IORING_CQE_F_MORE) == + (cqe->flags & IORING_CQE_F_NOTIF)) { + fprintf(stderr, "unexpected cflags %i res %i\n", + cqe->flags, cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + } + + for (i = 0; i < send_size; i++) { + if (buf[i] != rx_buffer[i]) { + fprintf(stderr, "botched data, first mismated byte %i, " + "%u vs %u\n", i, buf[i], rx_buffer[i]); + return 1; + } + } + return 0; +} + +static int test_inet_send(struct io_uring *ring) +{ + struct sockaddr_storage addr; + int sock_client = -1, sock_server = -1; + int ret, j, i; + + for (j = 0; j < 16; j++) { + bool ipv6 = j & 1; + bool client_connect = j & 2; + bool msg_zc_set = j & 4; + bool tcp = j & 8; + + if (tcp && !client_connect) + continue; + + ret = create_socketpair_ip(&addr, &sock_client, &sock_server, ipv6, + client_connect, msg_zc_set, tcp); + if (ret) { + fprintf(stderr, "sock prep failed %d\n", ret); + return 1; + } + + for (i = 0; i < 256; i++) { + int buf_flavour = i & 3; + bool fixed_buf = i & 4; + struct sockaddr_storage *addr_arg = (i & 8) ? &addr : NULL; + bool cork = i & 16; + bool mix_register = i & 32; + bool force_async = i & 64; + bool use_sendmsg = i & 128; + + if (buf_flavour == BUF_T_LARGE && !tcp) + continue; + if (!buffers_iov[buf_flavour].iov_base) + continue; + if (tcp && (cork || addr_arg)) + continue; + if (mix_register && (!cork || fixed_buf)) + continue; + if (!client_connect && addr_arg == NULL) + continue; + if (use_sendmsg && (mix_register || fixed_buf || !has_sendmsg)) + continue; + + ret = do_test_inet_send(ring, sock_client, sock_server, fixed_buf, + addr_arg, cork, mix_register, + buf_flavour, force_async, use_sendmsg); + if (ret) { + fprintf(stderr, "send failed fixed buf %i, conn %i, addr %i, " + "cork %i\n", + fixed_buf, client_connect, !!addr_arg, + cork); + return 1; + } + } + + close(sock_client); + close(sock_server); + } + return 0; +} + +static int test_async_addr(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct sockaddr_storage addr; + int sock_tx = -1, sock_rx = -1; + struct __kernel_timespec ts; + int ret; + + ts.tv_sec = 1; + ts.tv_nsec = 0; + ret = create_socketpair_ip(&addr, &sock_tx, &sock_rx, true, false, false, false); + if (ret) { + fprintf(stderr, "sock prep failed %d\n", ret); + return 1; + } + + sqe = io_uring_get_sqe(ring); + io_uring_prep_timeout(sqe, &ts, 0, IORING_TIMEOUT_ETIME_SUCCESS); + sqe->user_data = 1; + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_send_zc(sqe, sock_tx, tx_buffer, 1, 0, 0); + sqe->user_data = 2; + io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)&addr, + sizeof(struct sockaddr_in6)); + + ret = io_uring_submit(ring); + assert(ret == 2); + memset(&addr, 0, sizeof(addr)); + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret); + return 1; + } + if (cqe->user_data != 1 || cqe->res != -ETIME) { + fprintf(stderr, "invalid timeout res %i %i\n", + (int)cqe->user_data, cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret); + return 1; + } + if (cqe->user_data != 2 || cqe->res != 1) { + fprintf(stderr, "invalid send %i %i\n", + (int)cqe->user_data, cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + ret = recv(sock_rx, rx_buffer, 1, MSG_TRUNC); + assert(ret == 1); + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret); + return 1; + } + assert(cqe->flags & IORING_CQE_F_NOTIF); + io_uring_cqe_seen(ring, cqe); + + close(sock_tx); + close(sock_rx); + return 0; +} + +static bool io_check_zc_sendmsg(struct io_uring *ring) +{ + struct io_uring_probe *p; + int ret; + + p = t_calloc(1, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); + if (!p) { + fprintf(stderr, "probe allocation failed\n"); + return false; + } + ret = io_uring_register_probe(ring, p, 256); + if (ret) + return false; + return p->ops_len > IORING_OP_SENDMSG_ZC; +} + +/* see also send_recv.c:test_invalid */ +static int test_invalid_zc(int fds[2]) +{ + struct io_uring ring; + int ret; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + bool notif = false; + + if (!has_sendmsg) + return 0; + + ret = t_create_ring(8, &ring, 0); + if (ret) + return ret; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_sendmsg(sqe, fds[0], NULL, MSG_WAITALL); + sqe->opcode = IORING_OP_SENDMSG_ZC; + sqe->flags |= IOSQE_ASYNC; + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit failed %i\n", ret); + return ret; + } + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) + return 1; + if (cqe->flags & IORING_CQE_F_MORE) + notif = true; + io_uring_cqe_seen(&ring, cqe); + + if (notif) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) + return 1; + io_uring_cqe_seen(&ring, cqe); + } + io_uring_queue_exit(&ring); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct sockaddr_storage addr; + struct io_uring ring; + int i, ret, sp[2]; + size_t len; + + if (argc > 1) + return T_EXIT_SKIP; + + /* create TCP IPv6 pair */ + ret = create_socketpair_ip(&addr, &sp[0], &sp[1], true, true, false, true); + if (ret) { + fprintf(stderr, "sock prep failed %d\n", ret); + return T_EXIT_FAIL; + } + + len = 1U << 25; /* 32MB, should be enough to trigger a short send */ + tx_buffer = aligned_alloc(4096, len); + rx_buffer = aligned_alloc(4096, len); + if (tx_buffer && rx_buffer) { + buffers_iov[BUF_T_LARGE].iov_base = tx_buffer; + buffers_iov[BUF_T_LARGE].iov_len = len; + } else { + printf("skip large buffer tests, can't alloc\n"); + + len = 8192; + tx_buffer = aligned_alloc(4096, len); + rx_buffer = aligned_alloc(4096, len); + } + if (!tx_buffer || !rx_buffer) { + fprintf(stderr, "can't allocate buffers\n"); + return T_EXIT_FAIL; + } + + buffers_iov[BUF_T_NORMAL].iov_base = tx_buffer + 4096; + buffers_iov[BUF_T_NORMAL].iov_len = 4096; + buffers_iov[BUF_T_SMALL].iov_base = tx_buffer; + buffers_iov[BUF_T_SMALL].iov_len = 137; + buffers_iov[BUF_T_NONALIGNED].iov_base = tx_buffer + BUFFER_OFFSET; + buffers_iov[BUF_T_NONALIGNED].iov_len = 8192 - BUFFER_OFFSET - 13; + + ret = io_uring_queue_init(32, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return T_EXIT_FAIL; + } + + srand((unsigned)time(NULL)); + for (i = 0; i < len; i++) + tx_buffer[i] = i; + memset(rx_buffer, 0, len); + + ret = test_basic_send(&ring, sp[0], sp[1]); + if (ret == T_EXIT_SKIP) + return ret; + if (ret) { + fprintf(stderr, "test_basic_send() failed\n"); + return T_EXIT_FAIL; + } + + has_sendmsg = io_check_zc_sendmsg(&ring); + + ret = test_send_faults(&ring, sp[0], sp[1]); + if (ret) { + fprintf(stderr, "test_send_faults() failed\n"); + return T_EXIT_FAIL; + } + + ret = test_invalid_zc(sp); + if (ret) { + fprintf(stderr, "test_invalid_zc() failed\n"); + return T_EXIT_FAIL; + } + + close(sp[0]); + close(sp[1]); + + ret = test_async_addr(&ring); + if (ret) { + fprintf(stderr, "test_async_addr() failed\n"); + return T_EXIT_FAIL; + } + + ret = t_register_buffers(&ring, buffers_iov, ARRAY_SIZE(buffers_iov)); + if (ret == T_SETUP_SKIP) { + fprintf(stderr, "can't register bufs, skip\n"); + goto out; + } else if (ret != T_SETUP_OK) { + fprintf(stderr, "buffer registration failed %i\n", ret); + return T_EXIT_FAIL; + } + + ret = test_inet_send(&ring); + if (ret) { + fprintf(stderr, "test_inet_send() failed\n"); + return T_EXIT_FAIL; + } +out: + io_uring_queue_exit(&ring); + close(sp[0]); + close(sp[1]); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/send_recv.c b/contrib/libs/liburing/test/send_recv.c new file mode 100644 index 0000000000..5200494a6c --- /dev/null +++ b/contrib/libs/liburing/test/send_recv.c @@ -0,0 +1,334 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Simple test case showing using send and recv through io_uring + */ +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <arpa/inet.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <pthread.h> + +#include "liburing.h" +#include "helpers.h" + +static char str[] = "This is a test of send and recv over io_uring!"; + +#define MAX_MSG 128 + +#define PORT 10202 +#define HOST "127.0.0.1" + +static int recv_prep(struct io_uring *ring, struct iovec *iov, int *sock, + int registerfiles) +{ + struct sockaddr_in saddr; + struct io_uring_sqe *sqe; + int sockfd, ret, val, use_fd; + + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_addr.s_addr = htonl(INADDR_ANY); + saddr.sin_port = htons(PORT); + + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (sockfd < 0) { + perror("socket"); + return 1; + } + + val = 1; + setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + + ret = bind(sockfd, (struct sockaddr *)&saddr, sizeof(saddr)); + if (ret < 0) { + perror("bind"); + goto err; + } + + if (registerfiles) { + ret = io_uring_register_files(ring, &sockfd, 1); + if (ret) { + fprintf(stderr, "file reg failed\n"); + goto err; + } + use_fd = 0; + } else { + use_fd = sockfd; + } + + sqe = io_uring_get_sqe(ring); + io_uring_prep_recv(sqe, use_fd, iov->iov_base, iov->iov_len, 0); + if (registerfiles) + sqe->flags |= IOSQE_FIXED_FILE; + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + *sock = sockfd; + return 0; +err: + close(sockfd); + return 1; +} + +static int do_recv(struct io_uring *ring, struct iovec *iov) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stdout, "wait_cqe: %d\n", ret); + goto err; + } + if (cqe->res == -EINVAL) { + fprintf(stdout, "recv not supported, skipping\n"); + return 0; + } + if (cqe->res < 0) { + fprintf(stderr, "failed cqe: %d\n", cqe->res); + goto err; + } + + if (cqe->res -1 != strlen(str)) { + fprintf(stderr, "got wrong length: %d/%d\n", cqe->res, + (int) strlen(str) + 1); + goto err; + } + + if (strcmp(str, iov->iov_base)) { + fprintf(stderr, "string mismatch\n"); + goto err; + } + + return 0; +err: + return 1; +} + +struct recv_data { + pthread_mutex_t mutex; + int use_sqthread; + int registerfiles; +}; + +static void *recv_fn(void *data) +{ + struct recv_data *rd = data; + char buf[MAX_MSG + 1]; + struct iovec iov = { + .iov_base = buf, + .iov_len = sizeof(buf) - 1, + }; + struct io_uring_params p = { }; + struct io_uring ring; + int ret, sock; + + if (rd->use_sqthread) + p.flags = IORING_SETUP_SQPOLL; + ret = t_create_ring_params(1, &ring, &p); + if (ret == T_SETUP_SKIP) { + pthread_mutex_unlock(&rd->mutex); + ret = 0; + goto err; + } else if (ret < 0) { + pthread_mutex_unlock(&rd->mutex); + goto err; + } + + if (rd->use_sqthread && !rd->registerfiles) { + if (!(p.features & IORING_FEAT_SQPOLL_NONFIXED)) { + fprintf(stdout, "Non-registered SQPOLL not available, skipping\n"); + pthread_mutex_unlock(&rd->mutex); + goto err; + } + } + + ret = recv_prep(&ring, &iov, &sock, rd->registerfiles); + if (ret) { + fprintf(stderr, "recv_prep failed: %d\n", ret); + goto err; + } + pthread_mutex_unlock(&rd->mutex); + ret = do_recv(&ring, &iov); + + close(sock); + io_uring_queue_exit(&ring); +err: + return (void *)(intptr_t)ret; +} + +static int do_send(void) +{ + struct sockaddr_in saddr; + struct iovec iov = { + .iov_base = str, + .iov_len = sizeof(str), + }; + struct io_uring ring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int sockfd, ret; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return 1; + } + + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_port = htons(PORT); + inet_pton(AF_INET, HOST, &saddr.sin_addr); + + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (sockfd < 0) { + perror("socket"); + return 1; + } + + ret = connect(sockfd, (struct sockaddr *)&saddr, sizeof(saddr)); + if (ret < 0) { + perror("connect"); + return 1; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_send(sqe, sockfd, iov.iov_base, iov.iov_len, 0); + sqe->user_data = 1; + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (cqe->res == -EINVAL) { + fprintf(stdout, "send not supported, skipping\n"); + close(sockfd); + return 0; + } + if (cqe->res != iov.iov_len) { + fprintf(stderr, "failed cqe: %d\n", cqe->res); + goto err; + } + + close(sockfd); + return 0; +err: + close(sockfd); + return 1; +} + +static int test(int use_sqthread, int regfiles) +{ + pthread_mutexattr_t attr; + pthread_t recv_thread; + struct recv_data rd; + int ret; + void *retval; + + pthread_mutexattr_init(&attr); + pthread_mutexattr_setpshared(&attr, 1); + pthread_mutex_init(&rd.mutex, &attr); + pthread_mutex_lock(&rd.mutex); + rd.use_sqthread = use_sqthread; + rd.registerfiles = regfiles; + + ret = pthread_create(&recv_thread, NULL, recv_fn, &rd); + if (ret) { + fprintf(stderr, "Thread create failed: %d\n", ret); + pthread_mutex_unlock(&rd.mutex); + return 1; + } + + pthread_mutex_lock(&rd.mutex); + do_send(); + pthread_join(recv_thread, &retval); + return (intptr_t)retval; +} + +static int test_invalid(void) +{ + struct io_uring ring; + int ret, i; + int fds[2]; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + + ret = t_create_ring(8, &ring, 0); + if (ret) + return ret; + + ret = t_create_socket_pair(fds, true); + if (ret) + return ret; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_sendmsg(sqe, fds[0], NULL, MSG_WAITALL); + sqe->flags |= IOSQE_ASYNC; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_recvmsg(sqe, fds[1], NULL, 0); + sqe->flags |= IOSQE_ASYNC; + + ret = io_uring_submit_and_wait(&ring, 2); + if (ret != 2) + return ret; + + for (i = 0; i < 2; i++) { + ret = io_uring_peek_cqe(&ring, &cqe); + if (ret || cqe->res != -EFAULT) + return -1; + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + close(fds[0]); + close(fds[1]); + return 0; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test_invalid(); + if (ret) { + fprintf(stderr, "test_invalid failed\n"); + return ret; + } + + ret = test(0, 0); + if (ret) { + fprintf(stderr, "test sqthread=0 failed\n"); + return ret; + } + + ret = test(1, 1); + if (ret) { + fprintf(stderr, "test sqthread=1 reg=1 failed\n"); + return ret; + } + + ret = test(1, 0); + if (ret) { + fprintf(stderr, "test sqthread=1 reg=0 failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/send_recvmsg.c b/contrib/libs/liburing/test/send_recvmsg.c new file mode 100644 index 0000000000..da64a0fb06 --- /dev/null +++ b/contrib/libs/liburing/test/send_recvmsg.c @@ -0,0 +1,456 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Simple test case showing using sendmsg and recvmsg through io_uring + */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#include <arpa/inet.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <pthread.h> +#include <assert.h> + +#include "liburing.h" + +static char str[] = "This is a test of sendmsg and recvmsg over io_uring!"; + +static int ud; + +#define MAX_MSG 128 + +#define PORT 10203 +#define HOST "127.0.0.1" + +#define BUF_BGID 10 +#define BUF_BID 89 + +#define MAX_IOV_COUNT 10 + +static int no_pbuf_ring; + +static int recv_prep(struct io_uring *ring, int *sockfd, struct iovec iov[], + int iov_count, int bgid, int async) +{ + struct sockaddr_in saddr; + struct msghdr msg; + struct io_uring_sqe *sqe; + int ret, val = 1; + + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_addr.s_addr = htonl(INADDR_ANY); + saddr.sin_port = htons(PORT); + + *sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (*sockfd < 0) { + perror("socket"); + return 1; + } + + val = 1; + setsockopt(*sockfd, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + setsockopt(*sockfd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + + ret = bind(*sockfd, (struct sockaddr *)&saddr, sizeof(saddr)); + if (ret < 0) { + perror("bind"); + goto err; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "io_uring_get_sqe failed\n"); + return 1; + } + + io_uring_prep_recvmsg(sqe, *sockfd, &msg, 0); + if (bgid) { + iov->iov_base = NULL; + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = bgid; + iov_count = 1; + } + sqe->user_data = ++ud; + if (async) + sqe->flags |= IOSQE_ASYNC; + memset(&msg, 0, sizeof(msg)); + msg.msg_namelen = sizeof(struct sockaddr_in); + msg.msg_iov = iov; + msg.msg_iovlen = iov_count; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + return 0; +err: + close(*sockfd); + return 1; +} + +struct recv_data { + pthread_mutex_t *mutex; + int buf_select; + int buf_ring; + int no_buf_add; + int iov_count; + int async; +}; + +static int do_recvmsg(struct io_uring *ring, char buf[MAX_MSG + 1], + struct recv_data *rd) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stdout, "wait_cqe: %d\n", ret); + goto err; + } + if (cqe->res < 0) { + if (rd->no_buf_add && (rd->buf_select || rd->buf_ring)) + return 0; + fprintf(stderr, "%s: failed cqe: %d\n", __FUNCTION__, cqe->res); + goto err; + } + if (cqe->flags & IORING_CQE_F_BUFFER) { + int bid = cqe->flags >> 16; + if (bid != BUF_BID) + fprintf(stderr, "Buffer ID mismatch %d\n", bid); + } + + if (rd->no_buf_add && (rd->buf_ring || rd->buf_select)) { + fprintf(stderr, "Expected -ENOBUFS: %d\n", cqe->res); + goto err; + } + + if (cqe->res -1 != strlen(str)) { + fprintf(stderr, "got wrong length: %d/%d\n", cqe->res, + (int) strlen(str) + 1); + goto err; + } + + if (strncmp(str, buf, MAX_MSG + 1)) { + fprintf(stderr, "string mismatch\n"); + goto err; + } + + return 0; +err: + return 1; +} + +static void init_iov(struct iovec iov[MAX_IOV_COUNT], int iov_to_use, + char buf[MAX_MSG + 1]) +{ + int i, last_idx = iov_to_use - 1; + + assert(0 < iov_to_use && iov_to_use <= MAX_IOV_COUNT); + for (i = 0; i < last_idx; ++i) { + iov[i].iov_base = buf + i; + iov[i].iov_len = 1; + } + + iov[last_idx].iov_base = buf + last_idx; + iov[last_idx].iov_len = MAX_MSG - last_idx; +} + +static void *recv_fn(void *data) +{ + struct recv_data *rd = data; + pthread_mutex_t *mutex = rd->mutex; + struct io_uring_buf_ring *br = NULL; + char buf[MAX_MSG + 1]; + struct iovec iov[MAX_IOV_COUNT]; + struct io_uring ring; + int ret, sockfd; + + if (rd->buf_ring && no_pbuf_ring) + goto out_no_ring; + + init_iov(iov, rd->iov_count, buf); + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + goto err; + } + + if ((rd->buf_ring || rd->buf_select) && !rd->no_buf_add) { + if (rd->buf_ring) { + struct io_uring_buf_reg reg = { }; + void *ptr; + + if (posix_memalign(&ptr, 4096, 4096)) + goto err; + + reg.ring_addr = (unsigned long) ptr; + reg.ring_entries = 1; + reg.bgid = BUF_BGID; + if (io_uring_register_buf_ring(&ring, ®, 0)) { + no_pbuf_ring = 1; + goto out; + } + + br = ptr; + io_uring_buf_ring_init(br); + io_uring_buf_ring_add(br, buf, sizeof(buf), BUF_BID, + io_uring_buf_ring_mask(1), 0); + io_uring_buf_ring_advance(br, 1); + } else { + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_provide_buffers(sqe, buf, sizeof(buf) -1, + 1, BUF_BGID, BUF_BID); + sqe->user_data = ++ud; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit ret=%d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(&ring, cqe); + if (ret == -EINVAL) { + fprintf(stdout, "PROVIDE_BUFFERS not supported, skip\n"); + goto out; + } else if (ret < 0) { + fprintf(stderr, "PROVIDER_BUFFERS %d\n", ret); + goto err; + } + } + } + + ret = recv_prep(&ring, &sockfd, iov, rd->iov_count, + (rd->buf_ring || rd->buf_select) ? BUF_BGID : 0, + rd->async); + if (ret) { + fprintf(stderr, "recv_prep failed: %d\n", ret); + goto err; + } + + pthread_mutex_unlock(mutex); + ret = do_recvmsg(&ring, buf, rd); + close(sockfd); + + io_uring_queue_exit(&ring); + if (br) + free(br); +err: + return (void *)(intptr_t)ret; +out: + io_uring_queue_exit(&ring); +out_no_ring: + pthread_mutex_unlock(mutex); + if (br) + free(br); + return NULL; +} + +static int do_sendmsg(void) +{ + struct sockaddr_in saddr; + struct iovec iov = { + .iov_base = str, + .iov_len = sizeof(str), + }; + struct msghdr msg; + struct io_uring ring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int sockfd, ret; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return 1; + } + + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_port = htons(PORT); + inet_pton(AF_INET, HOST, &saddr.sin_addr); + + memset(&msg, 0, sizeof(msg)); + msg.msg_name = &saddr; + msg.msg_namelen = sizeof(struct sockaddr_in); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (sockfd < 0) { + perror("socket"); + return 1; + } + + usleep(10000); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_sendmsg(sqe, sockfd, &msg, 0); + sqe->user_data = ++ud; + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (cqe->res < 0) { + fprintf(stderr, "%s: failed cqe: %d\n", __FUNCTION__, cqe->res); + goto err; + } + + close(sockfd); + return 0; +err: + close(sockfd); + return 1; +} + +static int test(int buf_select, int buf_ring, int no_buf_add, int iov_count, + int async) +{ + struct recv_data rd; + pthread_mutexattr_t attr; + pthread_t recv_thread; + pthread_mutex_t mutex; + int ret; + void *retval; + + if (buf_select || buf_ring) + assert(iov_count == 1); + + pthread_mutexattr_init(&attr); + pthread_mutexattr_setpshared(&attr, 1); + pthread_mutex_init(&mutex, &attr); + pthread_mutex_lock(&mutex); + + rd.mutex = &mutex; + rd.buf_select = buf_select; + rd.buf_ring = buf_ring; + rd.no_buf_add = no_buf_add; + rd.iov_count = iov_count; + rd.async = async; + ret = pthread_create(&recv_thread, NULL, recv_fn, &rd); + if (ret) { + pthread_mutex_unlock(&mutex); + fprintf(stderr, "Thread create failed\n"); + return 1; + } + + pthread_mutex_lock(&mutex); + do_sendmsg(); + pthread_join(recv_thread, &retval); + ret = (intptr_t)retval; + + return ret; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test(0, 0, 0, 1, 0); + if (ret) { + fprintf(stderr, "send_recvmsg 0 0 0 1 0 failed\n"); + return 1; + } + + ret = test(0, 0, 0, 10, 0); + if (ret) { + fprintf(stderr, "send_recvmsg multi iov failed\n"); + return 1; + } + + ret = test(1, 0, 0, 1, 0); + if (ret) { + fprintf(stderr, "send_recvmsg 1 0 0 1 0 failed\n"); + return 1; + } + + ret = test(1, 0, 1, 1, 0); + if (ret) { + fprintf(stderr, "send_recvmsg 1 0 1 1 0 failed\n"); + return 1; + } + + ret = test(0, 1, 0, 1, 0); + if (ret) { + fprintf(stderr, "send_recvmsg 0 1 0 1 0 failed\n"); + return 1; + } + + ret = test(1, 1, 0, 1, 0); + if (ret) { + fprintf(stderr, "send_recvmsg 1 1 0 1 0 failed\n"); + return 1; + } + + ret = test(1, 1, 1, 1, 0); + if (ret) { + fprintf(stderr, "send_recvmsg 1 1 1 1 0 failed\n"); + return 1; + } + + ret = test(0, 0, 0, 1, 1); + if (ret) { + fprintf(stderr, "send_recvmsg async 0 0 0 1 1 failed\n"); + return 1; + } + + ret = test(0, 0, 0, 10, 1); + if (ret) { + fprintf(stderr, "send_recvmsg async multi iov failed\n"); + return 1; + } + + ret = test(1, 0, 0, 1, 1); + if (ret) { + fprintf(stderr, "send_recvmsg async 1 0 0 1 1 failed\n"); + return 1; + } + + ret = test(1, 0, 1, 1, 1); + if (ret) { + fprintf(stderr, "send_recvmsg async 1 0 1 1 1 failed\n"); + return 1; + } + + ret = test(0, 1, 0, 1, 1); + if (ret) { + fprintf(stderr, "send_recvmsg async 0 1 0 1 1 failed\n"); + return 1; + } + + ret = test(1, 1, 0, 1, 1); + if (ret) { + fprintf(stderr, "send_recvmsg async 1 1 0 1 1 failed\n"); + return 1; + } + + ret = test(1, 1, 1, 1, 1); + if (ret) { + fprintf(stderr, "send_recvmsg async 1 1 1 1 1 failed\n"); + return 1; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/sendmsg_fs_cve.c b/contrib/libs/liburing/test/sendmsg_fs_cve.c new file mode 100644 index 0000000000..3829a5c085 --- /dev/null +++ b/contrib/libs/liburing/test/sendmsg_fs_cve.c @@ -0,0 +1,201 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * repro-CVE-2020-29373 -- Reproducer for CVE-2020-29373. + * + * Copyright (c) 2021 SUSE + * Author: Nicolai Stange <nstange@suse.de> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include <unistd.h> +#include <stdio.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <fcntl.h> +#include <errno.h> +#include <inttypes.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/wait.h> +#include "liburing.h" + +/* + * This attempts to make the kernel issue a sendmsg() to + * path from io_uring's async io_sq_wq_submit_work(). + * + * Unfortunately, IOSQE_ASYNC is available only from kernel version + * 5.6 onwards. To still force io_uring to process the request + * asynchronously from io_sq_wq_submit_work(), queue a couple of + * auxiliary requests all failing with EAGAIN before. This is + * implemented by writing repeatedly to an auxiliary O_NONBLOCK + * AF_UNIX socketpair with a small SO_SNDBUF. + */ +static int try_sendmsg_async(const char * const path) +{ + int snd_sock, r; + struct io_uring ring; + char sbuf[16] = {}; + struct iovec siov = { .iov_base = &sbuf, .iov_len = sizeof(sbuf) }; + struct sockaddr_un addr = {}; + struct msghdr msg = { + .msg_name = &addr, + .msg_namelen = sizeof(addr), + .msg_iov = &siov, + .msg_iovlen = 1, + }; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + + snd_sock = socket(AF_UNIX, SOCK_DGRAM, 0); + if (snd_sock < 0) { + perror("socket(AF_UNIX)"); + return -1; + } + + addr.sun_family = AF_UNIX; + strcpy(addr.sun_path, path); + + r = io_uring_queue_init(512, &ring, 0); + if (r < 0) { + fprintf(stderr, "ring setup failed: %d\n", r); + goto close_iour; + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + r = -EFAULT; + goto close_iour; + } + + /* the actual one supposed to fail with -ENOENT. */ + io_uring_prep_sendmsg(sqe, snd_sock, &msg, 0); + sqe->flags = IOSQE_ASYNC; + sqe->user_data = 255; + + r = io_uring_submit(&ring); + if (r != 1) { + fprintf(stderr, "sqe submit failed: %d\n", r); + r = -EFAULT; + goto close_iour; + } + + r = io_uring_wait_cqe(&ring, &cqe); + if (r < 0) { + fprintf(stderr, "wait completion %d\n", r); + r = -EFAULT; + goto close_iour; + } + if (cqe->user_data != 255) { + fprintf(stderr, "user data %d\n", r); + r = -EFAULT; + goto close_iour; + } + if (cqe->res != -ENOENT) { + r = 3; + fprintf(stderr, + "error: cqe %i: res=%i, but expected -ENOENT\n", + (int)cqe->user_data, (int)cqe->res); + } + io_uring_cqe_seen(&ring, cqe); + +close_iour: + io_uring_queue_exit(&ring); + close(snd_sock); + return r; +} + +int main(int argc, char *argv[]) +{ + int r; + char tmpdir[] = "/tmp/tmp.XXXXXX"; + int rcv_sock; + struct sockaddr_un addr = {}; + pid_t c; + int wstatus; + + if (!mkdtemp(tmpdir)) { + perror("mkdtemp()"); + return 1; + } + + rcv_sock = socket(AF_UNIX, SOCK_DGRAM, 0); + if (rcv_sock < 0) { + perror("socket(AF_UNIX)"); + r = 1; + goto rmtmpdir; + } + + addr.sun_family = AF_UNIX; + snprintf(addr.sun_path, sizeof(addr.sun_path), "%s/sock", tmpdir); + + r = bind(rcv_sock, (struct sockaddr *)&addr, + sizeof(addr)); + if (r < 0) { + perror("bind()"); + close(rcv_sock); + r = 1; + goto rmtmpdir; + } + + c = fork(); + if (!c) { + close(rcv_sock); + + r = chroot(tmpdir); + if (r) { + if (errno == EPERM) { + fprintf(stderr, "chroot not allowed, skip\n"); + return 0; + } + + perror("chroot()"); + return 1; + } + + r = try_sendmsg_async(addr.sun_path); + if (r < 0) { + /* system call failure */ + r = 1; + } else if (r) { + /* test case failure */ + r += 1; + } + return r; + } + + if (waitpid(c, &wstatus, 0) == (pid_t)-1) { + perror("waitpid()"); + r = 1; + goto rmsock; + } + if (!WIFEXITED(wstatus)) { + fprintf(stderr, "child got terminated\n"); + r = 1; + goto rmsock; + } + r = WEXITSTATUS(wstatus); + if (r) + fprintf(stderr, "error: Test failed\n"); +rmsock: + close(rcv_sock); + unlink(addr.sun_path); +rmtmpdir: + rmdir(tmpdir); + return r; +} diff --git a/contrib/libs/liburing/test/shared-wq.c b/contrib/libs/liburing/test/shared-wq.c new file mode 100644 index 0000000000..d4b721c7e0 --- /dev/null +++ b/contrib/libs/liburing/test/shared-wq.c @@ -0,0 +1,85 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test wq sharing + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" + +static int test_attach_invalid(int ringfd) +{ + struct io_uring_params p; + struct io_uring ring; + int ret; + + memset(&p, 0, sizeof(p)); + p.flags = IORING_SETUP_ATTACH_WQ; + p.wq_fd = ringfd; + ret = io_uring_queue_init_params(1, &ring, &p); + if (ret != -EINVAL) { + fprintf(stderr, "Attach to zero: %d\n", ret); + goto err; + } + return 0; +err: + return 1; +} + +static int test_attach(int ringfd) +{ + struct io_uring_params p; + struct io_uring ring2; + int ret; + + memset(&p, 0, sizeof(p)); + p.flags = IORING_SETUP_ATTACH_WQ; + p.wq_fd = ringfd; + ret = io_uring_queue_init_params(1, &ring2, &p); + if (ret == -EINVAL) { + fprintf(stdout, "Sharing not supported, skipping\n"); + return 0; + } else if (ret) { + fprintf(stderr, "Attach to id: %d\n", ret); + goto err; + } + io_uring_queue_exit(&ring2); + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + /* stdout is definitely not an io_uring descriptor */ + ret = test_attach_invalid(2); + if (ret) { + fprintf(stderr, "test_attach_invalid failed\n"); + return ret; + } + + ret = test_attach(ring.ring_fd); + if (ret) { + fprintf(stderr, "test_attach failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/short-read.c b/contrib/libs/liburing/test/short-read.c new file mode 100644 index 0000000000..cd8c1644f8 --- /dev/null +++ b/contrib/libs/liburing/test/short-read.c @@ -0,0 +1,76 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <poll.h> + + +#include "helpers.h" +#include "liburing.h" + +#define BUF_SIZE 4096 +#define FILE_SIZE 1024 + +int main(int argc, char *argv[]) +{ + int ret, fd, save_errno; + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct iovec vec; + + if (argc > 1) + return 0; + + vec.iov_base = t_malloc(BUF_SIZE); + vec.iov_len = BUF_SIZE; + + t_create_file(".short-read", FILE_SIZE); + + fd = open(".short-read", O_RDONLY); + save_errno = errno; + unlink(".short-read"); + errno = save_errno; + if (fd < 0) { + perror("file open"); + return 1; + } + + ret = io_uring_queue_init(32, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return ret; + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + return 1; + } + io_uring_prep_readv(sqe, fd, &vec, 1, 0); + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit failed: %d\n", ret); + return 1; + } + + ret = io_uring_wait_cqes(&ring, &cqe, 1, 0, 0); + if (ret) { + fprintf(stderr, "wait_cqe failed: %d\n", ret); + return 1; + } + + if (cqe->res != FILE_SIZE) { + fprintf(stderr, "Read failed: %d\n", cqe->res); + return 1; + } + + io_uring_cqe_seen(&ring, cqe); + return 0; +} diff --git a/contrib/libs/liburing/test/shutdown.c b/contrib/libs/liburing/test/shutdown.c new file mode 100644 index 0000000000..e2c59c7666 --- /dev/null +++ b/contrib/libs/liburing/test/shutdown.c @@ -0,0 +1,165 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Check that writev on a socket that has been shutdown(2) fails + * + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <assert.h> + +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <netinet/tcp.h> +#include <netinet/in.h> +#include <arpa/inet.h> + +#include "liburing.h" +#include "helpers.h" + +static void sig_pipe(int sig) +{ +} + +int main(int argc, char *argv[]) +{ + int p_fd[2], ret; + int32_t recv_s0; + int32_t val = 1; + struct sockaddr_in addr = { }; + + if (argc > 1) + return 0; + + srand(getpid()); + + recv_s0 = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + + ret = setsockopt(recv_s0, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + assert(ret != -1); + ret = setsockopt(recv_s0, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + assert(ret != -1); + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr("127.0.0.1"); + + assert(!t_bind_ephemeral_port(recv_s0, &addr)); + ret = listen(recv_s0, 128); + assert(ret != -1); + + p_fd[1] = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + + val = 1; + ret = setsockopt(p_fd[1], IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)); + assert(ret != -1); + + int32_t flags = fcntl(p_fd[1], F_GETFL, 0); + assert(flags != -1); + + flags |= O_NONBLOCK; + ret = fcntl(p_fd[1], F_SETFL, flags); + assert(ret != -1); + + ret = connect(p_fd[1], (struct sockaddr*)&addr, sizeof(addr)); + assert(ret == -1); + + flags = fcntl(p_fd[1], F_GETFL, 0); + assert(flags != -1); + + flags &= ~O_NONBLOCK; + ret = fcntl(p_fd[1], F_SETFL, flags); + assert(ret != -1); + + p_fd[0] = accept(recv_s0, NULL, NULL); + assert(p_fd[0] != -1); + + signal(SIGPIPE, sig_pipe); + + while (1) { + int32_t code; + socklen_t code_len = sizeof(code); + + ret = getsockopt(p_fd[1], SOL_SOCKET, SO_ERROR, &code, &code_len); + assert(ret != -1); + + if (!code) + break; + } + + struct io_uring m_io_uring; + + ret = io_uring_queue_init(32, &m_io_uring, 0); + assert(ret >= 0); + + { + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int res; + + sqe = io_uring_get_sqe(&m_io_uring); + io_uring_prep_shutdown(sqe, p_fd[1], SHUT_WR); + sqe->user_data = 1; + + res = io_uring_submit_and_wait(&m_io_uring, 1); + assert(res != -1); + + res = io_uring_wait_cqe(&m_io_uring, &cqe); + if (res < 0) { + fprintf(stderr, "wait: %s\n", strerror(-ret)); + goto err; + } + + if (cqe->res) { + if (cqe->res == -EINVAL) { + fprintf(stdout, "Shutdown not supported, skipping\n"); + goto done; + } + fprintf(stderr, "writev: %d\n", cqe->res); + goto err; + } + + io_uring_cqe_seen(&m_io_uring, cqe); + } + + { + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct iovec iov[1]; + char send_buff[128]; + int res; + + iov[0].iov_base = send_buff; + iov[0].iov_len = sizeof(send_buff); + + sqe = io_uring_get_sqe(&m_io_uring); + assert(sqe != NULL); + + io_uring_prep_writev(sqe, p_fd[1], iov, 1, 0); + res = io_uring_submit_and_wait(&m_io_uring, 1); + assert(res != -1); + + res = io_uring_wait_cqe(&m_io_uring, &cqe); + if (res < 0) { + fprintf(stderr, "wait: %s\n", strerror(-ret)); + goto err; + } + + if (cqe->res != -EPIPE) { + fprintf(stderr, "writev: %d\n", cqe->res); + goto err; + } + io_uring_cqe_seen(&m_io_uring, cqe); + } + +done: + io_uring_queue_exit(&m_io_uring); + return 0; +err: + io_uring_queue_exit(&m_io_uring); + return 1; +} diff --git a/contrib/libs/liburing/test/sigfd-deadlock.c b/contrib/libs/liburing/test/sigfd-deadlock.c new file mode 100644 index 0000000000..a2a9ad9488 --- /dev/null +++ b/contrib/libs/liburing/test/sigfd-deadlock.c @@ -0,0 +1,89 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test that sigfd reading/polling works. A regression test for + * the upstream commit: + * + * fd7d6de22414 ("io_uring: don't recurse on tsk->sighand->siglock with signalfd") + */ +#include <unistd.h> +#include <sys/signalfd.h> +#include <sys/epoll.h> +#include <poll.h> +#include <stdio.h> +#include "liburing.h" +#include "helpers.h" + +static int setup_signal(void) +{ + sigset_t mask; + int sfd; + + sigemptyset(&mask); + sigaddset(&mask, SIGINT); + + sigprocmask(SIG_BLOCK, &mask, NULL); + sfd = signalfd(-1, &mask, SFD_NONBLOCK); + if (sfd < 0) + perror("signalfd"); + return sfd; +} + +static int test_uring(int sfd) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret; + + ret = io_uring_queue_init(32, &ring, 0); + if (ret) + return T_EXIT_FAIL; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, sfd, POLLIN); + ret = io_uring_submit(&ring); + if (ret < 0) { + ret = T_EXIT_FAIL; + goto err_exit; + } + + kill(getpid(), SIGINT); + + io_uring_wait_cqe(&ring, &cqe); + if (cqe->res == -EOPNOTSUPP) { + fprintf(stderr, "signalfd poll not supported\n"); + ret = T_EXIT_SKIP; + } else if (cqe->res < 0) { + fprintf(stderr, "poll failed: %d\n", cqe->res); + ret = T_EXIT_FAIL; + } else if (cqe->res & POLLIN) { + ret = T_EXIT_PASS; + } else { + fprintf(stderr, "Unexpected poll mask %x\n", cqe->res); + ret = T_EXIT_FAIL; + } + io_uring_cqe_seen(&ring, cqe); +err_exit: + io_uring_queue_exit(&ring); + return ret; +} + +int main(int argc, char *argv[]) +{ + int sfd, ret; + + if (argc > 1) + return T_EXIT_PASS; + + sfd = setup_signal(); + if (sfd < 0) + return T_EXIT_FAIL; + + ret = test_uring(sfd); + if (ret == T_EXIT_FAIL) + fprintf(stderr, "test_uring signalfd failed\n"); + + close(sfd); + return ret; +} diff --git a/contrib/libs/liburing/test/single-issuer.c b/contrib/libs/liburing/test/single-issuer.c new file mode 100644 index 0000000000..22f104e94e --- /dev/null +++ b/contrib/libs/liburing/test/single-issuer.c @@ -0,0 +1,172 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <error.h> +#include <sys/types.h> +#include <sys/wait.h> + +#include "liburing.h" +#include "test.h" +#include "helpers.h" + +static pid_t pid; + +static pid_t fork_t(void) +{ + pid = fork(); + if (pid == -1) { + fprintf(stderr, "fork failed\n"); + exit(T_EXIT_FAIL); + } + return pid; +} + +static void wait_child_t(void) +{ + int wstatus; + + if (waitpid(pid, &wstatus, 0) == (pid_t)-1) { + perror("waitpid()"); + exit(T_EXIT_FAIL); + } + if (!WIFEXITED(wstatus)) { + fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus)); + exit(T_EXIT_FAIL); + } + if (WEXITSTATUS(wstatus)) + exit(T_EXIT_FAIL); +} + +static int try_submit(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_nop(sqe); + sqe->user_data = 42; + + ret = io_uring_submit(ring); + if (ret < 0) + return ret; + + if (ret != 1) + error(1, ret, "submit %i", ret); + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) + error(1, ret, "wait fail %i", ret); + + if (cqe->res || cqe->user_data != 42) + error(1, ret, "invalid cqe"); + + io_uring_cqe_seen(ring, cqe); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER); + if (ret == -EINVAL) { + fprintf(stderr, "SETUP_SINGLE_ISSUER is not supported, skip\n"); + return T_EXIT_SKIP; + } else if (ret) { + fprintf(stderr, "io_uring_queue_init() failed %i\n", ret); + return T_EXIT_FAIL; + } + + /* test that the creator iw allowed to submit */ + ret = try_submit(&ring); + if (ret) { + fprintf(stderr, "the creator can't submit %i\n", ret); + return T_EXIT_FAIL; + } + + /* test that a second submitter doesn't succeed */ + if (!fork_t()) { + ret = try_submit(&ring); + if (ret != -EEXIST) + fprintf(stderr, "1: not owner child could submit %i\n", ret); + return ret != -EEXIST; + } + wait_child_t(); + io_uring_queue_exit(&ring); + + /* test that the first submitter but not creator can submit */ + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_R_DISABLED); + if (ret) + error(1, ret, "ring init (2) %i", ret); + + if (!fork_t()) { + io_uring_enable_rings(&ring); + ret = try_submit(&ring); + if (ret) + fprintf(stderr, "2: not owner child could submit %i\n", ret); + return !!ret; + } + wait_child_t(); + io_uring_queue_exit(&ring); + + /* test that only the first enabler can submit */ + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_R_DISABLED); + if (ret) + error(1, ret, "ring init (3) %i", ret); + + io_uring_enable_rings(&ring); + if (!fork_t()) { + ret = try_submit(&ring); + if (ret != -EEXIST) + fprintf(stderr, "3: not owner child could submit %i\n", ret); + return ret != -EEXIST; + } + wait_child_t(); + io_uring_queue_exit(&ring); + + /* test that anyone can submit to a SQPOLL|SINGLE_ISSUER ring */ + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_SQPOLL); + if (ret) + error(1, ret, "ring init (4) %i", ret); + + ret = try_submit(&ring); + if (ret) { + fprintf(stderr, "SQPOLL submit failed (creator) %i\n", ret); + return T_EXIT_FAIL; + } + + if (!fork_t()) { + ret = try_submit(&ring); + if (ret) + fprintf(stderr, "SQPOLL submit failed (child) %i\n", ret); + return !!ret; + } + wait_child_t(); + io_uring_queue_exit(&ring); + + /* test that IORING_ENTER_REGISTERED_RING doesn't break anything */ + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER); + if (ret) + error(1, ret, "ring init (5) %i", ret); + + if (!fork_t()) { + ret = try_submit(&ring); + if (ret != -EEXIST) + fprintf(stderr, "4: not owner child could submit %i\n", ret); + return ret != -EEXIST; + } + wait_child_t(); + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/skip-cqe.c b/contrib/libs/liburing/test/skip-cqe.c new file mode 100644 index 0000000000..c73e2b8dea --- /dev/null +++ b/contrib/libs/liburing/test/skip-cqe.c @@ -0,0 +1,430 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <assert.h> + +#include "liburing.h" + +#define LINK_SIZE 6 +#define TIMEOUT_USER_DATA (-1) + +static int fds[2]; + +/* should be successfully submitted but fails during execution */ +static void prep_exec_fail_req(struct io_uring_sqe *sqe) +{ + io_uring_prep_write(sqe, fds[1], NULL, 100, 0); +} + +static int test_link_success(struct io_uring *ring, int nr, bool skip_last) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + for (i = 0; i < nr; ++i) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_nop(sqe); + if (i != nr - 1 || skip_last) + sqe->flags |= IOSQE_IO_LINK | IOSQE_CQE_SKIP_SUCCESS; + sqe->user_data = i; + } + + ret = io_uring_submit(ring); + if (ret != nr) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + if (!skip_last) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret != 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res != 0) { + fprintf(stderr, "nop failed: res %d\n", cqe->res); + goto err; + } + if (cqe->user_data != nr - 1) { + fprintf(stderr, "invalid user_data %i\n", (int)cqe->user_data); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + if (io_uring_peek_cqe(ring, &cqe) >= 0) { + fprintf(stderr, "single CQE expected %i\n", (int)cqe->user_data); + goto err; + } + return 0; +err: + return 1; +} + +static int test_link_fail(struct io_uring *ring, int nr, int fail_idx) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + for (i = 0; i < nr; ++i) { + sqe = io_uring_get_sqe(ring); + if (i == fail_idx) + prep_exec_fail_req(sqe); + else + io_uring_prep_nop(sqe); + + if (i != nr - 1) + sqe->flags |= IOSQE_IO_LINK | IOSQE_CQE_SKIP_SUCCESS; + sqe->user_data = i; + } + + ret = io_uring_submit(ring); + if (ret != nr) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + ret = io_uring_wait_cqe(ring, &cqe); + if (ret != 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (!cqe->res || cqe->user_data != fail_idx) { + fprintf(stderr, "got: user_data %d res %d, expected data: %d\n", + (int)cqe->user_data, cqe->res, fail_idx); + goto err; + } + io_uring_cqe_seen(ring, cqe); + + if (io_uring_peek_cqe(ring, &cqe) >= 0) { + fprintf(stderr, "single CQE expected %i\n", (int)cqe->user_data); + goto err; + } + return 0; +err: + return 1; +} + +static int test_ltimeout_cancel(struct io_uring *ring, int nr, int tout_idx, + bool async, int fail_idx) +{ + struct __kernel_timespec ts = {.tv_sec = 1, .tv_nsec = 0}; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + int e_res = 0, e_idx = nr - 1; + + if (fail_idx >= 0) { + e_res = -EFAULT; + e_idx = fail_idx; + } + + for (i = 0; i < nr; ++i) { + sqe = io_uring_get_sqe(ring); + if (i == fail_idx) + prep_exec_fail_req(sqe); + else + io_uring_prep_nop(sqe); + sqe->user_data = i; + sqe->flags |= IOSQE_IO_LINK; + if (async) + sqe->flags |= IOSQE_ASYNC; + if (i != nr - 1) + sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; + + if (i == tout_idx) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->flags |= IOSQE_IO_LINK | IOSQE_CQE_SKIP_SUCCESS; + sqe->user_data = TIMEOUT_USER_DATA; + } + } + + ret = io_uring_submit(ring); + if (ret != nr + 1) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + ret = io_uring_wait_cqe(ring, &cqe); + if (ret != 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->user_data != e_idx) { + fprintf(stderr, "invalid user_data %i\n", (int)cqe->user_data); + goto err; + } + if (cqe->res != e_res) { + fprintf(stderr, "unexpected res: %d\n", cqe->res); + goto err; + } + io_uring_cqe_seen(ring, cqe); + + if (io_uring_peek_cqe(ring, &cqe) >= 0) { + fprintf(stderr, "single CQE expected %i\n", (int)cqe->user_data); + goto err; + } + return 0; +err: + return 1; +} + +static int test_ltimeout_fire(struct io_uring *ring, bool async, + bool skip_main, bool skip_tout) +{ + char buf[1]; + struct __kernel_timespec ts = {.tv_sec = 0, .tv_nsec = 1000000}; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + int nr = 1 + !skip_tout; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->flags |= async ? IOSQE_ASYNC : 0; + sqe->flags |= skip_main ? IOSQE_CQE_SKIP_SUCCESS : 0; + sqe->user_data = 0; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_link_timeout(sqe, &ts, 0); + sqe->flags |= skip_tout ? IOSQE_CQE_SKIP_SUCCESS : 0; + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret != 2) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return 1; + } + + for (i = 0; i < nr; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret != 0) { + fprintf(stderr, "wait completion %d\n", ret); + return 1; + } + switch (cqe->user_data) { + case 0: + if (cqe->res != -ECANCELED && cqe->res != -EINTR) { + fprintf(stderr, "unexpected read return: %d\n", cqe->res); + return 1; + } + break; + case 1: + if (skip_tout) { + fprintf(stderr, "extra timeout cqe, %d\n", cqe->res); + return 1; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + + if (io_uring_peek_cqe(ring, &cqe) >= 0) { + fprintf(stderr, "single CQE expected: got data: %i res: %i\n", + (int)cqe->user_data, cqe->res); + return 1; + } + return 0; +} + +static int test_hardlink(struct io_uring *ring, int nr, int fail_idx, + int skip_idx, bool hardlink_last) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret, i; + + assert(fail_idx < nr); + assert(skip_idx < nr); + + for (i = 0; i < nr; i++) { + sqe = io_uring_get_sqe(ring); + if (i == fail_idx) + prep_exec_fail_req(sqe); + else + io_uring_prep_nop(sqe); + if (i != nr - 1 || hardlink_last) + sqe->flags |= IOSQE_IO_HARDLINK; + if (i == skip_idx) + sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; + sqe->user_data = i; + } + + ret = io_uring_submit(ring); + if (ret != nr) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < nr; i++) { + if (i == skip_idx && fail_idx != skip_idx) + continue; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret != 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->user_data != i) { + fprintf(stderr, "invalid user_data %d (%i)\n", + (int)cqe->user_data, i); + goto err; + } + if (i == fail_idx) { + if (cqe->res >= 0) { + fprintf(stderr, "req should've failed %d %d\n", + (int)cqe->user_data, cqe->res); + goto err; + } + } else { + if (cqe->res) { + fprintf(stderr, "req error %d %d\n", + (int)cqe->user_data, cqe->res); + goto err; + } + } + + io_uring_cqe_seen(ring, cqe); + } + + if (io_uring_peek_cqe(ring, &cqe) >= 0) { + fprintf(stderr, "single CQE expected %i\n", (int)cqe->user_data); + goto err; + } + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret, i, j, k; + int mid_idx = LINK_SIZE / 2; + int last_idx = LINK_SIZE - 1; + + if (argc > 1) + return 0; + + if (pipe(fds)) { + fprintf(stderr, "pipe() failed\n"); + return 1; + } + ret = io_uring_queue_init(16, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + if (!(ring.features & IORING_FEAT_CQE_SKIP)) { + printf("IOSQE_CQE_SKIP_SUCCESS is not supported, skip\n"); + return 0; + } + + for (i = 0; i < 4; i++) { + bool skip_last = i & 1; + int sz = (i & 2) ? LINK_SIZE : 1; + + ret = test_link_success(&ring, sz, skip_last); + if (ret) { + fprintf(stderr, "test_link_success sz %d, %d last\n", + skip_last, sz); + return ret; + } + } + + ret = test_link_fail(&ring, LINK_SIZE, mid_idx); + if (ret) { + fprintf(stderr, "test_link_fail mid failed\n"); + return ret; + } + + ret = test_link_fail(&ring, LINK_SIZE, last_idx); + if (ret) { + fprintf(stderr, "test_link_fail last failed\n"); + return ret; + } + + for (i = 0; i < 2; i++) { + bool async = i & 1; + + ret = test_ltimeout_cancel(&ring, 1, 0, async, -1); + if (ret) { + fprintf(stderr, "test_ltimeout_cancel 1 failed, %i\n", + async); + return ret; + } + ret = test_ltimeout_cancel(&ring, LINK_SIZE, mid_idx, async, -1); + if (ret) { + fprintf(stderr, "test_ltimeout_cancel mid failed, %i\n", + async); + return ret; + } + ret = test_ltimeout_cancel(&ring, LINK_SIZE, last_idx, async, -1); + if (ret) { + fprintf(stderr, "test_ltimeout_cancel last failed, %i\n", + async); + return ret; + } + ret = test_ltimeout_cancel(&ring, LINK_SIZE, mid_idx, async, mid_idx); + if (ret) { + fprintf(stderr, "test_ltimeout_cancel fail mid failed, %i\n", + async); + return ret; + } + ret = test_ltimeout_cancel(&ring, LINK_SIZE, mid_idx, async, mid_idx - 1); + if (ret) { + fprintf(stderr, "test_ltimeout_cancel fail2 mid failed, %i\n", + async); + return ret; + } + ret = test_ltimeout_cancel(&ring, LINK_SIZE, mid_idx, async, mid_idx + 1); + if (ret) { + fprintf(stderr, "test_ltimeout_cancel fail3 mid failed, %i\n", + async); + return ret; + } + } + + for (i = 0; i < 8; i++) { + bool async = i & 1; + bool skip1 = i & 2; + bool skip2 = i & 4; + + ret = test_ltimeout_fire(&ring, async, skip1, skip2); + if (ret) { + fprintf(stderr, "test_ltimeout_fire failed\n"); + return ret; + } + } + + /* test 3 positions, start/middle/end of the link, i.e. indexes 0, 3, 6 */ + for (i = 0; i < 3; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + bool mark_last = k & 1; + + ret = test_hardlink(&ring, 7, i * 3, j * 3, mark_last); + if (ret) { + fprintf(stderr, "test_hardlink failed" + "fail %i skip %i mark last %i\n", + i * 3, j * 3, k); + return 1; + } + } + } + } + + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring); + return 0; +} diff --git a/contrib/libs/liburing/test/socket-rw-eagain.c b/contrib/libs/liburing/test/socket-rw-eagain.c new file mode 100644 index 0000000000..e762da531f --- /dev/null +++ b/contrib/libs/liburing/test/socket-rw-eagain.c @@ -0,0 +1,149 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Check that a readv on a nonblocking socket queued before a writev doesn't + * wait for data to arrive. + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <assert.h> + +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <netinet/tcp.h> +#include <netinet/in.h> +#include <arpa/inet.h> + +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + int p_fd[2], ret; + int32_t recv_s0; + int32_t val = 1; + struct sockaddr_in addr; + struct iovec iov_r[1], iov_w[1]; + + if (argc > 1) + return 0; + + srand(getpid()); + + recv_s0 = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + + ret = setsockopt(recv_s0, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + assert(ret != -1); + ret = setsockopt(recv_s0, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + assert(ret != -1); + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr("127.0.0.1"); + assert(!t_bind_ephemeral_port(recv_s0, &addr)); + ret = listen(recv_s0, 128); + assert(ret != -1); + + p_fd[1] = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + + val = 1; + ret = setsockopt(p_fd[1], IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)); + assert(ret != -1); + + int32_t flags = fcntl(p_fd[1], F_GETFL, 0); + assert(flags != -1); + + flags |= O_NONBLOCK; + ret = fcntl(p_fd[1], F_SETFL, flags); + assert(ret != -1); + + ret = connect(p_fd[1], (struct sockaddr*)&addr, sizeof(addr)); + assert(ret == -1); + + p_fd[0] = accept(recv_s0, NULL, NULL); + assert(p_fd[0] != -1); + + flags = fcntl(p_fd[0], F_GETFL, 0); + assert(flags != -1); + + flags |= O_NONBLOCK; + ret = fcntl(p_fd[0], F_SETFL, flags); + assert(ret != -1); + + while (1) { + int32_t code; + socklen_t code_len = sizeof(code); + + ret = getsockopt(p_fd[1], SOL_SOCKET, SO_ERROR, &code, &code_len); + assert(ret != -1); + + if (!code) + break; + } + + struct io_uring m_io_uring; + struct io_uring_params p = { }; + + ret = io_uring_queue_init_params(32, &m_io_uring, &p); + assert(ret >= 0); + + if (p.features & IORING_FEAT_FAST_POLL) + return 0; + + char recv_buff[128]; + char send_buff[128]; + + { + iov_r[0].iov_base = recv_buff; + iov_r[0].iov_len = sizeof(recv_buff); + + struct io_uring_sqe* sqe = io_uring_get_sqe(&m_io_uring); + assert(sqe != NULL); + + io_uring_prep_readv(sqe, p_fd[0], iov_r, 1, 0); + sqe->user_data = 1; + } + + { + iov_w[0].iov_base = send_buff; + iov_w[0].iov_len = sizeof(send_buff); + + struct io_uring_sqe* sqe = io_uring_get_sqe(&m_io_uring); + assert(sqe != NULL); + + io_uring_prep_writev(sqe, p_fd[1], iov_w, 1, 0); + sqe->user_data = 2; + } + + ret = io_uring_submit_and_wait(&m_io_uring, 2); + assert(ret != -1); + + struct io_uring_cqe* cqe; + uint32_t head; + uint32_t count = 0; + + while (count != 2) { + io_uring_for_each_cqe(&m_io_uring, head, cqe) { + if (cqe->user_data == 2 && cqe->res != 128) { + fprintf(stderr, "write=%d\n", cqe->res); + goto err; + } else if (cqe->user_data == 1 && cqe->res != -EAGAIN) { + fprintf(stderr, "read=%d\n", cqe->res); + goto err; + } + count++; + } + + assert(count <= 2); + io_uring_cq_advance(&m_io_uring, count); + } + + io_uring_queue_exit(&m_io_uring); + return 0; +err: + io_uring_queue_exit(&m_io_uring); + return 1; +} diff --git a/contrib/libs/liburing/test/socket-rw-offset.c b/contrib/libs/liburing/test/socket-rw-offset.c new file mode 100644 index 0000000000..26173876ef --- /dev/null +++ b/contrib/libs/liburing/test/socket-rw-offset.c @@ -0,0 +1,149 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Check that a readv on a socket queued before a writev doesn't hang + * the processing. + * + * From Hrvoje Zeba <zeba.hrvoje@gmail.com> + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <assert.h> + +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <netinet/tcp.h> +#include <netinet/in.h> +#include <arpa/inet.h> + +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + int p_fd[2], ret; + int32_t recv_s0; + int32_t val = 1; + struct sockaddr_in addr; + struct iovec iov_r[1], iov_w[1]; + + if (argc > 1) + return 0; + + srand(getpid()); + + recv_s0 = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + + ret = setsockopt(recv_s0, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + assert(ret != -1); + ret = setsockopt(recv_s0, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + assert(ret != -1); + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr("127.0.0.1"); + assert(!t_bind_ephemeral_port(recv_s0, &addr)); + ret = listen(recv_s0, 128); + assert(ret != -1); + + + p_fd[1] = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + + val = 1; + ret = setsockopt(p_fd[1], IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)); + assert(ret != -1); + + int32_t flags = fcntl(p_fd[1], F_GETFL, 0); + assert(flags != -1); + + flags |= O_NONBLOCK; + ret = fcntl(p_fd[1], F_SETFL, flags); + assert(ret != -1); + + ret = connect(p_fd[1], (struct sockaddr*)&addr, sizeof(addr)); + assert(ret == -1); + + flags = fcntl(p_fd[1], F_GETFL, 0); + assert(flags != -1); + + flags &= ~O_NONBLOCK; + ret = fcntl(p_fd[1], F_SETFL, flags); + assert(ret != -1); + + p_fd[0] = accept(recv_s0, NULL, NULL); + assert(p_fd[0] != -1); + + while (1) { + int32_t code; + socklen_t code_len = sizeof(code); + + ret = getsockopt(p_fd[1], SOL_SOCKET, SO_ERROR, &code, &code_len); + assert(ret != -1); + + if (!code) + break; + } + + struct io_uring m_io_uring; + struct io_uring_params p = { }; + + ret = io_uring_queue_init_params(32, &m_io_uring, &p); + assert(ret >= 0); + + /* skip for kernels without cur position read/write */ + if (!(p.features & IORING_FEAT_RW_CUR_POS)) + return 0; + + char recv_buff[128]; + char send_buff[128]; + + { + iov_r[0].iov_base = recv_buff; + iov_r[0].iov_len = sizeof(recv_buff); + + struct io_uring_sqe* sqe = io_uring_get_sqe(&m_io_uring); + assert(sqe != NULL); + + io_uring_prep_readv(sqe, p_fd[0], iov_r, 1, -1); + } + + { + iov_w[0].iov_base = send_buff; + iov_w[0].iov_len = sizeof(send_buff); + + struct io_uring_sqe* sqe = io_uring_get_sqe(&m_io_uring); + assert(sqe != NULL); + + io_uring_prep_writev(sqe, p_fd[1], iov_w, 1, 0); + } + + ret = io_uring_submit_and_wait(&m_io_uring, 2); + assert(ret != -1); + + struct io_uring_cqe* cqe; + uint32_t head; + uint32_t count = 0; + + ret = 0; + while (count != 2) { + io_uring_for_each_cqe(&m_io_uring, head, cqe) { + if (cqe->res != 128) { + fprintf(stderr, "Got %d, expected 128\n", cqe->res); + ret = 1; + goto err; + } + assert(cqe->res == 128); + count++; + } + + assert(count <= 2); + io_uring_cq_advance(&m_io_uring, count); + } + +err: + io_uring_queue_exit(&m_io_uring); + return ret; +} diff --git a/contrib/libs/liburing/test/socket-rw.c b/contrib/libs/liburing/test/socket-rw.c new file mode 100644 index 0000000000..5546714e65 --- /dev/null +++ b/contrib/libs/liburing/test/socket-rw.c @@ -0,0 +1,137 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Check that a readv on a socket queued before a writev doesn't hang + * the processing. + * + * From Hrvoje Zeba <zeba.hrvoje@gmail.com> + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <assert.h> + +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <netinet/tcp.h> +#include <netinet/in.h> +#include <arpa/inet.h> + +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + int p_fd[2], ret; + int32_t recv_s0; + int32_t val = 1; + struct sockaddr_in addr; + struct iovec iov_r[1], iov_w[1]; + + if (argc > 1) + return 0; + + srand(getpid()); + + recv_s0 = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + + ret = setsockopt(recv_s0, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); + assert(ret != -1); + ret = setsockopt(recv_s0, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + assert(ret != -1); + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr("127.0.0.1"); + assert(!t_bind_ephemeral_port(recv_s0, &addr)); + ret = listen(recv_s0, 128); + assert(ret != -1); + + + p_fd[1] = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + + val = 1; + ret = setsockopt(p_fd[1], IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)); + assert(ret != -1); + + int32_t flags = fcntl(p_fd[1], F_GETFL, 0); + assert(flags != -1); + + flags |= O_NONBLOCK; + ret = fcntl(p_fd[1], F_SETFL, flags); + assert(ret != -1); + + ret = connect(p_fd[1], (struct sockaddr*)&addr, sizeof(addr)); + assert(ret == -1); + + flags = fcntl(p_fd[1], F_GETFL, 0); + assert(flags != -1); + + flags &= ~O_NONBLOCK; + ret = fcntl(p_fd[1], F_SETFL, flags); + assert(ret != -1); + + p_fd[0] = accept(recv_s0, NULL, NULL); + assert(p_fd[0] != -1); + + while (1) { + int32_t code; + socklen_t code_len = sizeof(code); + + ret = getsockopt(p_fd[1], SOL_SOCKET, SO_ERROR, &code, &code_len); + assert(ret != -1); + + if (!code) + break; + } + + struct io_uring m_io_uring; + + ret = io_uring_queue_init(32, &m_io_uring, 0); + assert(ret >= 0); + + char recv_buff[128]; + char send_buff[128]; + + { + iov_r[0].iov_base = recv_buff; + iov_r[0].iov_len = sizeof(recv_buff); + + struct io_uring_sqe* sqe = io_uring_get_sqe(&m_io_uring); + assert(sqe != NULL); + + io_uring_prep_readv(sqe, p_fd[0], iov_r, 1, 0); + } + + { + iov_w[0].iov_base = send_buff; + iov_w[0].iov_len = sizeof(send_buff); + + struct io_uring_sqe* sqe = io_uring_get_sqe(&m_io_uring); + assert(sqe != NULL); + + io_uring_prep_writev(sqe, p_fd[1], iov_w, 1, 0); + } + + ret = io_uring_submit_and_wait(&m_io_uring, 2); + assert(ret != -1); + + struct io_uring_cqe* cqe; + uint32_t head; + uint32_t count = 0; + + while (count != 2) { + io_uring_for_each_cqe(&m_io_uring, head, cqe) { + assert(cqe->res == 128); + count++; + } + + assert(count <= 2); + io_uring_cq_advance(&m_io_uring, count); + } + + io_uring_queue_exit(&m_io_uring); + return 0; +} diff --git a/contrib/libs/liburing/test/socket.c b/contrib/libs/liburing/test/socket.c new file mode 100644 index 0000000000..c3250c8dc0 --- /dev/null +++ b/contrib/libs/liburing/test/socket.c @@ -0,0 +1,410 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Simple test case using the socket op + */ +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <arpa/inet.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <pthread.h> +#include <assert.h> + +#include "liburing.h" +#include "helpers.h" + +static char str[] = "This is a test of send and recv over io_uring!"; + +#define MAX_MSG 128 + +#define HOST "127.0.0.1" + +static int no_socket; +static __be32 g_port; + +static int recv_prep(struct io_uring *ring, struct iovec *iov, int *sock, + int registerfiles) +{ + struct sockaddr_in saddr; + struct io_uring_sqe *sqe; + int sockfd, ret, val, use_fd; + + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_addr.s_addr = htonl(INADDR_ANY); + + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (sockfd < 0) { + perror("socket"); + return 1; + } + + val = 1; + setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + + if (t_bind_ephemeral_port(sockfd, &saddr)) { + perror("bind"); + goto err; + } + g_port = saddr.sin_port; + + if (registerfiles) { + ret = io_uring_register_files(ring, &sockfd, 1); + if (ret) { + fprintf(stderr, "file reg failed\n"); + goto err; + } + use_fd = 0; + } else { + use_fd = sockfd; + } + + sqe = io_uring_get_sqe(ring); + io_uring_prep_recv(sqe, use_fd, iov->iov_base, iov->iov_len, 0); + if (registerfiles) + sqe->flags |= IOSQE_FIXED_FILE; + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + *sock = sockfd; + return 0; +err: + close(sockfd); + return 1; +} + +static int do_recv(struct io_uring *ring, struct iovec *iov) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stdout, "wait_cqe: %d\n", ret); + goto err; + } + if (cqe->res == -EINVAL) { + fprintf(stdout, "recv not supported, skipping\n"); + return 0; + } + if (cqe->res < 0) { + fprintf(stderr, "failed cqe: %d\n", cqe->res); + goto err; + } + + if (cqe->res -1 != strlen(str)) { + fprintf(stderr, "got wrong length: %d/%d\n", cqe->res, + (int) strlen(str) + 1); + goto err; + } + + if (strcmp(str, iov->iov_base)) { + fprintf(stderr, "string mismatch\n"); + goto err; + } + + return 0; +err: + return 1; +} + +struct recv_data { + pthread_mutex_t mutex; + int use_sqthread; + int registerfiles; +}; + +static void *recv_fn(void *data) +{ + struct recv_data *rd = data; + char buf[MAX_MSG + 1]; + struct iovec iov = { + .iov_base = buf, + .iov_len = sizeof(buf) - 1, + }; + struct io_uring_params p = { }; + struct io_uring ring; + int ret, sock; + + if (rd->use_sqthread) + p.flags = IORING_SETUP_SQPOLL; + ret = t_create_ring_params(1, &ring, &p); + if (ret == T_SETUP_SKIP) { + pthread_mutex_unlock(&rd->mutex); + ret = 0; + goto err; + } else if (ret < 0) { + pthread_mutex_unlock(&rd->mutex); + goto err; + } + + if (rd->use_sqthread && !rd->registerfiles) { + if (!(p.features & IORING_FEAT_SQPOLL_NONFIXED)) { + fprintf(stdout, "Non-registered SQPOLL not available, skipping\n"); + pthread_mutex_unlock(&rd->mutex); + goto err; + } + } + + ret = recv_prep(&ring, &iov, &sock, rd->registerfiles); + if (ret) { + fprintf(stderr, "recv_prep failed: %d\n", ret); + goto err; + } + pthread_mutex_unlock(&rd->mutex); + ret = do_recv(&ring, &iov); + + close(sock); + io_uring_queue_exit(&ring); +err: + return (void *)(intptr_t)ret; +} + +static int fallback_send(struct io_uring *ring, struct sockaddr_in *saddr) +{ + struct iovec iov = { + .iov_base = str, + .iov_len = sizeof(str), + }; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int sockfd, ret; + + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (sockfd < 0) { + perror("socket"); + return 1; + } + + ret = connect(sockfd, (struct sockaddr *)saddr, sizeof(*saddr)); + if (ret < 0) { + perror("connect"); + return 1; + } + + sqe = io_uring_get_sqe(ring); + io_uring_prep_send(sqe, sockfd, iov.iov_base, iov.iov_len, 0); + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (cqe->res == -EINVAL) { + fprintf(stdout, "send not supported, skipping\n"); + close(sockfd); + return 0; + } + if (cqe->res != iov.iov_len) { + fprintf(stderr, "failed cqe: %d\n", cqe->res); + goto err; + } + + close(sockfd); + return 0; +err: + close(sockfd); + return 1; +} + +static int do_send(int socket_direct, int alloc) +{ + struct sockaddr_in saddr; + struct iovec iov = { + .iov_base = str, + .iov_len = sizeof(str), + }; + struct io_uring ring; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int sockfd, ret, fd = -1; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return 1; + } + + if (socket_direct) { + ret = io_uring_register_files(&ring, &fd, 1); + if (ret) { + fprintf(stderr, "file register %d\n", ret); + return 1; + } + } + + assert(g_port != 0); + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_port = g_port; + inet_pton(AF_INET, HOST, &saddr.sin_addr); + + sqe = io_uring_get_sqe(&ring); + if (socket_direct) { + unsigned file_index = 0; + if (alloc) + file_index = IORING_FILE_INDEX_ALLOC - 1; + io_uring_prep_socket_direct(sqe, AF_INET, SOCK_DGRAM, 0, + file_index, 0); + } else { + io_uring_prep_socket(sqe, AF_INET, SOCK_DGRAM, 0, 0); + } + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "socket submit: %d\n", ret); + return 1; + } + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe: %d\n", ret); + return 1; + } + if (cqe->res < 0) { + if (cqe->res == -EINVAL) { + fprintf(stdout, "No socket support, skipping\n"); + no_socket = 1; + io_uring_cqe_seen(&ring, cqe); + return fallback_send(&ring, &saddr); + } + + fprintf(stderr, "socket res: %d\n", ret); + return 1; + } + + sockfd = cqe->res; + if (socket_direct && !alloc) + sockfd = 0; + io_uring_cqe_seen(&ring, cqe); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_connect(sqe, sockfd, (struct sockaddr *) &saddr, + sizeof(saddr)); + if (socket_direct) + sqe->flags |= IOSQE_FIXED_FILE; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "connect submit: %d\n", ret); + return 1; + } + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe: %d\n", ret); + return 1; + } + if (cqe->res < 0) { + fprintf(stderr, "connect res: %d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_send(sqe, sockfd, iov.iov_base, iov.iov_len, 0); + sqe->user_data = 1; + if (socket_direct) + sqe->flags |= IOSQE_FIXED_FILE; + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (cqe->res == -EINVAL) { + fprintf(stdout, "send not supported, skipping\n"); + close(sockfd); + return 0; + } + if (cqe->res != iov.iov_len) { + fprintf(stderr, "failed cqe: %d\n", cqe->res); + goto err; + } + + close(sockfd); + return 0; +err: + close(sockfd); + return 1; +} + +static int test(int use_sqthread, int regfiles, int socket_direct, int alloc) +{ + pthread_mutexattr_t attr; + pthread_t recv_thread; + struct recv_data rd; + int ret; + void *retval; + + pthread_mutexattr_init(&attr); + pthread_mutexattr_setpshared(&attr, 1); + pthread_mutex_init(&rd.mutex, &attr); + pthread_mutex_lock(&rd.mutex); + rd.use_sqthread = use_sqthread; + rd.registerfiles = regfiles; + + ret = pthread_create(&recv_thread, NULL, recv_fn, &rd); + if (ret) { + fprintf(stderr, "Thread create failed: %d\n", ret); + pthread_mutex_unlock(&rd.mutex); + return 1; + } + + pthread_mutex_lock(&rd.mutex); + do_send(socket_direct, alloc); + pthread_join(recv_thread, &retval); + return (intptr_t)retval; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test(0, 0, 0, 0); + if (ret) { + fprintf(stderr, "test sqthread=0 failed\n"); + return ret; + } + if (no_socket) + return 0; + + ret = test(1, 1, 0, 0); + if (ret) { + fprintf(stderr, "test sqthread=1 reg=1 failed\n"); + return ret; + } + + ret = test(1, 0, 0, 0); + if (ret) { + fprintf(stderr, "test sqthread=1 reg=0 failed\n"); + return ret; + } + + ret = test(0, 0, 1, 0); + if (ret) { + fprintf(stderr, "test sqthread=0 direct=1 failed\n"); + return ret; + } + + ret = test(0, 0, 1, 1); + if (ret) { + fprintf(stderr, "test sqthread=0 direct=alloc failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/splice.c b/contrib/libs/liburing/test/splice.c new file mode 100644 index 0000000000..d4042a8fdb --- /dev/null +++ b/contrib/libs/liburing/test/splice.c @@ -0,0 +1,513 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/mman.h> + +#include "helpers.h" +#include "liburing.h" + +#define BUF_SIZE (16 * 4096) + +struct test_ctx { + int real_pipe1[2]; + int real_pipe2[2]; + int real_fd_in; + int real_fd_out; + + /* fds or for registered files */ + int pipe1[2]; + int pipe2[2]; + int fd_in; + int fd_out; + + void *buf_in; + void *buf_out; +}; + +static unsigned int splice_flags = 0; +static unsigned int sqe_flags = 0; +static int has_splice = 0; +static int has_tee = 0; + +static int read_buf(int fd, void *buf, int len) +{ + int ret; + + while (len) { + ret = read(fd, buf, len); + if (ret < 0) + return ret; + len -= ret; + buf += ret; + } + return 0; +} + +static int write_buf(int fd, const void *buf, int len) +{ + int ret; + + while (len) { + ret = write(fd, buf, len); + if (ret < 0) + return ret; + len -= ret; + buf += ret; + } + return 0; +} + +static int check_content(int fd, void *buf, int len, const void *src) +{ + int ret; + + ret = read_buf(fd, buf, len); + if (ret) + return ret; + + ret = memcmp(buf, src, len); + return (ret != 0) ? -1 : 0; +} + +static int create_file(const char *filename) +{ + int fd, save_errno; + + fd = open(filename, O_RDWR | O_CREAT, 0644); + save_errno = errno; + unlink(filename); + errno = save_errno; + return fd; +} + +static int init_splice_ctx(struct test_ctx *ctx) +{ + int ret, rnd_fd; + + ctx->buf_in = t_calloc(BUF_SIZE, 1); + ctx->buf_out = t_calloc(BUF_SIZE, 1); + + ctx->fd_in = create_file(".splice-test-in"); + if (ctx->fd_in < 0) { + perror("file open"); + return 1; + } + + ctx->fd_out = create_file(".splice-test-out"); + if (ctx->fd_out < 0) { + perror("file open"); + return 1; + } + + /* get random data */ + rnd_fd = open("/dev/urandom", O_RDONLY); + if (rnd_fd < 0) + return 1; + + ret = read_buf(rnd_fd, ctx->buf_in, BUF_SIZE); + if (ret != 0) + return 1; + close(rnd_fd); + + /* populate file */ + ret = write_buf(ctx->fd_in, ctx->buf_in, BUF_SIZE); + if (ret) + return ret; + + if (pipe(ctx->pipe1) < 0) + return 1; + if (pipe(ctx->pipe2) < 0) + return 1; + + ctx->real_pipe1[0] = ctx->pipe1[0]; + ctx->real_pipe1[1] = ctx->pipe1[1]; + ctx->real_pipe2[0] = ctx->pipe2[0]; + ctx->real_pipe2[1] = ctx->pipe2[1]; + ctx->real_fd_in = ctx->fd_in; + ctx->real_fd_out = ctx->fd_out; + return 0; +} + +static int do_splice_op(struct io_uring *ring, + int fd_in, loff_t off_in, + int fd_out, loff_t off_out, + unsigned int len, + __u8 opcode) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret = -1; + + do { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return -1; + } + io_uring_prep_splice(sqe, fd_in, off_in, fd_out, off_out, + len, splice_flags); + sqe->flags |= sqe_flags; + sqe->user_data = 42; + sqe->opcode = opcode; + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return ret; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", cqe->res); + return ret; + } + + if (cqe->res <= 0) { + io_uring_cqe_seen(ring, cqe); + return cqe->res; + } + + len -= cqe->res; + if (off_in != -1) + off_in += cqe->res; + if (off_out != -1) + off_out += cqe->res; + io_uring_cqe_seen(ring, cqe); + } while (len); + + return 0; +} + +static int do_splice(struct io_uring *ring, + int fd_in, loff_t off_in, + int fd_out, loff_t off_out, + unsigned int len) +{ + return do_splice_op(ring, fd_in, off_in, fd_out, off_out, len, + IORING_OP_SPLICE); +} + +static int do_tee(struct io_uring *ring, int fd_in, int fd_out, + unsigned int len) +{ + return do_splice_op(ring, fd_in, 0, fd_out, 0, len, IORING_OP_TEE); +} + +static void check_splice_support(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = do_splice(ring, -1, 0, -1, 0, BUF_SIZE); + has_splice = (ret == -EBADF); +} + +static void check_tee_support(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = do_tee(ring, -1, -1, BUF_SIZE); + has_tee = (ret == -EBADF); +} + +static int check_zero_splice(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = do_splice(ring, ctx->fd_in, -1, ctx->pipe1[1], -1, 0); + if (ret) + return ret; + + ret = do_splice(ring, ctx->pipe2[0], -1, ctx->pipe1[1], -1, 0); + if (ret) + return ret; + + return 0; +} + +static int splice_to_pipe(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = lseek(ctx->real_fd_in, 0, SEEK_SET); + if (ret) + return ret; + + /* implicit file offset */ + ret = do_splice(ring, ctx->fd_in, -1, ctx->pipe1[1], -1, BUF_SIZE); + if (ret) + return ret; + + ret = check_content(ctx->real_pipe1[0], ctx->buf_out, BUF_SIZE, + ctx->buf_in); + if (ret) + return ret; + + /* explicit file offset */ + ret = do_splice(ring, ctx->fd_in, 0, ctx->pipe1[1], -1, BUF_SIZE); + if (ret) + return ret; + + return check_content(ctx->real_pipe1[0], ctx->buf_out, BUF_SIZE, + ctx->buf_in); +} + +static int splice_from_pipe(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = write_buf(ctx->real_pipe1[1], ctx->buf_in, BUF_SIZE); + if (ret) + return ret; + ret = do_splice(ring, ctx->pipe1[0], -1, ctx->fd_out, 0, BUF_SIZE); + if (ret) + return ret; + ret = check_content(ctx->real_fd_out, ctx->buf_out, BUF_SIZE, + ctx->buf_in); + if (ret) + return ret; + + ret = ftruncate(ctx->real_fd_out, 0); + if (ret) + return ret; + return lseek(ctx->real_fd_out, 0, SEEK_SET); +} + +static int splice_pipe_to_pipe(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = do_splice(ring, ctx->fd_in, 0, ctx->pipe1[1], -1, BUF_SIZE); + if (ret) + return ret; + ret = do_splice(ring, ctx->pipe1[0], -1, ctx->pipe2[1], -1, BUF_SIZE); + if (ret) + return ret; + + return check_content(ctx->real_pipe2[0], ctx->buf_out, BUF_SIZE, + ctx->buf_in); +} + +static int fail_splice_pipe_offset(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = do_splice(ring, ctx->fd_in, 0, ctx->pipe1[1], 0, BUF_SIZE); + if (ret != -ESPIPE && ret != -EINVAL) + return ret; + + ret = do_splice(ring, ctx->pipe1[0], 0, ctx->fd_out, 0, BUF_SIZE); + if (ret != -ESPIPE && ret != -EINVAL) + return ret; + + return 0; +} + +static int fail_tee_nonpipe(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = do_tee(ring, ctx->fd_in, ctx->pipe1[1], BUF_SIZE); + if (ret != -ESPIPE && ret != -EINVAL) + return ret; + + return 0; +} + +static int fail_tee_offset(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = do_splice_op(ring, ctx->pipe2[0], -1, ctx->pipe1[1], 0, + BUF_SIZE, IORING_OP_TEE); + if (ret != -ESPIPE && ret != -EINVAL) + return ret; + + ret = do_splice_op(ring, ctx->pipe2[0], 0, ctx->pipe1[1], -1, + BUF_SIZE, IORING_OP_TEE); + if (ret != -ESPIPE && ret != -EINVAL) + return ret; + + return 0; +} + +static int check_tee(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + ret = write_buf(ctx->real_pipe1[1], ctx->buf_in, BUF_SIZE); + if (ret) + return ret; + ret = do_tee(ring, ctx->pipe1[0], ctx->pipe2[1], BUF_SIZE); + if (ret) + return ret; + + ret = check_content(ctx->real_pipe1[0], ctx->buf_out, BUF_SIZE, + ctx->buf_in); + if (ret) { + fprintf(stderr, "tee(), invalid src data\n"); + return ret; + } + + ret = check_content(ctx->real_pipe2[0], ctx->buf_out, BUF_SIZE, + ctx->buf_in); + if (ret) { + fprintf(stderr, "tee(), invalid dst data\n"); + return ret; + } + + return 0; +} + +static int check_zero_tee(struct io_uring *ring, struct test_ctx *ctx) +{ + return do_tee(ring, ctx->pipe2[0], ctx->pipe1[1], 0); +} + +static int test_splice(struct io_uring *ring, struct test_ctx *ctx) +{ + int ret; + + if (has_splice) { + ret = check_zero_splice(ring, ctx); + if (ret) { + fprintf(stderr, "check_zero_splice failed %i %i\n", + ret, errno); + return ret; + } + + ret = splice_to_pipe(ring, ctx); + if (ret) { + fprintf(stderr, "splice_to_pipe failed %i %i\n", + ret, errno); + return ret; + } + + ret = splice_from_pipe(ring, ctx); + if (ret) { + fprintf(stderr, "splice_from_pipe failed %i %i\n", + ret, errno); + return ret; + } + + ret = splice_pipe_to_pipe(ring, ctx); + if (ret) { + fprintf(stderr, "splice_pipe_to_pipe failed %i %i\n", + ret, errno); + return ret; + } + + ret = fail_splice_pipe_offset(ring, ctx); + if (ret) { + fprintf(stderr, "fail_splice_pipe_offset failed %i %i\n", + ret, errno); + return ret; + } + } + + if (has_tee) { + ret = check_zero_tee(ring, ctx); + if (ret) { + fprintf(stderr, "check_zero_tee() failed %i %i\n", + ret, errno); + return ret; + } + + ret = fail_tee_nonpipe(ring, ctx); + if (ret) { + fprintf(stderr, "fail_tee_nonpipe() failed %i %i\n", + ret, errno); + return ret; + } + + ret = fail_tee_offset(ring, ctx); + if (ret) { + fprintf(stderr, "fail_tee_offset failed %i %i\n", + ret, errno); + return ret; + } + + ret = check_tee(ring, ctx); + if (ret) { + fprintf(stderr, "check_tee() failed %i %i\n", + ret, errno); + return ret; + } + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct io_uring_params p = { }; + struct test_ctx ctx; + int ret; + int reg_fds[6]; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init_params(8, &ring, &p); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + if (!(p.features & IORING_FEAT_FAST_POLL)) { + fprintf(stdout, "No splice support, skipping\n"); + return 0; + } + + ret = init_splice_ctx(&ctx); + if (ret) { + fprintf(stderr, "init failed %i %i\n", ret, errno); + return 1; + } + + check_splice_support(&ring, &ctx); + if (!has_splice) + fprintf(stdout, "skip, doesn't support splice()\n"); + check_tee_support(&ring, &ctx); + if (!has_tee) + fprintf(stdout, "skip, doesn't support tee()\n"); + + ret = test_splice(&ring, &ctx); + if (ret) { + fprintf(stderr, "basic splice tests failed\n"); + return ret; + } + + reg_fds[0] = ctx.real_pipe1[0]; + reg_fds[1] = ctx.real_pipe1[1]; + reg_fds[2] = ctx.real_pipe2[0]; + reg_fds[3] = ctx.real_pipe2[1]; + reg_fds[4] = ctx.real_fd_in; + reg_fds[5] = ctx.real_fd_out; + ret = io_uring_register_files(&ring, reg_fds, 6); + if (ret) { + fprintf(stderr, "%s: register ret=%d\n", __FUNCTION__, ret); + return 1; + } + + /* remap fds to registered */ + ctx.pipe1[0] = 0; + ctx.pipe1[1] = 1; + ctx.pipe2[0] = 2; + ctx.pipe2[1] = 3; + ctx.fd_in = 4; + ctx.fd_out = 5; + + splice_flags = SPLICE_F_FD_IN_FIXED; + sqe_flags = IOSQE_FIXED_FILE; + ret = test_splice(&ring, &ctx); + if (ret) { + fprintf(stderr, "registered fds splice tests failed\n"); + return ret; + } + return 0; +} diff --git a/contrib/libs/liburing/test/sq-full-cpp.cc b/contrib/libs/liburing/test/sq-full-cpp.cc new file mode 100644 index 0000000000..f825596bdf --- /dev/null +++ b/contrib/libs/liburing/test/sq-full-cpp.cc @@ -0,0 +1,46 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test SQ queue full condition + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_sqe *sqe; + struct io_uring ring; + int ret, i; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + + } + + i = 0; + while ((sqe = io_uring_get_sqe(&ring)) != NULL) + i++; + + if (i != 8) { + fprintf(stderr, "Got %d SQEs, wanted 8\n", i); + goto err; + } + + io_uring_queue_exit(&ring); + return 0; +err: + io_uring_queue_exit(&ring); + return 1; +} diff --git a/contrib/libs/liburing/test/sq-full.c b/contrib/libs/liburing/test/sq-full.c new file mode 100644 index 0000000000..f825596bdf --- /dev/null +++ b/contrib/libs/liburing/test/sq-full.c @@ -0,0 +1,46 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test SQ queue full condition + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_sqe *sqe; + struct io_uring ring; + int ret, i; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + + } + + i = 0; + while ((sqe = io_uring_get_sqe(&ring)) != NULL) + i++; + + if (i != 8) { + fprintf(stderr, "Got %d SQEs, wanted 8\n", i); + goto err; + } + + io_uring_queue_exit(&ring); + return 0; +err: + io_uring_queue_exit(&ring); + return 1; +} diff --git a/contrib/libs/liburing/test/sq-poll-dup.c b/contrib/libs/liburing/test/sq-poll-dup.c new file mode 100644 index 0000000000..bbeb63e6cb --- /dev/null +++ b/contrib/libs/liburing/test/sq-poll-dup.c @@ -0,0 +1,205 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test SQPOLL with IORING_SETUP_ATTACH_WQ and closing of + * the original ring descriptor. + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/eventfd.h> +#include <sys/resource.h> + +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE (128 * 1024 * 1024) +#define BS 4096 +#define BUFFERS 64 + +#define NR_RINGS 4 + +static struct iovec *vecs; +static struct io_uring rings[NR_RINGS]; + +static int wait_io(struct io_uring *ring, int nr_ios) +{ + struct io_uring_cqe *cqe; + int ret; + + while (nr_ios) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait_ret=%d\n", ret); + return 1; + } + if (cqe->res != BS) { + fprintf(stderr, "Unexpected ret %d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + nr_ios--; + } + + return 0; +} + +static int queue_io(struct io_uring *ring, int fd, int nr_ios) +{ + unsigned long off; + int i; + + i = 0; + off = 0; + while (nr_ios) { + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + if (!sqe) + break; + io_uring_prep_read(sqe, fd, vecs[i].iov_base, vecs[i].iov_len, off); + nr_ios--; + i++; + off += BS; + } + + io_uring_submit(ring); + return i; +} + +static int do_io(int fd, int ring_start, int ring_end) +{ + int i, rets[NR_RINGS]; + unsigned ios = 0; + + while (ios < 32) { + for (i = ring_start; i < ring_end; i++) { + int ret = queue_io(&rings[i], fd, BUFFERS); + if (ret < 0) + goto err; + rets[i] = ret; + } + for (i = ring_start; i < ring_end; i++) { + if (wait_io(&rings[i], rets[i])) + goto err; + } + ios += BUFFERS; + } + + return 0; +err: + return 1; +} + +static int test(int fd, int do_dup_and_close, int close_ring) +{ + int i, ret, ring_fd; + + for (i = 0; i < NR_RINGS; i++) { + struct io_uring_params p = { }; + + p.flags = IORING_SETUP_SQPOLL; + p.sq_thread_idle = 100; + if (i) { + p.wq_fd = rings[0].ring_fd; + p.flags |= IORING_SETUP_ATTACH_WQ; + } + ret = io_uring_queue_init_params(BUFFERS, &rings[i], &p); + if (ret) { + fprintf(stderr, "queue_init: %d/%d\n", ret, i); + goto err; + } + /* no sharing for non-fixed either */ + if (!(p.features & IORING_FEAT_SQPOLL_NONFIXED)) { + fprintf(stdout, "No SQPOLL sharing, skipping\n"); + return 0; + } + } + + /* test all rings */ + if (do_io(fd, 0, NR_RINGS)) + goto err; + + /* dup and close original ring fd */ + ring_fd = dup(rings[0].ring_fd); + if (close_ring) + close(rings[0].ring_fd); + rings[0].ring_fd = rings[0].enter_ring_fd = ring_fd; + if (do_dup_and_close) + goto done; + + /* test all but closed one */ + if (do_io(fd, 1, NR_RINGS)) + goto err; + + /* test closed one */ + if (do_io(fd, 0, 1)) + goto err; + + /* make sure thread is idle so we enter the kernel */ + usleep(200000); + + /* test closed one */ + if (do_io(fd, 0, 1)) + goto err; + + +done: + for (i = 0; i < NR_RINGS; i++) + io_uring_queue_exit(&rings[i]); + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + char *fname; + int ret, fd; + + if (argc > 1) { + fname = argv[1]; + } else { + fname = ".basic-rw-poll-dup"; + t_create_file(fname, FILE_SIZE); + } + + vecs = t_create_buffers(BUFFERS, BS); + + fd = open(fname, O_RDONLY | O_DIRECT); + if (fname != argv[1]) + unlink(fname); + + if (fd < 0) { + perror("open"); + return -1; + } + + ret = test(fd, 0, 0); + if (ret) { + fprintf(stderr, "test 0 0 failed\n"); + goto err; + } + + ret = test(fd, 0, 1); + if (ret) { + fprintf(stderr, "test 0 1 failed\n"); + goto err; + } + + + ret = test(fd, 1, 0); + if (ret) { + fprintf(stderr, "test 1 0 failed\n"); + goto err; + } + + return 0; +err: + return 1; +} diff --git a/contrib/libs/liburing/test/sq-poll-kthread.c b/contrib/libs/liburing/test/sq-poll-kthread.c new file mode 100644 index 0000000000..4ec43a9323 --- /dev/null +++ b/contrib/libs/liburing/test/sq-poll-kthread.c @@ -0,0 +1,170 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test if io_uring SQ poll kthread is stopped when the userspace + * process ended with or without closing the io_uring fd + * + */ +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <unistd.h> +#include <pthread.h> +#include <stdbool.h> +#include <stdlib.h> +#include <string.h> +#include <signal.h> +#include <poll.h> +#include <sys/wait.h> +#include <sys/epoll.h> + +#include "liburing.h" +#include "helpers.h" + +#define SQ_THREAD_IDLE 2000 +#define BUF_SIZE 128 +#define KTHREAD_NAME "io_uring-sq" + +enum { + TEST_OK = 0, + TEST_SKIPPED = 1, + TEST_FAILED = 2, +}; + +static int do_test_sq_poll_kthread_stopped(bool do_exit) +{ + int ret = 0, pipe1[2]; + struct io_uring_params param; + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + uint8_t buf[BUF_SIZE]; + struct iovec iov; + + if (pipe(pipe1) != 0) { + perror("pipe"); + return TEST_FAILED; + } + + memset(¶m, 0, sizeof(param)); + param.flags |= IORING_SETUP_SQPOLL; + param.sq_thread_idle = SQ_THREAD_IDLE; + + ret = t_create_ring_params(16, &ring, ¶m); + if (ret == T_SETUP_SKIP) { + ret = TEST_FAILED; + goto err_pipe; + } else if (ret != T_SETUP_OK) { + fprintf(stderr, "ring setup failed\n"); + ret = TEST_FAILED; + goto err_pipe; + } + + ret = io_uring_register_files(&ring, &pipe1[1], 1); + if (ret) { + fprintf(stderr, "file reg failed: %d\n", ret); + ret = TEST_FAILED; + goto err_uring; + } + + iov.iov_base = buf; + iov.iov_len = BUF_SIZE; + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "io_uring_get_sqe failed\n"); + ret = TEST_FAILED; + goto err_uring; + } + + io_uring_prep_writev(sqe, 0, &iov, 1, 0); + sqe->flags |= IOSQE_FIXED_FILE; + + ret = io_uring_submit(&ring); + if (ret < 0) { + fprintf(stderr, "io_uring_submit failed - ret: %d\n", + ret); + ret = TEST_FAILED; + goto err_uring; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "io_uring_wait_cqe - ret: %d\n", + ret); + ret = TEST_FAILED; + goto err_uring; + } + + if (cqe->res != BUF_SIZE) { + fprintf(stderr, "unexpected cqe->res %d [expected %d]\n", + cqe->res, BUF_SIZE); + ret = TEST_FAILED; + goto err_uring; + + } + + io_uring_cqe_seen(&ring, cqe); + + ret = TEST_OK; + +err_uring: + if (do_exit) + io_uring_queue_exit(&ring); +err_pipe: + close(pipe1[0]); + close(pipe1[1]); + + return ret; +} + +int test_sq_poll_kthread_stopped(bool do_exit) +{ + pid_t pid; + int status = 0; + + pid = fork(); + + if (pid == 0) { + int ret = do_test_sq_poll_kthread_stopped(do_exit); + exit(ret); + } + + pid = wait(&status); + if (status != 0) + return WEXITSTATUS(status); + + sleep(1); + if (system("ps --ppid 2 | grep " KTHREAD_NAME) == 0) { + fprintf(stderr, "%s kthread still running!\n", KTHREAD_NAME); + return TEST_FAILED; + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test_sq_poll_kthread_stopped(true); + if (ret == TEST_SKIPPED) { + printf("test_sq_poll_kthread_stopped_exit: skipped\n"); + } else if (ret == TEST_FAILED) { + fprintf(stderr, "test_sq_poll_kthread_stopped_exit failed\n"); + return ret; + } + + ret = test_sq_poll_kthread_stopped(false); + if (ret == TEST_SKIPPED) { + printf("test_sq_poll_kthread_stopped_noexit: skipped\n"); + } else if (ret == TEST_FAILED) { + fprintf(stderr, "test_sq_poll_kthread_stopped_noexit failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/sq-poll-share.c b/contrib/libs/liburing/test/sq-poll-share.c new file mode 100644 index 0000000000..a2af97543f --- /dev/null +++ b/contrib/libs/liburing/test/sq-poll-share.c @@ -0,0 +1,138 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test SQPOLL with IORING_SETUP_ATTACH_WQ + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <poll.h> +#include <sys/eventfd.h> +#include <sys/resource.h> + +#include "helpers.h" +#include "liburing.h" + +#define FILE_SIZE (128 * 1024 * 1024) +#define BS 4096 +#define BUFFERS 64 + +#define NR_RINGS 4 + +static struct iovec *vecs; + +static int wait_io(struct io_uring *ring, int nr_ios) +{ + struct io_uring_cqe *cqe; + + while (nr_ios) { + int ret = io_uring_wait_cqe(ring, &cqe); + + if (ret == -EAGAIN) { + continue; + } else if (ret) { + fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret); + return 1; + } + if (cqe->res != BS) { + fprintf(stderr, "Unexpected ret %d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + nr_ios--; + } + + return 0; +} + +static int queue_io(struct io_uring *ring, int fd, int nr_ios) +{ + unsigned long off; + int i; + + i = 0; + off = 0; + while (nr_ios) { + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(ring); + if (!sqe) + break; + io_uring_prep_read(sqe, fd, vecs[i].iov_base, vecs[i].iov_len, off); + nr_ios--; + i++; + off += BS; + } + + io_uring_submit(ring); + return i; +} + +int main(int argc, char *argv[]) +{ + struct io_uring rings[NR_RINGS]; + int rets[NR_RINGS]; + unsigned long ios; + int i, ret, fd; + char *fname; + + if (argc > 1) { + fname = argv[1]; + } else { + fname = ".basic-rw-poll-share"; + t_create_file(fname, FILE_SIZE); + } + + vecs = t_create_buffers(BUFFERS, BS); + + fd = open(fname, O_RDONLY | O_DIRECT); + if (fname != argv[1]) + unlink(fname); + if (fd < 0) { + perror("open"); + return -1; + } + + for (i = 0; i < NR_RINGS; i++) { + struct io_uring_params p = { }; + + p.flags = IORING_SETUP_SQPOLL; + if (i) { + p.wq_fd = rings[0].ring_fd; + p.flags |= IORING_SETUP_ATTACH_WQ; + } + ret = io_uring_queue_init_params(BUFFERS, &rings[i], &p); + if (ret) { + fprintf(stderr, "queue_init: %d/%d\n", ret, i); + goto err; + } + /* no sharing for non-fixed either */ + if (!(p.features & IORING_FEAT_SQPOLL_NONFIXED)) { + fprintf(stdout, "No SQPOLL sharing, skipping\n"); + return 0; + } + } + + ios = 0; + while (ios < (FILE_SIZE / BS)) { + for (i = 0; i < NR_RINGS; i++) { + ret = queue_io(&rings[i], fd, BUFFERS); + if (ret < 0) + goto err; + rets[i] = ret; + } + for (i = 0; i < NR_RINGS; i++) { + if (wait_io(&rings[i], rets[i])) + goto err; + } + ios += BUFFERS; + } + + return 0; +err: + return 1; +} diff --git a/contrib/libs/liburing/test/sq-space_left.c b/contrib/libs/liburing/test/sq-space_left.c new file mode 100644 index 0000000000..9402377df7 --- /dev/null +++ b/contrib/libs/liburing/test/sq-space_left.c @@ -0,0 +1,160 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test SQ queue space left + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" + +static int test_left(void) +{ + struct io_uring_sqe *sqe; + struct io_uring ring; + int ret, i = 0, s; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + + } + + if ((s = io_uring_sq_space_left(&ring)) != 8) { + fprintf(stderr, "Got %d SQEs left, expected %d\n", s, 8); + goto err; + } + + i = 0; + while ((sqe = io_uring_get_sqe(&ring)) != NULL) { + i++; + if ((s = io_uring_sq_space_left(&ring)) != 8 - i) { + fprintf(stderr, "Got %d SQEs left, expected %d\n", s, 8 - i); + goto err; + } + } + + if (i != 8) { + fprintf(stderr, "Got %d SQEs, expected %d\n", i, 8); + goto err; + } + + io_uring_queue_exit(&ring); + return 0; +err: + io_uring_queue_exit(&ring); + return 1; +} + +static int test_sync(void) +{ + struct io_uring_sqe *sqe; + struct io_uring ring; + int ret, i; + + ret = io_uring_queue_init(32, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + + } + + /* prep 8 NOPS */ + for (i = 0; i < 8; i++) { + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + } + + /* prep known bad command, this should terminate submission */ + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + sqe->opcode = 0xfe; + + /* prep 8 NOPS */ + for (i = 0; i < 8; i++) { + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_nop(sqe); + } + + /* we should have 8 + 1 + 8 pending now */ + ret = io_uring_sq_ready(&ring); + if (ret != 17) { + fprintf(stderr, "%d ready, wanted 17\n", ret); + goto err; + } + + ret = io_uring_submit(&ring); + + /* should submit 8 successfully, then error #9 and stop */ + if (ret != 9) { + fprintf(stderr, "submitted %d, wanted 9\n", ret); + goto err; + } + + /* should now have 8 ready, with 9 gone */ + ret = io_uring_sq_ready(&ring); + if (ret != 8) { + fprintf(stderr, "%d ready, wanted 8\n", ret); + goto err; + } + + ret = io_uring_submit(&ring); + + /* the last 8 should submit fine */ + if (ret != 8) { + fprintf(stderr, "submitted %d, wanted 8\n", ret); + goto err; + } + + ret = io_uring_sq_ready(&ring); + if (ret) { + fprintf(stderr, "%d ready, wanted 0\n", ret); + goto err; + } + + io_uring_queue_exit(&ring); + return 0; +err: + io_uring_queue_exit(&ring); + return 1; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test_left(); + if (ret) { + fprintf(stderr, "test_left failed\n"); + return ret; + } + + ret = test_sync(); + if (ret) { + fprintf(stderr, "test_sync failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/sqpoll-cancel-hang.c b/contrib/libs/liburing/test/sqpoll-cancel-hang.c new file mode 100644 index 0000000000..81a30e27de --- /dev/null +++ b/contrib/libs/liburing/test/sqpoll-cancel-hang.c @@ -0,0 +1,158 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <fcntl.h> +#include <signal.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/wait.h> +#include <time.h> +#include <unistd.h> +#include "liburing.h" +#include "../src/syscall.h" + +static uint64_t current_time_ms(void) +{ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts)) + exit(1); + return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000; +} + +#define SIZEOF_IO_URING_SQE 64 +#define SIZEOF_IO_URING_CQE 16 +#define SQ_TAIL_OFFSET 64 +#define SQ_RING_MASK_OFFSET 256 +#define SQ_RING_ENTRIES_OFFSET 264 +#define CQ_RING_ENTRIES_OFFSET 268 +#define CQ_CQES_OFFSET 320 + +#define IORING_OFF_SQES 0x10000000ULL + +static void kill_and_wait(int pid, int* status) +{ + kill(-pid, SIGKILL); + kill(pid, SIGKILL); + while (waitpid(-1, status, __WALL) != pid) { + } +} + +#define WAIT_FLAGS __WALL + +uint64_t r[3] = {0xffffffffffffffff, 0x0, 0x0}; + +static long syz_io_uring_setup(volatile long a0, volatile long a1, +volatile long a2, volatile long a3, volatile long a4, volatile long +a5) +{ + uint32_t entries = (uint32_t)a0; + struct io_uring_params* setup_params = (struct io_uring_params*)a1; + void* vma1 = (void*)a2; + void* vma2 = (void*)a3; + void** ring_ptr_out = (void**)a4; + void** sqes_ptr_out = (void**)a5; + uint32_t fd_io_uring = __sys_io_uring_setup(entries, setup_params); + uint32_t sq_ring_sz = setup_params->sq_off.array + +setup_params->sq_entries * sizeof(uint32_t); + uint32_t cq_ring_sz = setup_params->cq_off.cqes + +setup_params->cq_entries * SIZEOF_IO_URING_CQE; + uint32_t ring_sz = sq_ring_sz > cq_ring_sz ? sq_ring_sz : cq_ring_sz; + *ring_ptr_out = mmap(vma1, ring_sz, PROT_READ | PROT_WRITE, +MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, +IORING_OFF_SQ_RING); + uint32_t sqes_sz = setup_params->sq_entries * SIZEOF_IO_URING_SQE; + *sqes_ptr_out = mmap(vma2, sqes_sz, PROT_READ | PROT_WRITE, +MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, IORING_OFF_SQES); + return fd_io_uring; +} + +static long syz_io_uring_submit(volatile long a0, volatile long a1, +volatile long a2, volatile long a3) +{ + char* ring_ptr = (char*)a0; + char* sqes_ptr = (char*)a1; + char* sqe = (char*)a2; + uint32_t sqes_index = (uint32_t)a3; + uint32_t sq_ring_entries = *(uint32_t*)(ring_ptr + SQ_RING_ENTRIES_OFFSET); + uint32_t cq_ring_entries = *(uint32_t*)(ring_ptr + CQ_RING_ENTRIES_OFFSET); + uint32_t sq_array_off = (CQ_CQES_OFFSET + cq_ring_entries * +SIZEOF_IO_URING_CQE + 63) & ~63; + if (sq_ring_entries) + sqes_index %= sq_ring_entries; + char* sqe_dest = sqes_ptr + sqes_index * SIZEOF_IO_URING_SQE; + memcpy(sqe_dest, sqe, SIZEOF_IO_URING_SQE); + uint32_t sq_ring_mask = *(uint32_t*)(ring_ptr + SQ_RING_MASK_OFFSET); + uint32_t* sq_tail_ptr = (uint32_t*)(ring_ptr + SQ_TAIL_OFFSET); + uint32_t sq_tail = *sq_tail_ptr & sq_ring_mask; + uint32_t sq_tail_next = *sq_tail_ptr + 1; + uint32_t* sq_array = (uint32_t*)(ring_ptr + sq_array_off); + *(sq_array + sq_tail) = sqes_index; + __atomic_store_n(sq_tail_ptr, sq_tail_next, __ATOMIC_RELEASE); + return 0; +} + + +void trigger_bug(void) +{ + intptr_t res = 0; + *(uint32_t*)0x20000204 = 0; + *(uint32_t*)0x20000208 = 2; + *(uint32_t*)0x2000020c = 0; + *(uint32_t*)0x20000210 = 0; + *(uint32_t*)0x20000218 = -1; + memset((void*)0x2000021c, 0, 12); + res = -1; + res = syz_io_uring_setup(0x7987, 0x20000200, 0x20400000, 0x20ffd000, 0x200000c0, 0x200001c0); + if (res != -1) { + r[0] = res; + r[1] = *(uint64_t*)0x200000c0; + r[2] = *(uint64_t*)0x200001c0; + } + *(uint8_t*)0x20000180 = 0xb; + *(uint8_t*)0x20000181 = 1; + *(uint16_t*)0x20000182 = 0; + *(uint32_t*)0x20000184 = 0; + *(uint64_t*)0x20000188 = 4; + *(uint64_t*)0x20000190 = 0x20000140; + *(uint64_t*)0x20000140 = 0x77359400; + *(uint64_t*)0x20000148 = 0; + *(uint32_t*)0x20000198 = 1; + *(uint32_t*)0x2000019c = 0; + *(uint64_t*)0x200001a0 = 0; + *(uint16_t*)0x200001a8 = 0; + *(uint16_t*)0x200001aa = 0; + memset((void*)0x200001ac, 0, 20); + syz_io_uring_submit(r[1], r[2], 0x20000180, 1); + *(uint32_t*)0x20000544 = 0; + *(uint32_t*)0x20000548 = 0x36; + *(uint32_t*)0x2000054c = 0; + *(uint32_t*)0x20000550 = 0; + *(uint32_t*)0x20000558 = r[0]; + memset((void*)0x2000055c, 0, 12); + +} +int main(void) +{ + mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul); + int pid = fork(); + if (pid < 0) + exit(1); + if (pid == 0) { + trigger_bug(); + exit(0); + } + int status = 0; + uint64_t start = current_time_ms(); + for (;;) { + if (current_time_ms() - start < 1000) { + continue; + } + kill_and_wait(pid, &status); + break; + } + return 0; +} + + + diff --git a/contrib/libs/liburing/test/sqpoll-disable-exit.c b/contrib/libs/liburing/test/sqpoll-disable-exit.c new file mode 100644 index 0000000000..b2a4160c58 --- /dev/null +++ b/contrib/libs/liburing/test/sqpoll-disable-exit.c @@ -0,0 +1,197 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// https://syzkaller.appspot.com/bug?id=99f4ea77bb9b9ef24cefb66469be319f4aa9f162 +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <dirent.h> +#include <endian.h> +#include <errno.h> +#include <fcntl.h> +#include <signal.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/prctl.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <time.h> +#include <unistd.h> + +#include "liburing.h" +#include "../src/syscall.h" + +static void sleep_ms(uint64_t ms) +{ + usleep(ms * 1000); +} + +static uint64_t current_time_ms(void) +{ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts)) + exit(1); + return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000; +} + +static bool write_file(const char* file, const char* what, ...) +{ + char buf[1024]; + va_list args; + va_start(args, what); + vsnprintf(buf, sizeof(buf), what, args); + va_end(args); + buf[sizeof(buf) - 1] = 0; + int len = strlen(buf); + int fd = open(file, O_WRONLY | O_CLOEXEC); + if (fd == -1) + return false; + if (write(fd, buf, len) != len) { + int err = errno; + close(fd); + errno = err; + return false; + } + close(fd); + return true; +} + +#define SIZEOF_IO_URING_SQE 64 +#define SIZEOF_IO_URING_CQE 16 +#define SQ_HEAD_OFFSET 0 +#define SQ_TAIL_OFFSET 64 +#define SQ_RING_MASK_OFFSET 256 +#define SQ_RING_ENTRIES_OFFSET 264 +#define SQ_FLAGS_OFFSET 276 +#define SQ_DROPPED_OFFSET 272 +#define CQ_HEAD_OFFSET 128 +#define CQ_TAIL_OFFSET 192 +#define CQ_RING_MASK_OFFSET 260 +#define CQ_RING_ENTRIES_OFFSET 268 +#define CQ_RING_OVERFLOW_OFFSET 284 +#define CQ_FLAGS_OFFSET 280 +#define CQ_CQES_OFFSET 320 + +static long syz_io_uring_setup(volatile long a0, volatile long a1, + volatile long a2, volatile long a3, + volatile long a4, volatile long a5) +{ + uint32_t entries = (uint32_t)a0; + struct io_uring_params* setup_params = (struct io_uring_params*)a1; + void* vma1 = (void*)a2; + void* vma2 = (void*)a3; + void** ring_ptr_out = (void**)a4; + void** sqes_ptr_out = (void**)a5; + uint32_t fd_io_uring = __sys_io_uring_setup(entries, setup_params); + uint32_t sq_ring_sz = + setup_params->sq_off.array + setup_params->sq_entries * sizeof(uint32_t); + uint32_t cq_ring_sz = setup_params->cq_off.cqes + + setup_params->cq_entries * SIZEOF_IO_URING_CQE; + uint32_t ring_sz = sq_ring_sz > cq_ring_sz ? sq_ring_sz : cq_ring_sz; + *ring_ptr_out = mmap(vma1, ring_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, + IORING_OFF_SQ_RING); + uint32_t sqes_sz = setup_params->sq_entries * SIZEOF_IO_URING_SQE; + *sqes_ptr_out = + mmap(vma2, sqes_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, IORING_OFF_SQES); + return fd_io_uring; +} + +static void kill_and_wait(int pid, int* status) +{ + kill(-pid, SIGKILL); + kill(pid, SIGKILL); + for (int i = 0; i < 100; i++) { + if (waitpid(-1, status, WNOHANG | __WALL) == pid) + return; + usleep(1000); + } + DIR* dir = opendir("/sys/fs/fuse/connections"); + if (dir) { + for (;;) { + struct dirent* ent = readdir(dir); + if (!ent) + break; + if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) + continue; + char abort[300]; + snprintf(abort, sizeof(abort), "/sys/fs/fuse/connections/%s/abort", + ent->d_name); + int fd = open(abort, O_WRONLY); + if (fd == -1) { + continue; + } + if (write(fd, abort, 1) < 0) { + } + close(fd); + } + closedir(dir); + } else { + } + while (waitpid(-1, status, __WALL) != pid) { + } +} + +static void setup_test() +{ + prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); + setpgrp(); + write_file("/proc/self/oom_score_adj", "1000"); +} + +static void execute_one(void); + +#define WAIT_FLAGS __WALL + +static void loop(void) +{ + int iter = 0; + for (; iter < 100; iter++) { + int pid = fork(); + if (pid < 0) + exit(1); + if (pid == 0) { + setup_test(); + execute_one(); + exit(0); + } + int status = 0; + uint64_t start = current_time_ms(); + for (;;) { + if (waitpid(-1, &status, WNOHANG | WAIT_FLAGS) == pid) + break; + sleep_ms(1); + if (current_time_ms() - start < 5000) { + continue; + } + kill_and_wait(pid, &status); + break; + } + } +} + +void execute_one(void) +{ + *(uint32_t*)0x20000044 = 0; + *(uint32_t*)0x20000048 = 0x42; + *(uint32_t*)0x2000004c = 0; + *(uint32_t*)0x20000050 = 0; + *(uint32_t*)0x20000058 = -1; + *(uint32_t*)0x2000005c = 0; + *(uint32_t*)0x20000060 = 0; + *(uint32_t*)0x20000064 = 0; + syz_io_uring_setup(0x74bc, 0x20000040, 0x20ffb000, 0x20ffc000, 0, 0); +} +int main(void) +{ + mmap((void *)0x1ffff000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); + mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul); + mmap((void *)0x21000000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); + loop(); + return 0; +} diff --git a/contrib/libs/liburing/test/sqpoll-exit-hang.c b/contrib/libs/liburing/test/sqpoll-exit-hang.c new file mode 100644 index 0000000000..7f5e539903 --- /dev/null +++ b/contrib/libs/liburing/test/sqpoll-exit-hang.c @@ -0,0 +1,79 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test that we exit properly with SQPOLL and having a request that + * adds a circular reference to the ring itself. + */ +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/time.h> +#include <poll.h> +#include "liburing.h" + +static unsigned long long mtime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000; + usec /= 1000; + return sec + usec; +} + +static unsigned long long mtime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return mtime_since(tv, &end); +} + +int main(int argc, char *argv[]) +{ + struct io_uring_params p = {}; + struct timeval tv; + struct io_uring ring; + struct io_uring_sqe *sqe; + int ret; + + if (argc > 1) + return 0; + + p.flags = IORING_SETUP_SQPOLL; + p.sq_thread_idle = 100; + + ret = io_uring_queue_init_params(1, &ring, &p); + if (ret) { + if (geteuid()) { + printf("%s: skipped, not root\n", argv[0]); + return 0; + } + fprintf(stderr, "queue_init=%d\n", ret); + return 1; + } + + if (!(p.features & IORING_FEAT_SQPOLL_NONFIXED)) { + fprintf(stdout, "Skipping\n"); + return 0; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, ring.ring_fd, POLLIN); + io_uring_submit(&ring); + + gettimeofday(&tv, NULL); + do { + usleep(1000); + } while (mtime_since_now(&tv) < 1000); + + return 0; +} diff --git a/contrib/libs/liburing/test/sqpoll-sleep.c b/contrib/libs/liburing/test/sqpoll-sleep.c new file mode 100644 index 0000000000..f98d733ea4 --- /dev/null +++ b/contrib/libs/liburing/test/sqpoll-sleep.c @@ -0,0 +1,70 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test that the sqthread goes to sleep around the specified time, and that + * the NEED_WAKEUP flag is then set. + */ +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/time.h> +#include "liburing.h" + +static unsigned long long mtime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000; + usec /= 1000; + return sec + usec; +} + +static unsigned long long mtime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return mtime_since(tv, &end); +} + +int main(int argc, char *argv[]) +{ + struct io_uring_params p = {}; + struct timeval tv; + struct io_uring ring; + int ret; + + if (argc > 1) + return 0; + + p.flags = IORING_SETUP_SQPOLL; + p.sq_thread_idle = 100; + + ret = io_uring_queue_init_params(1, &ring, &p); + if (ret) { + if (geteuid()) { + printf("%s: skipped, not root\n", argv[0]); + return 0; + } + fprintf(stderr, "queue_init=%d\n", ret); + return 1; + } + + gettimeofday(&tv, NULL); + do { + usleep(1000); + if ((*ring.sq.kflags) & IORING_SQ_NEED_WAKEUP) + return 0; + } while (mtime_since_now(&tv) < 1000); + + return 1; +} diff --git a/contrib/libs/liburing/test/stdout.c b/contrib/libs/liburing/test/stdout.c new file mode 100644 index 0000000000..02b0456955 --- /dev/null +++ b/contrib/libs/liburing/test/stdout.c @@ -0,0 +1,233 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: check that STDOUT write works + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "helpers.h" +#include "liburing.h" + +static int test_pipe_io_fixed(struct io_uring *ring) +{ + const char str[] = "This is a fixed pipe test\n"; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct iovec vecs[2]; + char buffer[128]; + int i, ret, fds[2]; + + t_posix_memalign(&vecs[0].iov_base, 4096, 4096); + memcpy(vecs[0].iov_base, str, strlen(str)); + vecs[0].iov_len = strlen(str); + + if (pipe(fds) < 0) { + perror("pipe"); + return 1; + } + + ret = io_uring_register_buffers(ring, vecs, 1); + if (ret) { + fprintf(stderr, "Failed to register buffers: %d\n", ret); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_write_fixed(sqe, fds[1], vecs[0].iov_base, + vecs[0].iov_len, 0, 0); + sqe->user_data = 1; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + vecs[1].iov_base = buffer; + vecs[1].iov_len = sizeof(buffer); + io_uring_prep_readv(sqe, fds[0], &vecs[1], 1, 0); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret < 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } else if (ret != 2) { + fprintf(stderr, "Submitted only %d\n", ret); + goto err; + } + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res < 0) { + fprintf(stderr, "I/O write error on %lu: %s\n", + (unsigned long) cqe->user_data, + strerror(-cqe->res)); + goto err; + } + if (cqe->res != strlen(str)) { + fprintf(stderr, "Got %d bytes, wanted %d on %lu\n", + cqe->res, (int)strlen(str), + (unsigned long) cqe->user_data); + goto err; + } + if (cqe->user_data == 2 && memcmp(str, buffer, strlen(str))) { + fprintf(stderr, "read data mismatch\n"); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + io_uring_unregister_buffers(ring); + return 0; +err: + return 1; +} + +static int test_stdout_io_fixed(struct io_uring *ring) +{ + const char str[] = "This is a fixed pipe test\n"; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct iovec vecs; + int ret; + + t_posix_memalign(&vecs.iov_base, 4096, 4096); + memcpy(vecs.iov_base, str, strlen(str)); + vecs.iov_len = strlen(str); + + ret = io_uring_register_buffers(ring, &vecs, 1); + if (ret) { + fprintf(stderr, "Failed to register buffers: %d\n", ret); + return 1; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_write_fixed(sqe, STDOUT_FILENO, vecs.iov_base, vecs.iov_len, 0, 0); + + ret = io_uring_submit(ring); + if (ret < 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } else if (ret < 1) { + fprintf(stderr, "Submitted only %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res < 0) { + fprintf(stderr, "STDOUT write error: %s\n", strerror(-cqe->res)); + goto err; + } + if (cqe->res != vecs.iov_len) { + fprintf(stderr, "Got %d write, wanted %d\n", cqe->res, (int)vecs.iov_len); + goto err; + } + io_uring_cqe_seen(ring, cqe); + io_uring_unregister_buffers(ring); + return 0; +err: + return 1; +} + +static int test_stdout_io(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct iovec vecs; + int ret; + + vecs.iov_base = "This is a pipe test\n"; + vecs.iov_len = strlen(vecs.iov_base); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_writev(sqe, STDOUT_FILENO, &vecs, 1, 0); + + ret = io_uring_submit(ring); + if (ret < 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } else if (ret < 1) { + fprintf(stderr, "Submitted only %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res < 0) { + fprintf(stderr, "STDOUT write error: %s\n", + strerror(-cqe->res)); + goto err; + } + if (cqe->res != vecs.iov_len) { + fprintf(stderr, "Got %d write, wanted %d\n", cqe->res, + (int)vecs.iov_len); + goto err; + } + io_uring_cqe_seen(ring, cqe); + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + ret = test_stdout_io(&ring); + if (ret) { + fprintf(stderr, "test_pipe_io failed\n"); + return ret; + } + + ret = test_stdout_io_fixed(&ring); + if (ret) { + fprintf(stderr, "test_pipe_io_fixed failed\n"); + return ret; + } + + ret = test_pipe_io_fixed(&ring); + if (ret) { + fprintf(stderr, "test_pipe_io_fixed failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/submit-and-wait.c b/contrib/libs/liburing/test/submit-and-wait.c new file mode 100644 index 0000000000..ab841ca6b1 --- /dev/null +++ b/contrib/libs/liburing/test/submit-and-wait.c @@ -0,0 +1,109 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: Test that io_uring_submit_and_wait_timeout() returns the + * right value (submit count) and that it doesn't end up waiting twice. + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/time.h> + +#include "liburing.h" +#include "test.h" + +static unsigned long long mtime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000; + usec /= 1000; + return sec + usec; +} + +static unsigned long long mtime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return mtime_since(tv, &end); +} + +static int test(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + struct timeval tv; + int ret, i; + + for (i = 0; i < 1; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed at %d\n", i); + goto err; + } + io_uring_prep_nop(sqe); + } + + ts.tv_sec = 1; + ts.tv_nsec = 0; + gettimeofday(&tv, NULL); + ret = io_uring_submit_and_wait_timeout(ring, &cqe, 2, &ts, NULL); + if (ret < 0) { + fprintf(stderr, "submit_and_wait_timeout: %d\n", ret); + goto err; + } + ret = mtime_since_now(&tv); + /* allow some slack, should be around 1s */ + if (ret > 1200) { + fprintf(stderr, "wait took too long: %d\n", ret); + goto err; + } + return 0; +err: + return 1; +} + +static int test_ring(void) +{ + struct io_uring ring; + struct io_uring_params p = { }; + int ret; + + p.flags = 0; + ret = io_uring_queue_init_params(8, &ring, &p); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + ret = test(&ring); + if (ret) { + fprintf(stderr, "test failed\n"); + goto err; + } +err: + io_uring_queue_exit(&ring); + return ret; +} + +int main(int argc, char *argv[]) +{ + if (argc > 1) + return 0; + + return test_ring(); +} diff --git a/contrib/libs/liburing/test/submit-link-fail.c b/contrib/libs/liburing/test/submit-link-fail.c new file mode 100644 index 0000000000..3cbc4c1278 --- /dev/null +++ b/contrib/libs/liburing/test/submit-link-fail.c @@ -0,0 +1,157 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: tests linked requests failing during submission + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <assert.h> + +#include "liburing.h" + +#define DRAIN_USER_DATA 42 + +static int test_underprep_fail(bool hardlink, bool drain, bool link_last, + int link_size, int fail_idx) +{ + const int invalid_fd = 42; + int link_flags = IOSQE_IO_LINK; + int total_submit = link_size; + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + char buffer[1] = { }; + int i, ret, fds[2]; + + if (drain) + link_flags |= IOSQE_IO_DRAIN; + if (hardlink) + link_flags |= IOSQE_IO_HARDLINK; + + assert(fail_idx < link_size); + assert(link_size < 40); + + /* create a new ring as it leaves it dirty */ + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + printf("ring setup failed\n"); + return -1; + } + if (pipe(fds)) { + perror("pipe"); + return -1; + } + + if (drain) { + /* clog drain, so following reqs sent to draining */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, fds[0], buffer, sizeof(buffer), 0); + sqe->user_data = DRAIN_USER_DATA; + sqe->flags |= IOSQE_IO_DRAIN; + total_submit++; + } + + for (i = 0; i < link_size; i++) { + sqe = io_uring_get_sqe(&ring); + if (i == fail_idx) { + io_uring_prep_read(sqe, invalid_fd, buffer, 1, 0); + sqe->ioprio = (short) -1; + } else { + io_uring_prep_nop(sqe); + } + + if (i != link_size - 1 || !link_last) + sqe->flags |= link_flags; + sqe->user_data = i; + } + + ret = io_uring_submit(&ring); + if (ret != total_submit) { + /* Old behaviour, failed early and under-submitted */ + if (ret == fail_idx + 1 + drain) + goto out; + fprintf(stderr, "submit failed: %d\n", ret); + return -1; + } + + if (drain) { + /* unclog drain */ + ret = write(fds[1], buffer, sizeof(buffer)); + if (ret < 0) { + perror("write"); + return 1; + } + } + + for (i = 0; i < total_submit; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + + ret = cqe->res; + if (cqe->user_data == DRAIN_USER_DATA) { + if (ret != 1) { + fprintf(stderr, "drain failed %d\n", ret); + return 1; + } + } else if (cqe->user_data == fail_idx) { + if (ret == 0 || ret == -ECANCELED) { + fprintf(stderr, "half-prep req unexpected return %d\n", ret); + return 1; + } + } else { + if (ret != -ECANCELED) { + fprintf(stderr, "cancel failed %d, ud %d\n", ret, (int)cqe->user_data); + return 1; + } + } + io_uring_cqe_seen(&ring, cqe); + } +out: + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring); + return 0; +} + +int main(int argc, char *argv[]) +{ + int ret, link_size, fail_idx, i; + + if (argc > 1) + return 0; + + /* + * hardlink, size=3, fail_idx=1, drain=false -- kernel fault + * link, size=3, fail_idx=0, drain=true -- kernel fault + * link, size=3, fail_idx=1, drain=true -- invalid cqe->res + */ + for (link_size = 0; link_size < 3; link_size++) { + for (fail_idx = 0; fail_idx < link_size; fail_idx++) { + for (i = 0; i < 8; i++) { + bool hardlink = (i & 1) != 0; + bool drain = (i & 2) != 0; + bool link_last = (i & 4) != 0; + + ret = test_underprep_fail(hardlink, drain, link_last, + link_size, fail_idx); + if (!ret) + continue; + + fprintf(stderr, "failed %d, hard %d, drain %d," + "link_last %d, size %d, idx %d\n", + ret, hardlink, drain, link_last, + link_size, fail_idx); + return 1; + } + } + } + + return 0; +} diff --git a/contrib/libs/liburing/test/submit-reuse.c b/contrib/libs/liburing/test/submit-reuse.c new file mode 100644 index 0000000000..92f6b2f5d0 --- /dev/null +++ b/contrib/libs/liburing/test/submit-reuse.c @@ -0,0 +1,238 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test reads that will punt to blocking context, with immediate overwrite + * of iovec->iov_base to NULL. If the kernel doesn't properly handle + * reuse of the iovec, we should get -EFAULT. + */ +#include <unistd.h> +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <pthread.h> +#include <sys/time.h> + +#include "helpers.h" +#include "liburing.h" + +#define STR_SIZE 32768 +#define FILE_SIZE 65536 + +struct thread_data { + int fd1, fd2; + volatile int do_exit; +}; + +static void *flusher(void *__data) +{ + struct thread_data *data = __data; + + while (!data->do_exit) { + posix_fadvise(data->fd1, 0, FILE_SIZE, POSIX_FADV_DONTNEED); + posix_fadvise(data->fd2, 0, FILE_SIZE, POSIX_FADV_DONTNEED); + usleep(10); + } + + return NULL; +} + +static char str1[STR_SIZE]; +static char str2[STR_SIZE]; + +static struct io_uring ring; + +static int no_stable; + +static int prep(int fd, char *str, int split, int async) +{ + struct io_uring_sqe *sqe; + struct iovec iovs[16]; + int ret, i; + + if (split) { + int vsize = STR_SIZE / 16; + void *ptr = str; + + for (i = 0; i < 16; i++) { + iovs[i].iov_base = ptr; + iovs[i].iov_len = vsize; + ptr += vsize; + } + } else { + iovs[0].iov_base = str; + iovs[0].iov_len = STR_SIZE; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, fd, iovs, split ? 16 : 1, 0); + sqe->user_data = fd; + if (async) + sqe->flags = IOSQE_ASYNC; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit got %d\n", ret); + return 1; + } + if (split) { + for (i = 0; i < 16; i++) + iovs[i].iov_base = NULL; + } else { + iovs[0].iov_base = NULL; + } + return 0; +} + +static int wait_nr(int nr) +{ + int i, ret; + + for (i = 0; i < nr; i++) { + struct io_uring_cqe *cqe; + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) + return ret; + if (cqe->res < 0) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + } + + return 0; +} + +static unsigned long long mtime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000; + usec /= 1000; + return sec + usec; +} + +static unsigned long long mtime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return mtime_since(tv, &end); +} + +static int test_reuse(int argc, char *argv[], int split, int async) +{ + struct thread_data data; + struct io_uring_params p = { }; + int fd1, fd2, ret, i; + struct timeval tv; + pthread_t thread; + char *fname1 = ".reuse.1"; + int do_unlink = 1; + void *tret; + + ret = io_uring_queue_init_params(32, &ring, &p); + if (ret) { + fprintf(stderr, "io_uring_queue_init: %d\n", ret); + return 1; + } + + if (!(p.features & IORING_FEAT_SUBMIT_STABLE)) { + fprintf(stdout, "FEAT_SUBMIT_STABLE not there, skipping\n"); + io_uring_queue_exit(&ring); + no_stable = 1; + return 0; + } + + if (argc > 1) { + fname1 = argv[1]; + do_unlink = 0; + } else { + t_create_file(fname1, FILE_SIZE); + } + + fd1 = open(fname1, O_RDONLY); + if (do_unlink) + unlink(fname1); + if (fd1 < 0) { + perror("open fname1"); + goto err; + } + + t_create_file(".reuse.2", FILE_SIZE); + fd2 = open(".reuse.2", O_RDONLY); + unlink(".reuse.2"); + if (fd2 < 0) { + perror("open .reuse.2"); + goto err; + } + + data.fd1 = fd1; + data.fd2 = fd2; + data.do_exit = 0; + pthread_create(&thread, NULL, flusher, &data); + usleep(10000); + + gettimeofday(&tv, NULL); + for (i = 0; i < 1000; i++) { + ret = prep(fd1, str1, split, async); + if (ret) { + fprintf(stderr, "prep1 failed: %d\n", ret); + goto err; + } + ret = prep(fd2, str2, split, async); + if (ret) { + fprintf(stderr, "prep1 failed: %d\n", ret); + goto err; + } + ret = wait_nr(2); + if (ret) { + fprintf(stderr, "wait_nr: %d\n", ret); + goto err; + } + if (mtime_since_now(&tv) > 5000) + break; + } + + data.do_exit = 1; + pthread_join(thread, &tret); + + close(fd2); + close(fd1); + io_uring_queue_exit(&ring); + return 0; +err: + io_uring_queue_exit(&ring); + return 1; + +} + +int main(int argc, char *argv[]) +{ + int ret, i; + + for (i = 0; i < 4; i++) { + int split, async; + + split = (i & 1) != 0; + async = (i & 2) != 0; + + ret = test_reuse(argc, argv, split, async); + if (ret) { + fprintf(stderr, "test_reuse %d %d failed\n", split, async); + return ret; + } + if (no_stable) + break; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/symlink.c b/contrib/libs/liburing/test/symlink.c new file mode 100644 index 0000000000..755b51d152 --- /dev/null +++ b/contrib/libs/liburing/test/symlink.c @@ -0,0 +1,117 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring symlinkat handling + */ +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "liburing.h" + + +static int do_symlinkat(struct io_uring *ring, const char *oldname, const char *newname) +{ + int ret; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "sqe get failed\n"); + goto err; + } + io_uring_prep_symlinkat(sqe, oldname, AT_FDCWD, newname); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqes(ring, &cqe, 1, 0, 0); + if (ret) { + fprintf(stderr, "wait_cqe failed: %d\n", ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +err: + return 1; +} + +int test_link_contents(const char* linkname, const char *expected_contents) +{ + char buf[128]; + int ret = readlink(linkname, buf, 127); + if (ret < 0) { + perror("readlink"); + return ret; + } + buf[ret] = 0; + if (strncmp(buf, expected_contents, 128)) { + fprintf(stderr, "link contents differs from expected: '%s' vs '%s'", + buf, expected_contents); + return -1; + } + return 0; +} + +int main(int argc, char *argv[]) +{ + static const char target[] = "io_uring-symlinkat-test-target"; + static const char linkname[] = "io_uring-symlinkat-test-link"; + int ret; + struct io_uring ring; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return ret; + } + + ret = do_symlinkat(&ring, target, linkname); + if (ret < 0) { + if (ret == -EBADF || ret == -EINVAL) { + fprintf(stdout, "symlinkat not supported, skipping\n"); + goto out; + } + fprintf(stderr, "symlinkat: %s\n", strerror(-ret)); + goto err; + } else if (ret) { + goto err; + } + + ret = test_link_contents(linkname, target); + if (ret < 0) + goto err1; + + ret = do_symlinkat(&ring, target, linkname); + if (ret != -EEXIST) { + fprintf(stderr, "test_symlinkat linkname already exists failed: %d\n", ret); + goto err1; + } + + ret = do_symlinkat(&ring, target, "surely/this/does/not/exist"); + if (ret != -ENOENT) { + fprintf(stderr, "test_symlinkat no parent failed: %d\n", ret); + goto err1; + } + +out: + unlinkat(AT_FDCWD, linkname, 0); + io_uring_queue_exit(&ring); + return 0; +err1: + unlinkat(AT_FDCWD, linkname, 0); +err: + io_uring_queue_exit(&ring); + return 1; +} diff --git a/contrib/libs/liburing/test/sync-cancel.c b/contrib/libs/liburing/test/sync-cancel.c new file mode 100644 index 0000000000..096d210ffb --- /dev/null +++ b/contrib/libs/liburing/test/sync-cancel.c @@ -0,0 +1,236 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test io_uring_register_sync_cancel() + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +static int no_sync_cancel; + +static int test_sync_cancel_timeout(struct io_uring *ring, int async) +{ + struct io_uring_sync_cancel_reg reg = { }; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, fds[2], to_prep; + char buf[32]; + + if (pipe(fds) < 0) { + perror("pipe"); + return 1; + } + + to_prep = 1; + sqe = io_uring_get_sqe(ring); + io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0); + sqe->user_data = 0x89; + if (async) + sqe->flags |= IOSQE_ASYNC; + + ret = io_uring_submit(ring); + if (ret != to_prep) { + fprintf(stderr, "submit=%d\n", ret); + return 1; + } + + usleep(10000); + + reg.addr = 0x89; + reg.timeout.tv_nsec = 1; + ret = io_uring_register_sync_cancel(ring, ®); + if (async) { + /* we expect -ETIME here, but can race and get 0 */ + if (ret != -ETIME && ret != 0) { + fprintf(stderr, "sync_cancel=%d\n", ret); + return 1; + } + } else { + if (ret < 0) { + fprintf(stderr, "sync_cancel=%d\n", ret); + return 1; + } + } + + /* + * we could _almost_ use peek_cqe() here, but there is still + * a small gap where io-wq is done with the request and on + * its way to posting a completion, but hasn't done it just + * yet. the request is canceled and won't be doing any IO + * to buffers etc, but the cqe may not have quite arrived yet. + */ + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "peek=%d\n", ret); + return 1; + } + if (cqe->res >= 0) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + return 0; +} + +static int test_sync_cancel(struct io_uring *ring, int async, int nr_all, + int use_fd) +{ + struct io_uring_sync_cancel_reg reg = { }; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, fds[2], to_prep, i; + char buf[32]; + + if (pipe(fds) < 0) { + perror("pipe"); + return 1; + } + + to_prep = 1; + if (nr_all) + to_prep = 4; + for (i = 0; i < to_prep; i++) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0); + sqe->user_data = 0x89; + if (async) + sqe->flags |= IOSQE_ASYNC; + } + + ret = io_uring_submit(ring); + if (ret != to_prep) { + fprintf(stderr, "submit=%d\n", ret); + return 1; + } + + usleep(10000); + + if (!use_fd) + reg.addr = 0x89; + else + reg.fd = fds[0]; + reg.timeout.tv_sec = 200; + if (nr_all) + reg.flags |= IORING_ASYNC_CANCEL_ALL; + if (use_fd) + reg.flags |= IORING_ASYNC_CANCEL_FD; + ret = io_uring_register_sync_cancel(ring, ®); + if (ret < 0) { + if (ret == -EINVAL && !no_sync_cancel) { + no_sync_cancel = 1; + return 0; + } + fprintf(stderr, "sync_cancel=%d\n", ret); + return 1; + } + + for (i = 0; i < to_prep; i++) { + /* + * we could _almost_ use peek_cqe() here, but there is still + * a small gap where io-wq is done with the request and on + * its way to posting a completion, but hasn't done it just + * yet. the request is canceled and won't be doing any IO + * to buffers etc, but the cqe may not have quite arrived yet. + */ + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "peek=%d\n", ret); + return 1; + } + if (cqe->res >= 0) { + fprintf(stderr, "cqe->res=%d\n", cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = t_create_ring(7, &ring, 0); + if (ret == T_SETUP_SKIP) + return T_EXIT_SKIP; + else if (ret != T_SETUP_OK) + return ret; + + ret = test_sync_cancel(&ring, 0, 0, 0); + if (ret) { + fprintf(stderr, "test_sync_cancel 0 0 0 failed\n"); + return T_EXIT_FAIL; + } + if (no_sync_cancel) + return T_EXIT_SKIP; + + ret = test_sync_cancel(&ring, 1, 0, 0); + if (ret) { + fprintf(stderr, "test_sync_cancel 1 0 0 failed\n"); + return T_EXIT_FAIL; + } + + ret = test_sync_cancel(&ring, 0, 1, 0); + if (ret) { + fprintf(stderr, "test_sync_cancel 0 1 0 failed\n"); + return T_EXIT_FAIL; + } + + ret = test_sync_cancel(&ring, 1, 1, 0); + if (ret) { + fprintf(stderr, "test_sync_cancel 1 1 0 failed\n"); + return T_EXIT_FAIL; + } + + ret = test_sync_cancel(&ring, 0, 0, 1); + if (ret) { + fprintf(stderr, "test_sync_cancel 0 0 1 failed\n"); + return T_EXIT_FAIL; + } + + ret = test_sync_cancel(&ring, 1, 0, 1); + if (ret) { + fprintf(stderr, "test_sync_cancel 1 0 1 failed\n"); + return T_EXIT_FAIL; + } + + ret = test_sync_cancel(&ring, 0, 1, 1); + if (ret) { + fprintf(stderr, "test_sync_cancel 0 1 1 failed\n"); + return T_EXIT_FAIL; + } + + ret = test_sync_cancel(&ring, 1, 1, 1); + if (ret) { + fprintf(stderr, "test_sync_cancel 1 1 1 failed\n"); + return T_EXIT_FAIL; + } + + ret = test_sync_cancel_timeout(&ring, 0); + if (ret) { + fprintf(stderr, "test_sync_cancel_timeout 0\n"); + return T_EXIT_FAIL; + } + + /* must be last, leaves request */ + ret = test_sync_cancel_timeout(&ring, 1); + if (ret) { + fprintf(stderr, "test_sync_cancel_timeout 1\n"); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/teardowns.c b/contrib/libs/liburing/test/teardowns.c new file mode 100644 index 0000000000..4dfb20ef89 --- /dev/null +++ b/contrib/libs/liburing/test/teardowns.c @@ -0,0 +1,59 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> +#include <errno.h> + +#include "liburing.h" + +static void loop(void) +{ + int i, ret = 0; + + for (i = 0; i < 100; i++) { + struct io_uring ring; + int fd; + + memset(&ring, 0, sizeof(ring)); + fd = io_uring_queue_init(0xa4, &ring, 0); + if (fd >= 0) { + close(fd); + continue; + } + if (fd != -ENOMEM) + ret++; + } + exit(ret); +} + +int main(int argc, char *argv[]) +{ + int i, ret, status; + + if (argc > 1) + return 0; + + for (i = 0; i < 12; i++) { + if (!fork()) { + loop(); + break; + } + } + + ret = 0; + for (i = 0; i < 12; i++) { + if (waitpid(-1, &status, 0) < 0) { + perror("waitpid"); + return 1; + } + if (WEXITSTATUS(status)) + ret++; + } + + return ret; +} diff --git a/contrib/libs/liburing/test/test.h b/contrib/libs/liburing/test/test.h new file mode 100644 index 0000000000..3628163afe --- /dev/null +++ b/contrib/libs/liburing/test/test.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Description: Test configs for tests. + */ +#ifndef LIBURING_TEST_H +#define LIBURING_TEST_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct io_uring_test_config { + unsigned int flags; + const char *description; +} io_uring_test_config; + +io_uring_test_config io_uring_test_configs[] = { + { 0, "default" }, + { IORING_SETUP_SQE128, "large SQE"}, + { IORING_SETUP_CQE32, "large CQE"}, + { IORING_SETUP_SQE128 | IORING_SETUP_CQE32, "large SQE/CQE" }, +}; + +#define FOR_ALL_TEST_CONFIGS \ + for (int i = 0; i < sizeof(io_uring_test_configs) / sizeof(io_uring_test_configs[0]); i++) + +#define IORING_GET_TEST_CONFIG_FLAGS() (io_uring_test_configs[i].flags) +#define IORING_GET_TEST_CONFIG_DESCRIPTION() (io_uring_test_configs[i].description) + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/libs/liburing/test/thread-exit.c b/contrib/libs/liburing/test/thread-exit.c new file mode 100644 index 0000000000..282b1e8461 --- /dev/null +++ b/contrib/libs/liburing/test/thread-exit.c @@ -0,0 +1,144 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test that thread pool issued requests don't cancel on thread + * exit, but do get canceled once the parent exits. Do both + * writes that finish and a poll request that sticks around. + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <poll.h> +#include <pthread.h> + +#include "helpers.h" +#include "liburing.h" + +#define NR_IOS 8 +#define WSIZE 512 + +struct d { + int fd; + struct io_uring *ring; + unsigned long off; + int pipe_fd; + int err; + int i; +}; + +static char *g_buf[NR_IOS] = {NULL}; + +static void free_g_buf(void) +{ + int i; + for (i = 0; i < NR_IOS; i++) + free(g_buf[i]); +} + +static void *do_io(void *data) +{ + struct d *d = data; + struct io_uring_sqe *sqe; + char *buffer; + int ret; + + buffer = t_malloc(WSIZE); + g_buf[d->i] = buffer; + memset(buffer, 0x5a, WSIZE); + sqe = io_uring_get_sqe(d->ring); + if (!sqe) { + d->err++; + return NULL; + } + io_uring_prep_write(sqe, d->fd, buffer, WSIZE, d->off); + sqe->user_data = d->off; + + sqe = io_uring_get_sqe(d->ring); + if (!sqe) { + d->err++; + return NULL; + } + io_uring_prep_poll_add(sqe, d->pipe_fd, POLLIN); + + ret = io_uring_submit(d->ring); + if (ret != 2) + d->err++; + return NULL; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + const char *fname; + pthread_t thread; + int ret, do_unlink, i, fd; + struct d d; + int fds[2]; + + if (pipe(fds) < 0) { + perror("pipe"); + return 1; + } + + ret = io_uring_queue_init(32, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + if (argc > 1) { + fname = argv[1]; + do_unlink = 0; + } else { + fname = ".thread.exit"; + do_unlink = 1; + t_create_file(fname, 4096); + } + + fd = open(fname, O_WRONLY); + if (do_unlink) + unlink(fname); + if (fd < 0) { + perror("open"); + return 1; + } + + d.fd = fd; + d.ring = ˚ + d.off = 0; + d.pipe_fd = fds[0]; + d.err = 0; + for (i = 0; i < NR_IOS; i++) { + d.i = i; + memset(&thread, 0, sizeof(thread)); + pthread_create(&thread, NULL, do_io, &d); + pthread_join(thread, NULL); + d.off += WSIZE; + } + + for (i = 0; i < NR_IOS; i++) { + struct io_uring_cqe *cqe; + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe=%d\n", ret); + goto err; + } + if (cqe->res != WSIZE) { + fprintf(stderr, "cqe->res=%d, Expected %d\n", cqe->res, + WSIZE); + goto err; + } + io_uring_cqe_seen(&ring, cqe); + } + + free_g_buf(); + return d.err; +err: + free_g_buf(); + return 1; +} diff --git a/contrib/libs/liburing/test/timeout-new.c b/contrib/libs/liburing/test/timeout-new.c new file mode 100644 index 0000000000..d4aeced46b --- /dev/null +++ b/contrib/libs/liburing/test/timeout-new.c @@ -0,0 +1,253 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: tests for getevents timeout + * + */ +#include <stdio.h> +#include <sys/time.h> +#include <unistd.h> +#include <pthread.h> +#include "liburing.h" + +#define TIMEOUT_MSEC 200 +#define TIMEOUT_SEC 10 + +int thread_ret0, thread_ret1; +int cnt = 0; +pthread_mutex_t mutex; + +static void msec_to_ts(struct __kernel_timespec *ts, unsigned int msec) +{ + ts->tv_sec = msec / 1000; + ts->tv_nsec = (msec % 1000) * 1000000; +} + +static unsigned long long mtime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000; + usec /= 1000; + return sec + usec; +} + +static unsigned long long mtime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return mtime_since(tv, &end); +} + + +static int test_return_before_timeout(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + bool retried = false; + struct __kernel_timespec ts; + + msec_to_ts(&ts, TIMEOUT_MSEC); + + sqe = io_uring_get_sqe(ring); + io_uring_prep_nop(sqe); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + return 1; + } + +again: + ret = io_uring_wait_cqe_timeout(ring, &cqe, &ts); + if (ret == -ETIME && (ring->flags & IORING_SETUP_SQPOLL) && !retried) { + /* + * there is a small chance SQPOLL hasn't been waked up yet, + * give it one more try. + */ + printf("warning: funky SQPOLL timing\n"); + sleep(1); + retried = true; + goto again; + } else if (ret < 0) { + fprintf(stderr, "%s: timeout error: %d\n", __FUNCTION__, ret); + return 1; + } + io_uring_cqe_seen(ring, cqe); + return 0; +} + +static int test_return_after_timeout(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int ret; + struct __kernel_timespec ts; + struct timeval tv; + unsigned long long exp; + + msec_to_ts(&ts, TIMEOUT_MSEC); + gettimeofday(&tv, NULL); + ret = io_uring_wait_cqe_timeout(ring, &cqe, &ts); + exp = mtime_since_now(&tv); + if (ret != -ETIME) { + fprintf(stderr, "%s: timeout error: %d\n", __FUNCTION__, ret); + return 1; + } + + if (exp < TIMEOUT_MSEC / 2 || exp > (TIMEOUT_MSEC * 3) / 2) { + fprintf(stderr, "%s: Timeout seems wonky (got %llu)\n", __FUNCTION__, exp); + return 1; + } + + return 0; +} + +int __reap_thread_fn(void *data) { + struct io_uring *ring = (struct io_uring *)data; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + + msec_to_ts(&ts, TIMEOUT_SEC); + pthread_mutex_lock(&mutex); + cnt++; + pthread_mutex_unlock(&mutex); + return io_uring_wait_cqe_timeout(ring, &cqe, &ts); +} + +void *reap_thread_fn0(void *data) { + thread_ret0 = __reap_thread_fn(data); + return NULL; +} + +void *reap_thread_fn1(void *data) { + thread_ret1 = __reap_thread_fn(data); + return NULL; +} + +/* + * This is to test issuing a sqe in main thread and reaping it in two child-thread + * at the same time. To see if timeout feature works or not. + */ +int test_multi_threads_timeout() { + struct io_uring ring; + int ret; + bool both_wait = false; + pthread_t reap_thread0, reap_thread1; + struct io_uring_sqe *sqe; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "%s: ring setup failed: %d\n", __FUNCTION__, ret); + return 1; + } + + pthread_create(&reap_thread0, NULL, reap_thread_fn0, &ring); + pthread_create(&reap_thread1, NULL, reap_thread_fn1, &ring); + + /* + * make two threads both enter io_uring_wait_cqe_timeout() before issuing the sqe + * as possible as we can. So that there are two threads in the ctx->wait queue. + * In this way, we can test if a cqe wakes up two threads at the same time. + */ + while(!both_wait) { + pthread_mutex_lock(&mutex); + if (cnt == 2) + both_wait = true; + pthread_mutex_unlock(&mutex); + sleep(1); + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + io_uring_prep_nop(sqe); + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + pthread_join(reap_thread0, NULL); + pthread_join(reap_thread1, NULL); + + if ((thread_ret0 && thread_ret0 != -ETIME) || (thread_ret1 && thread_ret1 != -ETIME)) { + fprintf(stderr, "%s: thread wait cqe timeout failed: %d %d\n", + __FUNCTION__, thread_ret0, thread_ret1); + goto err; + } + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring_normal, ring_sq; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(8, &ring_normal, 0); + if (ret) { + fprintf(stderr, "ring_normal setup failed: %d\n", ret); + return 1; + } + if (!(ring_normal.features & IORING_FEAT_EXT_ARG)) { + fprintf(stderr, "feature IORING_FEAT_EXT_ARG not supported, skipping.\n"); + return 0; + } + + ret = test_return_before_timeout(&ring_normal); + if (ret) { + fprintf(stderr, "ring_normal: test_return_before_timeout failed\n"); + return ret; + } + + ret = test_return_after_timeout(&ring_normal); + if (ret) { + fprintf(stderr, "ring_normal: test_return_after_timeout failed\n"); + return ret; + } + + ret = io_uring_queue_init(8, &ring_sq, IORING_SETUP_SQPOLL); + if (ret) { + fprintf(stderr, "ring_sq setup failed: %d\n", ret); + return 1; + } + + ret = test_return_before_timeout(&ring_sq); + if (ret) { + fprintf(stderr, "ring_sq: test_return_before_timeout failed\n"); + return ret; + } + + ret = test_return_after_timeout(&ring_sq); + if (ret) { + fprintf(stderr, "ring_sq: test_return_after_timeout failed\n"); + return ret; + } + + ret = test_multi_threads_timeout(); + if (ret) { + fprintf(stderr, "test_multi_threads_timeout failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/timeout-overflow.c b/contrib/libs/liburing/test/timeout-overflow.c new file mode 100644 index 0000000000..8c57ff31b1 --- /dev/null +++ b/contrib/libs/liburing/test/timeout-overflow.c @@ -0,0 +1,205 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run timeout overflow test + * + */ +#include <errno.h> +#include <stdio.h> +#include <limits.h> +#include <string.h> +#include <sys/time.h> + +#include "liburing.h" +#include "helpers.h" + +#define TIMEOUT_MSEC 200 +static int not_supported; + +static void msec_to_ts(struct __kernel_timespec *ts, unsigned int msec) +{ + ts->tv_sec = msec / 1000; + ts->tv_nsec = (msec % 1000) * 1000000; +} + +static int check_timeout_support(void) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + struct io_uring_params p; + struct io_uring ring; + int ret; + + memset(&p, 0, sizeof(p)); + ret = io_uring_queue_init_params(1, &ring, &p); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + + /* not really a match, but same kernel added batched completions */ + if (p.features & IORING_FEAT_POLL_32BITS) { + not_supported = 1; + return T_EXIT_SKIP; + } + + sqe = io_uring_get_sqe(&ring); + msec_to_ts(&ts, TIMEOUT_MSEC); + io_uring_prep_timeout(sqe, &ts, 1, 0); + + ret = io_uring_submit(&ring); + if (ret < 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + + if (cqe->res == -EINVAL) { + not_supported = 1; + fprintf(stdout, "Timeout not supported, ignored\n"); + return 0; + } + + io_uring_cqe_seen(&ring, cqe); + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +err: + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} + +/* + * We first setup 4 timeout requests, which require a count value of 1, 1, 2, + * UINT_MAX, so the sequence is 1, 2, 4, 2. Before really timeout, this 4 + * requests will not lead the change of cq_cached_tail, so as sq_dropped. + * + * And before this patch. The order of this four requests will be req1->req2-> + * req4->req3. Actually, it should be req1->req2->req3->req4. + * + * Then, if there is 2 nop req. All timeout requests expect req4 will completed + * successful after the patch. And req1/req2 will completed successful with + * req3/req4 return -ETIME without this patch! + */ +static int test_timeout_overflow(void) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + struct io_uring ring; + int i, ret; + + ret = io_uring_queue_init(16, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + msec_to_ts(&ts, TIMEOUT_MSEC); + for (i = 0; i < 4; i++) { + unsigned num = 0; + sqe = io_uring_get_sqe(&ring); + switch (i) { + case 0: + case 1: + num = 1; + break; + case 2: + num = 2; + break; + case 3: + num = UINT_MAX; + break; + } + io_uring_prep_timeout(sqe, &ts, num, 0); + } + + for (i = 0; i < 2; i++) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + io_uring_sqe_set_data(sqe, (void *) 1); + } + ret = io_uring_submit(&ring); + if (ret < 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + i = 0; + while (i < 6) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + + /* + * cqe1: first nop req + * cqe2: first timeout req, because of cqe1 + * cqe3: second timeout req because of cqe1 + cqe2 + * cqe4: second nop req + * cqe5~cqe6: the left three timeout req + */ + switch (i) { + case 0: + case 3: + if (io_uring_cqe_get_data(cqe) != (void *) 1) { + fprintf(stderr, "nop not seen as 1 or 2\n"); + goto err; + } + break; + case 1: + case 2: + case 4: + if (cqe->res == -ETIME) { + fprintf(stderr, "expected not return -ETIME " + "for the #%d timeout req\n", i - 1); + goto err; + } + break; + case 5: + if (cqe->res != -ETIME) { + fprintf(stderr, "expected return -ETIME for " + "the #%d timeout req\n", i - 1); + goto err; + } + break; + } + io_uring_cqe_seen(&ring, cqe); + i++; + } + + return 0; +err: + return 1; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = check_timeout_support(); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "check_timeout_support failed: %d\n", ret); + return T_EXIT_FAIL; + } + + if (not_supported) + return T_EXIT_SKIP; + + ret = test_timeout_overflow(); + if (ret) { + fprintf(stderr, "test_timeout_overflow failed\n"); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/timeout.c b/contrib/libs/liburing/test/timeout.c new file mode 100644 index 0000000000..41d8627938 --- /dev/null +++ b/contrib/libs/liburing/test/timeout.c @@ -0,0 +1,1524 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various timeout tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/time.h> +#include <sys/wait.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include "liburing.h" +#include "../src/syscall.h" + +#define TIMEOUT_MSEC 200 +static int not_supported; +static int no_modify; + +static void msec_to_ts(struct __kernel_timespec *ts, unsigned int msec) +{ + ts->tv_sec = msec / 1000; + ts->tv_nsec = (msec % 1000) * 1000000; +} + +static unsigned long long mtime_since(const struct timeval *s, + const struct timeval *e) +{ + long long sec, usec; + + sec = e->tv_sec - s->tv_sec; + usec = (e->tv_usec - s->tv_usec); + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + sec *= 1000; + usec /= 1000; + return sec + usec; +} + +static unsigned long long mtime_since_now(struct timeval *tv) +{ + struct timeval end; + + gettimeofday(&end, NULL); + return mtime_since(tv, &end); +} + +/* + * Test that we return to userspace if a timeout triggers, even if we + * don't satisfy the number of events asked for. + */ +static int test_single_timeout_many(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + unsigned long long exp; + struct __kernel_timespec ts; + struct timeval tv; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + msec_to_ts(&ts, TIMEOUT_MSEC); + io_uring_prep_timeout(sqe, &ts, 0, 0); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + gettimeofday(&tv, NULL); + ret = __sys_io_uring_enter(ring->ring_fd, 0, 4, IORING_ENTER_GETEVENTS, + NULL); + if (ret < 0) { + fprintf(stderr, "%s: io_uring_enter %d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (ret == -EINVAL) { + fprintf(stdout, "Timeout not supported, ignored\n"); + not_supported = 1; + return 0; + } else if (ret != -ETIME) { + fprintf(stderr, "Timeout: %s\n", strerror(-ret)); + goto err; + } + + exp = mtime_since_now(&tv); + if (exp >= TIMEOUT_MSEC / 2 && exp <= (TIMEOUT_MSEC * 3) / 2) + return 0; + fprintf(stderr, "%s: Timeout seems wonky (got %llu)\n", __FUNCTION__, exp); +err: + return 1; +} + +/* + * Test numbered trigger of timeout + */ +static int test_single_timeout_nr(struct io_uring *ring, int nr) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + int i, ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + msec_to_ts(&ts, TIMEOUT_MSEC); + io_uring_prep_timeout(sqe, &ts, nr, 0); + + sqe = io_uring_get_sqe(ring); + io_uring_prep_nop(sqe); + io_uring_sqe_set_data(sqe, (void *) 1); + sqe = io_uring_get_sqe(ring); + io_uring_prep_nop(sqe); + io_uring_sqe_set_data(sqe, (void *) 1); + + ret = io_uring_submit_and_wait(ring, 3); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + i = 0; + while (i < 3) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + ret = cqe->res; + + /* + * NOP commands have user_data as 1. Check that we get the + * at least 'nr' NOPs first, then the successfully removed timeout. + */ + if (io_uring_cqe_get_data(cqe) == NULL) { + if (i < nr) { + fprintf(stderr, "%s: timeout received too early\n", __FUNCTION__); + goto err; + } + if (ret) { + fprintf(stderr, "%s: timeout triggered by passage of" + " time, not by events completed\n", __FUNCTION__); + goto err; + } + } + + io_uring_cqe_seen(ring, cqe); + if (ret) { + fprintf(stderr, "res: %d\n", ret); + goto err; + } + i++; + }; + + return 0; +err: + return 1; +} + +static int test_single_timeout_wait(struct io_uring *ring, + struct io_uring_params *p) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + int i, ret; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_nop(sqe); + io_uring_sqe_set_data(sqe, (void *) 1); + + sqe = io_uring_get_sqe(ring); + io_uring_prep_nop(sqe); + io_uring_sqe_set_data(sqe, (void *) 1); + + /* no implied submit for newer kernels */ + if (p->features & IORING_FEAT_EXT_ARG) { + ret = io_uring_submit(ring); + if (ret != 2) { + fprintf(stderr, "%s: submit %d\n", __FUNCTION__, ret); + return 1; + } + } + + msec_to_ts(&ts, 1000); + + i = 0; + do { + ret = io_uring_wait_cqes(ring, &cqe, 2, &ts, NULL); + if (ret == -ETIME) + break; + if (ret < 0) { + fprintf(stderr, "%s: wait timeout failed: %d\n", __FUNCTION__, ret); + goto err; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (ret < 0) { + fprintf(stderr, "res: %d\n", ret); + goto err; + } + i++; + } while (1); + + if (i != 2) { + fprintf(stderr, "got %d completions\n", i); + goto err; + } + return 0; +err: + return 1; +} + +/* + * Test single timeout waking us up + */ +static int test_single_timeout(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + unsigned long long exp; + struct __kernel_timespec ts; + struct timeval tv; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + msec_to_ts(&ts, TIMEOUT_MSEC); + io_uring_prep_timeout(sqe, &ts, 0, 0); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + gettimeofday(&tv, NULL); + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (ret == -EINVAL) { + fprintf(stdout, "%s: Timeout not supported, ignored\n", __FUNCTION__); + not_supported = 1; + return 0; + } else if (ret != -ETIME) { + fprintf(stderr, "%s: Timeout: %s\n", __FUNCTION__, strerror(-ret)); + goto err; + } + + exp = mtime_since_now(&tv); + if (exp >= TIMEOUT_MSEC / 2 && exp <= (TIMEOUT_MSEC * 3) / 2) + return 0; + fprintf(stderr, "%s: Timeout seems wonky (got %llu)\n", __FUNCTION__, exp); +err: + return 1; +} + +static int test_single_timeout_remove_notfound(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + int ret, i; + + if (no_modify) + return 0; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + msec_to_ts(&ts, TIMEOUT_MSEC); + io_uring_prep_timeout(sqe, &ts, 2, 0); + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + io_uring_prep_timeout_remove(sqe, 2, 0); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + /* + * We should get two completions. One is our modify request, which should + * complete with -ENOENT. The other is the timeout that will trigger after + * TIMEOUT_MSEC. + */ + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + if (cqe->user_data == 2) { + if (cqe->res != -ENOENT) { + fprintf(stderr, "%s: modify ret %d, wanted ENOENT\n", __FUNCTION__, cqe->res); + break; + } + } else if (cqe->user_data == 1) { + if (cqe->res != -ETIME) { + fprintf(stderr, "%s: timeout ret %d, wanted -ETIME\n", __FUNCTION__, cqe->res); + break; + } + } + io_uring_cqe_seen(ring, cqe); + } + return 0; +err: + return 1; +} + +static int test_single_timeout_remove(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + int ret, i; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + msec_to_ts(&ts, TIMEOUT_MSEC); + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + io_uring_prep_timeout_remove(sqe, 1, 0); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + /* + * We should have two completions ready. One is for the original timeout + * request, user_data == 1, that should have a ret of -ECANCELED. The other + * is for our modify request, user_data == 2, that should have a ret of 0. + */ + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + if (no_modify) + goto seen; + if (cqe->res == -EINVAL && cqe->user_data == 2) { + fprintf(stdout, "Timeout modify not supported, ignoring\n"); + no_modify = 1; + goto seen; + } + if (cqe->user_data == 1) { + if (cqe->res != -ECANCELED) { + fprintf(stderr, "%s: timeout ret %d, wanted canceled\n", __FUNCTION__, cqe->res); + break; + } + } else if (cqe->user_data == 2) { + if (cqe->res) { + fprintf(stderr, "%s: modify ret %d, wanted 0\n", __FUNCTION__, cqe->res); + break; + } + } +seen: + io_uring_cqe_seen(ring, cqe); + } + return 0; +err: + return 1; +} + +/* + * Test single absolute timeout waking us up + */ +static int test_single_timeout_abs(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + unsigned long long exp; + struct __kernel_timespec ts; + struct timespec abs_ts; + struct timeval tv; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + clock_gettime(CLOCK_MONOTONIC, &abs_ts); + ts.tv_sec = abs_ts.tv_sec + 1; + ts.tv_nsec = abs_ts.tv_nsec; + io_uring_prep_timeout(sqe, &ts, 0, IORING_TIMEOUT_ABS); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + gettimeofday(&tv, NULL); + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (ret == -EINVAL) { + fprintf(stdout, "Absolute timeouts not supported, ignored\n"); + return 0; + } else if (ret != -ETIME) { + fprintf(stderr, "Timeout: %s\n", strerror(-ret)); + goto err; + } + + exp = mtime_since_now(&tv); + if (exp >= 1000 / 2 && exp <= (1000 * 3) / 2) + return 0; + fprintf(stderr, "%s: Timeout seems wonky (got %llu)\n", __FUNCTION__, exp); +err: + return 1; +} + +/* + * Test that timeout is canceled on exit + */ +static int test_single_timeout_exit(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + msec_to_ts(&ts, 30000); + io_uring_prep_timeout(sqe, &ts, 0, 0); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + io_uring_queue_exit(ring); + return 0; +err: + io_uring_queue_exit(ring); + return 1; +} + +/* + * Test multi timeouts waking us up + */ +static int test_multi_timeout(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts[2]; + unsigned int timeout[2]; + unsigned long long exp; + struct timeval tv; + int ret, i; + + /* req_1: timeout req, count = 1, time = (TIMEOUT_MSEC * 2) */ + timeout[0] = TIMEOUT_MSEC * 2; + msec_to_ts(&ts[0], timeout[0]); + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts[0], 1, 0); + sqe->user_data = 1; + + /* req_2: timeout req, count = 1, time = TIMEOUT_MSEC */ + timeout[1] = TIMEOUT_MSEC; + msec_to_ts(&ts[1], timeout[1]); + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts[1], 1, 0); + sqe->user_data = 2; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + gettimeofday(&tv, NULL); + for (i = 0; i < 2; i++) { + unsigned int time = 0; + __u64 user_data = 0; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + /* + * Both of these two reqs should timeout, but req_2 should + * return before req_1. + */ + switch (i) { + case 0: + user_data = 2; + time = timeout[1]; + break; + case 1: + user_data = 1; + time = timeout[0]; + break; + } + + if (cqe->user_data != user_data) { + fprintf(stderr, "%s: unexpected timeout req %d sequence\n", + __FUNCTION__, i+1); + goto err; + } + if (cqe->res != -ETIME) { + fprintf(stderr, "%s: Req %d timeout: %s\n", + __FUNCTION__, i+1, strerror(cqe->res)); + goto err; + } + exp = mtime_since_now(&tv); + if (exp < time / 2 || exp > (time * 3) / 2) { + fprintf(stderr, "%s: Req %d timeout seems wonky (got %llu)\n", + __FUNCTION__, i+1, exp); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test multi timeout req with different count + */ +static int test_multi_timeout_nr(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + int ret, i; + + msec_to_ts(&ts, TIMEOUT_MSEC); + + /* req_1: timeout req, count = 2 */ + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 2, 0); + sqe->user_data = 1; + + /* req_2: timeout req, count = 1 */ + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 1, 0); + sqe->user_data = 2; + + /* req_3: nop req */ + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_nop(sqe); + io_uring_sqe_set_data(sqe, (void *) 1); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + /* + * req_2 (count=1) should return without error and req_1 (count=2) + * should timeout. + */ + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + switch (i) { + case 0: + /* Should be nop req */ + if (io_uring_cqe_get_data(cqe) != (void *) 1) { + fprintf(stderr, "%s: nop not seen as 1 or 2\n", __FUNCTION__); + goto err; + } + break; + case 1: + /* Should be timeout req_2 */ + if (cqe->user_data != 2) { + fprintf(stderr, "%s: unexpected timeout req %d sequence\n", + __FUNCTION__, i+1); + goto err; + } + if (cqe->res < 0) { + fprintf(stderr, "%s: Req %d res %d\n", + __FUNCTION__, i+1, cqe->res); + goto err; + } + break; + case 2: + /* Should be timeout req_1 */ + if (cqe->user_data != 1) { + fprintf(stderr, "%s: unexpected timeout req %d sequence\n", + __FUNCTION__, i+1); + goto err; + } + if (cqe->res != -ETIME) { + fprintf(stderr, "%s: Req %d timeout: %s\n", + __FUNCTION__, i+1, strerror(cqe->res)); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test timeout <link> timeout <drain> timeout + */ +static int test_timeout_flags1(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + int ret, i; + + msec_to_ts(&ts, TIMEOUT_MSEC); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 1; + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 2; + sqe->flags |= IOSQE_IO_DRAIN; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 3; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + if (cqe->res == -EINVAL) { + if (!i) + fprintf(stdout, "%s: timeout flags not supported\n", + __FUNCTION__); + io_uring_cqe_seen(ring, cqe); + continue; + } + + switch (cqe->user_data) { + case 1: + if (cqe->res != -ETIME) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, -ETIME); + goto err; + } + break; + case 2: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, + -ECANCELED); + goto err; + } + break; + case 3: + if (cqe->res != -ETIME) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, -ETIME); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test timeout <link> timeout <link> timeout + */ +static int test_timeout_flags2(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + int ret, i; + + msec_to_ts(&ts, TIMEOUT_MSEC); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 1; + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 2; + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 3; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + if (cqe->res == -EINVAL) { + if (!i) + fprintf(stdout, "%s: timeout flags not supported\n", + __FUNCTION__); + io_uring_cqe_seen(ring, cqe); + continue; + } + + switch (cqe->user_data) { + case 1: + if (cqe->res != -ETIME) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, -ETIME); + goto err; + } + break; + case 2: + case 3: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, + -ECANCELED); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +/* + * Test timeout <drain> timeout <link> timeout + */ +static int test_timeout_flags3(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + int ret, i; + + msec_to_ts(&ts, TIMEOUT_MSEC); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 1; + sqe->flags |= IOSQE_IO_DRAIN; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 2; + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 3; + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + for (i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + if (cqe->res == -EINVAL) { + if (!i) + fprintf(stdout, "%s: timeout flags not supported\n", + __FUNCTION__); + io_uring_cqe_seen(ring, cqe); + continue; + } + + switch (cqe->user_data) { + case 1: + case 2: + if (cqe->res != -ETIME) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, -ETIME); + goto err; + } + break; + case 3: + if (cqe->res != -ECANCELED) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, + -ECANCELED); + goto err; + } + break; + } + io_uring_cqe_seen(ring, cqe); + } + + return 0; +err: + return 1; +} + +static int test_update_timeout(struct io_uring *ring, unsigned long ms, + bool abs, bool async, bool linked) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts, ts_upd; + unsigned long long exp_ms, base_ms = 10000; + struct timeval tv; + int ret, i, nr = 2; + __u32 mode = abs ? IORING_TIMEOUT_ABS : 0; + + msec_to_ts(&ts_upd, ms); + gettimeofday(&tv, NULL); + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + msec_to_ts(&ts, base_ms); + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->user_data = 1; + + if (linked) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_nop(sqe); + sqe->user_data = 3; + sqe->flags = IOSQE_IO_LINK; + if (async) + sqe->flags |= IOSQE_ASYNC; + nr++; + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout_update(sqe, &ts_upd, 1, mode); + sqe->user_data = 2; + if (async) + sqe->flags |= IOSQE_ASYNC; + + ret = io_uring_submit(ring); + if (ret != nr) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + for (i = 0; i < nr; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + switch (cqe->user_data) { + case 1: + if (cqe->res != -ETIME) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, -ETIME); + goto err; + } + break; + case 2: + if (cqe->res != 0) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, + 0); + goto err; + } + break; + case 3: + if (cqe->res != 0) { + fprintf(stderr, "nop failed\n"); + goto err; + } + break; + default: + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + exp_ms = mtime_since_now(&tv); + if (exp_ms >= base_ms / 2) { + fprintf(stderr, "too long, timeout wasn't updated\n"); + goto err; + } + if (ms >= 1000 && !abs && exp_ms < ms / 2) { + fprintf(stderr, "fired too early, potentially updated to 0 ms" + "instead of %lu\n", ms); + goto err; + } + return 0; +err: + return 1; +} + +static int test_update_nonexistent_timeout(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + msec_to_ts(&ts, 0); + io_uring_prep_timeout_update(sqe, &ts, 42, 0); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + ret = cqe->res; + if (ret == -ENOENT) + ret = 0; + io_uring_cqe_seen(ring, cqe); + return ret; +err: + return 1; +} + +static int test_update_invalid_flags(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + io_uring_prep_timeout_remove(sqe, 0, IORING_TIMEOUT_ABS); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + if (cqe->res != -EINVAL) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, -EINVAL); + goto err; + } + io_uring_cqe_seen(ring, cqe); + + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + msec_to_ts(&ts, 0); + io_uring_prep_timeout_update(sqe, &ts, 0, -1); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + if (cqe->res != -EINVAL) { + fprintf(stderr, "%s: got %d, wanted %d\n", + __FUNCTION__, cqe->res, -EINVAL); + goto err; + } + io_uring_cqe_seen(ring, cqe); + + return 0; +err: + return 1; +} + +static int fill_exec_target(char *dst, char *path) +{ + struct stat sb; + + /* + * Should either be ./exec-target.t or test/exec-target.t + */ + sprintf(dst, "%s", path); + return stat(dst, &sb); +} + +static int test_timeout_link_cancel(void) +{ + struct io_uring ring; + struct io_uring_cqe *cqe; + char prog_path[PATH_MAX]; + pid_t p; + int ret, i, wstatus; + + if (fill_exec_target(prog_path, "./exec-target.t") && + fill_exec_target(prog_path, "test/exec-target.t")) { + fprintf(stdout, "Can't find exec-target, skipping\n"); + return 0; + } + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + p = fork(); + if (p == -1) { + fprintf(stderr, "fork() failed\n"); + return 1; + } + + if (p == 0) { + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + + msec_to_ts(&ts, 10000); + sqe = io_uring_get_sqe(&ring); + io_uring_prep_timeout(sqe, &ts, 0, 0); + sqe->flags |= IOSQE_IO_LINK; + sqe->user_data = 0; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + sqe->user_data = 1; + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "%s: got %d, wanted 1\n", __FUNCTION__, ret); + exit(1); + } + + /* trigger full cancellation */ + ret = execl(prog_path, prog_path, NULL); + if (ret) { + fprintf(stderr, "exec failed %i\n", errno); + exit(1); + } + exit(0); + } + + if (waitpid(p, &wstatus, 0) == (pid_t)-1) { + perror("waitpid()"); + return 1; + } + if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus)) { + fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus)); + return 1; + } + + for (i = 0; i < 2; ++i) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + if (cqe->res != -ECANCELED) { + fprintf(stderr, "invalid result, user_data: %i res: %i\n", + (int)cqe->user_data, cqe->res); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + } + + io_uring_queue_exit(&ring); + return 0; +} + + +static int test_not_failing_links(void) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct __kernel_timespec ts; + int ret; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + msec_to_ts(&ts, 1); + sqe = io_uring_get_sqe(&ring); + io_uring_prep_timeout(sqe, &ts, 0, IORING_TIMEOUT_ETIME_SUCCESS); + sqe->user_data = 1; + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + sqe->user_data = 2; + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + return 1; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + return 1; + } else if (cqe->user_data == 1 && cqe->res == -EINVAL) { + fprintf(stderr, "ETIME_SUCCESS is not supported, skip\n"); + goto done; + } else if (cqe->res != -ETIME || cqe->user_data != 1) { + fprintf(stderr, "timeout failed %i %i\n", cqe->res, + (int)cqe->user_data); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + return 1; + } else if (cqe->res || cqe->user_data != 2) { + fprintf(stderr, "nop failed %i %i\n", cqe->res, + (int)cqe->user_data); + return 1; + } +done: + io_uring_cqe_seen(&ring, cqe); + io_uring_queue_exit(&ring); + return 0; +} + + +int main(int argc, char *argv[]) +{ + struct io_uring ring, sqpoll_ring; + bool has_timeout_update, sqpoll; + struct io_uring_params p = { }; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init_params(8, &ring, &p); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + ret = io_uring_queue_init(8, &sqpoll_ring, IORING_SETUP_SQPOLL); + sqpoll = !ret; + + ret = test_single_timeout(&ring); + if (ret) { + fprintf(stderr, "test_single_timeout failed\n"); + return ret; + } + if (not_supported) + return 0; + + ret = test_multi_timeout(&ring); + if (ret) { + fprintf(stderr, "test_multi_timeout failed\n"); + return ret; + } + + ret = test_single_timeout_abs(&ring); + if (ret) { + fprintf(stderr, "test_single_timeout_abs failed\n"); + return ret; + } + + ret = test_single_timeout_remove(&ring); + if (ret) { + fprintf(stderr, "test_single_timeout_remove failed\n"); + return ret; + } + + ret = test_single_timeout_remove_notfound(&ring); + if (ret) { + fprintf(stderr, "test_single_timeout_remove_notfound failed\n"); + return ret; + } + + ret = test_single_timeout_many(&ring); + if (ret) { + fprintf(stderr, "test_single_timeout_many failed\n"); + return ret; + } + + ret = test_single_timeout_nr(&ring, 1); + if (ret) { + fprintf(stderr, "test_single_timeout_nr(1) failed\n"); + return ret; + } + ret = test_single_timeout_nr(&ring, 2); + if (ret) { + fprintf(stderr, "test_single_timeout_nr(2) failed\n"); + return ret; + } + + ret = test_multi_timeout_nr(&ring); + if (ret) { + fprintf(stderr, "test_multi_timeout_nr failed\n"); + return ret; + } + + ret = test_timeout_flags1(&ring); + if (ret) { + fprintf(stderr, "test_timeout_flags1 failed\n"); + return ret; + } + + ret = test_timeout_flags2(&ring); + if (ret) { + fprintf(stderr, "test_timeout_flags2 failed\n"); + return ret; + } + + ret = test_timeout_flags3(&ring); + if (ret) { + fprintf(stderr, "test_timeout_flags3 failed\n"); + return ret; + } + + ret = test_single_timeout_wait(&ring, &p); + if (ret) { + fprintf(stderr, "test_single_timeout_wait failed\n"); + return ret; + } + + /* io_uring_wait_cqes() may have left a timeout, reinit ring */ + io_uring_queue_exit(&ring); + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + ret = test_update_nonexistent_timeout(&ring); + has_timeout_update = (ret != -EINVAL); + if (has_timeout_update) { + if (ret) { + fprintf(stderr, "test_update_nonexistent_timeout failed\n"); + return ret; + } + + ret = test_update_invalid_flags(&ring); + if (ret) { + fprintf(stderr, "test_update_invalid_flags failed\n"); + return ret; + } + + ret = test_update_timeout(&ring, 0, false, false, false); + if (ret) { + fprintf(stderr, "test_update_timeout failed\n"); + return ret; + } + + ret = test_update_timeout(&ring, 1, false, false, false); + if (ret) { + fprintf(stderr, "test_update_timeout 1ms failed\n"); + return ret; + } + + ret = test_update_timeout(&ring, 1000, false, false, false); + if (ret) { + fprintf(stderr, "test_update_timeout 1s failed\n"); + return ret; + } + + ret = test_update_timeout(&ring, 0, true, true, false); + if (ret) { + fprintf(stderr, "test_update_timeout abs failed\n"); + return ret; + } + + + ret = test_update_timeout(&ring, 0, false, true, false); + if (ret) { + fprintf(stderr, "test_update_timeout async failed\n"); + return ret; + } + + ret = test_update_timeout(&ring, 0, false, false, true); + if (ret) { + fprintf(stderr, "test_update_timeout linked failed\n"); + return ret; + } + + if (sqpoll) { + ret = test_update_timeout(&sqpoll_ring, 0, false, false, + false); + if (ret) { + fprintf(stderr, "test_update_timeout sqpoll" + "failed\n"); + return ret; + } + } + } + + /* + * this test must go last, it kills the ring + */ + ret = test_single_timeout_exit(&ring); + if (ret) { + fprintf(stderr, "test_single_timeout_exit failed\n"); + return ret; + } + + ret = test_timeout_link_cancel(); + if (ret) { + fprintf(stderr, "test_timeout_link_cancel failed\n"); + return ret; + } + + ret = test_not_failing_links(); + if (ret) { + fprintf(stderr, "test_not_failing_links failed\n"); + return ret; + } + + if (sqpoll) + io_uring_queue_exit(&sqpoll_ring); + return 0; +} diff --git a/contrib/libs/liburing/test/tty-write-dpoll.c b/contrib/libs/liburing/test/tty-write-dpoll.c new file mode 100644 index 0000000000..62a5f155a7 --- /dev/null +++ b/contrib/libs/liburing/test/tty-write-dpoll.c @@ -0,0 +1,61 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test double poll tty write. A test case for the regression fixed by: + * + * commit 6e295a664efd083ac9a5c1a8130c45be1db0cde7 + * Author: Jens Axboe <axboe@kernel.dk> + * Date: Tue Mar 22 13:11:28 2022 -0600 + * + * io_uring: fix assuming triggered poll waitqueue is the single poll + * + */ +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +#define SQES 128 +#define BUFSIZE 512 + +int main(int argc, char *argv[]) +{ + static char buf[BUFSIZE]; + struct iovec vecs[SQES]; + struct io_uring ring; + int ret, i, fd; + + if (argc > 1) + return 0; + + fd = open("/dev/ttyS0", O_RDWR | O_NONBLOCK); + if (fd < 0) + return 0; + + ret = t_create_ring(SQES, &ring, 0); + if (ret == T_SETUP_SKIP) + return 0; + else if (ret < 0) + return 1; + + for (i = 0; i < SQES; i++) { + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(&ring); + vecs[i].iov_base = buf; + vecs[i].iov_len = sizeof(buf); + io_uring_prep_writev(sqe, fd, &vecs[i], 1, 0); + } + + ret = io_uring_submit(&ring); + if (ret != SQES) { + fprintf(stderr, "submit: %d\n", ret); + return 1; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/unlink.c b/contrib/libs/liburing/test/unlink.c new file mode 100644 index 0000000000..b5094ccb59 --- /dev/null +++ b/contrib/libs/liburing/test/unlink.c @@ -0,0 +1,113 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: run various nop tests + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/stat.h> + +#include "liburing.h" + +static int test_unlink(struct io_uring *ring, const char *old) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + io_uring_prep_unlink(sqe, old, 0); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + return ret; +err: + return 1; +} + +static int stat_file(const char *buf) +{ + struct stat sb; + + if (!stat(buf, &sb)) + return 0; + + return errno; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + char buf[32] = "./XXXXXX"; + int ret; + + if (argc > 1) + return 0; + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + ret = mkstemp(buf); + if (ret < 0) { + perror("mkstemp"); + return 1; + } + close(ret); + + if (stat_file(buf) != 0) { + perror("stat"); + return 1; + } + + ret = test_unlink(&ring, buf); + if (ret < 0) { + if (ret == -EBADF || ret == -EINVAL) { + fprintf(stdout, "Unlink not supported, skipping\n"); + unlink(buf); + return 0; + } + fprintf(stderr, "rename: %s\n", strerror(-ret)); + goto err; + } else if (ret) + goto err; + + ret = stat_file(buf); + if (ret != ENOENT) { + fprintf(stderr, "stat got %s\n", strerror(ret)); + return 1; + } + + ret = test_unlink(&ring, "/3/2/3/1/z/y"); + if (ret != -ENOENT) { + fprintf(stderr, "invalid unlink got %s\n", strerror(-ret)); + return 1; + } + + return 0; +err: + unlink(buf); + return 1; +} diff --git a/contrib/libs/liburing/test/wakeup-hang.c b/contrib/libs/liburing/test/wakeup-hang.c new file mode 100644 index 0000000000..1d1140c15e --- /dev/null +++ b/contrib/libs/liburing/test/wakeup-hang.c @@ -0,0 +1,163 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +#include <sys/eventfd.h> +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pthread.h> +#include <liburing.h> +#include <fcntl.h> +#include <poll.h> +#include <sys/time.h> + +struct thread_data { + struct io_uring *ring; + int write_fd; +}; + +static void error_exit(char *message) +{ + perror(message); + exit(1); +} + +static void *listener_thread(void *data) +{ + struct thread_data *td = data; + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_wait_cqe(td->ring, &cqe); + if (ret < 0) { + fprintf(stderr, "Error waiting for completion: %s\n", + strerror(-ret)); + goto err; + } + if (cqe->res < 0) { + fprintf(stderr, "Error in async operation: %s\n", strerror(-cqe->res)); + goto err; + } + io_uring_cqe_seen(td->ring, cqe); + return NULL; +err: + return (void *) 1; +} + +static void *wakeup_io_uring(void *data) +{ + struct thread_data *td = data; + int res; + + res = eventfd_write(td->write_fd, (eventfd_t) 1L); + if (res < 0) { + perror("eventfd_write"); + return (void *) 1; + } + return NULL; +} + +static int test_pipes(void) +{ + struct io_uring_sqe *sqe; + struct thread_data td; + struct io_uring ring; + pthread_t t1, t2; + int ret, fds[2]; + void *pret; + + if (pipe(fds) < 0) + error_exit("eventfd"); + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "Unable to setup io_uring: %s\n", strerror(-ret)); + return 1; + } + + td.write_fd = fds[1]; + td.ring = ˚ + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, fds[0], POLLIN); + sqe->user_data = 2; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "ring_submit=%d\n", ret); + return 1; + } + + pthread_create(&t1, NULL, listener_thread, &td); + + sleep(1); + + pthread_create(&t2, NULL, wakeup_io_uring, &td); + pthread_join(t1, &pret); + + io_uring_queue_exit(&ring); + return pret != NULL; +} + +static int test_eventfd(void) +{ + struct io_uring_sqe *sqe; + struct thread_data td; + struct io_uring ring; + pthread_t t1, t2; + int efd, ret; + void *pret; + + efd = eventfd(0, 0); + if (efd < 0) + error_exit("eventfd"); + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "Unable to setup io_uring: %s\n", strerror(-ret)); + return 1; + } + + td.write_fd = efd; + td.ring = ˚ + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, efd, POLLIN); + sqe->user_data = 2; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "ring_submit=%d\n", ret); + return 1; + } + + pthread_create(&t1, NULL, listener_thread, &td); + + sleep(1); + + pthread_create(&t2, NULL, wakeup_io_uring, &td); + pthread_join(t1, &pret); + + io_uring_queue_exit(&ring); + return pret != NULL; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return 0; + + ret = test_pipes(); + if (ret) { + fprintf(stderr, "test_pipe failed\n"); + return ret; + } + + ret = test_eventfd(); + if (ret) { + fprintf(stderr, "test_eventfd failed\n"); + return ret; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/xattr.c b/contrib/libs/liburing/test/xattr.c new file mode 100644 index 0000000000..e1f97e5829 --- /dev/null +++ b/contrib/libs/liburing/test/xattr.c @@ -0,0 +1,426 @@ +#include "../config-host.h" +#include <assert.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/xattr.h> +#include <unistd.h> + +#include "helpers.h" +#include "liburing.h" + +static int no_xattr; + +/* Define constants. */ +#define XATTR_SIZE 255 +#define QUEUE_DEPTH 32 + +#define FILENAME "xattr.test" +#define KEY1 "user.val1" +#define KEY2 "user.val2" +#define VALUE1 "value1" +#define VALUE2 "value2-a-lot-longer" + + +/* Call fsetxattr. */ +static int io_uring_fsetxattr(struct io_uring *ring, int fd, const char *name, + const void *value, size_t size, int flags) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "Error cannot get sqe\n"); + return -1; + } + + io_uring_prep_fsetxattr(sqe, fd, name, value, flags, size); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "Error io_uring_submit_and_wait: ret=%d\n", ret); + return -1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "Error io_uring_wait_cqe: ret=%d\n", ret); + return -1; + } + + ret = cqe->res; + if (ret == -EINVAL) + no_xattr = 1; + io_uring_cqe_seen(ring, cqe); + + return ret; +} + +/* Submit fgetxattr request. */ +static int io_uring_fgetxattr(struct io_uring *ring, int fd, const char *name, + void *value, size_t size) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "Error cannot get sqe\n"); + return -1; + } + + io_uring_prep_fgetxattr(sqe, fd, name, value, size); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "Error io_uring_submit_and_wait: ret=%d\n", ret); + return -1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "Error io_uring_wait_cqe: ret=%d\n", ret); + return -1; + } + + ret = cqe->res; + if (ret == -1) { + fprintf(stderr, "Error couldn'tget value\n"); + return -1; + } + + io_uring_cqe_seen(ring, cqe); + return ret; +} + +/* Call setxattr. */ +static int io_uring_setxattr(struct io_uring *ring, const char *path, + const char *name, const void *value, size_t size, + int flags) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "Error cannot get sqe\n"); + return -1; + } + + io_uring_prep_setxattr(sqe, name, value, path, flags, size); + + ret = io_uring_submit_and_wait(ring, 1); + if (ret != 1) { + fprintf(stderr, "Error io_uring_submit_and_wait: ret=%d\n", ret); + return -1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "Error io_uring_wait_cqe: ret=%d\n", ret); + return -1; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + + return ret; +} + +/* Submit getxattr request. */ +static int io_uring_getxattr(struct io_uring *ring, const char *path, + const char *name, void *value, size_t size) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "Error cannot get sqe\n"); + return -1; + } + + io_uring_prep_getxattr(sqe, name, value, path, size); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "Error io_uring_submit_and_wait: ret=%d\n", ret); + return -1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "Error io_uring_wait_cqe: ret=%d\n", ret); + return -1; + } + + ret = cqe->res; + if (ret == -1) { + fprintf(stderr, "Error couldn'tget value\n"); + return -1; + } + + io_uring_cqe_seen(ring, cqe); + return ret; +} + +/* Test driver for fsetxattr and fgetxattr. */ +static int test_fxattr(void) +{ + int rc = 0; + size_t value_len; + struct io_uring ring; + char value[XATTR_SIZE]; + + /* Init io-uring queue. */ + int ret = io_uring_queue_init(QUEUE_DEPTH, &ring, 0); + if (ret) { + fprintf(stderr, "child: ring setup failed: %d\n", ret); + return -1; + } + + /* Create the test file. */ + int fd = open(FILENAME, O_CREAT | O_RDWR, 0644); + if (fd < 0) { + fprintf(stderr, "Error: cannot open file: ret=%d\n", fd); + return -1; + } + + /* Test writing attributes. */ + if (io_uring_fsetxattr(&ring, fd, KEY1, VALUE1, strlen(VALUE1), 0) < 0) { + if (no_xattr) { + fprintf(stdout, "No xattr support, skipping\n"); + goto Exit; + } + fprintf(stderr, "Error fsetxattr cannot write key1\n"); + rc = -1; + goto Exit; + } + + if (io_uring_fsetxattr(&ring, fd, KEY2, VALUE2, strlen(VALUE2), 0) < 0) { + fprintf(stderr, "Error fsetxattr cannot write key1\n"); + rc = -1; + goto Exit; + } + + /* Test reading attributes. */ + value_len = io_uring_fgetxattr(&ring, fd, KEY1, value, XATTR_SIZE); + if (value_len != strlen(VALUE1) || strncmp(value, VALUE1, value_len)) { + fprintf(stderr, "Error: fgetxattr expected value: %s, returned value: %s\n", VALUE1, value); + rc = -1; + goto Exit; + } + + value_len = io_uring_fgetxattr(&ring, fd, KEY2, value, XATTR_SIZE); + if (value_len != strlen(VALUE2) || strncmp(value, VALUE2, value_len)) { + fprintf(stderr, "Error: fgetxattr expected value: %s, returned value: %s\n", VALUE2, value); + rc = -1; + goto Exit; + } + + /* Cleanup. */ +Exit: + close(fd); + unlink(FILENAME); + + io_uring_queue_exit(&ring); + + return rc; +} + +/* Test driver for setxattr and getxattr. */ +static int test_xattr(void) +{ + int rc = 0; + int value_len; + struct io_uring ring; + char value[XATTR_SIZE]; + + /* Init io-uring queue. */ + int ret = io_uring_queue_init(QUEUE_DEPTH, &ring, 0); + if (ret) { + fprintf(stderr, "child: ring setup failed: %d\n", ret); + return -1; + } + + /* Create the test file. */ + t_create_file(FILENAME, 0); + + /* Test writing attributes. */ + if (io_uring_setxattr(&ring, FILENAME, KEY1, VALUE1, strlen(VALUE1), 0) < 0) { + fprintf(stderr, "Error setxattr cannot write key1\n"); + rc = -1; + goto Exit; + } + + if (io_uring_setxattr(&ring, FILENAME, KEY2, VALUE2, strlen(VALUE2), 0) < 0) { + fprintf(stderr, "Error setxattr cannot write key1\n"); + rc = -1; + goto Exit; + } + + /* Test reading attributes. */ + value_len = io_uring_getxattr(&ring, FILENAME, KEY1, value, XATTR_SIZE); + if (value_len != strlen(VALUE1) || strncmp(value, VALUE1, value_len)) { + fprintf(stderr, "Error: getxattr expected value: %s, returned value: %s\n", VALUE1, value); + rc = -1; + goto Exit; + } + + value_len = io_uring_getxattr(&ring, FILENAME, KEY2, value, XATTR_SIZE); + if (value_len != strlen(VALUE2) || strncmp(value, VALUE2, value_len)) { + fprintf(stderr, "Error: getxattr expected value: %s, returned value: %s\n", VALUE2, value); + rc = -1; + goto Exit; + } + + /* Cleanup. */ +Exit: + io_uring_queue_exit(&ring); + unlink(FILENAME); + + return rc; +} + +/* Test driver for failure cases of fsetxattr and fgetxattr. */ +static int test_failure_fxattr(void) +{ + int rc = 0; + struct io_uring ring; + char value[XATTR_SIZE]; + + /* Init io-uring queue. */ + int ret = io_uring_queue_init(QUEUE_DEPTH, &ring, 0); + if (ret) { + fprintf(stderr, "child: ring setup failed: %d\n", ret); + return -1; + } + + /* Create the test file. */ + int fd = open(FILENAME, O_CREAT | O_RDWR, 0644); + if (fd < 0) { + fprintf(stderr, "Error: cannot open file: ret=%d\n", fd); + return -1; + } + + /* Test writing attributes. */ + assert(io_uring_fsetxattr(&ring, -1, KEY1, VALUE1, strlen(VALUE1), 0) < 0); + assert(io_uring_fsetxattr(&ring, fd, NULL, VALUE1, strlen(VALUE1), 0) < 0); + assert(io_uring_fsetxattr(&ring, fd, KEY1, NULL, strlen(VALUE1), 0) < 0); + assert(io_uring_fsetxattr(&ring, fd, KEY1, VALUE1, 0, 0) == 0); + assert(io_uring_fsetxattr(&ring, fd, KEY1, VALUE1, -1, 0) < 0); + + /* Test reading attributes. */ + assert(io_uring_fgetxattr(&ring, -1, KEY1, value, XATTR_SIZE) < 0); + assert(io_uring_fgetxattr(&ring, fd, NULL, value, XATTR_SIZE) < 0); + assert(io_uring_fgetxattr(&ring, fd, KEY1, value, 0) == 0); + + /* Cleanup. */ + close(fd); + unlink(FILENAME); + + io_uring_queue_exit(&ring); + + return rc; +} + + +/* Test driver for failure cases for setxattr and getxattr. */ +static int test_failure_xattr(void) +{ + int rc = 0; + struct io_uring ring; + char value[XATTR_SIZE]; + + /* Init io-uring queue. */ + int ret = io_uring_queue_init(QUEUE_DEPTH, &ring, 0); + if (ret) { + fprintf(stderr, "child: ring setup failed: %d\n", ret); + return -1; + } + + /* Create the test file. */ + t_create_file(FILENAME, 0); + + /* Test writing attributes. */ + assert(io_uring_setxattr(&ring, "complete garbage", KEY1, VALUE1, strlen(VALUE1), 0) < 0); + assert(io_uring_setxattr(&ring, NULL, KEY1, VALUE1, strlen(VALUE1), 0) < 0); + assert(io_uring_setxattr(&ring, FILENAME, NULL, VALUE1, strlen(VALUE1), 0) < 0); + assert(io_uring_setxattr(&ring, FILENAME, KEY1, NULL, strlen(VALUE1), 0) < 0); + assert(io_uring_setxattr(&ring, FILENAME, KEY1, VALUE1, 0, 0) == 0); + + /* Test reading attributes. */ + assert(io_uring_getxattr(&ring, "complete garbage", KEY1, value, XATTR_SIZE) < 0); + assert(io_uring_getxattr(&ring, NULL, KEY1, value, XATTR_SIZE) < 0); + assert(io_uring_getxattr(&ring, FILENAME, NULL, value, XATTR_SIZE) < 0); + assert(io_uring_getxattr(&ring, FILENAME, KEY1, NULL, XATTR_SIZE) == 0); + assert(io_uring_getxattr(&ring, FILENAME, KEY1, value, 0) == 0); + + /* Cleanup. */ + io_uring_queue_exit(&ring); + unlink(FILENAME); + + return rc; +} + +/* Test for invalid SQE, this will cause a segmentation fault if enabled. */ +static int test_invalid_sqe(void) +{ +#ifdef DESTRUCTIVE_TEST + struct io_uring_sqe *sqe = NULL; + struct io_uring_cqe *cqe = NULL; + struct io_uring ring; + + /* Init io-uring queue. */ + int ret = io_uring_queue_init(QUEUE_DEPTH, &ring, 0); + if (ret) { + fprintf(stderr, "child: ring setup failed: %d\n", ret); + return -1; + } + + /* Pass invalid SQE. */ + io_uring_prep_setxattr(sqe, FILENAME, KEY1, VALUE1, strlen(VALUE1), 0); + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "Error io_uring_submit_and_wait: ret=%d\n", ret); + return -1; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "Error io_uring_wait_cqe: ret=%d\n", ret); + return -1; + } + + ret = cqe->res; + io_uring_cqe_seen(&ring, cqe); + + return ret; +#else + return 0; +#endif +} + +/* Test driver. */ +int main(int argc, char *argv[]) +{ + if (argc > 1) + return 0; + + if (test_fxattr()) + return EXIT_FAILURE; + if (no_xattr) + return EXIT_SUCCESS; + if (test_xattr() || test_failure_fxattr() || test_failure_xattr() || + test_invalid_sqe()) + return EXIT_FAILURE; + + return EXIT_SUCCESS; +} diff --git a/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/alter_table.md b/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/alter_table.md index 58e52b9262..100b4ca438 100644 --- a/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/alter_table.md +++ b/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/alter_table.md @@ -64,7 +64,7 @@ ALTER TABLE `series` DROP INDEX `title_index`; * `NEW_AND_OLD_IMAGES` - комбинация режимов `NEW_IMAGE` и `OLD_IMAGE`. Будут записаны значения всех столбцов _до_ и _в результате_ изменения. * `FORMAT` — формат данных, в котором будут записаны данные. * `JSON` — структура записи приведена на странице [описания changefeed](../../../../concepts/cdc#record-structure). -* `RETENTION_PERIOD` — время хранения записей. Тип значения — `Interval`, значение по умолчанию — 24 часа (`Interval('PT24H')`). +* `RETENTION_PERIOD` — [время хранения записей](../../../../concepts/cdc#retention-period). Тип значения — `Interval`, значение по умолчанию — 24 часа (`Interval('PT24H')`). Приведенный ниже код добавит поток изменений с именем `updates_feed`, в который будут выгружаться значения изменившихся столбцов таблицы в формате JSON: diff --git a/ydb/library/pdisk_io/CMakeLists.linux-aarch64.txt b/ydb/library/pdisk_io/CMakeLists.linux-aarch64.txt index 5680d4da42..02ae81be74 100644 --- a/ydb/library/pdisk_io/CMakeLists.linux-aarch64.txt +++ b/ydb/library/pdisk_io/CMakeLists.linux-aarch64.txt @@ -7,7 +7,6 @@ find_package(AIO REQUIRED) -find_package(liburing REQUIRED) add_subdirectory(protos) add_library(ydb-library-pdisk_io) @@ -16,7 +15,7 @@ target_link_libraries(ydb-library-pdisk_io PUBLIC yutil tools-enum_parser-enum_serialization_runtime AIO::aio - liburing::liburing + contrib-libs-liburing cpp-actors-core cpp-actors-wilson cpp-monlib-dynamic_counters diff --git a/ydb/library/pdisk_io/CMakeLists.linux.txt b/ydb/library/pdisk_io/CMakeLists.linux.txt index 5680d4da42..02ae81be74 100644 --- a/ydb/library/pdisk_io/CMakeLists.linux.txt +++ b/ydb/library/pdisk_io/CMakeLists.linux.txt @@ -7,7 +7,6 @@ find_package(AIO REQUIRED) -find_package(liburing REQUIRED) add_subdirectory(protos) add_library(ydb-library-pdisk_io) @@ -16,7 +15,7 @@ target_link_libraries(ydb-library-pdisk_io PUBLIC yutil tools-enum_parser-enum_serialization_runtime AIO::aio - liburing::liburing + contrib-libs-liburing cpp-actors-core cpp-actors-wilson cpp-monlib-dynamic_counters |