diff options
| author | Alexander Smirnov <alex@ydb.tech> | 2025-04-16 21:15:40 +0000 |
|---|---|---|
| committer | Alexander Smirnov <alex@ydb.tech> | 2025-04-16 21:15:40 +0000 |
| commit | 86cc1622df88a9a9d1372a0db64faf4db6eb760c (patch) | |
| tree | 2f91a47403dc034a0603698db757a656cce74d4d | |
| parent | a30e65bc695ebb642c9a25ad7b1c12134171ce30 (diff) | |
| parent | 1da0c06adbe3fddcd5cf81c3ff0922a38644c62e (diff) | |
| download | ydb-86cc1622df88a9a9d1372a0db64faf4db6eb760c.tar.gz | |
Merge branch 'rightlib' into merge-libs-250416-2114
112 files changed, 3529 insertions, 1872 deletions
diff --git a/build/plugins/nots.py b/build/plugins/nots.py index a24a5ca2e59..3d5122b6285 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -26,7 +26,8 @@ if TYPE_CHECKING: # 1 is 60 files per chunk for TIMEOUT(60) - default timeout for SIZE(SMALL) # 0.5 is 120 files per chunk for TIMEOUT(60) - default timeout for SIZE(SMALL) # 0.2 is 300 files per chunk for TIMEOUT(60) - default timeout for SIZE(SMALL) -ESLINT_FILE_PROCESSING_TIME_DEFAULT = 0.2 # seconds per file +# 0.0 - not to use chunks +ESLINT_FILE_PROCESSING_TIME_DEFAULT = 0.0 # seconds per file REQUIRED_MISSING = "~~required~~" diff --git a/contrib/libs/lzma/.yandex_meta/__init__.py b/contrib/libs/lzma/.yandex_meta/__init__.py index 29e2a09e7e9..3705027de73 100644 --- a/contrib/libs/lzma/.yandex_meta/__init__.py +++ b/contrib/libs/lzma/.yandex_meta/__init__.py @@ -13,6 +13,7 @@ lzma = GNUMakeNixProject( flags=["--localedir=/var/empty"], disable_includes=[ "crc32_arm64.h", + "crc32_loongarch.h", "crc32_table_be.h", "crc64_table_be.h", "dpmi.h", diff --git a/contrib/libs/lzma/.yandex_meta/devtools.copyrights.report b/contrib/libs/lzma/.yandex_meta/devtools.copyrights.report index f182796fbda..4b168d8f34a 100644 --- a/contrib/libs/lzma/.yandex_meta/devtools.copyrights.report +++ b/contrib/libs/lzma/.yandex_meta/devtools.copyrights.report @@ -55,7 +55,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - COPYING [46:46] + COPYING [52:52] KEEP COPYRIGHT_SERVICE_LABEL 8cf91579d8cb4087b6bd775cd2b2bbed BELONGS ya.make diff --git a/contrib/libs/lzma/.yandex_meta/devtools.licenses.report b/contrib/libs/lzma/.yandex_meta/devtools.licenses.report index aed21538703..ac3196e2aa9 100644 --- a/contrib/libs/lzma/.yandex_meta/devtools.licenses.report +++ b/contrib/libs/lzma/.yandex_meta/devtools.licenses.report @@ -46,24 +46,11 @@ BELONGS ya.make Match type : TAG Links : http://landley.net/toybox/license.html, https://spdx.org/licenses/0BSD Files with this license: - COPYING [43:43] - -SKIP 0BSD 14de854251593dd36ab8a222793a5b8f -BELONGS ya.make - # Not a license - License text: - domain; 0BSD-licensed code is copyrighted but available under - Scancode info: - Original SPDX id: 0BSD - Score : 50.00 - Match type : REFERENCE - Links : http://landley.net/toybox/license.html, https://spdx.org/licenses/0BSD - Files with this license: - COPYING [74:74] + COPYING [49:49] SKIP LGPL-2.1-only 181bd2f3201dc6ff4c508cb04d170817 BELONGS ya.make -FILE_INCLUDE COPYING.LGPLv2.1 found in files: README at line 81 +FILE_INCLUDE COPYING.LGPLv2.1 found in files: README at line 82 # Not a license by itself License text: COPYING.LGPLv2.1 GNU Lesser General Public License version 2.1 @@ -73,20 +60,7 @@ FILE_INCLUDE COPYING.LGPLv2.1 found in files: README at line 81 Match type : REFERENCE Links : http://www.gnu.org/licenses/lgpl-2.1.html, http://www.gnu.org/licenses/lgpl-2.1.txt, https://spdx.org/licenses/LGPL-2.1-only Files with this license: - README [81:81] - -KEEP Public-Domain 19d6afb7fb41001fbab2151dd27a7fb9 -BELONGS ya.make - License text: - There is very little *practical* difference between public - domain and 0BSD. The main difference likely is that one - Scancode info: - Original SPDX id: LicenseRef-scancode-public-domain - Score : 70.00 - Match type : REFERENCE - Links : http://www.linfo.org/publicdomain.html, https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/public-domain.LICENSE - Files with this license: - COPYING [71:72] + README [82:82] SKIP 0BSD 1a459a2067895b8576617d7b67fe52ca BELONGS ya.make @@ -112,19 +86,7 @@ BELONGS ya.make Match type : REFERENCE Links : http://landley.net/toybox/license.html, https://spdx.org/licenses/0BSD Files with this license: - COPYING [48:48] - -KEEP 0BSD 21b027dd1b59c23a56e34a32360d0c10 -BELONGS ya.make - License text: - 0BSD for newer releases was made in Febrary 2024 because - Scancode info: - Original SPDX id: 0BSD - Score : 50.00 - Match type : REFERENCE - Links : http://landley.net/toybox/license.html, https://spdx.org/licenses/0BSD - Files with this license: - COPYING [67:67] + COPYING [54:54] SKIP LGPL-2.1-only 23c2a5e0106b99d75238986559bb5fc6 BELONGS ya.make @@ -141,7 +103,7 @@ FILE_INCLUDE COPYING found in files: COPYING.LGPLv2.1 at line 116 SKIP GPL-2.0-only 3220ce24cf5698918173f671259bb375 BELONGS ya.make -FILE_INCLUDE COPYING.GPLv2 found in files: README at line 79 +FILE_INCLUDE COPYING.GPLv2 found in files: README at line 80 # Not a license by itself License text: COPYING.GPLv2 GNU General Public License version 2 @@ -151,7 +113,7 @@ FILE_INCLUDE COPYING.GPLv2 found in files: README at line 79 Match type : REFERENCE Links : http://www.gnu.org/licenses/gpl-2.0.html, http://www.gnu.org/licenses/gpl-2.0.txt, https://spdx.org/licenses/GPL-2.0-only Files with this license: - README [79:79] + README [80:80] KEEP 0BSD 327b128c3f16957fab69efc9b095e368 BELONGS ya.make @@ -221,7 +183,7 @@ BELONGS ya.make Match type : REFERENCE Links : http://landley.net/toybox/license.html, https://spdx.org/licenses/0BSD Files with this license: - COPYING [57:57] + COPYING [63:63] SKIP LGPL-2.0-only AND LGPL-2.1-or-later 493b7c38175021ac13d1589215e13989 BELONGS ya.make @@ -282,18 +244,6 @@ BELONGS ya.make liblzma/api/lzma/vli.h [1:1] liblzma/liblzma_linux.map [1:1] -KEEP Public-Domain 65b77ffe5d577fa84afe9f017beede5c -BELONGS ya.make - License text: - public domain has (real or perceived) legal ambiguities in - Scancode info: - Original SPDX id: LicenseRef-scancode-public-domain - Score : 70.00 - Match type : REFERENCE - Links : http://www.linfo.org/publicdomain.html, https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/public-domain.LICENSE - Files with this license: - COPYING [68:68] - SKIP GPL-3.0-only 6dbb0879a53d93955d47d50ed23aa325 BELONGS ya.make # This is a reference to the file with GPL-3.0 license @@ -305,7 +255,7 @@ BELONGS ya.make Match type : REFERENCE Links : http://www.gnu.org/licenses/gpl-3.0-standalone.html, http://www.gnu.org/licenses/gpl-3.0.html, https://spdx.org/licenses/GPL-3.0-only Files with this license: - COPYING [60:60] + COPYING [66:66] KEEP Public-Domain 702cbf9589b30951b6a2b94913b66332 BELONGS ya.make @@ -330,7 +280,7 @@ BELONGS ya.make Match type : REFERENCE Links : http://www.gnu.org/licenses/gpl-2.0.html, http://www.gnu.org/licenses/gpl-2.0.txt, https://spdx.org/licenses/GPL-2.0-only Files with this license: - COPYING [59:59] + COPYING [65:65] KEEP 0BSD 79430bd4c5e1b494780b3fb35b24fd7d BELONGS ya.make @@ -364,37 +314,6 @@ BELONGS ya.make Files with this license: COPYING [25:25] -KEEP 0BSD AND Public-Domain 83aabd12f73b5065603bf2054fb6d54f -BELONGS ya.make - License text: - an extremely permissive license. Neither 0BSD nor public domain - Scancode info: - Original SPDX id: 0BSD - Score : 50.00 - Match type : REFERENCE - Links : http://landley.net/toybox/license.html, https://spdx.org/licenses/0BSD - Files with this license: - COPYING [75:75] - Scancode info: - Original SPDX id: LicenseRef-scancode-public-domain - Score : 70.00 - Match type : REFERENCE - Links : http://www.linfo.org/publicdomain.html, https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/public-domain.LICENSE - Files with this license: - COPYING [75:75] - -KEEP Public-Domain 90705824d045bd99f4cb1ef6f42a57ae -BELONGS ya.make - License text: - significant amount of code put into the public domain and - Scancode info: - Original SPDX id: LicenseRef-scancode-public-domain - Score : 100.00 - Match type : TEXT - Links : http://www.linfo.org/publicdomain.html, https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/public-domain.LICENSE - Files with this license: - COPYING [65:65] - KEEP BSD-3-Clause AND 0BSD a675dbcfcecb36b880f01b1c31049149 BELONGS ya.make License text: @@ -414,33 +333,9 @@ BELONGS ya.make Files with this license: liblzma/api/lzma.h [23:23] -KEEP Public-Domain a7c077779fe1982e666c1f33e396f1b5 -BELONGS ya.make - License text: - that obviously remains so. The switch from public domain to - Scancode info: - Original SPDX id: LicenseRef-scancode-public-domain - Score : 70.00 - Match type : REFERENCE - Links : http://www.linfo.org/publicdomain.html, https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/public-domain.LICENSE - Files with this license: - COPYING [66:66] - -KEEP 0BSD a8a6073ae588092bed4e746ed49b52f5 -BELONGS ya.make - License text: - domain and 0BSD. The main difference likely is that one - Scancode info: - Original SPDX id: 0BSD - Score : 50.00 - Match type : REFERENCE - Links : http://landley.net/toybox/license.html, https://spdx.org/licenses/0BSD - Files with this license: - COPYING [72:72] - SKIP GPL-3.0-only ad3e81fa9e595e378d10023b1bf20d51 BELONGS ya.make -FILE_INCLUDE COPYING.GPLv3 found in files: README at line 80 +FILE_INCLUDE COPYING.GPLv3 found in files: README at line 81 # Not a license by itself License text: COPYING.GPLv3 GNU General Public License version 3 @@ -450,31 +345,7 @@ FILE_INCLUDE COPYING.GPLv3 found in files: README at line 80 Match type : REFERENCE Links : http://www.gnu.org/licenses/gpl-3.0-standalone.html, http://www.gnu.org/licenses/gpl-3.0.html, https://spdx.org/licenses/GPL-3.0-only Files with this license: - README [80:80] - -KEEP BSD-2-Clause b3129c96ef9c28cff49bffaf9d804a88 -BELONGS ya.make - License text: - for example, BSD 2-Clause "Simplified" License which does have - Scancode info: - Original SPDX id: BSD-2-Clause - Score : 100.00 - Match type : NOTICE - Links : http://opensource.org/licenses/bsd-license.php, http://www.opensource.org/licenses/BSD-2-Clause, https://spdx.org/licenses/BSD-2-Clause - Files with this license: - COPYING [78:78] - -KEEP Public-Domain c1a07f044cdb7e843b741e6236100ea3 -BELONGS ya.make - License text: - /// and the public domain code from https://github.com/rawrunprotected/crc - Scancode info: - Original SPDX id: LicenseRef-scancode-public-domain - Score : 100.00 - Match type : NOTICE - Links : http://www.linfo.org/publicdomain.html, https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/public-domain.LICENSE - Files with this license: - liblzma/check/crc_x86_clmul.h [13:13] + README [81:81] KEEP Public-Domain cc574f0a684665d6c5ae08f28aec98dd BELONGS ya.make @@ -511,20 +382,7 @@ BELONGS ya.make Match type : REFERENCE Links : http://www.gnu.org/licenses/lgpl-2.1.html, http://www.gnu.org/licenses/lgpl-2.1.txt, https://spdx.org/licenses/LGPL-2.1-only Files with this license: - COPYING [58:58] - -KEEP Public-Domain d2d5f589792b8c7a8679d6c69614d2e9 -BELONGS ya.make - License text: - shouldn't claim that 0BSD-licensed code is in the public - domain; 0BSD-licensed code is copyrighted but available under - Scancode info: - Original SPDX id: LicenseRef-scancode-public-domain - Score : 100.00 - Match type : TEXT - Links : http://www.linfo.org/publicdomain.html, https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/public-domain.LICENSE - Files with this license: - COPYING [73:74] + COPYING [64:64] KEEP 0BSD deae85a8d274ed0cce9af4ffbf7f1265 BELONGS ya.make @@ -552,7 +410,7 @@ BELONGS ya.make KEEP 0BSD e5a555c3368d6625b4f901cafe6caf46 BELONGS ya.make -FILE_INCLUDE COPYING.0BSD found in files: README at line 78 +FILE_INCLUDE COPYING.0BSD found in files: README at line 79 License text: COPYING.0BSD BSD Zero Clause License Scancode info: @@ -561,7 +419,7 @@ FILE_INCLUDE COPYING.0BSD found in files: README at line 78 Match type : REFERENCE Links : http://landley.net/toybox/license.html, https://spdx.org/licenses/0BSD Files with this license: - README [78:78] + README [79:79] KEEP 0BSD e7ecdbf411fbdc32d3df058d1c7bf661 BELONGS ya.make @@ -585,10 +443,8 @@ BELONGS ya.make liblzma/check/check.c [1:1] liblzma/check/check.h [1:1] liblzma/check/crc32_fast.c [1:1] - liblzma/check/crc32_table.c [1:1] liblzma/check/crc32_table_le.h [1:1] liblzma/check/crc64_fast.c [1:1] - liblzma/check/crc64_table.c [1:1] liblzma/check/crc64_table_le.h [1:1] liblzma/check/crc_common.h [1:1] liblzma/check/crc_x86_clmul.h [1:1] @@ -706,18 +562,6 @@ BELONGS ya.make liblzma/simple/sparc.c [1:1] liblzma/simple/x86.c [1:1] -KEEP 0BSD f3ff7259b99521ac278b6ea15c6f179f -BELONGS ya.make - License text: - shouldn't claim that 0BSD-licensed code is in the public - Scancode info: - Original SPDX id: 0BSD - Score : 50.00 - Match type : REFERENCE - Links : http://landley.net/toybox/license.html, https://spdx.org/licenses/0BSD - Files with this license: - COPYING [73:73] - KEEP 0BSD f6a343d8de811f538c36fa8681cb7078 BELONGS ya.make License text: diff --git a/contrib/libs/lzma/.yandex_meta/licenses.list.txt b/contrib/libs/lzma/.yandex_meta/licenses.list.txt index e1c02738279..3120ceffdf1 100644 --- a/contrib/libs/lzma/.yandex_meta/licenses.list.txt +++ b/contrib/libs/lzma/.yandex_meta/licenses.list.txt @@ -1,8 +1,4 @@ ====================0BSD==================== - 0BSD for newer releases was made in Febrary 2024 because - - -====================0BSD==================== and ChangeLog) are under 0BSD unless stated otherwise in @@ -11,14 +7,6 @@ ====================0BSD==================== - domain and 0BSD. The main difference likely is that one - - -====================0BSD==================== - shouldn't claim that 0BSD-licensed code is in the public - - -====================0BSD==================== the BSD Zero Clause License (0BSD). @@ -59,14 +47,6 @@ LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -====================0BSD AND Public-Domain==================== - an extremely permissive license. Neither 0BSD nor public domain - - -====================BSD-2-Clause==================== - for example, BSD 2-Clause "Simplified" License which does have - - ====================BSD-3-Clause AND 0BSD==================== * liblzma is distributed under the BSD Zero Clause License (0BSD). @@ -117,38 +97,12 @@ CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ====================Public-Domain==================== - There is very little *practical* difference between public - domain and 0BSD. The main difference likely is that one - - -====================Public-Domain==================== lzma-file-format.xt are in the public domain but may ====================Public-Domain==================== - public domain has (real or perceived) legal ambiguities in - - -====================Public-Domain==================== - shouldn't claim that 0BSD-licensed code is in the public - domain; 0BSD-licensed code is copyrighted but available under - - -====================Public-Domain==================== - significant amount of code put into the public domain and - - -====================Public-Domain==================== some old translations are in the public domain. ====================Public-Domain==================== - that obviously remains so. The switch from public domain to - - -====================Public-Domain==================== // The C code is based on the public domain SHA-256 code found from - - -====================Public-Domain==================== -/// and the public domain code from https://github.com/rawrunprotected/crc diff --git a/contrib/libs/lzma/.yandex_meta/override.nix b/contrib/libs/lzma/.yandex_meta/override.nix index 32dfe8f429c..3acfc409579 100644 --- a/contrib/libs/lzma/.yandex_meta/override.nix +++ b/contrib/libs/lzma/.yandex_meta/override.nix @@ -1,12 +1,16 @@ pkgs: attrs: with pkgs; with attrs; rec { - version = "5.6.4"; + version = "5.8.1"; src = fetchFromGitHub { owner = "tukaani-project"; repo = "xz"; rev = "v${version}"; - hash = "sha256-Xp1uLtQIoOG/qVLpq5D/KFmTOJ0+mNkNclyuJsvlUbE="; + hash = "sha256-vGUNoX5VTM0aQ5GmBPXip97WGN9vaVrQLE9msToZyKs="; }; nativeBuildInputs = [ autoreconfHook ]; + + configureFlags = [ + "--build=x86_64-pc-linux-gnu" + ]; } diff --git a/contrib/libs/lzma/AUTHORS b/contrib/libs/lzma/AUTHORS index 5eff238ae41..f805a204ecb 100644 --- a/contrib/libs/lzma/AUTHORS +++ b/contrib/libs/lzma/AUTHORS @@ -24,7 +24,7 @@ Authors of XZ Utils by Michał Górny. Architecture-specific CRC optimizations were contributed by - Ilya Kurdyukov, Hans Jansen, and Chenxi Mao. + Ilya Kurdyukov, Chenxi Mao, and Xi Ruoyao. Other authors: - Jonathan Nieder diff --git a/contrib/libs/lzma/COPYING b/contrib/libs/lzma/COPYING index aed21531497..ef3371389d7 100644 --- a/contrib/libs/lzma/COPYING +++ b/contrib/libs/lzma/COPYING @@ -40,6 +40,12 @@ XZ Utils Licensing free software licenses. These aren't built or installed as part of XZ Utils. + The following command may be helpful in finding per-file license + information. It works on xz.git and on a clean file tree extracted + from a release tarball. + + sh build-aux/license-check.sh -v + For the files under the BSD Zero Clause License (0BSD), if a copyright notice is needed, the following is sufficient: @@ -59,25 +65,6 @@ XZ Utils Licensing - COPYING.GPLv2: GNU General Public License version 2 - COPYING.GPLv3: GNU General Public License version 3 - A note about old XZ Utils releases: - - XZ Utils releases 5.4.6 and older and 5.5.1alpha have a - significant amount of code put into the public domain and - that obviously remains so. The switch from public domain to - 0BSD for newer releases was made in Febrary 2024 because - public domain has (real or perceived) legal ambiguities in - some jurisdictions. - - There is very little *practical* difference between public - domain and 0BSD. The main difference likely is that one - shouldn't claim that 0BSD-licensed code is in the public - domain; 0BSD-licensed code is copyrighted but available under - an extremely permissive license. Neither 0BSD nor public domain - require retaining or reproducing author, copyright holder, or - license notices when distributing the software. (Compare to, - for example, BSD 2-Clause "Simplified" License which does have - such requirements.) - If you have questions, don't hesitate to ask for more information. The contact information is in the README file. diff --git a/contrib/libs/lzma/INSTALL b/contrib/libs/lzma/INSTALL index f7422817716..ec890472746 100644 --- a/contrib/libs/lzma/INSTALL +++ b/contrib/libs/lzma/INSTALL @@ -16,7 +16,7 @@ XZ Utils Installation 1.2.8. DOS 1.2.9. z/OS 1.3. Adding support for new platforms - 2. configure options + 2. configure and CMake options 2.1. Static vs. dynamic linking of liblzma 2.2. Optimizing xzdec and lzmadec 3. xzgrep and other scripts @@ -76,6 +76,11 @@ XZ Utils Installation you use CC=xlc instead, you must disable threading support with --disable-threads (usually not recommended). + If building a 32-bit executable, the address space available to xz + might be limited to 256 MiB by default. To increase the address + space to 2 GiB, pass LDFLAGS=-Wl,-bmaxdata:0x80000000 as an argument + to configure. + 1.2.2. IRIX @@ -213,19 +218,53 @@ XZ Utils Installation in C89 or C++. -2. configure options --------------------- +2. configure and CMake options +------------------------------ In most cases, the defaults are what you want. Many of the options below are useful only when building a size-optimized version of liblzma or command line tools. + configure options are those that begin with two dashes "--" + or "gl_". + + CMake options begin with "XZ_", "TUKLIB_", or "CMAKE_". To use + them on the command line, prefix them with "-D", for example, + "cmake -DCMAKE_COMPILE_WARNING_AS_ERROR=ON". + + CMAKE_BUILD_TYPE=TYPE + CMake only: + + For release builds, CMAKE_BUILD_TYPE=Release is fine. + On targets where CMake defaults to -O3, the default + value is overridden to -O2. + + Empty value (CMAKE_BUILD_TYPE=) is fine if using custom + optimization options. *In this package* the empty build + type also disables debugging code just like "Release" + does. To enable debugging code with empty build type, + use -UNDEBUG in the CFLAGS environment variable or in + the CMAKE_C_FLAGS CMake variable to override -DNDEBUG. + + Non-standard build types like "None" do NOT disable + debugging code! Such non-standard build types should + be avoided for production builds! + --enable-encoders=LIST --disable-encoders - Specify a comma-separated LIST of filter encoders to - build. See "./configure --help" for exact list of - available filter encoders. The default is to build all - supported encoders. + XZ_ENCODERS=LIST + Specify a LIST of filter encoders to build. In the + configure option the list is comma separated. + CMake lists are semicolon separated. + + To see the exact list of available filter encoders: + + - Autotools: ./configure --help + + - CMake: Configure the tree normally first, then use + "cmake -LH ." to list the cache variables. + + The default is to build all supported encoders. If LIST is empty or --disable-encoders is used, no filter encoders will be built and also the code shared between @@ -237,10 +276,12 @@ XZ Utils Installation --enable-decoders=LIST --disable-decoders + XZ_DECODERS=LIST This is like --enable-encoders but for decoders. The default is to build all supported decoders. --enable-match-finders=LIST + XZ_MATCH_FINDERS=LIST liblzma includes two categories of match finders: hash chains and binary trees. Hash chains (hc3 and hc4) are quite fast but they don't provide the best compression @@ -257,9 +298,11 @@ XZ Utils Installation or LZMA2 filter encoders are being built. --enable-checks=LIST + XZ_CHECKS=LIST liblzma support multiple integrity checks. CRC32 is - mandatory, and cannot be omitted. See "./configure --help" - for exact list of available integrity check types. + mandatory, and cannot be omitted. Supported check + types are "crc32", "crc64", and "sha256". By default + all supported check types are enabled. liblzma and the command line tools can decompress files which use unsupported integrity check type, but naturally @@ -270,6 +313,7 @@ XZ Utils Installation it is known to not cause problems. --enable-external-sha256 + XZ_EXTERNAL_SHA256=ON Try to use SHA-256 code from the operating system libc or similar base system libraries. This doesn't try to use OpenSSL or libgcrypt or such libraries. @@ -306,6 +350,8 @@ XZ Utils Installation time xz --test foo.xz --disable-microlzma + XZ_MICROLZMA_ENCODER=OFF + XZ_MICROLZMA_DECODER=OFF Don't build MicroLZMA encoder and decoder. This omits lzma_microlzma_encoder() and lzma_microlzma_decoder() API functions from liblzma. These functions are needed @@ -313,6 +359,7 @@ XZ Utils Installation erofs-utils but they may be used by others too. --disable-lzip-decoder + XZ_LZIP_DECODER=OFF Disable decompression support for .lz (lzip) files. This omits the API function lzma_lzip_decoder() from liblzma and .lz support from the xz tool. @@ -321,6 +368,10 @@ XZ Utils Installation --disable-xzdec --disable-lzmadec --disable-lzmainfo + XZ_TOOL_XZ=OFF + XZ_TOOL_XZDEC=OFF + XZ_TOOL_LZMADEC=OFF + XZ_TOOL_LZMAINFO=OFF Don't build and install the command line tool mentioned in the option name. @@ -330,29 +381,40 @@ XZ Utils Installation a dangling man page symlink lzmadec.1 -> xzdec.1 is created. + XZ_TOOL_SYMLINKS=OFF + Don't create the unxz and xzcat symlinks. (There is + no "configure" option to disable these symlinks.) + --disable-lzma-links + XZ_TOOL_SYMLINKS_LZMA=OFF Don't create symlinks for LZMA Utils compatibility. This includes lzma, unlzma, and lzcat. If scripts are installed, also lzdiff, lzcmp, lzgrep, lzegrep, lzfgrep, lzmore, and lzless will be omitted if this option is used. --disable-scripts + XZ_TOOL_SCRIPTS=OFF Don't install the scripts xzdiff, xzgrep, xzmore, xzless, and their symlinks. --disable-doc + XZ_DOC=OFF Don't install the documentation files to $docdir (often /usr/doc/xz or /usr/local/doc/xz). Man pages will still be installed. The $docdir can be changed with --docdir=DIR. --enable-doxygen + XZ_DOXYGEN=ON Enable generation of the HTML version of the liblzma API documentation using Doxygen. The resulting files are installed to $docdir/api. This option assumes that the 'doxygen' tool is available. + NOTE: --disable-doc or XZ_DOC=OFF don't affect this. + --disable-assembler + XZ_ASM_I386=OFF This disables CRC32 and CRC64 assembly code on 32-bit x86. This option currently does nothing on other architectures (not even on x86-64). @@ -365,7 +427,16 @@ XZ Utils Installation pre-i686 systems, you may want to disable the assembler code. + The assembly code is compatible with only certain OSes + and toolchains (it's not compatible with MSVC). + + Since XZ Utils 5.7.1alpha, the 32-bit x86 assembly code + co-exists with the modern CLMUL code: CLMUL is used if + support for it is detected at runtime. On old processors + the assembly code is used. + --disable-clmul-crc + XZ_CLMUL_CRC=OFF Disable the use of carryless multiplication for CRC calculation even if compiler support for it is detected. The code uses runtime detection of SSSE3, SSE4.1, and @@ -378,6 +449,7 @@ XZ Utils Installation detection isn't used and the generic code is omitted. --disable-arm64-crc32 + XZ_ARM64_CRC32=OFF Disable the use of the ARM64 CRC32 instruction extension even if compiler support for it is detected. The code will detect support for the instruction at runtime. @@ -387,7 +459,16 @@ XZ Utils Installation and later) then runtime detection isn't used and the generic code is omitted. + --disable-loongarch-crc32 + XZ_LOONGARCH_CRC32=OFF + Disable the use of the 64-bit LoongArch CRC32 + instruction extension even if compiler support for + it is detected. There is no runtime detection because + all 64-bit LoongArch processors should support + the CRC32 instructions. + --enable-unaligned-access + TUKLIB_FAST_UNALIGNED_ACCESS=ON Allow liblzma to use unaligned memory access for 16-bit, 32-bit, and 64-bit loads and stores. This should be enabled only when the hardware supports this, that is, @@ -435,6 +516,7 @@ XZ Utils Installation how unaligned access is done in the C code. --enable-unsafe-type-punning + TUKLIB_USE_UNSAFE_TYPE_PUNNING=ON This enables use of code like uint8_t *buf8 = ...; @@ -451,6 +533,7 @@ XZ Utils Installation GCC 3 and early 4.x on x86, GCC < 6 on ARMv6 and ARMv7). --enable-small + XZ_SMALL=ON Reduce the size of liblzma by selecting smaller but semantically equivalent version of some functions, and omit precomputed lookup tables. This option tends to @@ -467,6 +550,7 @@ XZ Utils Installation flag(s) to CFLAGS manually. --enable-assume-ram=SIZE + XZ_ASSUME_RAM=SIZE On the most common operating systems, XZ Utils is able to detect the amount of physical memory on the system. This information is used by the options --memlimit-compress, @@ -483,6 +567,7 @@ XZ Utils Installation src/common/tuklib_physmem.c for details. --enable-threads=METHOD + XZ_THREADS=METHOD Threading support is enabled by default so normally there is no need to specify this option. @@ -519,6 +604,7 @@ XZ Utils Installation one thread, something bad may happen. --enable-sandbox=METHOD + XZ_SANDBOX=METHOD There is limited sandboxing support in the xz and xzdec tools. If built with sandbox support, xz uses it automatically when (de)compressing exactly one file to @@ -554,6 +640,7 @@ XZ Utils Installation is found, configure will give an error. --enable-symbol-versions[=VARIANT] + XZ_SYMBOL_VERSIONING=VARIANT Use symbol versioning for liblzma shared library. This is enabled by default on GNU/Linux (glibc only), other GNU-based systems, and FreeBSD. @@ -598,13 +685,25 @@ XZ Utils Installation run-time consistency checks. It makes the code slower, so you normally don't want to have this enabled. + In CMake, the build type (CMAKE_BUILD_TYPE) controls if + -DNDEBUG is passed to the compiler. *In this package*, + an empty build type disables debugging code too. + Non-standard build types like "None" do NOT disable + debugging code! + + To enable debugging code with empty build type in CMake, + use -UNDEBUG in the CFLAGS environment variable or in + the CMAKE_C_FLAGS CMake variable to override -DNDEBUG. + --enable-werror + CMAKE_COMPILE_WARNING_AS_ERROR=ON (CMake >= 3.24) If building with GCC, make all compiler warnings an error, that abort the compilation. This may help catching bugs, and should work on most systems. This has no effect on the resulting binaries. --enable-path-for-scripts=PREFIX + (CMake determines this from the path of XZ_POSIX_SHELL) If PREFIX isn't empty, PATH=PREFIX:$PATH will be set in the beginning of the scripts (xzgrep and others). The default is empty except on Solaris the default is @@ -621,6 +720,36 @@ XZ Utils Installation the PATH for the scripts. It is described in section 3.2 and is supported in this xz version too. + gl_cv_posix_shell=/path/to/bin/sh + XZ_POSIX_SHELL=/path/to/bin/sh + POSIX shell to use for xzgrep and other scripts. + + - configure should autodetect this well enough. + Typically it's /bin/sh but in some cases, like + Solaris, something else is used. + + - CMake build uses /bin/sh except on Solaris the + default is /usr/xpg4/bin/sh. + + CMAKE_DLL_NAME_WITH_SOVERSION=ON + CMake on native Windows (not Cygwin) only: + + This changes the filename liblzma.dll to liblzma-5.dll. + + The unversioned filename liblzma.dll has been used + since XZ Utils 5.0.0 when creating binary packages + using the included windows/build.bash. The same + unversioned filename is the default with CMake. + However, there are popular builds that, very + understandably and reasonably, use the versioned + filename produced by GNU Libtool. + + This option should usually be left to its default value + (OFF). It can be set to ON if the liblzma DLL filename + must be compatible with the versioned filename + produced by GNU Libtool. For example, binaries + distributed in MSYS2 use a versioned DLL filename. + 2.1. Static vs. dynamic linking of liblzma diff --git a/contrib/libs/lzma/NEWS b/contrib/libs/lzma/NEWS index f260a332f77..978ef54b915 100644 --- a/contrib/libs/lzma/NEWS +++ b/contrib/libs/lzma/NEWS @@ -2,6 +2,240 @@ XZ Utils Release Notes ====================== +5.8.1 (2025-04-03) + + IMPORTANT: This includes a security fix for CVE-2025-31115 which + affects XZ Utils from 5.3.3alpha to 5.8.0. No new 5.4.x or 5.6.x + releases will be made, but the fix is in the v5.4 and v5.6 branches + in the xz Git repository. A standalone patch for all affected + versions is available as well. + + * Multithreaded .xz decoder (lzma_stream_decoder_mt()): + + - Fix a bug that could at least result in a crash with + invalid input. (CVE-2025-31115) + + - Fix a performance bug: Only one thread was used if the whole + input file was provided at once to lzma_code(), the output + buffer was big enough, timeout was disabled, and LZMA_FINISH + was used. There are no bug reports about this, thus it's + possible that no real-world application was affected. + + * Avoid <stdalign.h> even with C11/C17 compilers. This fixes the + build with Oracle Developer Studio 12.6 on Solaris 10 when the + compiler is in C11 mode (the header doesn't exist). + + * Autotools: Restore compatibility with GNU make versions older + than 4.0 by creating the package using GNU gettext 0.23.1 + infrastructure instead of 0.24. + + * Update Croatian translation. + + +5.8.0 (2025-03-25) + + This bumps the minor version of liblzma because new features were + added. The API and ABI are still backward compatible with liblzma + 5.6.x, 5.4.x, 5.2.x, and 5.0.x. + + * liblzma on 32/64-bit x86: When possible, use SSE2 intrinsics + instead of memcpy() in the LZMA/LZMA2 decoder. In typical cases, + this may reduce decompression time by 0-5 %. However, when built + against musl libc, over 15 % time reduction was observed with + highly compressed files. + + * CMake: Make the feature test macros match the Autotools-based + build on NetBSD, Darwin, and mingw-w64. + + * Update the Croatian, Italian, Portuguese, and Romanian + translations. + + * Update the German, Italian, Korean, Romanian, Serbian, and + Ukrainian man page translations. + + Summary of changes in the 5.7.x development releases: + + * Mark the following LZMA Utils script aliases as deprecated: + lzcmp, lzdiff, lzless, lzmore, lzgrep, lzegrep, and lzfgrep. + + * liblzma: + + - Improve LZMA/LZMA2 encoder speed on 64-bit PowerPC (both + endiannesses) and those 64-bit RISC-V processors that + support fast unaligned access. + + - Add low-level APIs for RISC-V, ARM64, and x86 BCJ filters + to lzma/bcj.h. These are primarily for erofs-utils. + + - x86/x86-64/E2K CLMUL CRC code was rewritten. + + - Use the CRC32 instructions on LoongArch. + + * xz: + + - Synchronize the output file and its directory using fsync() + before deleting the input file. No syncing is done when xz + isn't going to delete the input file. + + - Add --no-sync to disable the sync-before-delete behavior. + + - Make --single-stream imply --keep. + + * xz, xzdec, lzmainfo: When printing messages, replace + non-printable characters with question marks. + + * xz and xzdec on Linux: Support Landlock ABI versions 5 and 6. + + * CMake: Revise the configuration variables and some of their + options, and document them in the file INSTALL. CMake support + is no longer experimental. (It was already not experimental + when building for native Windows.) + + * Add build-aux/license-check.sh. + + +5.7.2beta (2025-03-08) + + * On the man pages, mark the following LZMA Utils script aliases as + deprecated: lzcmp, lzdiff, lzless, lzmore, lzgrep, lzegrep, and + lzfgrep. The commands that start with xz* instead of lz* have + identical behavior. + + The LZMA Utils aliases lzma, unlzma, and lzcat aren't deprecated + because some of these aliases are still in common use. lzmadec + and lzmainfo aren't deprecated either. + + * xz: In the ENVIRONMENT section of the man page, warn about + problems that some uses of XZ_DEFAULTS and XZ_OPT may create. + + * Windows (native builds, not Cygwin): In xz, xzdec, and lzmadec, + avoid an error message on broken pipe. + + * Autotools: Fix out-of-tree builds when using the bundled + getopt_long. + + * Translations: + + - Updated: Chinese (traditional), Croatian, Finnish, Georgian, + German, Korean, Polish, Romanian, Serbian, Spanish, Swedish, + Turkish, and Ukrainian + + - Added: Dutch + + * Man page translations: + + - Updated: German, Korean, Romanian, and Ukrainian + + - Added: Italian and Serbian + + +5.7.1alpha (2025-01-23) + + * All fixes from 5.6.4. + + * liblzma: + + - Improve LZMA/LZMA2 encoder speed on 64-bit PowerPC (both + endiannesses) and those 64-bit RISC-V processors that + support fast unaligned access. + + - x86/x86-64/E2K CLMUL CRC code was rewritten. It's faster and + doesn't cause false positives from sanitizers. Attributes + like __attribute__((__no_sanitize_address__)) are no longer + present. + + - On 32-bit x86, CLMUL CRC and the old (but still good) + assembly versions now co-exist with runtime detection. + Both Autotools and CMake build systems handle this + automatically now. + + - Use the CRC32 instructions on LoongArch to make CRC32 + calculation faster. + + - Add low-level APIs for RISC-V, ARM64, and x86 BCJ filters + to lzma/bcj.h. These are primarily for erofs-utils. + + - Minor tweaks to ARM64 CRC32 code and BCJ filters were made. + + * xz: + + - Synchronize the output file and its directory before deleting + the input file using fsync(). This reduces the probability of + data loss after a system crash. However, it can be a major + performance hit if processing many small files. + + NOTE: No syncing is done when xz isn't going to delete + the input file. + + - Add a new option --no-sync to disable the sync-before-delete + behavior. It's useful when compressing many small files and + one doesn't worry about data loss in case of a system crash. + + - Make --single-stream imply --keep. + + - Use automatic word wrapping for the text in --help and + similar situations to hopefully make the strings easier for + majority of translators (no need to count spaces anymore). + + * xz, xzdec, lzmainfo: When printing messages, replace + non-printable characters with question marks. This way + malicious filenames cannot be used to send escape sequences + to a terminal. This change is also applied to filenames shown + in "xz --robot --list". + + * xz and xzdec on Linux: Add support for Landlock ABI versions 5 + and 6. + + * CMake updates: + + - Increase the minimum required CMake version to 3.20. + + - Revise the configuration variables and some of their options. + Document them in the file INSTALL. + + - Attempt to produce liblzma.pc so that the paths are based on + ${prefix}, which makes it simpler to override the paths + if the liblzma files have been moved. + + - To enable translations, gettext-tools is now required. The + CMake build no longer supports installing pre-compiled + message catalog binary files (po/*.gmo). + + - Apple: Use Mach-O shared library versioning that is + compatible with GNU Libtool. This should make it easier to + switch between the build systems on Apple OSes that enforce + the correct compatibility_version (macOS >= 12 doesn't?). + This change is backward compatible: binaries linked against + old CMake-built liblzma will run with liblzma that uses + Libtool style versioning. + + - Windows (not Cygwin): Document CMAKE_DLL_NAME_WITH_SOVERSION + (CMake >= 3.27) in the file INSTALL. This option should + usually be left to its default value (OFF). It can be set + to ON if the liblzma DLL filename must be compatible with + the versioned filename produced by GNU Libtool. For example, + binaries distributed in MSYS2 use a versioned DLL filename. + + - CMake support is no longer experimental. (It was already + not experimental when building for native Windows.) + + * Windows: Building liblzma with Visual Studio 2013 is no longer + supported. Visual Studio 2015 or later (with CMake) can be used + to build liblzma and the command line tools. + + * Add preliminary Georgian translation. This already contains + translations of most of the strings that are now automatically + word wrapped. + + * Add build-aux/license-check.sh. Without arguments, it checks that + no license information has been forgotten. With the -v argument, + it shows the license info (or the lack of it) for each file. + + If the .git directory is available, only the files in the + repository are checked. Without the .git directory, a clean tree + from an extracted release tarball is expected. + + 5.6.4 (2025-01-23) * liblzma: Fix LZMA/LZMA2 encoder on big endian ARM64. diff --git a/contrib/libs/lzma/README b/contrib/libs/lzma/README index 9d097deff37..41671676a51 100644 --- a/contrib/libs/lzma/README +++ b/contrib/libs/lzma/README @@ -10,6 +10,7 @@ XZ Utils 2. Version numbering 3. Reporting bugs 4. Translations + 4.1. Testing translations 5. Other implementations of the .xz format 6. Contact information @@ -203,77 +204,47 @@ XZ Utils https://translationproject.org/html/translators.html - Below are notes and testing instructions specific to xz - translations. + Updates to translations won't be accepted by methods that bypass + the Translation Project because there is a risk of duplicate work: + translation updates made in the xz repository aren't seen by the + translators in the Translation Project. If you have found bugs in + a translation, please report them to the Language-Team address + which can be found near the beginning of the PO file. - Testing can be done by installing xz into a temporary directory: + If you find language problems in the original English strings, + feel free to suggest improvements. Ask if something is unclear. + + +4.1. Testing translations + + Testing can be done by installing xz into a temporary directory. + + If building from Git repository (not tarball), generate the + Autotools files: + + ./autogen.sh + + Create a subdirectory for the build files. The tmp-build directory + can be deleted after testing. + + mkdir tmp-build + cd tmp-build + ../configure --disable-shared --enable-debug --prefix=$PWD/inst + + Edit the .po file in the po directory. Then build and install to + the "tmp-build/inst" directory, and use translations.bash to see + how some of the messages look. Repeat these steps if needed: - ./configure --disable-shared --prefix=/tmp/xz-test - # <Edit the .po file in the po directory.> make -C po update-po - make install - bash debug/translation.bash | less - bash debug/translation.bash | less -S # For --list outputs - - Repeat the above as needed (no need to re-run configure though). - - Note especially the following: - - - The output of --help and --long-help must look nice on - an 80-column terminal. It's OK to add extra lines if needed. - - - In contrast, don't add extra lines to error messages and such. - They are often preceded with e.g. a filename on the same line, - so you have no way to predict where to put a \n. Let the terminal - do the wrapping even if it looks ugly. Adding new lines will be - even uglier in the generic case even if it looks nice in a few - limited examples. - - - Be careful with column alignment in tables and table-like output - (--list, --list --verbose --verbose, --info-memory, --help, and - --long-help): - - * All descriptions of options in --help should start in the - same column (but it doesn't need to be the same column as - in the English messages; just be consistent if you change it). - Check that both --help and --long-help look OK, since they - share several strings. - - * --list --verbose and --info-memory print lines that have - the format "Description: %s". If you need a longer - description, you can put extra space between the colon - and %s. Then you may need to add extra space to other - strings too so that the result as a whole looks good (all - values start at the same column). - - * The columns of the actual tables in --list --verbose --verbose - should be aligned properly. Abbreviate if necessary. It might - be good to keep at least 2 or 3 spaces between column headings - and avoid spaces in the headings so that the columns stand out - better, but this is a matter of opinion. Do what you think - looks best. - - - Be careful to put a period at the end of a sentence when the - original version has it, and don't put it when the original - doesn't have it. Similarly, be careful with \n characters - at the beginning and end of the strings. - - - Read the TRANSLATORS comments that have been extracted from the - source code and included in xz.pot. Some comments suggest - testing with a specific command which needs an .xz file. You - may use e.g. any tests/files/good-*.xz. However, these test - commands are included in translations.bash output, so reading - translations.bash output carefully can be enough. - - - If you find language problems in the original English strings, - feel free to suggest improvements. Ask if something is unclear. - - - The translated messages should be understandable (sometimes this - may be a problem with the original English messages too). Don't - make a direct word-by-word translation from English especially if - the result doesn't sound good in your language. - - Thanks for your help! + make -j"$(nproc)" install + bash ../debug/translation.bash | less + bash ../debug/translation.bash | less -S # For --list outputs + + To test other languages, set the LANGUAGE environment variable + before running translations.bash. The value should match the PO file + name without the .po suffix. Example: + + export LANGUAGE=fi 5. Other implementations of the .xz format diff --git a/contrib/libs/lzma/THANKS b/contrib/libs/lzma/THANKS index 3326e9712e0..a6a7a672107 100644 --- a/contrib/libs/lzma/THANKS +++ b/contrib/libs/lzma/THANKS @@ -20,6 +20,7 @@ has been important. :-) In alphabetical order: - Jakub Bogusz - Adam Borowski - Maarten Bosmans + - Roel Bouckaert - Lukas Braune - Benjamin Buch - Trent W. Buck @@ -29,13 +30,18 @@ has been important. :-) In alphabetical order: - Frank Busse - Daniel Mealha Cabrita - Milo Casagrande + - Cristiano Ceglia - Marek Černocký - Tomer Chachamu - Vitaly Chikunov - Antoine Cœur + - Elijah Almeida Coimbra - Felix Collin + - Ryan Colyer - Marcus Comstedt + - Vincent Cruz - Gabi Davar + - Ron Desmond - İhsan Doğan - Chris Donawa - Andrew Dudman @@ -48,9 +54,11 @@ has been important. :-) In alphabetical order: - Denis Excoffier - Vincent Fazio - Michael Felt + - Sean Fenian - Michael Fox - Andres Freund - Mike Frysinger + - Collin Funk - Daniel Richard G. - Tomasz Gajc - Bjarni Ingi Gislason @@ -59,10 +67,14 @@ has been important. :-) In alphabetical order: - Matthew Good - Michał Górny - Jason Gorski + - Alexander M. Greenham - Juan Manuel Guerrero - Gabriela Gutierrez - Diederik de Haas + - Jan Terje Hansen + - Tobias Lahrmann Hansen - Joachim Henke + - Lizandro Heredia - Christian Hesse - Vincenzo Innocente - Peter Ivanov @@ -78,9 +90,11 @@ has been important. :-) In alphabetical order: - Per Øyvind Karlsen - Firas Khalil Khana - Iouri Kharon + - Kim Jinyeong - Thomas Klausner - Richard Koch - Anton Kochkov + - Harri K. Koskinen - Ville Koskinen - Sergey Kosukhin - Marcin Kowalczyk @@ -105,14 +119,20 @@ has been important. :-) In alphabetical order: - Chenxi Mao - Gregory Margo - Julien Marrec + - Pierre-Yves Martin - Ed Maste - Martin Matuška + - Scott McAllister + - Chris McCrohan + - Derwin McGeary - Ivan A. Melnikov - Jim Meyering - Arkadiusz Miskiewicz - Nathan Moinvaziri - Étienne Mollier - Conley Moorhous + - Dirk Müller + - Rainer Müller - Andrew Murray - Rafał Mużyło - Adrien Nader @@ -122,6 +142,7 @@ has been important. :-) In alphabetical order: - Jonathan Nieder - Asgeir Storesund Nilsen - Andre Noll + - Ruarí Ødegaard - Peter O'Gorman - Dimitri Papadopoulos Orfanos - Daniel Packard @@ -133,17 +154,20 @@ has been important. :-) In alphabetical order: - Igor Pavlov - Diego Elio Pettenò - Elbert Pol + - Guiorgy Potskhishvili - Mikko Pouru - Frank Prochnow - Rich Prohaska - Trần Ngọc Quân - Pavel Raiskup + - Matthieu Rakotojaona - Ole André Vadla Ravnås - Eric S. Raymond - Robert Readman - Bernhard Reutner-Fischer - Markus Rickert - Cristian Rodríguez + - Jeroen Roovers - Christian von Roques - Boud Roukema - Torsten Rupp @@ -160,6 +184,7 @@ has been important. :-) In alphabetical order: - Dan Shechter - Stuart Shelton - Sebastian Andrzej Siewior + - Andrej Skenderija - Ville Skyttä - Brad Smith - Bruce Stark @@ -191,15 +216,22 @@ has been important. :-) In alphabetical order: - Ralf Wildenhues - Charles Wilson - Lars Wirzenius + - Vincent Wixsom - Pilorz Wojciech - Chien Wong + - Xi Ruoyao - Ryan Young - Andreas Zieringer + - 榆柳松 (ZhengSen Wang) Companies: - Google - Sandfly Security +Other credits: + - cleemy desu wayo working with Trend Micro Zero Day Initiative + - Orange Tsai and splitline from DEVCORE Research Team + Also thanks to all the people who have participated in the Tukaani project. I have probably forgot to add some names to the above list. Sorry about diff --git a/contrib/libs/lzma/TODO b/contrib/libs/lzma/TODO index ad37f3f559a..7a0bf16ed86 100644 --- a/contrib/libs/lzma/TODO +++ b/contrib/libs/lzma/TODO @@ -5,12 +5,7 @@ XZ Utils To-Do List Known bugs ---------- - The test suite is too incomplete. - - If the memory usage limit is less than about 13 MiB, xz is unable to - automatically scale down the compression settings enough even though - it would be possible by switching from BT2/BT3/BT4 match finder to - HC3/HC4. + The test suite is incomplete. XZ Utils compress some files significantly worse than LZMA Utils. This is due to faster compression presets used by XZ Utils, and @@ -19,9 +14,6 @@ Known bugs compress extremely well, so going from compression ratio of 0.003 to 0.004 means big relative increase in the compressed file size. - xz doesn't quote unprintable characters when it displays file names - given on the command line. - tuklib_exit() doesn't block signals => EINTR is possible. If liblzma has created threads and fork() gets called, liblzma @@ -41,9 +33,6 @@ Missing features be mostly useful when using a preset dictionary in LZMA2, but it may have other uses too. Compare to deflateCopy() in zlib. - Support LZMA_FINISH in raw decoder to indicate end of LZMA1 and - other streams that don't have an end of payload marker. - Adjust dictionary size when the input file size is known. Maybe do this only if an option is given. @@ -67,9 +56,9 @@ Missing features Support LZMA_FULL_FLUSH for lzma_stream_decoder() to stop at Block and Stream boundaries. - lzma_strerror() to convert lzma_ret to human readable form? - This is tricky, because the same error codes are used with - slightly different meanings, and this cannot be fixed anymore. + Error codes from lzma_code() aren't very specific. A more detailed + error message (string) could be provided too. It could be returned + by a new function or use a currently-reserved member of lzma_stream. Make it possible to adjust LZMA2 options in the middle of a Block so that the encoding speed vs. compression ratio can be optimized @@ -97,9 +86,3 @@ Documentation Document the LZMA1 and LZMA2 algorithms. - -Miscellaneous ------------- - - Try to get the media type for .xz registered at IANA. - diff --git a/contrib/libs/lzma/common/sysdefs.h b/contrib/libs/lzma/common/sysdefs.h index 1c2405dcd83..b10ffa7c3b1 100644 --- a/contrib/libs/lzma/common/sysdefs.h +++ b/contrib/libs/lzma/common/sysdefs.h @@ -168,17 +168,26 @@ typedef unsigned char _Bool; # define __bool_true_false_are_defined 1 #endif +// We may need alignas from C11/C17/C23. +#if __STDC_VERSION__ >= 202311 + // alignas is a keyword in C23. Do nothing. +#elif __STDC_VERSION__ >= 201112 + // Oracle Developer Studio 12.6 lacks <stdalign.h>. + // For simplicity, avoid the header with all C11/C17 compilers. +# define alignas _Alignas +#elif defined(__GNUC__) || defined(__clang__) +# define alignas(n) __attribute__((__aligned__(n))) +#else +# define alignas(n) +#endif + #include <string.h> -// Visual Studio 2013 update 2 supports only __inline, not inline. -// MSVC v19.0 / VS 2015 and newer support both. +// MSVC v19.00 (VS 2015 version 14.0) and later should work. // // MSVC v19.27 (VS 2019 version 16.7) added support for restrict. // Older ones support only __restrict. #ifdef _MSC_VER -# if _MSC_VER < 1900 && !defined(inline) -# define inline __inline -# endif # if _MSC_VER < 1927 && !defined(restrict) # define restrict __restrict # endif @@ -208,4 +217,13 @@ typedef unsigned char _Bool; # define lzma_attr_alloc_size(x) #endif +#if __STDC_VERSION__ >= 202311 +# define FALLTHROUGH [[__fallthrough__]] +#elif (defined(__GNUC__) && __GNUC__ >= 7) \ + || (defined(__clang_major__) && __clang_major__ >= 10) +# define FALLTHROUGH __attribute__((__fallthrough__)) +#else +# define FALLTHROUGH ((void)0) +#endif + #endif diff --git a/contrib/libs/lzma/common/tuklib_common.h b/contrib/libs/lzma/common/tuklib_common.h index 7554dfc86fb..d73f07255e4 100644 --- a/contrib/libs/lzma/common/tuklib_common.h +++ b/contrib/libs/lzma/common/tuklib_common.h @@ -56,6 +56,13 @@ # define TUKLIB_GNUC_REQ(major, minor) 0 #endif +#if defined(__GNUC__) || defined(__clang__) +# define tuklib_attr_format_printf(fmt_index, args_index) \ + __attribute__((__format__(__printf__, fmt_index, args_index))) +#else +# define tuklib_attr_format_printf(fmt_index, args_index) +#endif + // tuklib_attr_noreturn attribute is used to mark functions as non-returning. // We cannot use "noreturn" as the macro name because then C23 code that // uses [[noreturn]] would break as it would expand to [[ [[noreturn]] ]]. @@ -68,9 +75,7 @@ // __attribute__((nonnull(1))) // extern void foo(const char *s); // -// FIXME: Update __STDC_VERSION__ for the final C23 version. 202000 is used -// by GCC 13 and Clang 15 with -std=c2x. -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202000 +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311 # define tuklib_attr_noreturn [[noreturn]] #elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112 # define tuklib_attr_noreturn _Noreturn diff --git a/contrib/libs/lzma/common/tuklib_physmem.c b/contrib/libs/lzma/common/tuklib_physmem.c index 0a4fcb3871a..2b686d86588 100644 --- a/contrib/libs/lzma/common/tuklib_physmem.c +++ b/contrib/libs/lzma/common/tuklib_physmem.c @@ -91,18 +91,11 @@ tuklib_physmem(void) // supports reporting values greater than 4 GiB. To keep the // code working also on older Windows versions, use // GlobalMemoryStatusEx() conditionally. - HMODULE kernel32 = GetModuleHandle(TEXT("kernel32.dll")); + HMODULE kernel32 = GetModuleHandleA("kernel32.dll"); if (kernel32 != NULL) { typedef BOOL (WINAPI *gmse_type)(LPMEMORYSTATUSEX); -#ifdef CAN_DISABLE_WCAST_FUNCTION_TYPE -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wcast-function-type" -#endif gmse_type gmse = (gmse_type)GetProcAddress( kernel32, "GlobalMemoryStatusEx"); -#ifdef CAN_DISABLE_WCAST_FUNCTION_TYPE -# pragma GCC diagnostic pop -#endif if (gmse != NULL) { MEMORYSTATUSEX meminfo; meminfo.dwLength = sizeof(meminfo); @@ -155,7 +148,7 @@ tuklib_physmem(void) ret += entries[i].end - entries[i].start + 1; #elif defined(TUKLIB_PHYSMEM_AIX) - ret = _system_configuration.physmem; + ret = (uint64_t)_system_configuration.physmem; #elif defined(TUKLIB_PHYSMEM_SYSCONF) const long pagesize = sysconf(_SC_PAGESIZE); diff --git a/contrib/libs/lzma/liblzma/api/lzma/bcj.h b/contrib/libs/lzma/liblzma/api/lzma/bcj.h index 7f6611feb32..fb737cbba49 100644 --- a/contrib/libs/lzma/liblzma/api/lzma/bcj.h +++ b/contrib/libs/lzma/liblzma/api/lzma/bcj.h @@ -96,3 +96,100 @@ typedef struct { uint32_t start_offset; } lzma_options_bcj; + + +/** + * \brief Raw ARM64 BCJ encoder + * + * This is for special use cases only. + * + * \param start_offset The lowest 32 bits of the offset in the + * executable being filtered. For the ARM64 + * filter, this must be a multiple of four. + * For the very best results, this should also + * be in sync with 4096-byte page boundaries + * in the executable due to how ARM64's ADRP + * instruction works. + * \param buf Buffer to be filtered in place + * \param size Size of the buffer + * + * \return Number of bytes that were processed in `buf`. This is at most + * `size`. With the ARM64 filter, the return value is always + * a multiple of 4, and at most 3 bytes are left unfiltered. + * + * \since 5.7.1alpha + */ +extern LZMA_API(size_t) lzma_bcj_arm64_encode( + uint32_t start_offset, uint8_t *buf, size_t size) lzma_nothrow; + +/** + * \brief Raw ARM64 BCJ decoder + * + * See lzma_bcj_arm64_encode(). + * + * \since 5.7.1alpha + */ +extern LZMA_API(size_t) lzma_bcj_arm64_decode( + uint32_t start_offset, uint8_t *buf, size_t size) lzma_nothrow; + + +/** + * \brief Raw RISC-V BCJ encoder + * + * This is for special use cases only. + * + * \param start_offset The lowest 32 bits of the offset in the + * executable being filtered. For the RISC-V + * filter, this must be a multiple of 2. + * \param buf Buffer to be filtered in place + * \param size Size of the buffer + * + * \return Number of bytes that were processed in `buf`. This is at most + * `size`. With the RISC-V filter, the return value is always + * a multiple of 2, and at most 7 bytes are left unfiltered. + * + * \since 5.7.1alpha + */ +extern LZMA_API(size_t) lzma_bcj_riscv_encode( + uint32_t start_offset, uint8_t *buf, size_t size) lzma_nothrow; + +/** + * \brief Raw RISC-V BCJ decoder + * + * See lzma_bcj_riscv_encode(). + * + * \since 5.7.1alpha + */ +extern LZMA_API(size_t) lzma_bcj_riscv_decode( + uint32_t start_offset, uint8_t *buf, size_t size) lzma_nothrow; + + +/** + * \brief Raw x86 BCJ encoder + * + * This is for special use cases only. + * + * \param start_offset The lowest 32 bits of the offset in the + * executable being filtered. For the x86 + * filter, all values are valid. + * \param buf Buffer to be filtered in place + * \param size Size of the buffer + * + * \return Number of bytes that were processed in `buf`. This is at most + * `size`. For the x86 filter, the return value is always + * a multiple of 1, and at most 4 bytes are left unfiltered. + * + * \since 5.7.1alpha + */ +extern LZMA_API(size_t) lzma_bcj_x86_encode( + uint32_t start_offset, uint8_t *buf, size_t size) lzma_nothrow; + +/** + * \brief Raw x86 BCJ decoder + * + * See lzma_bcj_x86_encode(). + * + * \since 5.7.1alpha + */ +extern LZMA_API(size_t) lzma_bcj_x86_decode( + uint32_t start_offset, uint8_t *buf, size_t size) lzma_nothrow; diff --git a/contrib/libs/lzma/liblzma/api/lzma/container.h b/contrib/libs/lzma/liblzma/api/lzma/container.h index ee5d77e4f1a..dbd414cbf8c 100644 --- a/contrib/libs/lzma/liblzma/api/lzma/container.h +++ b/contrib/libs/lzma/liblzma/api/lzma/container.h @@ -573,7 +573,7 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( * The action argument must be LZMA_FINISH and the return value will never be * LZMA_OK. Thus the encoding is always done with a single lzma_code() after * the initialization. The benefit of the combination of initialization - * function and lzma_code() is that memory allocations can be re-used for + * function and lzma_code() is that memory allocations can be reused for * better performance. * * lzma_code() will try to encode as much input as is possible to fit into diff --git a/contrib/libs/lzma/liblzma/api/lzma/version.h b/contrib/libs/lzma/liblzma/api/lzma/version.h index 86c8b199f55..86b35563596 100644 --- a/contrib/libs/lzma/liblzma/api/lzma/version.h +++ b/contrib/libs/lzma/liblzma/api/lzma/version.h @@ -19,10 +19,10 @@ #define LZMA_VERSION_MAJOR 5 /** \brief Minor version number of the liblzma release. */ -#define LZMA_VERSION_MINOR 6 +#define LZMA_VERSION_MINOR 8 /** \brief Patch version number of the liblzma release. */ -#define LZMA_VERSION_PATCH 4 +#define LZMA_VERSION_PATCH 1 /** * \brief Version stability marker diff --git a/contrib/libs/lzma/liblzma/check/check.h b/contrib/libs/lzma/liblzma/check/check.h index 08627e783a6..724246570c6 100644 --- a/contrib/libs/lzma/liblzma/check/check.h +++ b/contrib/libs/lzma/liblzma/check/check.h @@ -95,24 +95,6 @@ typedef struct { } lzma_check_state; -/// lzma_crc32_table[0] is needed by LZ encoder so we need to keep -/// the array two-dimensional. -#ifdef HAVE_SMALL -lzma_attr_visibility_hidden -extern uint32_t lzma_crc32_table[1][256]; - -extern void lzma_crc32_init(void); - -#else - -lzma_attr_visibility_hidden -extern const uint32_t lzma_crc32_table[8][256]; - -lzma_attr_visibility_hidden -extern const uint64_t lzma_crc64_table[4][256]; -#endif - - /// \brief Initialize *check depending on type extern void lzma_check_init(lzma_check_state *check, lzma_check type); diff --git a/contrib/libs/lzma/liblzma/check/crc32_fast.c b/contrib/libs/lzma/liblzma/check/crc32_fast.c index fce1af6119b..caaa4710afd 100644 --- a/contrib/libs/lzma/liblzma/check/crc32_fast.c +++ b/contrib/libs/lzma/liblzma/check/crc32_fast.c @@ -7,7 +7,6 @@ // // Authors: Lasse Collin // Ilya Kurdyukov -// Hans Jansen // /////////////////////////////////////////////////////////////////////////////// @@ -15,10 +14,12 @@ #include "crc_common.h" #if defined(CRC_X86_CLMUL) -# define BUILDING_CRC32_CLMUL +# define BUILDING_CRC_CLMUL 32 # include "crc_x86_clmul.h" #elif defined(CRC32_ARM64) # error #include "crc32_arm64.h" +#elif defined(CRC32_LOONGARCH) +# error #include "crc32_loongarch.h" #endif @@ -28,8 +29,19 @@ // Generic CRC32 // /////////////////// +#ifdef WORDS_BIGENDIAN +# error #include "crc32_table_be.h" +#else +# include "crc32_table_le.h" +#endif + + +#ifdef HAVE_CRC_X86_ASM +extern uint32_t lzma_crc32_generic( + const uint8_t *buf, size_t size, uint32_t crc); +#else static uint32_t -crc32_generic(const uint8_t *buf, size_t size, uint32_t crc) +lzma_crc32_generic(const uint8_t *buf, size_t size, uint32_t crc) { crc = ~crc; @@ -85,7 +97,8 @@ crc32_generic(const uint8_t *buf, size_t size, uint32_t crc) return ~crc; } -#endif +#endif // HAVE_CRC_X86_ASM +#endif // CRC32_GENERIC #if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) @@ -119,7 +132,7 @@ static crc32_func_type crc32_resolve(void) { return is_arch_extension_supported() - ? &crc32_arch_optimized : &crc32_generic; + ? &crc32_arch_optimized : &lzma_crc32_generic; } @@ -164,27 +177,6 @@ extern LZMA_API(uint32_t) lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) { #if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) - // On x86-64, if CLMUL is available, it is the best for non-tiny - // inputs, being over twice as fast as the generic slice-by-four - // version. However, for size <= 16 it's different. In the extreme - // case of size == 1 the generic version can be five times faster. - // At size >= 8 the CLMUL starts to become reasonable. It - // varies depending on the alignment of buf too. - // - // The above doesn't include the overhead of mythread_once(). - // At least on x86-64 GNU/Linux, pthread_once() is very fast but - // it still makes lzma_crc32(buf, 1, crc) 50-100 % slower. When - // size reaches 12-16 bytes the overhead becomes negligible. - // - // So using the generic version for size <= 16 may give better - // performance with tiny inputs but if such inputs happen rarely - // it's not so obvious because then the lookup table of the - // generic version may not be in the processor cache. -#ifdef CRC_USE_GENERIC_FOR_SMALL_INPUTS - if (size <= 16) - return crc32_generic(buf, size, crc); -#endif - /* #ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR // See crc32_dispatch(). This would be the alternative which uses @@ -199,6 +191,6 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) return crc32_arch_optimized(buf, size, crc); #else - return crc32_generic(buf, size, crc); + return lzma_crc32_generic(buf, size, crc); #endif } diff --git a/contrib/libs/lzma/liblzma/check/crc32_table.c b/contrib/libs/lzma/liblzma/check/crc32_table.c deleted file mode 100644 index db8d9d58312..00000000000 --- a/contrib/libs/lzma/liblzma/check/crc32_table.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: 0BSD - -/////////////////////////////////////////////////////////////////////////////// -// -/// \file crc32_table.c -/// \brief Precalculated CRC32 table with correct endianness -// -// Author: Lasse Collin -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" - - -// FIXME: Compared to crc_common.h this has to check for __x86_64__ too -// so that in 32-bit builds crc32_x86.S won't break due to a missing table. -#if defined(HAVE_USABLE_CLMUL) && ((defined(__x86_64__) && defined(__SSSE3__) \ - && defined(__SSE4_1__) && defined(__PCLMUL__)) \ - || (defined(__e2k__) && __iset__ >= 6)) -# define NO_CRC32_TABLE - -#elif defined(HAVE_ARM64_CRC32) \ - && !defined(WORDS_BIGENDIAN) \ - && defined(__ARM_FEATURE_CRC32) -# define NO_CRC32_TABLE -#endif - - -#if !defined(HAVE_ENCODERS) && defined(NO_CRC32_TABLE) -// No table needed. Use a typedef to avoid an empty translation unit. -typedef void lzma_crc32_dummy; - -#else -// Having the declaration here silences clang -Wmissing-variable-declarations. -extern const uint32_t lzma_crc32_table[8][256]; - -# ifdef WORDS_BIGENDIAN -# error #include "crc32_table_be.h" -# else -# include "crc32_table_le.h" -# endif -#endif diff --git a/contrib/libs/lzma/liblzma/check/crc64_fast.c b/contrib/libs/lzma/liblzma/check/crc64_fast.c index 0ce83fe4ad3..2c767bdcfde 100644 --- a/contrib/libs/lzma/liblzma/check/crc64_fast.c +++ b/contrib/libs/lzma/liblzma/check/crc64_fast.c @@ -14,7 +14,7 @@ #include "crc_common.h" #if defined(CRC_X86_CLMUL) -# define BUILDING_CRC64_CLMUL +# define BUILDING_CRC_CLMUL 64 # include "crc_x86_clmul.h" #endif @@ -25,6 +25,18 @@ // Generic slice-by-four CRC64 // ///////////////////////////////// +#if defined(WORDS_BIGENDIAN) +# error #include "crc64_table_be.h" +#else +# include "crc64_table_le.h" +#endif + + +#ifdef HAVE_CRC_X86_ASM +extern uint64_t lzma_crc64_generic( + const uint8_t *buf, size_t size, uint64_t crc); +#else + #ifdef WORDS_BIGENDIAN # define A1(x) ((x) >> 56) #else @@ -34,7 +46,7 @@ // See the comments in crc32_fast.c. They aren't duplicated here. static uint64_t -crc64_generic(const uint8_t *buf, size_t size, uint64_t crc) +lzma_crc64_generic(const uint8_t *buf, size_t size, uint64_t crc) { crc = ~crc; @@ -78,7 +90,8 @@ crc64_generic(const uint8_t *buf, size_t size, uint64_t crc) return ~crc; } -#endif +#endif // HAVE_CRC_X86_ASM +#endif // CRC64_GENERIC #if defined(CRC64_GENERIC) && defined(CRC64_ARCH_OPTIMIZED) @@ -97,7 +110,7 @@ static crc64_func_type crc64_resolve(void) { return is_arch_extension_supported() - ? &crc64_arch_optimized : &crc64_generic; + ? &crc64_arch_optimized : &lzma_crc64_generic; } #ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR @@ -133,24 +146,24 @@ crc64_dispatch(const uint8_t *buf, size_t size, uint64_t crc) extern LZMA_API(uint64_t) lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) { -#if defined(CRC64_GENERIC) && defined(CRC64_ARCH_OPTIMIZED) - -#ifdef CRC_USE_GENERIC_FOR_SMALL_INPUTS - if (size <= 16) - return crc64_generic(buf, size, crc); +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ + && defined(_M_IX86) && defined(CRC64_ARCH_OPTIMIZED) + // VS2015-2022 might corrupt the ebx register on 32-bit x86 when + // the CLMUL code is enabled. This hack forces MSVC to store and + // restore ebx. This is only needed here, not in lzma_crc32(). + __asm mov ebx, ebx #endif + +#if defined(CRC64_GENERIC) && defined(CRC64_ARCH_OPTIMIZED) return crc64_func(buf, size, crc); #elif defined(CRC64_ARCH_OPTIMIZED) // If arch-optimized version is used unconditionally without runtime // CPU detection then omitting the generic version and its 8 KiB // lookup table makes the library smaller. - // - // FIXME: Lookup table isn't currently omitted on 32-bit x86, - // see crc64_table.c. return crc64_arch_optimized(buf, size, crc); #else - return crc64_generic(buf, size, crc); + return lzma_crc64_generic(buf, size, crc); #endif } diff --git a/contrib/libs/lzma/liblzma/check/crc64_table.c b/contrib/libs/lzma/liblzma/check/crc64_table.c deleted file mode 100644 index e7ceb0276fa..00000000000 --- a/contrib/libs/lzma/liblzma/check/crc64_table.c +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: 0BSD - -/////////////////////////////////////////////////////////////////////////////// -// -/// \file crc64_table.c -/// \brief Precalculated CRC64 table with correct endianness -// -// Author: Lasse Collin -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" - - -// FIXME: Compared to crc_common.h this has to check for __x86_64__ too -// so that in 32-bit builds crc64_x86.S won't break due to a missing table. -#if defined(HAVE_USABLE_CLMUL) && ((defined(__x86_64__) && defined(__SSSE3__) \ - && defined(__SSE4_1__) && defined(__PCLMUL__)) \ - || (defined(__e2k__) && __iset__ >= 6)) -# define NO_CRC64_TABLE -#endif - - -#ifdef NO_CRC64_TABLE -// No table needed. Use a typedef to avoid an empty translation unit. -typedef void lzma_crc64_dummy; - -#else -// Having the declaration here silences clang -Wmissing-variable-declarations. -extern const uint64_t lzma_crc64_table[4][256]; - -# if defined(WORDS_BIGENDIAN) -# error #include "crc64_table_be.h" -# else -# include "crc64_table_le.h" -# endif -#endif diff --git a/contrib/libs/lzma/liblzma/check/crc_common.h b/contrib/libs/lzma/liblzma/check/crc_common.h index c15d4c675c8..7ea1e60b043 100644 --- a/contrib/libs/lzma/liblzma/check/crc_common.h +++ b/contrib/libs/lzma/liblzma/check/crc_common.h @@ -3,11 +3,10 @@ /////////////////////////////////////////////////////////////////////////////// // /// \file crc_common.h -/// \brief Some functions and macros for CRC32 and CRC64 +/// \brief Macros and declarations for CRC32 and CRC64 // // Authors: Lasse Collin // Ilya Kurdyukov -// Hans Jansen // Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -18,6 +17,10 @@ #include "common.h" +///////////// +// Generic // +///////////// + #ifdef WORDS_BIGENDIAN # define A(x) ((x) >> 24) # define B(x) (((x) >> 16) & 0xFF) @@ -38,43 +41,63 @@ #endif -// CRC CLMUL code needs this because accessing input buffers that aren't -// aligned to the vector size will inherently trip the address sanitizer. -#if lzma_has_attribute(__no_sanitize_address__) -# define crc_attr_no_sanitize_address \ - __attribute__((__no_sanitize_address__)) +/// lzma_crc32_table[0] is needed by LZ encoder so we need to keep +/// the array two-dimensional. +#ifdef HAVE_SMALL +lzma_attr_visibility_hidden +extern uint32_t lzma_crc32_table[1][256]; + +extern void lzma_crc32_init(void); + #else -# define crc_attr_no_sanitize_address -#endif -// Keep this in sync with changes to crc32_arm64.h -#if defined(_WIN32) || defined(HAVE_GETAUXVAL) \ - || defined(HAVE_ELF_AUX_INFO) \ - || (defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME)) -# define ARM64_RUNTIME_DETECTION 1 +lzma_attr_visibility_hidden +extern const uint32_t lzma_crc32_table[8][256]; + +lzma_attr_visibility_hidden +extern const uint64_t lzma_crc64_table[4][256]; #endif +/////////////////// +// Configuration // +/////////////////// + +// NOTE: This config isn't used if HAVE_SMALL is defined! + +// These are defined if the generic slicing-by-n implementations and their +// lookup tables are built. #undef CRC32_GENERIC #undef CRC64_GENERIC +// These are defined if an arch-specific version is built. If both this +// and matching _GENERIC is defined then runtime detection must be used. #undef CRC32_ARCH_OPTIMIZED #undef CRC64_ARCH_OPTIMIZED // The x86 CLMUL is used for both CRC32 and CRC64. #undef CRC_X86_CLMUL +// Many ARM64 processor have CRC32 instructions. +// CRC64 could be done with CLMUL but it's not implemented yet. #undef CRC32_ARM64 -#undef CRC64_ARM64_CLMUL -#undef CRC_USE_GENERIC_FOR_SMALL_INPUTS +// 64-bit LoongArch has CRC32 instructions. +#undef CRC32_LOONGARCH + + +// ARM64 +// +// Keep this in sync with changes to crc32_arm64.h +#if defined(_WIN32) || defined(HAVE_GETAUXVAL) \ + || defined(HAVE_ELF_AUX_INFO) \ + || (defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME)) +# define CRC_ARM64_RUNTIME_DETECTION 1 +#endif // ARM64 CRC32 instruction is only useful for CRC32. Currently, only // little endian is supported since we were unable to test on a big // endian machine. -// -// NOTE: Keep this and the next check in sync with the macro -// NO_CRC32_TABLE in crc32_table.c #if defined(HAVE_ARM64_CRC32) && !defined(WORDS_BIGENDIAN) // Allow ARM64 CRC32 instruction without a runtime check if // __ARM_FEATURE_CRC32 is defined. GCC and Clang only define @@ -82,21 +105,40 @@ # if defined(__ARM_FEATURE_CRC32) # define CRC32_ARCH_OPTIMIZED 1 # define CRC32_ARM64 1 -# elif defined(ARM64_RUNTIME_DETECTION) +# elif defined(CRC_ARM64_RUNTIME_DETECTION) # define CRC32_ARCH_OPTIMIZED 1 # define CRC32_ARM64 1 # define CRC32_GENERIC 1 # endif #endif -#if defined(HAVE_USABLE_CLMUL) -// If CLMUL is allowed unconditionally in the compiler options then the -// generic version can be omitted. Note that this doesn't work with MSVC -// as I don't know how to detect the features here. + +// LoongArch // -// NOTE: Keep this in sync with the NO_CRC32_TABLE macro in crc32_table.c -// and NO_CRC64_TABLE in crc64_table.c. -# if (defined(__SSSE3__) && defined(__SSE4_1__) && defined(__PCLMUL__)) \ +// Only 64-bit LoongArch is supported for now. No runtime detection +// is needed because the LoongArch specification says that the CRC32 +// instructions are a part of the Basic Integer Instructions and +// they shall be implemented by 64-bit LoongArch implementations. +#ifdef HAVE_LOONGARCH_CRC32 +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC32_LOONGARCH 1 +#endif + + +// x86 and E2K +#if defined(HAVE_USABLE_CLMUL) + // If CLMUL is allowed unconditionally in the compiler options then + // the generic version and the tables can be omitted. Exceptions: + // + // - If 32-bit x86 assembly files are enabled then those are always + // built and runtime detection is used even if compiler flags + // were set to allow CLMUL unconditionally. + // + // - This doesn't work with MSVC as I don't know how to detect + // the features here. + // +# if (defined(__SSSE3__) && defined(__SSE4_1__) && defined(__PCLMUL__) \ + && !defined(HAVE_CRC_X86_ASM)) \ || (defined(__e2k__) && __iset__ >= 6) # define CRC32_ARCH_OPTIMIZED 1 # define CRC64_ARCH_OPTIMIZED 1 @@ -107,21 +149,12 @@ # define CRC32_ARCH_OPTIMIZED 1 # define CRC64_ARCH_OPTIMIZED 1 # define CRC_X86_CLMUL 1 - -/* - // The generic code is much faster with 1-8-byte inputs and - // has similar performance up to 16 bytes at least in - // microbenchmarks (it depends on input buffer alignment - // too). If both versions are built, this #define will use - // the generic version for inputs up to 16 bytes and CLMUL - // for bigger inputs. It saves a little in code size since - // the special cases for 0-16-byte inputs will be omitted - // from the CLMUL code. -# define CRC_USE_GENERIC_FOR_SMALL_INPUTS 1 -*/ # endif #endif + +// Fallback configuration +// // For CRC32 use the generic slice-by-eight implementation if no optimized // version is available. #if !defined(CRC32_ARCH_OPTIMIZED) && !defined(CRC32_GENERIC) diff --git a/contrib/libs/lzma/liblzma/check/crc_x86_clmul.h b/contrib/libs/lzma/liblzma/check/crc_x86_clmul.h index 50306e49a72..b302d6cf7f5 100644 --- a/contrib/libs/lzma/liblzma/check/crc_x86_clmul.h +++ b/contrib/libs/lzma/liblzma/check/crc_x86_clmul.h @@ -8,26 +8,20 @@ /// The CRC32 and CRC64 implementations use 32/64-bit x86 SSSE3, SSE4.1, and /// CLMUL instructions. This is compatible with Elbrus 2000 (E2K) too. /// -/// They were derived from +/// See the Intel white paper "Fast CRC Computation for Generic Polynomials +/// Using PCLMULQDQ Instruction" from 2009. The original file seems to be +/// gone from Intel's website but a version is available here: /// https://www.researchgate.net/publication/263424619_Fast_CRC_computation -/// and the public domain code from https://github.com/rawrunprotected/crc -/// (URLs were checked on 2023-10-14). +/// (The link was checked on 2024-06-11.) /// /// While this file has both CRC32 and CRC64 implementations, only one -/// should be built at a time to ensure that crc_simd_body() is inlined -/// even with compilers with which lzma_always_inline expands to plain inline. -/// The version to build is selected by defining BUILDING_CRC32_CLMUL or -/// BUILDING_CRC64_CLMUL before including this file. +/// can be built at a time. The version to build is selected by defining +/// BUILDING_CRC_CLMUL to 32 or 64 before including this file. /// -/// FIXME: Builds for 32-bit x86 use the assembly .S files by default -/// unless configured with --disable-assembler. Even then the lookup table -/// isn't omitted in crc64_table.c since it doesn't know that assembly -/// code has been disabled. +/// NOTE: The x86 CLMUL CRC implementation was rewritten for XZ Utils 5.8.0. // -// Authors: Ilya Kurdyukov -// Hans Jansen -// Lasse Collin -// Jia Tan +// Authors: Lasse Collin +// Ilya Kurdyukov // /////////////////////////////////////////////////////////////////////////////// @@ -37,6 +31,10 @@ #endif #define LZMA_CRC_X86_CLMUL_H +#if BUILDING_CRC_CLMUL != 32 && BUILDING_CRC_CLMUL != 64 +# error BUILDING_CRC_CLMUL is undefined or has an invalid value +#endif + #include <immintrin.h> #if defined(_MSC_VER) @@ -59,330 +57,277 @@ #endif -#define MASK_L(in, mask, r) r = _mm_shuffle_epi8(in, mask) +// GCC and Clang would produce good code with _mm_set_epi64x +// but MSVC needs _mm_cvtsi64_si128 on x86-64. +#if defined(__i386__) || defined(_M_IX86) +# define my_set_low64(a) _mm_set_epi64x(0, (a)) +#else +# define my_set_low64(a) _mm_cvtsi64_si128(a) +#endif -#define MASK_H(in, mask, r) \ - r = _mm_shuffle_epi8(in, _mm_xor_si128(mask, vsign)) -#define MASK_LH(in, mask, low, high) \ - MASK_L(in, mask, low); \ - MASK_H(in, mask, high) +// Align it so that the whole array is within the same cache line. +// More than one unaligned load can be done from this during the +// same CRC function call. +// +// The bytes [0] to [31] are used with AND to clear the low bytes. (With ANDN +// those could be used to clear the high bytes too but it's not needed here.) +// +// The bytes [16] to [47] are for left shifts. +// The bytes [32] to [63] are for right shifts. +alignas(64) +static uint8_t vmasks[64] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, +}; + + +// *Unaligned* 128-bit load +crc_attr_target +static inline __m128i +my_load128(const uint8_t *p) +{ + return _mm_loadu_si128((const __m128i *)p); +} +// Keep the highest "count" bytes as is and clear the remaining low bytes. crc_attr_target -crc_attr_no_sanitize_address -static lzma_always_inline void -crc_simd_body(const uint8_t *buf, const size_t size, __m128i *v0, __m128i *v1, - const __m128i vfold16, const __m128i initial_crc) +static inline __m128i +keep_high_bytes(__m128i v, size_t count) { - // Create a vector with 8-bit values 0 to 15. This is used to - // construct control masks for _mm_blendv_epi8 and _mm_shuffle_epi8. - const __m128i vramp = _mm_setr_epi32( - 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c); - - // This is used to inverse the control mask of _mm_shuffle_epi8 - // so that bytes that wouldn't be picked with the original mask - // will be picked and vice versa. - const __m128i vsign = _mm_set1_epi8(-0x80); + return _mm_and_si128(my_load128((vmasks + count)), v); +} - // Memory addresses A to D and the distances between them: - // - // A B C D - // [skip_start][size][skip_end] - // [ size2 ] - // - // A and D are 16-byte aligned. B and C are 1-byte aligned. - // skip_start and skip_end are 0-15 bytes. size is at least 1 byte. - // - // A = aligned_buf will initially point to this address. - // B = The address pointed by the caller-supplied buf. - // C = buf + size == aligned_buf + size2 - // D = buf + size + skip_end == aligned_buf + size2 + skip_end - const size_t skip_start = (size_t)((uintptr_t)buf & 15); - const size_t skip_end = (size_t)((0U - (uintptr_t)(buf + size)) & 15); - const __m128i *aligned_buf = (const __m128i *)( - (uintptr_t)buf & ~(uintptr_t)15); - - // If size2 <= 16 then the whole input fits into a single 16-byte - // vector. If size2 > 16 then at least two 16-byte vectors must - // be processed. If size2 > 16 && size <= 16 then there is only - // one 16-byte vector's worth of input but it is unaligned in memory. - // - // NOTE: There is no integer overflow here if the arguments - // are valid. If this overflowed, buf + size would too. - const size_t size2 = skip_start + size; - - // Masks to be used with _mm_blendv_epi8 and _mm_shuffle_epi8: - // The first skip_start or skip_end bytes in the vectors will have - // the high bit (0x80) set. _mm_blendv_epi8 and _mm_shuffle_epi8 - // will produce zeros for these positions. (Bitwise-xor of these - // masks with vsign will produce the opposite behavior.) - const __m128i mask_start - = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_start)); - const __m128i mask_end - = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_end)); - - // Get the first 1-16 bytes into data0. If loading less than 16 - // bytes, the bytes are loaded to the high bits of the vector and - // the least significant positions are filled with zeros. - const __m128i data0 = _mm_blendv_epi8(_mm_load_si128(aligned_buf), - _mm_setzero_si128(), mask_start); - aligned_buf++; - - __m128i v2, v3; - -#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS - if (size <= 16) { - // Right-shift initial_crc by 1-16 bytes based on "size" - // and store the result in v1 (high bytes) and v0 (low bytes). - // - // NOTE: The highest 8 bytes of initial_crc are zeros so - // v1 will be filled with zeros if size >= 8. The highest - // 8 bytes of v1 will always become zeros. - // - // [ v1 ][ v0 ] - // [ initial_crc ] size == 1 - // [ initial_crc ] size == 2 - // [ initial_crc ] size == 15 - // [ initial_crc ] size == 16 (all in v0) - const __m128i mask_low = _mm_add_epi8( - vramp, _mm_set1_epi8((char)(size - 16))); - MASK_LH(initial_crc, mask_low, *v0, *v1); - - if (size2 <= 16) { - // There are 1-16 bytes of input and it is all - // in data0. Copy the input bytes to v3. If there - // are fewer than 16 bytes, the low bytes in v3 - // will be filled with zeros. That is, the input - // bytes are stored to the same position as - // (part of) initial_crc is in v0. - MASK_L(data0, mask_end, v3); - } else { - // There are 2-16 bytes of input but not all bytes - // are in data0. - const __m128i data1 = _mm_load_si128(aligned_buf); - - // Collect the 2-16 input bytes from data0 and data1 - // to v2 and v3, and bitwise-xor them with the - // low bits of initial_crc in v0. Note that the - // the second xor is below this else-block as it - // is shared with the other branch. - MASK_H(data0, mask_end, v2); - MASK_L(data1, mask_end, v3); - *v0 = _mm_xor_si128(*v0, v2); - } - *v0 = _mm_xor_si128(*v0, v3); - *v1 = _mm_alignr_epi8(*v1, *v0, 8); - } else -#endif - { - // There is more than 16 bytes of input. - const __m128i data1 = _mm_load_si128(aligned_buf); - const __m128i *end = (const __m128i*)( - (const char *)aligned_buf - 16 + size2); - aligned_buf++; - - MASK_LH(initial_crc, mask_start, *v0, *v1); - *v0 = _mm_xor_si128(*v0, data0); - *v1 = _mm_xor_si128(*v1, data1); - - while (aligned_buf < end) { - *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( - *v0, vfold16, 0x00)); - *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( - *v0, vfold16, 0x11)); - *v1 = _mm_load_si128(aligned_buf++); - } +// Shift the 128-bit value left by "amount" bytes (not bits). +crc_attr_target +static inline __m128i +shift_left(__m128i v, size_t amount) +{ + return _mm_shuffle_epi8(v, my_load128((vmasks + 32 - amount))); +} - if (aligned_buf != end) { - MASK_H(*v0, mask_end, v2); - MASK_L(*v0, mask_end, *v0); - MASK_L(*v1, mask_end, v3); - *v1 = _mm_or_si128(v2, v3); - } - *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( - *v0, vfold16, 0x00)); - *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( - *v0, vfold16, 0x11)); - *v1 = _mm_srli_si128(*v0, 8); - } +// Shift the 128-bit value right by "amount" bytes (not bits). +crc_attr_target +static inline __m128i +shift_right(__m128i v, size_t amount) +{ + return _mm_shuffle_epi8(v, my_load128((vmasks + 32 + amount))); } -///////////////////// -// x86 CLMUL CRC32 // -///////////////////// - -/* -// These functions were used to generate the constants -// at the top of crc32_arch_optimized(). -static uint64_t -calc_lo(uint64_t p, uint64_t a, int n) +crc_attr_target +static inline __m128i +fold(__m128i v, __m128i k) { - uint64_t b = 0; int i; - for (i = 0; i < n; i++) { - b = b >> 1 | (a & 1) << (n - 1); - a = (a >> 1) ^ ((0 - (a & 1)) & p); - } - return b; + __m128i a = _mm_clmulepi64_si128(v, k, 0x00); + __m128i b = _mm_clmulepi64_si128(v, k, 0x11); + return _mm_xor_si128(a, b); } -// same as ~crc(&a, sizeof(a), ~0) -static uint64_t -calc_hi(uint64_t p, uint64_t a, int n) + +crc_attr_target +static inline __m128i +fold_xor(__m128i v, __m128i k, const uint8_t *buf) { - int i; - for (i = 0; i < n; i++) - a = (a >> 1) ^ ((0 - (a & 1)) & p); - return a; + return _mm_xor_si128(my_load128(buf), fold(v, k)); } -*/ -#ifdef BUILDING_CRC32_CLMUL +#if BUILDING_CRC_CLMUL == 32 crc_attr_target -crc_attr_no_sanitize_address static uint32_t crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) +#else +crc_attr_target +static uint64_t +crc64_arch_optimized(const uint8_t *buf, size_t size, uint64_t crc) +#endif { -#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS - // The code assumes that there is at least one byte of input. + // We will assume that there is at least one byte of input. if (size == 0) return crc; -#endif - // uint32_t poly = 0xedb88320; - const int64_t p = 0x1db710640; // p << 1 - const int64_t mu = 0x1f7011641; // calc_lo(p, p, 32) << 1 | 1 - const int64_t k5 = 0x163cd6124; // calc_hi(p, p, 32) << 1 - const int64_t k4 = 0x0ccaa009e; // calc_hi(p, p, 64) << 1 - const int64_t k3 = 0x1751997d0; // calc_hi(p, p, 128) << 1 - - const __m128i vfold4 = _mm_set_epi64x(mu, p); - const __m128i vfold8 = _mm_set_epi64x(0, k5); - const __m128i vfold16 = _mm_set_epi64x(k4, k3); - - __m128i v0, v1, v2; - - crc_simd_body(buf, size, &v0, &v1, vfold16, - _mm_cvtsi32_si128((int32_t)~crc)); - - v1 = _mm_xor_si128( - _mm_clmulepi64_si128(v0, vfold16, 0x10), v1); // xxx0 - v2 = _mm_shuffle_epi32(v1, 0xe7); // 0xx0 - v0 = _mm_slli_epi64(v1, 32); // [0] - v0 = _mm_clmulepi64_si128(v0, vfold8, 0x00); - v0 = _mm_xor_si128(v0, v2); // [1] [2] - v2 = _mm_clmulepi64_si128(v0, vfold4, 0x10); - v2 = _mm_clmulepi64_si128(v2, vfold4, 0x00); - v0 = _mm_xor_si128(v0, v2); // [2] - return ~(uint32_t)_mm_extract_epi32(v0, 2); -} -#endif // BUILDING_CRC32_CLMUL + // See crc_clmul_consts_gen.c. +#if BUILDING_CRC_CLMUL == 32 + const __m128i fold512 = _mm_set_epi64x(0x1d9513d7, 0x8f352d95); + const __m128i fold128 = _mm_set_epi64x(0xccaa009e, 0xae689191); + const __m128i mu_p = _mm_set_epi64x( + (int64_t)0xb4e5b025f7011641, 0x1db710640); +#else + const __m128i fold512 = _mm_set_epi64x( + (int64_t)0x081f6054a7842df4, (int64_t)0x6ae3efbb9dd441f3); + const __m128i fold128 = _mm_set_epi64x( + (int64_t)0xdabe95afc7875f40, (int64_t)0xe05dd497ca393ae4); -///////////////////// -// x86 CLMUL CRC64 // -///////////////////// + const __m128i mu_p = _mm_set_epi64x( + (int64_t)0x9c3e466c172963d5, (int64_t)0x92d8af2baf0e1e84); +#endif -/* -// These functions were used to generate the constants -// at the top of crc64_arch_optimized(). -static uint64_t -calc_lo(uint64_t poly) -{ - uint64_t a = poly; - uint64_t b = 0; + __m128i v0, v1, v2, v3; - for (unsigned i = 0; i < 64; ++i) { - b = (b >> 1) | (a << 63); - a = (a >> 1) ^ (a & 1 ? poly : 0); - } + crc = ~crc; - return b; -} + if (size < 8) { + uint64_t x = crc; + size_t i = 0; -static uint64_t -calc_hi(uint64_t poly, uint64_t a) -{ - for (unsigned i = 0; i < 64; ++i) - a = (a >> 1) ^ (a & 1 ? poly : 0); + // Checking the bit instead of comparing the size means + // that we don't need to update the size between the steps. + if (size & 4) { + x ^= read32le(buf); + buf += 4; + i = 32; + } - return a; -} -*/ + if (size & 2) { + x ^= (uint64_t)read16le(buf) << i; + buf += 2; + i += 16; + } -#ifdef BUILDING_CRC64_CLMUL + if (size & 1) + x ^= (uint64_t)*buf << i; -// MSVC (VS2015 - VS2022) produces bad 32-bit x86 code from the CLMUL CRC -// code when optimizations are enabled (release build). According to the bug -// report, the ebx register is corrupted and the calculated result is wrong. -// Trying to workaround the problem with "__asm mov ebx, ebx" didn't help. -// The following pragma works and performance is still good. x86-64 builds -// and CRC32 CLMUL aren't affected by this problem. The problem does not -// happen in crc_simd_body() either (which is shared with CRC32 CLMUL anyway). -// -// NOTE: Another pragma after crc64_arch_optimized() restores -// the optimizations. If the #if condition here is updated, -// the other one must be updated too. -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ - && defined(_M_IX86) -# pragma optimize("g", off) -#endif + v0 = my_set_low64((int64_t)x); + v0 = shift_left(v0, 8 - size); -crc_attr_target -crc_attr_no_sanitize_address -static uint64_t -crc64_arch_optimized(const uint8_t *buf, size_t size, uint64_t crc) -{ -#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS - // The code assumes that there is at least one byte of input. - if (size == 0) - return crc; -#endif - - // const uint64_t poly = 0xc96c5795d7870f42; // CRC polynomial - const uint64_t p = 0x92d8af2baf0e1e85; // (poly << 1) | 1 - const uint64_t mu = 0x9c3e466c172963d5; // (calc_lo(poly) << 1) | 1 - const uint64_t k2 = 0xdabe95afc7875f40; // calc_hi(poly, 1) - const uint64_t k1 = 0xe05dd497ca393ae4; // calc_hi(poly, k2) + } else if (size < 16) { + v0 = my_set_low64((int64_t)(crc ^ read64le(buf))); - const __m128i vfold8 = _mm_set_epi64x((int64_t)p, (int64_t)mu); - const __m128i vfold16 = _mm_set_epi64x((int64_t)k2, (int64_t)k1); + // NOTE: buf is intentionally left 8 bytes behind so that + // we can read the last 1-7 bytes with read64le(buf + size). + size -= 8; - __m128i v0, v1, v2; + // Handling 8-byte input specially is a speed optimization + // as the clmul can be skipped. A branch is also needed to + // avoid a too high shift amount. + if (size > 0) { + const size_t padding = 8 - size; + uint64_t high = read64le(buf + size) >> (padding * 8); #if defined(__i386__) || defined(_M_IX86) - crc_simd_body(buf, size, &v0, &v1, vfold16, - _mm_set_epi64x(0, (int64_t)~crc)); + // Simple but likely not the best code for 32-bit x86. + v0 = _mm_insert_epi32(v0, (int32_t)high, 2); + v0 = _mm_insert_epi32(v0, (int32_t)(high >> 32), 3); #else - // GCC and Clang would produce good code with _mm_set_epi64x - // but MSVC needs _mm_cvtsi64_si128 on x86-64. - crc_simd_body(buf, size, &v0, &v1, vfold16, - _mm_cvtsi64_si128((int64_t)~crc)); + v0 = _mm_insert_epi64(v0, (int64_t)high, 1); #endif - v1 = _mm_xor_si128(_mm_clmulepi64_si128(v0, vfold16, 0x10), v1); - v0 = _mm_clmulepi64_si128(v1, vfold8, 0x00); - v2 = _mm_clmulepi64_si128(v0, vfold8, 0x10); - v0 = _mm_xor_si128(_mm_xor_si128(v1, _mm_slli_si128(v0, 8)), v2); + v0 = shift_left(v0, padding); + + v1 = _mm_srli_si128(v0, 8); + v0 = _mm_clmulepi64_si128(v0, fold128, 0x10); + v0 = _mm_xor_si128(v0, v1); + } + } else { + v0 = my_set_low64((int64_t)crc); + + // To align or not to align the buf pointer? If the end of + // the buffer isn't aligned, aligning the pointer here would + // make us do an extra folding step with the associated byte + // shuffling overhead. The cost of that would need to be + // lower than the benefit of aligned reads. Testing on an old + // Intel Ivy Bridge processor suggested that aligning isn't + // worth the cost but it likely depends on the processor and + // buffer size. Unaligned loads (MOVDQU) should be fast on + // x86 processors that support PCLMULQDQ, so we don't align + // the buf pointer here. + + // Read the first (and possibly the only) full 16 bytes. + v0 = _mm_xor_si128(v0, my_load128(buf)); + buf += 16; + size -= 16; + + if (size >= 48) { + v1 = my_load128(buf); + v2 = my_load128(buf + 16); + v3 = my_load128(buf + 32); + buf += 48; + size -= 48; + + while (size >= 64) { + v0 = fold_xor(v0, fold512, buf); + v1 = fold_xor(v1, fold512, buf + 16); + v2 = fold_xor(v2, fold512, buf + 32); + v3 = fold_xor(v3, fold512, buf + 48); + buf += 64; + size -= 64; + } + + v0 = _mm_xor_si128(v1, fold(v0, fold128)); + v0 = _mm_xor_si128(v2, fold(v0, fold128)); + v0 = _mm_xor_si128(v3, fold(v0, fold128)); + } + + while (size >= 16) { + v0 = fold_xor(v0, fold128, buf); + buf += 16; + size -= 16; + } + + if (size > 0) { + // We want the last "size" number of input bytes to + // be at the high bits of v1. First do a full 16-byte + // load and then mask the low bytes to zeros. + v1 = my_load128(buf + size - 16); + v1 = keep_high_bytes(v1, size); + + // Shift high bytes from v0 to the low bytes of v1. + // + // Alternatively we could replace the combination + // keep_high_bytes + shift_right + _mm_or_si128 with + // _mm_shuffle_epi8 + _mm_blendv_epi8 but that would + // require larger tables for the masks. Now there are + // three loads (instead of two) from the mask tables + // but they all are from the same cache line. + v1 = _mm_or_si128(v1, shift_right(v0, size)); + + // Shift high bytes of v0 away, padding the + // low bytes with zeros. + v0 = shift_left(v0, 16 - size); + + v0 = _mm_xor_si128(v1, fold(v0, fold128)); + } + v1 = _mm_srli_si128(v0, 8); + v0 = _mm_clmulepi64_si128(v0, fold128, 0x10); + v0 = _mm_xor_si128(v0, v1); + } + + // Barrett reduction + +#if BUILDING_CRC_CLMUL == 32 + v1 = _mm_clmulepi64_si128(v0, mu_p, 0x10); // v0 * mu + v1 = _mm_clmulepi64_si128(v1, mu_p, 0x00); // v1 * p + v0 = _mm_xor_si128(v0, v1); + return ~(uint32_t)_mm_extract_epi32(v0, 2); +#else + // Because p is 65 bits but one bit doesn't fit into the 64-bit + // half of __m128i, finish the second clmul by shifting v1 left + // by 64 bits and xorring it to the final result. + v1 = _mm_clmulepi64_si128(v0, mu_p, 0x10); // v0 * mu + v2 = _mm_slli_si128(v1, 8); + v1 = _mm_clmulepi64_si128(v1, mu_p, 0x00); // v1 * p + v0 = _mm_xor_si128(v0, v2); + v0 = _mm_xor_si128(v0, v1); #if defined(__i386__) || defined(_M_IX86) return ~(((uint64_t)(uint32_t)_mm_extract_epi32(v0, 3) << 32) | (uint64_t)(uint32_t)_mm_extract_epi32(v0, 2)); #else return ~(uint64_t)_mm_extract_epi64(v0, 1); #endif -} - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ - && defined(_M_IX86) -# pragma optimize("", on) #endif - -#endif // BUILDING_CRC64_CLMUL +} // Even though this is an inline function, compile it only when needed. diff --git a/contrib/libs/lzma/liblzma/common/alone_decoder.c b/contrib/libs/lzma/liblzma/common/alone_decoder.c index 78af651578f..e2b58e1f375 100644 --- a/contrib/libs/lzma/liblzma/common/alone_decoder.c +++ b/contrib/libs/lzma/liblzma/common/alone_decoder.c @@ -134,8 +134,7 @@ alone_decode(void *coder_ptr, const lzma_allocator *allocator, coder->pos = 0; coder->sequence = SEQ_CODER_INIT; - - // Fall through + FALLTHROUGH; case SEQ_CODER_INIT: { if (coder->memusage > coder->memlimit) diff --git a/contrib/libs/lzma/liblzma/common/auto_decoder.c b/contrib/libs/lzma/liblzma/common/auto_decoder.c index fdd520f905c..da49345f909 100644 --- a/contrib/libs/lzma/liblzma/common/auto_decoder.c +++ b/contrib/libs/lzma/liblzma/common/auto_decoder.c @@ -79,7 +79,7 @@ auto_decode(void *coder_ptr, const lzma_allocator *allocator, return LZMA_GET_CHECK; } - // Fall through + FALLTHROUGH; case SEQ_CODE: { const lzma_ret ret = coder->next.code( @@ -91,10 +91,9 @@ auto_decode(void *coder_ptr, const lzma_allocator *allocator, return ret; coder->sequence = SEQ_FINISH; + FALLTHROUGH; } - // Fall through - case SEQ_FINISH: // When LZMA_CONCATENATED was used and we were decoding // a LZMA_Alone file, we need to check that there is no diff --git a/contrib/libs/lzma/liblzma/common/block_decoder.c b/contrib/libs/lzma/liblzma/common/block_decoder.c index 2e369d316bd..bbc9f5566c8 100644 --- a/contrib/libs/lzma/liblzma/common/block_decoder.c +++ b/contrib/libs/lzma/liblzma/common/block_decoder.c @@ -146,10 +146,9 @@ block_decode(void *coder_ptr, const lzma_allocator *allocator, coder->block->uncompressed_size = coder->uncompressed_size; coder->sequence = SEQ_PADDING; + FALLTHROUGH; } - // Fall through - case SEQ_PADDING: // Compressed Data is padded to a multiple of four bytes. while (coder->compressed_size & 3) { @@ -173,8 +172,7 @@ block_decode(void *coder_ptr, const lzma_allocator *allocator, lzma_check_finish(&coder->check, coder->block->check); coder->sequence = SEQ_CHECK; - - // Fall through + FALLTHROUGH; case SEQ_CHECK: { const size_t check_size = lzma_check_size(coder->block->check); diff --git a/contrib/libs/lzma/liblzma/common/block_encoder.c b/contrib/libs/lzma/liblzma/common/block_encoder.c index ce8c1de6944..eb7997a72ae 100644 --- a/contrib/libs/lzma/liblzma/common/block_encoder.c +++ b/contrib/libs/lzma/liblzma/common/block_encoder.c @@ -94,10 +94,9 @@ block_encode(void *coder_ptr, const lzma_allocator *allocator, coder->block->uncompressed_size = coder->uncompressed_size; coder->sequence = SEQ_PADDING; + FALLTHROUGH; } - // Fall through - case SEQ_PADDING: // Pad Compressed Data to a multiple of four bytes. We can // use coder->compressed_size for this since we don't need @@ -117,8 +116,7 @@ block_encode(void *coder_ptr, const lzma_allocator *allocator, lzma_check_finish(&coder->check, coder->block->check); coder->sequence = SEQ_CHECK; - - // Fall through + FALLTHROUGH; case SEQ_CHECK: { const size_t check_size = lzma_check_size(coder->block->check); diff --git a/contrib/libs/lzma/liblzma/common/common.c b/contrib/libs/lzma/liblzma/common/common.c index cc0e06a51be..6e031a56c88 100644 --- a/contrib/libs/lzma/liblzma/common/common.c +++ b/contrib/libs/lzma/liblzma/common/common.c @@ -96,6 +96,12 @@ lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { + assert(in != NULL || *in_pos == in_size); + assert(out != NULL || *out_pos == out_size); + + assert(*in_pos <= in_size); + assert(*out_pos <= out_size); + const size_t in_avail = in_size - *in_pos; const size_t out_avail = out_size - *out_pos; const size_t copy_size = my_min(in_avail, out_avail); @@ -348,7 +354,7 @@ lzma_code(lzma_stream *strm, lzma_action action) else strm->internal->sequence = ISEQ_END; - // Fall through + FALLTHROUGH; case LZMA_NO_CHECK: case LZMA_UNSUPPORTED_CHECK: diff --git a/contrib/libs/lzma/liblzma/common/file_info.c b/contrib/libs/lzma/liblzma/common/file_info.c index 7c85084a706..4b2eb5d0400 100644 --- a/contrib/libs/lzma/liblzma/common/file_info.c +++ b/contrib/libs/lzma/liblzma/common/file_info.c @@ -298,15 +298,13 @@ file_info_decode(void *coder_ptr, const lzma_allocator *allocator, // Start looking for Stream Padding and Stream Footer // at the end of the file. coder->file_target_pos = coder->file_size; - - // Fall through + FALLTHROUGH; case SEQ_PADDING_SEEK: coder->sequence = SEQ_PADDING_DECODE; return_if_error(reverse_seek( coder, in_start, in_pos, in_size)); - - // Fall through + FALLTHROUGH; case SEQ_PADDING_DECODE: { // Copy to coder->temp first. This keeps the code simpler if @@ -356,9 +354,9 @@ file_info_decode(void *coder_ptr, const lzma_allocator *allocator, if (coder->temp_size < LZMA_STREAM_HEADER_SIZE) return_if_error(reverse_seek( coder, in_start, in_pos, in_size)); - } - // Fall through + FALLTHROUGH; + } case SEQ_FOOTER: // Copy the Stream Footer field into coder->temp. @@ -414,7 +412,7 @@ file_info_decode(void *coder_ptr, const lzma_allocator *allocator, return LZMA_SEEK_NEEDED; } - // Fall through + FALLTHROUGH; case SEQ_INDEX_INIT: { // Calculate the amount of memory already used by the earlier @@ -444,10 +442,9 @@ file_info_decode(void *coder_ptr, const lzma_allocator *allocator, coder->index_remaining = coder->footer_flags.backward_size; coder->sequence = SEQ_INDEX_DECODE; + FALLTHROUGH; } - // Fall through - case SEQ_INDEX_DECODE: { // Decode (a part of) the Index. If the whole Index is already // in coder->temp, read it from there. Otherwise read from @@ -574,9 +571,9 @@ file_info_decode(void *coder_ptr, const lzma_allocator *allocator, return_if_error(reverse_seek(coder, in_start, in_pos, in_size)); } - } - // Fall through + FALLTHROUGH; + } case SEQ_HEADER_DECODE: // Copy the Stream Header field into coder->temp. @@ -596,8 +593,7 @@ file_info_decode(void *coder_ptr, const lzma_allocator *allocator, coder->temp + coder->temp_size))); coder->sequence = SEQ_HEADER_COMPARE; - - // Fall through + FALLTHROUGH; case SEQ_HEADER_COMPARE: // Compare Stream Header against Stream Footer. They must diff --git a/contrib/libs/lzma/liblzma/common/index_decoder.c b/contrib/libs/lzma/liblzma/common/index_decoder.c index 4bcb3069211..4eab56d942e 100644 --- a/contrib/libs/lzma/liblzma/common/index_decoder.c +++ b/contrib/libs/lzma/liblzma/common/index_decoder.c @@ -93,8 +93,7 @@ index_decode(void *coder_ptr, const lzma_allocator *allocator, coder->pos = 0; coder->sequence = SEQ_MEMUSAGE; - - // Fall through + FALLTHROUGH; case SEQ_MEMUSAGE: if (lzma_index_memusage(1, coder->count) > coder->memlimit) { @@ -153,8 +152,7 @@ index_decode(void *coder_ptr, const lzma_allocator *allocator, case SEQ_PADDING_INIT: coder->pos = lzma_index_padding_size(coder->index); coder->sequence = SEQ_PADDING; - - // Fall through + FALLTHROUGH; case SEQ_PADDING: if (coder->pos > 0) { @@ -170,8 +168,7 @@ index_decode(void *coder_ptr, const lzma_allocator *allocator, *in_pos - in_start, coder->crc32); coder->sequence = SEQ_CRC32; - - // Fall through + FALLTHROUGH; case SEQ_CRC32: do { diff --git a/contrib/libs/lzma/liblzma/common/index_encoder.c b/contrib/libs/lzma/liblzma/common/index_encoder.c index ecc299c0159..80f1be1e3ae 100644 --- a/contrib/libs/lzma/liblzma/common/index_encoder.c +++ b/contrib/libs/lzma/liblzma/common/index_encoder.c @@ -93,8 +93,7 @@ index_encode(void *coder_ptr, } coder->sequence = SEQ_UNPADDED; - - // Fall through + FALLTHROUGH; case SEQ_UNPADDED: case SEQ_UNCOMPRESSED: { @@ -127,8 +126,7 @@ index_encode(void *coder_ptr, *out_pos - out_start, coder->crc32); coder->sequence = SEQ_CRC32; - - // Fall through + FALLTHROUGH; case SEQ_CRC32: // We don't use the main loop, because we don't want diff --git a/contrib/libs/lzma/liblzma/common/index_hash.c b/contrib/libs/lzma/liblzma/common/index_hash.c index caa5967ca49..b7f1b6b58d1 100644 --- a/contrib/libs/lzma/liblzma/common/index_hash.c +++ b/contrib/libs/lzma/liblzma/common/index_hash.c @@ -267,9 +267,9 @@ lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, index_hash->pos = (LZMA_VLI_C(4) - index_size_unpadded( index_hash->records.count, index_hash->records.index_list_size)) & 3; - index_hash->sequence = SEQ_PADDING; - // Fall through + index_hash->sequence = SEQ_PADDING; + FALLTHROUGH; case SEQ_PADDING: if (index_hash->pos > 0) { @@ -302,8 +302,7 @@ lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, *in_pos - in_start, index_hash->crc32); index_hash->sequence = SEQ_CRC32; - - // Fall through + FALLTHROUGH; case SEQ_CRC32: do { diff --git a/contrib/libs/lzma/liblzma/common/lzip_decoder.c b/contrib/libs/lzma/liblzma/common/lzip_decoder.c index 651a0ae712c..4dff2d5889e 100644 --- a/contrib/libs/lzma/liblzma/common/lzip_decoder.c +++ b/contrib/libs/lzma/liblzma/common/lzip_decoder.c @@ -150,10 +150,9 @@ lzip_decode(void *coder_ptr, const lzma_allocator *allocator, coder->member_size = sizeof(lzip_id_string); coder->sequence = SEQ_VERSION; + FALLTHROUGH; } - // Fall through - case SEQ_VERSION: if (*in_pos >= in_size) return LZMA_OK; @@ -173,7 +172,7 @@ lzip_decode(void *coder_ptr, const lzma_allocator *allocator, if (coder->tell_any_check) return LZMA_GET_CHECK; - // Fall through + FALLTHROUGH; case SEQ_DICT_SIZE: { if (*in_pos >= in_size) @@ -220,10 +219,9 @@ lzip_decode(void *coder_ptr, const lzma_allocator *allocator, // LZMA_MEMLIMIT_ERROR we need to be able to restart after // the memlimit has been increased. coder->sequence = SEQ_CODER_INIT; + FALLTHROUGH; } - // Fall through - case SEQ_CODER_INIT: { if (coder->memusage > coder->memlimit) return LZMA_MEMLIMIT_ERROR; @@ -243,10 +241,9 @@ lzip_decode(void *coder_ptr, const lzma_allocator *allocator, coder->crc32 = 0; coder->sequence = SEQ_LZMA_STREAM; + FALLTHROUGH; } - // Fall through - case SEQ_LZMA_STREAM: { const size_t in_start = *in_pos; const size_t out_start = *out_pos; @@ -273,10 +270,9 @@ lzip_decode(void *coder_ptr, const lzma_allocator *allocator, return ret; coder->sequence = SEQ_MEMBER_FOOTER; + FALLTHROUGH; } - // Fall through - case SEQ_MEMBER_FOOTER: { // The footer of .lz version 0 lacks the Member size field. // This is the only difference between version 0 and diff --git a/contrib/libs/lzma/liblzma/common/memcmplen.h b/contrib/libs/lzma/liblzma/common/memcmplen.h index 86b5d6f37eb..82e90854229 100644 --- a/contrib/libs/lzma/liblzma/common/memcmplen.h +++ b/contrib/libs/lzma/liblzma/common/memcmplen.h @@ -58,8 +58,7 @@ lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2, #if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ && (((TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) \ - && (defined(__x86_64__) \ - || defined(__aarch64__))) \ + && SIZE_MAX == UINT64_MAX) \ || (defined(__INTEL_COMPILER) && defined(__x86_64__)) \ || (defined(__INTEL_COMPILER) && defined(_M_X64)) \ || (defined(_MSC_VER) && (defined(_M_X64) \ diff --git a/contrib/libs/lzma/liblzma/common/stream_decoder.c b/contrib/libs/lzma/liblzma/common/stream_decoder.c index 7f426841366..94004b74a16 100644 --- a/contrib/libs/lzma/liblzma/common/stream_decoder.c +++ b/contrib/libs/lzma/liblzma/common/stream_decoder.c @@ -154,9 +154,9 @@ stream_decode(void *coder_ptr, const lzma_allocator *allocator, if (coder->tell_any_check) return LZMA_GET_CHECK; - } - // Fall through + FALLTHROUGH; + } case SEQ_BLOCK_HEADER: { if (*in_pos >= in_size) @@ -187,10 +187,9 @@ stream_decode(void *coder_ptr, const lzma_allocator *allocator, coder->pos = 0; coder->sequence = SEQ_BLOCK_INIT; + FALLTHROUGH; } - // Fall through - case SEQ_BLOCK_INIT: { // Checking memusage and doing the initialization needs // its own sequence point because we need to be able to @@ -252,10 +251,9 @@ stream_decode(void *coder_ptr, const lzma_allocator *allocator, return ret; coder->sequence = SEQ_BLOCK_RUN; + FALLTHROUGH; } - // Fall through - case SEQ_BLOCK_RUN: { const lzma_ret ret = coder->block_decoder.code( coder->block_decoder.coder, allocator, @@ -291,10 +289,9 @@ stream_decode(void *coder_ptr, const lzma_allocator *allocator, return ret; coder->sequence = SEQ_STREAM_FOOTER; + FALLTHROUGH; } - // Fall through - case SEQ_STREAM_FOOTER: { // Copy the Stream Footer to the internal buffer. lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, @@ -331,10 +328,9 @@ stream_decode(void *coder_ptr, const lzma_allocator *allocator, return LZMA_STREAM_END; coder->sequence = SEQ_STREAM_PADDING; + FALLTHROUGH; } - // Fall through - case SEQ_STREAM_PADDING: assert(coder->concatenated); diff --git a/contrib/libs/lzma/liblzma/common/stream_decoder_mt.c b/contrib/libs/lzma/liblzma/common/stream_decoder_mt.c index 244624a4790..271f9b07c4b 100644 --- a/contrib/libs/lzma/liblzma/common/stream_decoder_mt.c +++ b/contrib/libs/lzma/liblzma/common/stream_decoder_mt.c @@ -23,15 +23,10 @@ typedef enum { THR_IDLE, /// Decoding is in progress. - /// Main thread may change this to THR_STOP or THR_EXIT. + /// Main thread may change this to THR_IDLE or THR_EXIT. /// The worker thread may change this to THR_IDLE. THR_RUN, - /// The main thread wants the thread to stop whatever it was doing - /// but not exit. Main thread may change this to THR_EXIT. - /// The worker thread may change this to THR_IDLE. - THR_STOP, - /// The main thread wants the thread to exit. THR_EXIT, @@ -346,27 +341,6 @@ worker_enable_partial_update(void *thr_ptr) } -/// Things do to at THR_STOP or when finishing a Block. -/// This is called with thr->mutex locked. -static void -worker_stop(struct worker_thread *thr) -{ - // Update memory usage counters. - thr->coder->mem_in_use -= thr->in_size; - thr->in_size = 0; // thr->in was freed above. - - thr->coder->mem_in_use -= thr->mem_filters; - thr->coder->mem_cached += thr->mem_filters; - - // Put this thread to the stack of free threads. - thr->next = thr->coder->threads_free; - thr->coder->threads_free = thr; - - mythread_cond_signal(&thr->coder->cond); - return; -} - - static MYTHREAD_RET_TYPE worker_decoder(void *thr_ptr) { @@ -397,17 +371,6 @@ next_loop_unlocked: return MYTHREAD_RET_VALUE; } - if (thr->state == THR_STOP) { - thr->state = THR_IDLE; - mythread_mutex_unlock(&thr->mutex); - - mythread_sync(thr->coder->mutex) { - worker_stop(thr); - } - - goto next_loop_lock; - } - assert(thr->state == THR_RUN); // Update progress info for get_progress(). @@ -472,8 +435,7 @@ next_loop_unlocked: } // Either we finished successfully (LZMA_STREAM_END) or an error - // occurred. Both cases are handled almost identically. The error - // case requires updating thr->coder->thread_error. + // occurred. // // The sizes are in the Block Header and the Block decoder // checks that they match, thus we know these: @@ -481,16 +443,30 @@ next_loop_unlocked: assert(ret != LZMA_STREAM_END || thr->out_pos == thr->block_options.uncompressed_size); - // Free the input buffer. Don't update in_size as we need - // it later to update thr->coder->mem_in_use. - lzma_free(thr->in, thr->allocator); - thr->in = NULL; - mythread_sync(thr->mutex) { + // Block decoder ensures this, but do a sanity check anyway + // because thr->in_filled < thr->in_size means that the main + // thread is still writing to thr->in. + if (ret == LZMA_STREAM_END && thr->in_filled != thr->in_size) { + assert(0); + ret = LZMA_PROG_ERROR; + } + if (thr->state != THR_EXIT) thr->state = THR_IDLE; } + // Free the input buffer. Don't update in_size as we need + // it later to update thr->coder->mem_in_use. + // + // This step is skipped if an error occurred because the main thread + // might still be writing to thr->in. The memory will be freed after + // threads_end() sets thr->state = THR_EXIT. + if (ret == LZMA_STREAM_END) { + lzma_free(thr->in, thr->allocator); + thr->in = NULL; + } + mythread_sync(thr->coder->mutex) { // Move our progress info to the main thread. thr->coder->progress_in += thr->in_pos; @@ -510,7 +486,20 @@ next_loop_unlocked: && thr->coder->thread_error == LZMA_OK) thr->coder->thread_error = ret; - worker_stop(thr); + // Return the worker thread to the stack of available + // threads only if no errors occurred. + if (ret == LZMA_STREAM_END) { + // Update memory usage counters. + thr->coder->mem_in_use -= thr->in_size; + thr->coder->mem_in_use -= thr->mem_filters; + thr->coder->mem_cached += thr->mem_filters; + + // Put this thread to the stack of free threads. + thr->next = thr->coder->threads_free; + thr->coder->threads_free = thr; + } + + mythread_cond_signal(&thr->coder->cond); } goto next_loop_lock; @@ -544,17 +533,22 @@ threads_end(struct lzma_stream_coder *coder, const lzma_allocator *allocator) } +/// Tell worker threads to stop without doing any cleaning up. +/// The clean up will be done when threads_exit() is called; +/// it's not possible to reuse the threads after threads_stop(). +/// +/// This is called before returning an unrecoverable error code +/// to the application. It would be waste of processor time +/// to keep the threads running in such a situation. static void threads_stop(struct lzma_stream_coder *coder) { for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + // The threads that are in the THR_RUN state will stop + // when they check the state the next time. There's no + // need to signal coder->threads[i].cond. mythread_sync(coder->threads[i].mutex) { - // The state must be changed conditionally because - // THR_IDLE -> THR_STOP is not a valid state change. - if (coder->threads[i].state != THR_IDLE) { - coder->threads[i].state = THR_STOP; - mythread_cond_signal(&coder->threads[i].cond); - } + coder->threads[i].state = THR_IDLE; } } @@ -1077,9 +1071,9 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, if (coder->tell_any_check) return LZMA_GET_CHECK; - } - // Fall through + FALLTHROUGH; + } case SEQ_BLOCK_HEADER: { const size_t in_old = *in_pos; @@ -1214,10 +1208,9 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, } coder->sequence = SEQ_BLOCK_INIT; + FALLTHROUGH; } - // Fall through - case SEQ_BLOCK_INIT: { // Check if decoding is possible at all with the current // memlimit_stop which we must never exceed. @@ -1303,10 +1296,9 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, } coder->sequence = SEQ_BLOCK_THR_INIT; + FALLTHROUGH; } - // Fall through - case SEQ_BLOCK_THR_INIT: { // We need to wait for a multiple conditions to become true // until we can initialize the Block decoder and let a worker @@ -1508,10 +1500,9 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, } coder->sequence = SEQ_BLOCK_THR_RUN; + FALLTHROUGH; } - // Fall through - case SEQ_BLOCK_THR_RUN: { if (action == LZMA_FINISH && coder->fail_fast) { // We know that we won't get more input and that @@ -1549,10 +1540,17 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, // Read output from the output queue. Just like in // SEQ_BLOCK_HEADER, we wait to fill the output buffer // only if waiting_allowed was set to true in the beginning - // of this function (see the comment there). + // of this function (see the comment there) and there is + // no input available. In SEQ_BLOCK_HEADER, there is never + // input available when read_output_and_wait() is called, + // but here there can be when LZMA_FINISH is used, thus we + // need to check if *in_pos == in_size. Otherwise we would + // wait here instead of using the available input to start + // a new thread. return_if_error(read_output_and_wait(coder, allocator, out, out_pos, out_size, - NULL, waiting_allowed, + NULL, + waiting_allowed && *in_pos == in_size, &wait_abs, &has_blocked)); if (coder->pending_error != LZMA_OK) { @@ -1561,6 +1559,10 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, } // Return if the input didn't contain the whole Block. + // + // NOTE: When we updated coder->thr->in_filled a few lines + // above, the worker thread might by now have finished its + // work and returned itself back to the stack of free threads. if (coder->thr->in_filled < coder->thr->in_size) { assert(*in_pos == in_size); return LZMA_OK; @@ -1613,10 +1615,9 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, coder->mem_direct_mode = coder->mem_next_filters; coder->sequence = SEQ_BLOCK_DIRECT_RUN; + FALLTHROUGH; } - // Fall through - case SEQ_BLOCK_DIRECT_RUN: { const size_t in_old = *in_pos; const size_t out_old = *out_pos; @@ -1652,8 +1653,7 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, return LZMA_OK; coder->sequence = SEQ_INDEX_DECODE; - - // Fall through + FALLTHROUGH; case SEQ_INDEX_DECODE: { // If we don't have any input, don't call @@ -1672,10 +1672,9 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, return ret; coder->sequence = SEQ_STREAM_FOOTER; + FALLTHROUGH; } - // Fall through - case SEQ_STREAM_FOOTER: { // Copy the Stream Footer to the internal buffer. const size_t in_old = *in_pos; @@ -1714,10 +1713,9 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, return LZMA_STREAM_END; coder->sequence = SEQ_STREAM_PADDING; + FALLTHROUGH; } - // Fall through - case SEQ_STREAM_PADDING: assert(coder->concatenated); @@ -1948,7 +1946,7 @@ stream_decoder_mt_init(lzma_next_coder *next, const lzma_allocator *allocator, // accounting from scratch, too. Changes in filter and block sizes may // affect number of threads. // - // FIXME? Reusing should be easy but unlike the single-threaded + // Reusing threads doesn't seem worth it. Unlike the single-threaded // decoder, with some types of input file combinations reusing // could leave quite a lot of memory allocated but unused (first // file could allocate a lot, the next files could use fewer diff --git a/contrib/libs/lzma/liblzma/common/stream_encoder_mt.c b/contrib/libs/lzma/liblzma/common/stream_encoder_mt.c index f0fef152331..fd0eb98df68 100644 --- a/contrib/libs/lzma/liblzma/common/stream_encoder_mt.c +++ b/contrib/libs/lzma/liblzma/common/stream_encoder_mt.c @@ -731,8 +731,7 @@ stream_encode_mt(void *coder_ptr, const lzma_allocator *allocator, coder->header_pos = 0; coder->sequence = SEQ_BLOCK; - - // Fall through + FALLTHROUGH; case SEQ_BLOCK: { // Initialized to silence warnings. @@ -851,9 +850,9 @@ stream_encode_mt(void *coder_ptr, const lzma_allocator *allocator, // to be ready to be copied out. coder->progress_out += lzma_index_size(coder->index) + LZMA_STREAM_HEADER_SIZE; - } - // Fall through + FALLTHROUGH; + } case SEQ_INDEX: { // Call the Index encoder. It doesn't take any input, so @@ -873,10 +872,9 @@ stream_encode_mt(void *coder_ptr, const lzma_allocator *allocator, return LZMA_PROG_ERROR; coder->sequence = SEQ_STREAM_FOOTER; + FALLTHROUGH; } - // Fall through - case SEQ_STREAM_FOOTER: lzma_bufcpy(coder->header, &coder->header_pos, sizeof(coder->header), diff --git a/contrib/libs/lzma/liblzma/common/string_conversion.c b/contrib/libs/lzma/liblzma/common/string_conversion.c index 3a08486a1f6..015acf22585 100644 --- a/contrib/libs/lzma/liblzma/common/string_conversion.c +++ b/contrib/libs/lzma/liblzma/common/string_conversion.c @@ -12,6 +12,11 @@ #include "filter_common.h" +// liblzma itself doesn't use gettext to translate messages. +// Mark the strings still so that xz can translate them. +#define N_(msgid) msgid + + ///////////////////// // String building // ///////////////////// @@ -319,7 +324,7 @@ parse_lzma12_preset(const char **const str, const char *str_end, assert(*str < str_end); if (!(**str >= '0' && **str <= '9')) - return "Unsupported preset"; + return N_("Unsupported preset"); *preset = (uint32_t)(**str - '0'); @@ -331,7 +336,7 @@ parse_lzma12_preset(const char **const str, const char *str_end, break; default: - return "Unsupported preset flag"; + return N_("Unsupported flag in the preset"); } } @@ -350,7 +355,7 @@ set_lzma12_preset(const char **const str, const char *str_end, lzma_options_lzma *opts = filter_options; if (lzma_lzma_preset(opts, preset)) - return "Unsupported preset"; + return N_("Unsupported preset"); return NULL; } @@ -442,7 +447,7 @@ parse_lzma12(const char **const str, const char *str_end, void *filter_options) return errmsg; if (opts->lc + opts->lp > LZMA_LCLP_MAX) - return "The sum of lc and lp must not exceed 4"; + return N_("The sum of lc and lp must not exceed 4"); return NULL; } @@ -578,21 +583,21 @@ parse_options(const char **const str, const char *str_end, // Fail if the '=' wasn't found or the option name is missing // (the first char is '='). if (equals_sign == NULL || **str == '=') - return "Options must be 'name=value' pairs separated " - "with commas"; + return N_("Options must be 'name=value' pairs " + "separated with commas"); // Reject a too long option name so that the memcmp() // in the loop below won't read past the end of the // string in optmap[i].name. const size_t name_len = (size_t)(equals_sign - *str); if (name_len > NAME_LEN_MAX) - return "Unknown option name"; + return N_("Unknown option name"); // Find the option name from optmap[]. size_t i = 0; while (true) { if (i == optmap_size) - return "Unknown option name"; + return N_("Unknown option name"); if (memcmp(*str, optmap[i].name, name_len) == 0 && optmap[i].name[name_len] == '\0') @@ -609,7 +614,7 @@ parse_options(const char **const str, const char *str_end, // string so check it here. const size_t value_len = (size_t)(name_eq_value_end - *str); if (value_len == 0) - return "Option value cannot be empty"; + return N_("Option value cannot be empty"); // LZMA1/2 preset has its own parsing function. if (optmap[i].type == OPTMAP_TYPE_LZMA_PRESET) { @@ -630,14 +635,14 @@ parse_options(const char **const str, const char *str_end, // in the loop below won't read past the end of the // string in optmap[i].u.map[j].name. if (value_len > NAME_LEN_MAX) - return "Invalid option value"; + return N_("Invalid option value"); const name_value_map *map = optmap[i].u.map; size_t j = 0; while (true) { // The array is terminated with an empty name. if (map[j].name[0] == '\0') - return "Invalid option value"; + return N_("Invalid option value"); if (memcmp(*str, map[j].name, value_len) == 0 && map[j].name[value_len] @@ -651,7 +656,8 @@ parse_options(const char **const str, const char *str_end, } else if (**str < '0' || **str > '9') { // Note that "max" isn't supported while it is // supported in xz. It's not useful here. - return "Value is not a non-negative decimal integer"; + return N_("Value is not a non-negative " + "decimal integer"); } else { // strtoul() has locale-specific behavior so it cannot // be relied on to get reproducible results since we @@ -665,13 +671,13 @@ parse_options(const char **const str, const char *str_end, v = 0; do { if (v > UINT32_MAX / 10) - return "Value out of range"; + return N_("Value out of range"); v *= 10; const uint32_t add = (uint32_t)(*p - '0'); if (UINT32_MAX - add < v) - return "Value out of range"; + return N_("Value out of range"); v += add; ++p; @@ -696,8 +702,9 @@ parse_options(const char **const str, const char *str_end, if ((optmap[i].flags & OPTMAP_USE_BYTE_SUFFIX) == 0) { *str = multiplier_start; - return "This option does not support " - "any integer suffixes"; + return N_("This option does not " + "support any multiplier " + "suffixes"); } uint32_t shift; @@ -720,8 +727,13 @@ parse_options(const char **const str, const char *str_end, default: *str = multiplier_start; - return "Invalid multiplier suffix " - "(KiB, MiB, or GiB)"; + + // TRANSLATORS: Don't translate the + // suffixes "KiB", "MiB", or "GiB" + // because a user can only specify + // untranslated suffixes. + return N_("Invalid multiplier suffix " + "(KiB, MiB, or GiB)"); } ++p; @@ -740,19 +752,19 @@ parse_options(const char **const str, const char *str_end, // Now we must have no chars remaining. if (p < name_eq_value_end) { *str = multiplier_start; - return "Invalid multiplier suffix " - "(KiB, MiB, or GiB)"; + return N_("Invalid multiplier suffix " + "(KiB, MiB, or GiB)"); } if (v > (UINT32_MAX >> shift)) - return "Value out of range"; + return N_("Value out of range"); v <<= shift; } if (v < optmap[i].u.range.min || v > optmap[i].u.range.max) - return "Value out of range"; + return N_("Value out of range"); } // Set the value in filter_options. Enums are handled @@ -814,15 +826,15 @@ parse_filter(const char **const str, const char *str_end, lzma_filter *filter, // string in filter_name_map[i].name. const size_t name_len = (size_t)(name_end - *str); if (name_len > NAME_LEN_MAX) - return "Unknown filter name"; + return N_("Unknown filter name"); for (size_t i = 0; i < ARRAY_SIZE(filter_name_map); ++i) { if (memcmp(*str, filter_name_map[i].name, name_len) == 0 && filter_name_map[i].name[name_len] == '\0') { if (only_xz && filter_name_map[i].id >= LZMA_FILTER_RESERVED_START) - return "This filter cannot be used in " - "the .xz format"; + return N_("This filter cannot be used in " + "the .xz format"); // Allocate the filter-specific options and // initialize the memory with zeros. @@ -830,7 +842,7 @@ parse_filter(const char **const str, const char *str_end, lzma_filter *filter, filter_name_map[i].opts_size, allocator); if (options == NULL) - return "Memory allocation failed"; + return N_("Memory allocation failed"); // Filter name was found so the input string is good // at least this far. @@ -850,7 +862,7 @@ parse_filter(const char **const str, const char *str_end, lzma_filter *filter, } } - return "Unknown filter name"; + return N_("Unknown filter name"); } @@ -869,8 +881,8 @@ str_to_filters(const char **const str, lzma_filter *filters, uint32_t flags, ++*str; if (**str == '\0') - return "Empty string is not allowed, " - "try \"6\" if a default value is needed"; + return N_("Empty string is not allowed, " + "try '6' if a default value is needed"); // Detect the type of the string. // @@ -893,7 +905,7 @@ str_to_filters(const char **const str, lzma_filter *filters, uint32_t flags, // there are no chars other than spaces. for (size_t i = 1; str_end[i] != '\0'; ++i) if (str_end[i] != ' ') - return "Unsupported preset"; + return N_("Unsupported preset"); } else { // There are no trailing spaces. Use the whole string. str_end = *str + str_len; @@ -906,11 +918,11 @@ str_to_filters(const char **const str, lzma_filter *filters, uint32_t flags, lzma_options_lzma *opts = lzma_alloc(sizeof(*opts), allocator); if (opts == NULL) - return "Memory allocation failed"; + return N_("Memory allocation failed"); if (lzma_lzma_preset(opts, preset)) { lzma_free(opts, allocator); - return "Unsupported preset"; + return N_("Unsupported preset"); } filters[0].id = LZMA_FILTER_LZMA2; @@ -934,7 +946,7 @@ str_to_filters(const char **const str, lzma_filter *filters, uint32_t flags, size_t i = 0; do { if (i == LZMA_FILTERS_MAX) { - errmsg = "The maximum number of filters is four"; + errmsg = N_("The maximum number of filters is four"); goto error; } @@ -956,7 +968,7 @@ str_to_filters(const char **const str, lzma_filter *filters, uint32_t flags, // Inputs that have "--" at the end or "-- " in the middle // will result in an empty filter name. if (filter_end == *str) { - errmsg = "Filter name is missing"; + errmsg = N_("Filter name is missing"); goto error; } @@ -983,8 +995,8 @@ str_to_filters(const char **const str, lzma_filter *filters, uint32_t flags, const lzma_ret ret = lzma_validate_chain(temp_filters, &dummy); assert(ret == LZMA_OK || ret == LZMA_OPTIONS_ERROR); if (ret != LZMA_OK) { - errmsg = "Invalid filter chain " - "('lzma2' missing at the end?)"; + errmsg = N_("Invalid filter chain " + "('lzma2' missing at the end?)"); goto error; } } @@ -1012,17 +1024,26 @@ lzma_str_to_filters(const char *str, int *error_pos, lzma_filter *filters, if (error_pos != NULL) *error_pos = 0; - if (str == NULL || filters == NULL) + if (str == NULL || filters == NULL) { + // Don't translate this because it's only shown in case of + // a programming error. return "Unexpected NULL pointer argument(s) " "to lzma_str_to_filters()"; + } // Validate the flags. const uint32_t supported_flags = LZMA_STR_ALL_FILTERS | LZMA_STR_NO_VALIDATION; - if (flags & ~supported_flags) + if (flags & ~supported_flags) { + // This message is possible only if the caller uses flags + // that are only supported in a newer liblzma version (or + // the flags are simply buggy). Don't translate this at least + // when liblzma itself doesn't use gettext; xz and liblzma + // are usually upgraded at the same time. return "Unsupported flags to lzma_str_to_filters()"; + } const char *used = str; const char *errmsg = str_to_filters(&used, filters, flags, allocator); diff --git a/contrib/libs/lzma/liblzma/liblzma_linux.map b/contrib/libs/lzma/liblzma/liblzma_linux.map index 7e4b25e1762..50f1571de21 100644 --- a/contrib/libs/lzma/liblzma/liblzma_linux.map +++ b/contrib/libs/lzma/liblzma/liblzma_linux.map @@ -141,3 +141,13 @@ XZ_5.6.0 { global: lzma_mt_block_size; } XZ_5.4; + +XZ_5.8 { +global: + lzma_bcj_arm64_encode; + lzma_bcj_arm64_decode; + lzma_bcj_riscv_encode; + lzma_bcj_riscv_decode; + lzma_bcj_x86_encode; + lzma_bcj_x86_decode; +} XZ_5.6.0; diff --git a/contrib/libs/lzma/liblzma/lz/lz_decoder.c b/contrib/libs/lzma/liblzma/lz/lz_decoder.c index 92913f225a0..1cb120ab3b0 100644 --- a/contrib/libs/lzma/liblzma/lz/lz_decoder.c +++ b/contrib/libs/lzma/liblzma/lz/lz_decoder.c @@ -53,9 +53,9 @@ typedef struct { static void lz_decoder_reset(lzma_coder *coder) { - coder->dict.pos = 2 * LZ_DICT_REPEAT_MAX; + coder->dict.pos = LZ_DICT_INIT_POS; coder->dict.full = 0; - coder->dict.buf[2 * LZ_DICT_REPEAT_MAX - 1] = '\0'; + coder->dict.buf[LZ_DICT_INIT_POS - 1] = '\0'; coder->dict.has_wrapped = false; coder->dict.need_reset = false; return; @@ -261,10 +261,12 @@ lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, // recommended to give aligned buffers to liblzma. // // Reserve 2 * LZ_DICT_REPEAT_MAX bytes of extra space which is - // needed for alloc_size. + // needed for alloc_size. Reserve also LZ_DICT_EXTRA bytes of extra + // space which is *not* counted in alloc_size or coder->dict.size. // // Avoid integer overflow. - if (lz_options.dict_size > SIZE_MAX - 15 - 2 * LZ_DICT_REPEAT_MAX) + if (lz_options.dict_size > SIZE_MAX - 15 - 2 * LZ_DICT_REPEAT_MAX + - LZ_DICT_EXTRA) return LZMA_MEM_ERROR; lz_options.dict_size = (lz_options.dict_size + 15) & ~((size_t)(15)); @@ -277,7 +279,13 @@ lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, // Allocate and initialize the dictionary. if (coder->dict.size != alloc_size) { lzma_free(coder->dict.buf, allocator); - coder->dict.buf = lzma_alloc(alloc_size, allocator); + + // The LZ_DICT_EXTRA bytes at the end of the buffer aren't + // included in alloc_size. These extra bytes allow + // dict_repeat() to read and write more data than requested. + // Otherwise this extra space is ignored. + coder->dict.buf = lzma_alloc(alloc_size + LZ_DICT_EXTRA, + allocator); if (coder->dict.buf == NULL) return LZMA_MEM_ERROR; @@ -320,5 +328,6 @@ lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size) { - return sizeof(lzma_coder) + (uint64_t)(dictionary_size); + return sizeof(lzma_coder) + (uint64_t)(dictionary_size) + + 2 * LZ_DICT_REPEAT_MAX + LZ_DICT_EXTRA; } diff --git a/contrib/libs/lzma/liblzma/lz/lz_decoder.h b/contrib/libs/lzma/liblzma/lz/lz_decoder.h index cb61b6e24c7..2698e0167fc 100644 --- a/contrib/libs/lzma/liblzma/lz/lz_decoder.h +++ b/contrib/libs/lzma/liblzma/lz/lz_decoder.h @@ -15,10 +15,40 @@ #include "common.h" +#ifdef HAVE_IMMINTRIN_H +# include <immintrin.h> +#endif + + +// dict_repeat() implementation variant: +// 0 = Byte-by-byte copying only. +// 1 = Use memcpy() for non-overlapping copies. +// 2 = Use x86 SSE2 for non-overlapping copies. +#ifndef LZMA_LZ_DECODER_CONFIG +# if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && defined(HAVE_IMMINTRIN_H) \ + && (defined(__SSE2__) || defined(_M_X64) \ + || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)) +# define LZMA_LZ_DECODER_CONFIG 2 +# else +# define LZMA_LZ_DECODER_CONFIG 1 +# endif +#endif -/// Maximum length of a match rounded up to a nice power of 2 which is -/// a good size for aligned memcpy(). The allocated dictionary buffer will -/// be 2 * LZ_DICT_REPEAT_MAX bytes larger than the actual dictionary size: +/// Byte-by-byte and memcpy() copy exactly the amount needed. Other methods +/// can copy up to LZ_DICT_EXTRA bytes more than requested, and this amount +/// of extra space is needed at the end of the allocated dictionary buffer. +/// +/// NOTE: If this is increased, update LZMA_DICT_REPEAT_MAX too. +#if LZMA_LZ_DECODER_CONFIG >= 2 +# define LZ_DICT_EXTRA 32 +#else +# define LZ_DICT_EXTRA 0 +#endif + +/// Maximum number of bytes that dict_repeat() may copy. The allocated +/// dictionary buffer will be 2 * LZ_DICT_REPEAT_MAX + LZMA_DICT_EXTRA bytes +/// larger than the actual dictionary size: /// /// (1) Every time the decoder reaches the end of the dictionary buffer, /// the last LZ_DICT_REPEAT_MAX bytes will be copied to the beginning. @@ -27,14 +57,26 @@ /// /// (2) The other LZ_DICT_REPEAT_MAX bytes is kept as a buffer between /// the oldest byte still in the dictionary and the current write -/// position. This way dict_repeat(dict, dict->size - 1, &len) +/// position. This way dict_repeat() with the maximum valid distance /// won't need memmove() as the copying cannot overlap. /// +/// (3) LZ_DICT_EXTRA bytes are required at the end of the dictionary buffer +/// so that extra copying done by dict_repeat() won't write or read past +/// the end of the allocated buffer. This amount is *not* counted as part +/// of lzma_dict.size. +/// /// Note that memcpy() still cannot be used if distance < len. /// -/// LZMA's longest match length is 273 so pick a multiple of 16 above that. +/// LZMA's longest match length is 273 bytes. The LZMA decoder looks at +/// the lowest four bits of the dictionary position, thus 273 must be +/// rounded up to the next multiple of 16 (288). In addition, optimized +/// dict_repeat() copies 32 bytes at a time, thus this must also be +/// a multiple of 32. #define LZ_DICT_REPEAT_MAX 288 +/// Initial position in lzma_dict.buf when the dictionary is empty. +#define LZ_DICT_INIT_POS (2 * LZ_DICT_REPEAT_MAX) + typedef struct { /// Pointer to the dictionary buffer. @@ -158,7 +200,8 @@ dict_is_distance_valid(const lzma_dict *const dict, const size_t distance) /// Repeat *len bytes at distance. static inline bool -dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) +dict_repeat(lzma_dict *restrict dict, + uint32_t distance, uint32_t *restrict len) { // Don't write past the end of the dictionary. const size_t dict_avail = dict->limit - dict->pos; @@ -169,9 +212,17 @@ dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) if (distance >= dict->pos) back += dict->size - LZ_DICT_REPEAT_MAX; - // Repeat a block of data from the history. Because memcpy() is faster - // than copying byte by byte in a loop, the copying process gets split - // into two cases. +#if LZMA_LZ_DECODER_CONFIG == 0 + // Minimal byte-by-byte method. This might be the least bad choice + // if memcpy() isn't fast and there's no replacement for it below. + while (left-- > 0) { + dict->buf[dict->pos++] = dict->buf[back++]; + } + +#else + // Because memcpy() or a similar method can be faster than copying + // byte by byte in a loop, the copying process is split into + // two cases. if (distance < left) { // Source and target areas overlap, thus we can't use // memcpy() nor even memmove() safely. @@ -179,32 +230,56 @@ dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) dict->buf[dict->pos++] = dict->buf[back++]; } while (--left > 0); } else { +# if LZMA_LZ_DECODER_CONFIG == 1 memcpy(dict->buf + dict->pos, dict->buf + back, left); dict->pos += left; + +# elif LZMA_LZ_DECODER_CONFIG == 2 + // This can copy up to 32 bytes more than required. + // (If left == 0, we still copy 32 bytes.) + size_t pos = dict->pos; + dict->pos += left; + do { + const __m128i x0 = _mm_loadu_si128( + (__m128i *)(dict->buf + back)); + const __m128i x1 = _mm_loadu_si128( + (__m128i *)(dict->buf + back + 16)); + back += 32; + _mm_storeu_si128( + (__m128i *)(dict->buf + pos), x0); + _mm_storeu_si128( + (__m128i *)(dict->buf + pos + 16), x1); + pos += 32; + } while (pos < dict->pos); + +# else +# error "Invalid LZMA_LZ_DECODER_CONFIG value" +# endif } +#endif // Update how full the dictionary is. if (!dict->has_wrapped) - dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; + dict->full = dict->pos - LZ_DICT_INIT_POS; return *len != 0; } static inline void -dict_put(lzma_dict *dict, uint8_t byte) +dict_put(lzma_dict *restrict dict, uint8_t byte) { dict->buf[dict->pos++] = byte; if (!dict->has_wrapped) - dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; + dict->full = dict->pos - LZ_DICT_INIT_POS; } /// Puts one byte into the dictionary. Returns true if the dictionary was /// already full and the byte couldn't be added. static inline bool -dict_put_safe(lzma_dict *dict, uint8_t byte) +dict_put_safe(lzma_dict *restrict dict, uint8_t byte) { if (unlikely(dict->pos == dict->limit)) return true; @@ -234,7 +309,7 @@ dict_write(lzma_dict *restrict dict, const uint8_t *restrict in, dict->buf, &dict->pos, dict->limit); if (!dict->has_wrapped) - dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; + dict->full = dict->pos - LZ_DICT_INIT_POS; return; } diff --git a/contrib/libs/lzma/liblzma/lz/lz_encoder.c b/contrib/libs/lzma/liblzma/lz/lz_encoder.c index 9e34ab4cd7f..d2c179d9ec6 100644 --- a/contrib/libs/lzma/liblzma/lz/lz_encoder.c +++ b/contrib/libs/lzma/liblzma/lz/lz_encoder.c @@ -15,7 +15,7 @@ // See lz_encoder_hash.h. This is a bit hackish but avoids making // endianness a conditional in makefiles. -#if defined(WORDS_BIGENDIAN) && !defined(HAVE_SMALL) +#ifdef LZMA_LZ_HASH_TABLE_IS_NEEDED # error #include "lz_encoder_hash_table.h" #endif diff --git a/contrib/libs/lzma/liblzma/lz/lz_encoder_hash.h b/contrib/libs/lzma/liblzma/lz/lz_encoder_hash.h index 8ace82b04c5..6d4bf837fd1 100644 --- a/contrib/libs/lzma/liblzma/lz/lz_encoder_hash.h +++ b/contrib/libs/lzma/liblzma/lz/lz_encoder_hash.h @@ -5,23 +5,37 @@ /// \file lz_encoder_hash.h /// \brief Hash macros for match finders // -// Author: Igor Pavlov +// Authors: Igor Pavlov +// Lasse Collin // /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_ENCODER_HASH_H #define LZMA_LZ_ENCODER_HASH_H -#if defined(WORDS_BIGENDIAN) && !defined(HAVE_SMALL) - // This is to make liblzma produce the same output on big endian - // systems that it does on little endian systems. lz_encoder.c - // takes care of including the actual table. +// We need to know if CRC32_GENERIC is defined and we may need the declaration +// of lzma_crc32_table[][]. +#include "crc_common.h" + +// If HAVE_SMALL is defined, then lzma_crc32_table[][] exists and +// it's little endian even on big endian systems. +// +// If HAVE_SMALL isn't defined, lzma_crc32_table[][] is in native endian +// but we want a little endian one so that the compressed output won't +// depend on the processor endianness. Big endian systems are less common +// so those get the burden of an extra 1 KiB table. +// +// If HAVE_SMALL isn't defined and CRC32_GENERIC isn't defined either, +// then lzma_crc32_table[][] doesn't exist. +#if defined(HAVE_SMALL) \ + || (defined(CRC32_GENERIC) && !defined(WORDS_BIGENDIAN)) +# define hash_table lzma_crc32_table[0] +#else + // lz_encoder.c takes care of including the actual table. lzma_attr_visibility_hidden extern const uint32_t lzma_lz_hash_table[256]; # define hash_table lzma_lz_hash_table -#else -# include "check.h" -# define hash_table lzma_crc32_table[0] +# define LZMA_LZ_HASH_TABLE_IS_NEEDED 1 #endif #define HASH_2_SIZE (UINT32_C(1) << 10) diff --git a/contrib/libs/lzma/liblzma/lzma/lzma2_encoder.c b/contrib/libs/lzma/liblzma/lzma/lzma2_encoder.c index e20b75b3003..71cfd9b4114 100644 --- a/contrib/libs/lzma/liblzma/lzma/lzma2_encoder.c +++ b/contrib/libs/lzma/liblzma/lzma/lzma2_encoder.c @@ -159,8 +159,7 @@ lzma2_encode(void *coder_ptr, lzma_mf *restrict mf, coder->uncompressed_size = 0; coder->compressed_size = 0; coder->sequence = SEQ_LZMA_ENCODE; - - // Fall through + FALLTHROUGH; case SEQ_LZMA_ENCODE: { // Calculate how much more uncompressed data this chunk @@ -219,10 +218,9 @@ lzma2_encode(void *coder_ptr, lzma_mf *restrict mf, lzma2_header_lzma(coder); coder->sequence = SEQ_LZMA_COPY; + FALLTHROUGH; } - // Fall through - case SEQ_LZMA_COPY: // Copy the compressed chunk along its headers to the // output buffer. @@ -244,8 +242,7 @@ lzma2_encode(void *coder_ptr, lzma_mf *restrict mf, return LZMA_OK; coder->sequence = SEQ_UNCOMPRESSED_COPY; - - // Fall through + FALLTHROUGH; case SEQ_UNCOMPRESSED_COPY: // Copy the uncompressed data as is from the dictionary diff --git a/contrib/libs/lzma/liblzma/simple/arm.c b/contrib/libs/lzma/liblzma/simple/arm.c index 58acb2d11ad..f9d9c08b3c4 100644 --- a/contrib/libs/lzma/liblzma/simple/arm.c +++ b/contrib/libs/lzma/liblzma/simple/arm.c @@ -18,8 +18,10 @@ arm_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { + size &= ~(size_t)3; + size_t i; - for (i = 0; i + 4 <= size; i += 4) { + for (i = 0; i < size; i += 4) { if (buffer[i + 3] == 0xEB) { uint32_t src = ((uint32_t)(buffer[i + 2]) << 16) | ((uint32_t)(buffer[i + 1]) << 8) diff --git a/contrib/libs/lzma/liblzma/simple/arm64.c b/contrib/libs/lzma/liblzma/simple/arm64.c index 16c2f565f73..2ec10d937fb 100644 --- a/contrib/libs/lzma/liblzma/simple/arm64.c +++ b/contrib/libs/lzma/liblzma/simple/arm64.c @@ -28,6 +28,8 @@ arm64_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { + size &= ~(size_t)3; + size_t i; // Clang 14.0.6 on x86-64 makes this four times bigger and 40 % slower @@ -37,7 +39,7 @@ arm64_code(void *simple lzma_attribute((__unused__)), #ifdef __clang__ # pragma clang loop vectorize(disable) #endif - for (i = 0; i + 4 <= size; i += 4) { + for (i = 0; i < size; i += 4) { uint32_t pc = (uint32_t)(now_pos + i); uint32_t instr = read32le(buffer + i); @@ -122,6 +124,15 @@ lzma_simple_arm64_encoder_init(lzma_next_coder *next, { return arm64_coder_init(next, allocator, filters, true); } + + +extern LZMA_API(size_t) +lzma_bcj_arm64_encode(uint32_t start_offset, uint8_t *buf, size_t size) +{ + // start_offset must be a multiple of four. + start_offset &= ~UINT32_C(3); + return arm64_code(NULL, start_offset, true, buf, size); +} #endif @@ -133,4 +144,13 @@ lzma_simple_arm64_decoder_init(lzma_next_coder *next, { return arm64_coder_init(next, allocator, filters, false); } + + +extern LZMA_API(size_t) +lzma_bcj_arm64_decode(uint32_t start_offset, uint8_t *buf, size_t size) +{ + // start_offset must be a multiple of four. + start_offset &= ~UINT32_C(3); + return arm64_code(NULL, start_offset, false, buf, size); +} #endif diff --git a/contrib/libs/lzma/liblzma/simple/armthumb.c b/contrib/libs/lzma/liblzma/simple/armthumb.c index f1eeca9b80f..368b51c7fea 100644 --- a/contrib/libs/lzma/liblzma/simple/armthumb.c +++ b/contrib/libs/lzma/liblzma/simple/armthumb.c @@ -18,8 +18,13 @@ armthumb_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { + if (size < 4) + return 0; + + size -= 4; + size_t i; - for (i = 0; i + 4 <= size; i += 2) { + for (i = 0; i <= size; i += 2) { if ((buffer[i + 1] & 0xF8) == 0xF0 && (buffer[i + 3] & 0xF8) == 0xF8) { uint32_t src = (((uint32_t)(buffer[i + 1]) & 7) << 19) diff --git a/contrib/libs/lzma/liblzma/simple/ia64.c b/contrib/libs/lzma/liblzma/simple/ia64.c index 50250140997..2a4aaebb472 100644 --- a/contrib/libs/lzma/liblzma/simple/ia64.c +++ b/contrib/libs/lzma/liblzma/simple/ia64.c @@ -25,8 +25,10 @@ ia64_code(void *simple lzma_attribute((__unused__)), 4, 4, 0, 0, 4, 4, 0, 0 }; + size &= ~(size_t)15; + size_t i; - for (i = 0; i + 16 <= size; i += 16) { + for (i = 0; i < size; i += 16) { const uint32_t instr_template = buffer[i] & 0x1F; const uint32_t mask = BRANCH_TABLE[instr_template]; uint32_t bit_pos = 5; diff --git a/contrib/libs/lzma/liblzma/simple/powerpc.c b/contrib/libs/lzma/liblzma/simple/powerpc.c index ba6cfbef3ab..ea47d14d4c3 100644 --- a/contrib/libs/lzma/liblzma/simple/powerpc.c +++ b/contrib/libs/lzma/liblzma/simple/powerpc.c @@ -18,8 +18,10 @@ powerpc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { + size &= ~(size_t)3; + size_t i; - for (i = 0; i + 4 <= size; i += 4) { + for (i = 0; i < size; i += 4) { // PowerPC branch 6(48) 24(Offset) 1(Abs) 1(Link) if ((buffer[i] >> 2) == 0x12 && ((buffer[i + 3] & 3) == 1)) { diff --git a/contrib/libs/lzma/liblzma/simple/riscv.c b/contrib/libs/lzma/liblzma/simple/riscv.c index b18df8b637d..bc97ebdbb0f 100644 --- a/contrib/libs/lzma/liblzma/simple/riscv.c +++ b/contrib/libs/lzma/liblzma/simple/riscv.c @@ -617,6 +617,15 @@ lzma_simple_riscv_encoder_init(lzma_next_coder *next, return lzma_simple_coder_init(next, allocator, filters, &riscv_encode, 0, 8, 2, true); } + + +extern LZMA_API(size_t) +lzma_bcj_riscv_encode(uint32_t start_offset, uint8_t *buf, size_t size) +{ + // start_offset must be a multiple of two. + start_offset &= ~UINT32_C(1); + return riscv_encode(NULL, start_offset, true, buf, size); +} #endif @@ -752,4 +761,13 @@ lzma_simple_riscv_decoder_init(lzma_next_coder *next, return lzma_simple_coder_init(next, allocator, filters, &riscv_decode, 0, 8, 2, false); } + + +extern LZMA_API(size_t) +lzma_bcj_riscv_decode(uint32_t start_offset, uint8_t *buf, size_t size) +{ + // start_offset must be a multiple of two. + start_offset &= ~UINT32_C(1); + return riscv_decode(NULL, start_offset, false, buf, size); +} #endif diff --git a/contrib/libs/lzma/liblzma/simple/sparc.c b/contrib/libs/lzma/liblzma/simple/sparc.c index e8ad285a192..1fa4850458e 100644 --- a/contrib/libs/lzma/liblzma/simple/sparc.c +++ b/contrib/libs/lzma/liblzma/simple/sparc.c @@ -18,9 +18,10 @@ sparc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { - size_t i; - for (i = 0; i + 4 <= size; i += 4) { + size &= ~(size_t)3; + size_t i; + for (i = 0; i < size; i += 4) { if ((buffer[i] == 0x40 && (buffer[i + 1] & 0xC0) == 0x00) || (buffer[i] == 0x7F && (buffer[i + 1] & 0xC0) == 0xC0)) { diff --git a/contrib/libs/lzma/liblzma/simple/x86.c b/contrib/libs/lzma/liblzma/simple/x86.c index f216231f2d1..dffa7863131 100644 --- a/contrib/libs/lzma/liblzma/simple/x86.c +++ b/contrib/libs/lzma/liblzma/simple/x86.c @@ -143,6 +143,18 @@ lzma_simple_x86_encoder_init(lzma_next_coder *next, { return x86_coder_init(next, allocator, filters, true); } + + +extern LZMA_API(size_t) +lzma_bcj_x86_encode(uint32_t start_offset, uint8_t *buf, size_t size) +{ + lzma_simple_x86 simple = { + .prev_mask = 0, + .prev_pos = (uint32_t)(-5), + }; + + return x86_code(&simple, start_offset, true, buf, size); +} #endif @@ -154,4 +166,16 @@ lzma_simple_x86_decoder_init(lzma_next_coder *next, { return x86_coder_init(next, allocator, filters, false); } + + +extern LZMA_API(size_t) +lzma_bcj_x86_decode(uint32_t start_offset, uint8_t *buf, size_t size) +{ + lzma_simple_x86 simple = { + .prev_mask = 0, + .prev_pos = (uint32_t)(-5), + }; + + return x86_code(&simple, start_offset, false, buf, size); +} #endif diff --git a/contrib/libs/lzma/ya.make b/contrib/libs/lzma/ya.make index 60d612bae15..b49400efaa5 100644 --- a/contrib/libs/lzma/ya.make +++ b/contrib/libs/lzma/ya.make @@ -1,19 +1,18 @@ -# Generated by devtools/yamaker from nixpkgs 22.11. +# Generated by devtools/yamaker from nixpkgs 24.05. LIBRARY() LICENSE( 0BSD AND - BSD-2-Clause AND BSD-3-Clause AND Public-Domain ) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(5.6.4) +VERSION(5.8.1) -ORIGINAL_SOURCE(https://github.com/tukaani-project/xz/archive/v5.6.4.tar.gz) +ORIGINAL_SOURCE(https://github.com/tukaani-project/xz/archive/v5.8.1.tar.gz) ADDINCL( GLOBAL contrib/libs/lzma/liblzma/api @@ -43,9 +42,7 @@ SRCS( common/tuklib_physmem.c liblzma/check/check.c liblzma/check/crc32_fast.c - liblzma/check/crc32_table.c liblzma/check/crc64_fast.c - liblzma/check/crc64_table.c liblzma/check/sha256.c liblzma/common/alone_decoder.c liblzma/common/alone_encoder.c diff --git a/contrib/python/multidict/.dist-info/METADATA b/contrib/python/multidict/.dist-info/METADATA index 3cd254428e2..9ee0987b9db 100644 --- a/contrib/python/multidict/.dist-info/METADATA +++ b/contrib/python/multidict/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: multidict -Version: 6.3.0 +Version: 6.4.3 Summary: multidict implementation Home-page: https://github.com/aio-libs/multidict Author: Andrew Svetlov @@ -137,7 +137,12 @@ e.g.: Please note, the pure Python (uncompiled) version is about 20-50 times slower depending on the usage scenario!!! +For extension development, set the ``MULTIDICT_DEBUG_BUILD`` environment variable to compile +the extensions in debug mode: +.. code-block:: console + + $ MULTIDICT_DEBUG_BUILD=1 pip install multidict Changelog --------- diff --git a/contrib/python/multidict/README.rst b/contrib/python/multidict/README.rst index fbab818a979..50a7f041b59 100644 --- a/contrib/python/multidict/README.rst +++ b/contrib/python/multidict/README.rst @@ -104,7 +104,12 @@ e.g.: Please note, the pure Python (uncompiled) version is about 20-50 times slower depending on the usage scenario!!! +For extension development, set the ``MULTIDICT_DEBUG_BUILD`` environment variable to compile +the extensions in debug mode: +.. code-block:: console + + $ MULTIDICT_DEBUG_BUILD=1 pip install multidict Changelog --------- diff --git a/contrib/python/multidict/multidict/__init__.py b/contrib/python/multidict/multidict/__init__.py index 7159a2d6c0f..31b077f58c0 100644 --- a/contrib/python/multidict/multidict/__init__.py +++ b/contrib/python/multidict/multidict/__init__.py @@ -22,7 +22,7 @@ __all__ = ( "getversion", ) -__version__ = "6.3.0" +__version__ = "6.4.3" if TYPE_CHECKING or not USE_EXTENSIONS: diff --git a/contrib/python/multidict/multidict/_multidict.c b/contrib/python/multidict/multidict/_multidict.c index 64d3c395e96..04af12cc416 100644 --- a/contrib/python/multidict/multidict/_multidict.c +++ b/contrib/python/multidict/multidict/_multidict.c @@ -3,38 +3,43 @@ #include "_multilib/pythoncapi_compat.h" -// Include order important -#include "_multilib/defs.h" -#include "_multilib/istr.h" -#include "_multilib/pair_list.h" #include "_multilib/dict.h" +#include "_multilib/istr.h" #include "_multilib/iter.h" +#include "_multilib/pair_list.h" #include "_multilib/parser.h" +#include "_multilib/state.h" #include "_multilib/views.h" -static PyTypeObject multidict_type; -static PyTypeObject cimultidict_type; -static PyTypeObject multidict_proxy_type; -static PyTypeObject cimultidict_proxy_type; - -#define MultiDict_CheckExact(o) (Py_TYPE(o) == &multidict_type) -#define CIMultiDict_CheckExact(o) (Py_TYPE(o) == &cimultidict_type) -#define MultiDictProxy_CheckExact(o) (Py_TYPE(o) == &multidict_proxy_type) -#define CIMultiDictProxy_CheckExact(o) (Py_TYPE(o) == &cimultidict_proxy_type) - -/* Helper macro for something like isinstance(obj, Base) */ -#define _MultiDict_Check(o) \ - ((MultiDict_CheckExact(o)) || \ - (CIMultiDict_CheckExact(o)) || \ - (MultiDictProxy_CheckExact(o)) || \ - (CIMultiDictProxy_CheckExact(o))) +#define MultiDict_CheckExact(state, obj) Py_IS_TYPE(obj, state->MultiDictType) +#define MultiDict_Check(state, obj) \ + (MultiDict_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->MultiDictType)) +#define CIMultiDict_CheckExact(state, obj) Py_IS_TYPE(obj, state->CIMultiDictType) +#define CIMultiDict_Check(state, obj) \ + (CIMultiDict_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->CIMultiDictType)) +#define AnyMultiDict_Check(state, obj) \ + (MultiDict_CheckExact(state, obj) \ + || CIMultiDict_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->MultiDictType)) +#define MultiDictProxy_CheckExact(state, obj) Py_IS_TYPE(obj, state->MultiDictProxyType) +#define MultiDictProxy_Check(state, obj) \ + (MultiDictProxy_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->MultiDictProxyType)) +#define CIMultiDictProxy_CheckExact(state, obj) \ + Py_IS_TYPE(obj, state->CIMultiDictProxyType) +#define CIMultiDictProxy_Check(state, obj) \ + (CIMultiDictProxy_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->CIMultiDictProxyType)) +#define AnyMultiDictProxy_Check(state, obj) \ + (MultiDictProxy_CheckExact(state, obj) \ + || CIMultiDictProxy_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->MultiDictProxyType)) /******************** Internal Methods ********************/ -/* Forward declaration */ -static PyObject *multidict_items(MultiDictObject *self); - static inline PyObject * _multidict_getone(MultiDictObject *self, PyObject *key, PyObject *_default) { @@ -62,6 +67,7 @@ static inline int _multidict_extend(MultiDictObject *self, PyObject *arg, PyObject *kwds, const char *name, int do_add) { + mod_state *state = self->pairs.state; PyObject *used = NULL; PyObject *seq = NULL; pair_list_t *list; @@ -78,12 +84,12 @@ _multidict_extend(MultiDictObject *self, PyObject *arg, } if (arg != NULL) { - if (MultiDict_CheckExact(arg) || CIMultiDict_CheckExact(arg)) { + if (AnyMultiDict_Check(state, arg)) { list = &((MultiDictObject*)arg)->pairs; if (pair_list_update_from_pair_list(&self->pairs, used, list) < 0) { goto fail; } - } else if (MultiDictProxy_CheckExact(arg) || CIMultiDictProxy_CheckExact(arg)) { + } else if (AnyMultiDictProxy_Check(state, arg)) { list = &((MultiDictProxyObject*)arg)->md->pairs; if (pair_list_update_from_pair_list(&self->pairs, used, list) < 0) { goto fail; @@ -219,12 +225,8 @@ fail: /******************** Base Methods ********************/ static inline PyObject * -multidict_getall( - MultiDictObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_getall(MultiDictObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *list = NULL, *key = NULL, @@ -252,12 +254,8 @@ multidict_getall( } static inline PyObject * -multidict_getone( - MultiDictObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_getone(MultiDictObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *key = NULL, *_default = NULL; @@ -270,12 +268,8 @@ multidict_getone( } static inline PyObject * -multidict_get( - MultiDictObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_get(MultiDictObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *key = NULL, *_default = NULL, @@ -319,7 +313,7 @@ multidict_reduce(MultiDictObject *self) *args = NULL, *result = NULL; - items = multidict_items(self); + items = multidict_itemsview_new(self); if (items == NULL) { goto ret; } @@ -335,7 +329,6 @@ multidict_reduce(MultiDictObject *self) } result = PyTuple_Pack(2, Py_TYPE(self), args); - ret: Py_XDECREF(args); Py_XDECREF(items_list); @@ -347,10 +340,20 @@ ret: static inline PyObject * multidict_repr(MultiDictObject *self) { - PyObject *name = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "__name__"); - if (name == NULL) + int tmp = Py_ReprEnter((PyObject *)self); + if (tmp < 0) { + return NULL; + } + if (tmp > 0) { + return PyUnicode_FromString("..."); + } + PyObject *name = PyObject_GetAttrString((PyObject *)Py_TYPE(self), "__name__"); + if (name == NULL) { + Py_ReprLeave((PyObject *)self); return NULL; + } PyObject *ret = pair_list_repr(&self->pairs, name, true, true); + Py_ReprLeave((PyObject *)self); Py_CLEAR(name); return ret; } @@ -406,12 +409,13 @@ multidict_tp_richcompare(PyObject *self, PyObject *other, int op) return PyBool_FromLong(cmp); } - if (MultiDict_CheckExact(other) || CIMultiDict_CheckExact(other)) { + mod_state *state = ((MultiDictObject*)self)->pairs.state; + if (AnyMultiDict_Check(state, other)) { cmp = pair_list_eq( &((MultiDictObject*)self)->pairs, &((MultiDictObject*)other)->pairs ); - } else if (MultiDictProxy_CheckExact(other) || CIMultiDictProxy_CheckExact(other)) { + } else if (AnyMultiDictProxy_Check(state, other)) { cmp = pair_list_eq( &((MultiDictObject*)self)->pairs, &((MultiDictProxyObject*)other)->md->pairs @@ -430,7 +434,8 @@ multidict_tp_richcompare(PyObject *self, PyObject *other, int op) Py_CLEAR(keys); } if (fits) { - cmp = pair_list_eq_to_mapping(&((MultiDictObject*)self)->pairs, other); + cmp = pair_list_eq_to_mapping(&((MultiDictObject*)self)->pairs, + other); } else { cmp = 0; // e.g., multidict is not equal to a list } @@ -449,9 +454,7 @@ multidict_tp_dealloc(MultiDictObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, multidict_tp_dealloc) - if (self->weaklist != NULL) { - PyObject_ClearWeakRefs((PyObject *)self); - }; + PyObject_ClearWeakRefs((PyObject *)self); pair_list_dealloc(&self->pairs); Py_TYPE(self)->tp_free((PyObject *)self); Py_TRASHCAN_END // there should be no code after this @@ -460,6 +463,7 @@ multidict_tp_dealloc(MultiDictObject *self) static inline int multidict_tp_traverse(MultiDictObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); return pair_list_traverse(&self->pairs, visit, arg); } @@ -492,27 +496,28 @@ PyDoc_STRVAR(multidict_values_doc, static inline int multidict_tp_init(MultiDictObject *self, PyObject *args, PyObject *kwds) { + mod_state *state = get_mod_state_by_def((PyObject *)self); PyObject *arg = NULL; Py_ssize_t size = _multidict_extend_parse_args(args, kwds, "MultiDict", &arg); if (size < 0) { - return -1; + goto fail; } - if (pair_list_init(&self->pairs, size) < 0) { - return -1; + if (pair_list_init(&self->pairs, state, size) < 0) { + goto fail; } if (_multidict_extend(self, arg, kwds, "MultiDict", 1) < 0) { - return -1; + goto fail; } + Py_CLEAR(arg); return 0; +fail: + Py_CLEAR(arg); + return -1; } static inline PyObject * -multidict_add( - MultiDictObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_add(MultiDictObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *key = NULL, *val = NULL; @@ -534,14 +539,17 @@ multidict_extend(MultiDictObject *self, PyObject *args, PyObject *kwds) PyObject *arg = NULL; Py_ssize_t size = _multidict_extend_parse_args(args, kwds, "extend", &arg); if (size < 0) { - return NULL; + goto fail; } pair_list_grow(&self->pairs, size); if (_multidict_extend(self, arg, kwds, "extend", 1) < 0) { - return NULL; + goto fail; } - + Py_CLEAR(arg); Py_RETURN_NONE; +fail: + Py_CLEAR(arg); + return NULL; } static inline PyObject * @@ -555,12 +563,8 @@ multidict_clear(MultiDictObject *self) } static inline PyObject * -multidict_setdefault( - MultiDictObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_setdefault(MultiDictObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *key = NULL, *_default = NULL; @@ -573,12 +577,8 @@ multidict_setdefault( } static inline PyObject * -multidict_popone( - MultiDictObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_popone(MultiDictObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *key = NULL, *_default = NULL, @@ -639,12 +639,8 @@ multidict_pop( } static inline PyObject * -multidict_popall( - MultiDictObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_popall(MultiDictObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *key = NULL, *_default = NULL, @@ -682,12 +678,16 @@ multidict_update(MultiDictObject *self, PyObject *args, PyObject *kwds) { PyObject *arg = NULL; if (_multidict_extend_parse_args(args, kwds, "update", &arg) < 0) { - return NULL; + goto fail; } if (_multidict_extend(self, arg, kwds, "update", 0) < 0) { - return NULL; + goto fail; } + Py_CLEAR(arg); Py_RETURN_NONE; +fail: + Py_CLEAR(arg); + return NULL; } PyDoc_STRVAR(multidict_add_doc, @@ -741,16 +741,6 @@ _multidict_sizeof(MultiDictObject *self) } -static PySequenceMethods multidict_sequence = { - .sq_contains = (objobjproc)multidict_sq_contains, -}; - -static PyMappingMethods multidict_mapping = { - .mp_length = (lenfunc)multidict_mp_len, - .mp_subscript = (binaryfunc)multidict_mp_subscript, - .mp_ass_subscript = (objobjargproc)multidict_mp_as_subscript, -}; - static PyMethodDef multidict_methods[] = { { "getall", @@ -876,70 +866,100 @@ static PyMethodDef multidict_methods[] = { PyDoc_STRVAR(MultDict_doc, "Dictionary with the support for duplicate keys."); +#ifndef MANAGED_WEAKREFS +static PyMemberDef multidict_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, + offsetof(MultiDictObject, weaklist), Py_READONLY}, + {NULL} /* Sentinel */ +}; +#endif + +static PyType_Slot multidict_slots[] = { + {Py_tp_dealloc, multidict_tp_dealloc}, + {Py_tp_repr, multidict_repr}, + {Py_tp_doc, (void *)MultDict_doc}, + + {Py_sq_contains, multidict_sq_contains}, + {Py_mp_length, multidict_mp_len}, + {Py_mp_subscript, multidict_mp_subscript}, + {Py_mp_ass_subscript, multidict_mp_as_subscript}, + + {Py_tp_traverse, multidict_tp_traverse}, + {Py_tp_clear, multidict_tp_clear}, + {Py_tp_richcompare, multidict_tp_richcompare}, + {Py_tp_iter, multidict_tp_iter}, + {Py_tp_methods, multidict_methods}, + {Py_tp_init, multidict_tp_init}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_free, PyObject_GC_Del}, + +#ifndef MANAGED_WEAKREFS + {Py_tp_members, multidict_members}, +#endif + {0, NULL}, +}; -static PyTypeObject multidict_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "multidict._multidict.MultiDict", /* tp_name */ - sizeof(MultiDictObject), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_tp_dealloc, - .tp_repr = (reprfunc)multidict_repr, - .tp_as_sequence = &multidict_sequence, - .tp_as_mapping = &multidict_mapping, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .tp_doc = MultDict_doc, - .tp_traverse = (traverseproc)multidict_tp_traverse, - .tp_clear = (inquiry)multidict_tp_clear, - .tp_richcompare = (richcmpfunc)multidict_tp_richcompare, - .tp_weaklistoffset = offsetof(MultiDictObject, weaklist), - .tp_iter = (getiterfunc)multidict_tp_iter, - .tp_methods = multidict_methods, - .tp_init = (initproc)multidict_tp_init, - .tp_alloc = PyType_GenericAlloc, - .tp_new = PyType_GenericNew, - .tp_free = PyObject_GC_Del, +static PyType_Spec multidict_spec = { + .name = "multidict._multidict.MultiDict", + .basicsize = sizeof(MultiDictObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE +#if PY_VERSION_HEX >= 0x030a00f0 + | Py_TPFLAGS_IMMUTABLETYPE +#endif +#ifdef MANAGED_WEAKREFS + | Py_TPFLAGS_MANAGED_WEAKREF +#endif + | Py_TPFLAGS_HAVE_GC), + .slots = multidict_slots, }; + /******************** CIMultiDict ********************/ static inline int cimultidict_tp_init(MultiDictObject *self, PyObject *args, PyObject *kwds) { + mod_state *state = get_mod_state_by_def((PyObject *)self); PyObject *arg = NULL; Py_ssize_t size = _multidict_extend_parse_args(args, kwds, "CIMultiDict", &arg); if (size < 0) { - return -1; + goto fail; } - if (ci_pair_list_init(&self->pairs, size) < 0) { - return -1; + if (ci_pair_list_init(&self->pairs, state, size) < 0) { + goto fail; } if (_multidict_extend(self, arg, kwds, "CIMultiDict", 1) < 0) { - return -1; + goto fail; } + Py_CLEAR(arg); return 0; +fail: + Py_CLEAR(arg); + return -1; } PyDoc_STRVAR(CIMultDict_doc, "Dictionary with the support for duplicate case-insensitive keys."); +static PyType_Slot cimultidict_slots[] = { + {Py_tp_doc, (void *)CIMultDict_doc}, + {Py_tp_init, cimultidict_tp_init}, + {0, NULL}, +}; -static PyTypeObject cimultidict_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "multidict._multidict.CIMultiDict", /* tp_name */ - sizeof(MultiDictObject), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_tp_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .tp_doc = CIMultDict_doc, - .tp_traverse = (traverseproc)multidict_tp_traverse, - .tp_clear = (inquiry)multidict_tp_clear, - .tp_weaklistoffset = offsetof(MultiDictObject, weaklist), - .tp_base = &multidict_type, - .tp_init = (initproc)cimultidict_tp_init, - .tp_alloc = PyType_GenericAlloc, - .tp_new = PyType_GenericNew, - .tp_free = PyObject_GC_Del, +static PyType_Spec cimultidict_spec = { + .name = "multidict._multidict.CIMultiDict", + .basicsize = sizeof(MultiDictObject), + .flags = (Py_TPFLAGS_DEFAULT +#if PY_VERSION_HEX >= 0x030a00f0 + | Py_TPFLAGS_IMMUTABLETYPE +#endif + | Py_TPFLAGS_BASETYPE), + .slots = cimultidict_slots, }; /******************** MultiDictProxy ********************/ @@ -948,6 +968,7 @@ static inline int multidict_proxy_tp_init(MultiDictProxyObject *self, PyObject *args, PyObject *kwds) { + mod_state *state = get_mod_state_by_def((PyObject *)self); PyObject *arg = NULL; MultiDictObject *md = NULL; @@ -963,9 +984,8 @@ multidict_proxy_tp_init(MultiDictProxyObject *self, PyObject *args, ); return -1; } - if (!MultiDictProxy_CheckExact(arg) && - !CIMultiDict_CheckExact(arg) && - !MultiDict_CheckExact(arg)) + if (!AnyMultiDictProxy_Check(state, arg) && + !AnyMultiDict_Check(state, arg)) { PyErr_Format( PyExc_TypeError, @@ -976,9 +996,10 @@ multidict_proxy_tp_init(MultiDictProxyObject *self, PyObject *args, return -1; } - md = (MultiDictObject*)arg; - if (MultiDictProxy_CheckExact(arg)) { + if (AnyMultiDictProxy_Check(state, arg)) { md = ((MultiDictProxyObject*)arg)->md; + } else { + md = (MultiDictObject*)arg; } Py_INCREF(md); self->md = md; @@ -987,49 +1008,24 @@ multidict_proxy_tp_init(MultiDictProxyObject *self, PyObject *args, } static inline PyObject * -multidict_proxy_getall( - MultiDictProxyObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_proxy_getall(MultiDictProxyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { - return multidict_getall( - self->md, - args, - nargs, - kwnames - ); + return multidict_getall(self->md, args, nargs, kwnames); } static inline PyObject * -multidict_proxy_getone( - MultiDictProxyObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_proxy_getone(MultiDictProxyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { - return multidict_getone( - self->md, args, - nargs, kwnames - ); + return multidict_getone(self->md, args, nargs, kwnames); } static inline PyObject * -multidict_proxy_get( - MultiDictProxyObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames -) +multidict_proxy_get(MultiDictProxyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames) { - return multidict_get( - self->md, - args, - nargs, - kwnames - ); + return multidict_get(self->md, args, nargs, kwnames); } static inline PyObject * @@ -1053,7 +1049,7 @@ multidict_proxy_values(MultiDictProxyObject *self) static inline PyObject * multidict_proxy_copy(MultiDictProxyObject *self) { - return _multidict_proxy_copy(self, &multidict_type); + return _multidict_proxy_copy(self, self->md->pairs.state->MultiDictType); } static inline PyObject * @@ -1102,9 +1098,7 @@ static inline void multidict_proxy_tp_dealloc(MultiDictProxyObject *self) { PyObject_GC_UnTrack(self); - if (self->weaklist != NULL) { - PyObject_ClearWeakRefs((PyObject *)self); - }; + PyObject_ClearWeakRefs((PyObject *)self); Py_XDECREF(self->md); Py_TYPE(self)->tp_free((PyObject *)self); } @@ -1113,6 +1107,7 @@ static inline int multidict_proxy_tp_traverse(MultiDictProxyObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->md); return 0; } @@ -1136,15 +1131,6 @@ multidict_proxy_repr(MultiDictProxyObject *self) } -static PySequenceMethods multidict_proxy_sequence = { - .sq_contains = (objobjproc)multidict_proxy_sq_contains, -}; - -static PyMappingMethods multidict_proxy_mapping = { - .mp_length = (lenfunc)multidict_proxy_mp_len, - .mp_subscript = (binaryfunc)multidict_proxy_mp_subscript, -}; - static PyMethodDef multidict_proxy_methods[] = { { "getall", @@ -1210,27 +1196,51 @@ static PyMethodDef multidict_proxy_methods[] = { PyDoc_STRVAR(MultDictProxy_doc, "Read-only proxy for MultiDict instance."); +#ifndef MANAGED_WEAKREFS +static PyMemberDef multidict_proxy_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, + offsetof(MultiDictProxyObject, weaklist), Py_READONLY}, + {NULL} /* Sentinel */ +}; +#endif + +static PyType_Slot multidict_proxy_slots[] = { + {Py_tp_dealloc, multidict_proxy_tp_dealloc}, + {Py_tp_repr, multidict_proxy_repr}, + {Py_tp_doc, (void *)MultDictProxy_doc}, + + {Py_sq_contains, multidict_proxy_sq_contains}, + {Py_mp_length, multidict_proxy_mp_len}, + {Py_mp_subscript, multidict_proxy_mp_subscript}, + + {Py_tp_traverse, multidict_proxy_tp_traverse}, + {Py_tp_clear, multidict_proxy_tp_clear}, + {Py_tp_richcompare, multidict_proxy_tp_richcompare}, + {Py_tp_iter, multidict_proxy_tp_iter}, + {Py_tp_methods, multidict_proxy_methods}, + {Py_tp_init, multidict_proxy_tp_init}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_free, PyObject_GC_Del}, + +#ifndef MANAGED_WEAKREFS + {Py_tp_members, multidict_proxy_members}, +#endif + {0, NULL}, +}; -static PyTypeObject multidict_proxy_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "multidict._multidict.MultiDictProxy", /* tp_name */ - sizeof(MultiDictProxyObject), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_proxy_tp_dealloc, - .tp_repr = (reprfunc)multidict_proxy_repr, - .tp_as_sequence = &multidict_proxy_sequence, - .tp_as_mapping = &multidict_proxy_mapping, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .tp_doc = MultDictProxy_doc, - .tp_traverse = (traverseproc)multidict_proxy_tp_traverse, - .tp_clear = (inquiry)multidict_proxy_tp_clear, - .tp_richcompare = (richcmpfunc)multidict_proxy_tp_richcompare, - .tp_weaklistoffset = offsetof(MultiDictProxyObject, weaklist), - .tp_iter = (getiterfunc)multidict_proxy_tp_iter, - .tp_methods = multidict_proxy_methods, - .tp_init = (initproc)multidict_proxy_tp_init, - .tp_alloc = PyType_GenericAlloc, - .tp_new = PyType_GenericNew, - .tp_free = PyObject_GC_Del, +static PyType_Spec multidict_proxy_spec = { + .name = "multidict._multidict.MultiDictProxy", + .basicsize = sizeof(MultiDictProxyObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE +#if PY_VERSION_HEX >= 0x030a00f0 + | Py_TPFLAGS_IMMUTABLETYPE +#endif +#ifdef MANAGED_WEAKREFS + | Py_TPFLAGS_MANAGED_WEAKREF +#endif + | Py_TPFLAGS_HAVE_GC), + .slots = multidict_proxy_slots, }; /******************** CIMultiDictProxy ********************/ @@ -1239,6 +1249,7 @@ static inline int cimultidict_proxy_tp_init(MultiDictProxyObject *self, PyObject *args, PyObject *kwds) { + mod_state *state = get_mod_state_by_def((PyObject *)self); PyObject *arg = NULL; MultiDictObject *md = NULL; @@ -1254,7 +1265,8 @@ cimultidict_proxy_tp_init(MultiDictProxyObject *self, PyObject *args, ); return -1; } - if (!CIMultiDictProxy_CheckExact(arg) && !CIMultiDict_CheckExact(arg)) { + if (!CIMultiDictProxy_Check(state, arg) + && !CIMultiDict_Check(state, arg)) { PyErr_Format( PyExc_TypeError, "ctor requires CIMultiDict or CIMultiDictProxy instance, " @@ -1264,9 +1276,10 @@ cimultidict_proxy_tp_init(MultiDictProxyObject *self, PyObject *args, return -1; } - md = (MultiDictObject*)arg; - if (CIMultiDictProxy_CheckExact(arg)) { + if (CIMultiDictProxy_Check(state, arg)) { md = ((MultiDictProxyObject*)arg)->md; + } else { + md = (MultiDictObject*)arg; } Py_INCREF(md); self->md = md; @@ -1277,7 +1290,7 @@ cimultidict_proxy_tp_init(MultiDictProxyObject *self, PyObject *args, static inline PyObject * cimultidict_proxy_copy(MultiDictProxyObject *self) { - return _multidict_proxy_copy(self, &cimultidict_type); + return _multidict_proxy_copy(self, self->md->pairs.state->CIMultiDictType); } @@ -1300,23 +1313,22 @@ static PyMethodDef cimultidict_proxy_methods[] = { } /* sentinel */ }; -static PyTypeObject cimultidict_proxy_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "multidict._multidict.CIMultiDictProxy", /* tp_name */ - sizeof(MultiDictProxyObject), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_proxy_tp_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .tp_doc = CIMultDictProxy_doc, - .tp_traverse = (traverseproc)multidict_proxy_tp_traverse, - .tp_clear = (inquiry)multidict_proxy_tp_clear, - .tp_richcompare = (richcmpfunc)multidict_proxy_tp_richcompare, - .tp_weaklistoffset = offsetof(MultiDictProxyObject, weaklist), - .tp_methods = cimultidict_proxy_methods, - .tp_base = &multidict_proxy_type, - .tp_init = (initproc)cimultidict_proxy_tp_init, - .tp_alloc = PyType_GenericAlloc, - .tp_new = PyType_GenericNew, - .tp_free = PyObject_GC_Del, +static PyType_Slot cimultidict_proxy_slots[] = { + {Py_tp_doc, (void *)CIMultDictProxy_doc}, + {Py_tp_methods, cimultidict_proxy_methods}, + {Py_tp_init, cimultidict_proxy_tp_init}, + {0, NULL}, +}; + +static PyType_Spec cimultidict_proxy_spec = { + .name = "multidict._multidict.CIMultiDictProxy", + .basicsize = sizeof(MultiDictProxyObject), + .flags = (Py_TPFLAGS_DEFAULT +#if PY_VERSION_HEX >= 0x030a00f0 + | Py_TPFLAGS_IMMUTABLETYPE +#endif + | Py_TPFLAGS_BASETYPE), + .slots = cimultidict_proxy_slots, }; /******************** Other functions ********************/ @@ -1324,10 +1336,11 @@ static PyTypeObject cimultidict_proxy_type = { static inline PyObject * getversion(PyObject *self, PyObject *md) { + mod_state *state = get_mod_state(self); pair_list_t *pairs = NULL; - if (MultiDict_CheckExact(md) || CIMultiDict_CheckExact(md)) { + if (AnyMultiDict_Check(state, md)) { pairs = &((MultiDictObject*)md)->pairs; - } else if (MultiDictProxy_CheckExact(md) || CIMultiDictProxy_CheckExact(md)) { + } else if (AnyMultiDictProxy_Check(state, md)) { pairs = &((MultiDictProxyObject*)md)->md->pairs; } else { PyErr_Format(PyExc_TypeError, "unexpected type"); @@ -1338,131 +1351,189 @@ getversion(PyObject *self, PyObject *md) /******************** Module ********************/ +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + mod_state *state = get_mod_state(mod); + + Py_VISIT(state->IStrType); + + Py_VISIT(state->MultiDictType); + Py_VISIT(state->CIMultiDictType); + Py_VISIT(state->MultiDictProxyType); + Py_VISIT(state->CIMultiDictProxyType); + + Py_VISIT(state->KeysViewType); + Py_VISIT(state->ItemsViewType); + Py_VISIT(state->ValuesViewType); + + Py_VISIT(state->KeysIterType); + Py_VISIT(state->ItemsIterType); + Py_VISIT(state->ValuesIterType); + + Py_VISIT(state->str_lower); + Py_VISIT(state->str_canonical); + + return 0; +} + +static int +module_clear(PyObject *mod) +{ + mod_state *state = get_mod_state(mod); + + Py_CLEAR(state->IStrType); + + Py_CLEAR(state->MultiDictType); + Py_CLEAR(state->CIMultiDictType); + Py_CLEAR(state->MultiDictProxyType); + Py_CLEAR(state->CIMultiDictProxyType); + + Py_CLEAR(state->KeysViewType); + Py_CLEAR(state->ItemsViewType); + Py_CLEAR(state->ValuesViewType); + + Py_CLEAR(state->KeysIterType); + Py_CLEAR(state->ItemsIterType); + Py_CLEAR(state->ValuesIterType); + + Py_CLEAR(state->str_lower); + Py_CLEAR(state->str_canonical); + + return 0; +} + static inline void -module_free(void *m) +module_free(void *mod) { - Py_CLEAR(multidict_str_lower); - Py_CLEAR(multidict_str_canonical); + (void)module_clear((PyObject *)mod); } -static PyMethodDef multidict_module_methods[] = { +static PyMethodDef module_methods[] = { {"getversion", (PyCFunction)getversion, METH_O}, {NULL, NULL} /* sentinel */ }; -static PyModuleDef multidict_module = { - PyModuleDef_HEAD_INIT, /* m_base */ - "_multidict", /* m_name */ - .m_size = -1, - .m_methods = multidict_module_methods, - .m_free = (freefunc)module_free, -}; -PyMODINIT_FUNC -PyInit__multidict(void) +static int +module_exec(PyObject *mod) { - multidict_str_lower = PyUnicode_InternFromString("lower"); - if (multidict_str_lower == NULL) { + mod_state *state = get_mod_state(mod); + PyObject *tmp; + PyObject *tpl = NULL; + + state->str_lower = PyUnicode_InternFromString("lower"); + if (state->str_lower == NULL) { goto fail; } - multidict_str_canonical = PyUnicode_InternFromString("_canonical"); - if (multidict_str_canonical == NULL) { + state->str_canonical = PyUnicode_InternFromString("_canonical"); + if (state->str_canonical == NULL) { goto fail; } - PyObject *module = NULL; - - if (multidict_views_init() < 0) { + if (multidict_views_init(mod, state) < 0) { goto fail; } - if (multidict_iter_init() < 0) { + if (multidict_iter_init(mod, state) < 0) { goto fail; } - if (istr_init() < 0) { + if (istr_init(mod, state) < 0) { goto fail; } - if (PyType_Ready(&multidict_type) < 0 || - PyType_Ready(&cimultidict_type) < 0 || - PyType_Ready(&multidict_proxy_type) < 0 || - PyType_Ready(&cimultidict_proxy_type) < 0) - { + tmp = PyType_FromModuleAndSpec(mod, &multidict_spec, NULL); + if (tmp == NULL) { goto fail; } + state->MultiDictType = (PyTypeObject *)tmp; - /* Instantiate this module */ - module = PyModule_Create(&multidict_module); - if (module == NULL) { + tpl = PyTuple_Pack(1, (PyObject *)state->MultiDictType); + if (tpl == NULL) { goto fail; } - -#ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); -#endif - - Py_INCREF(&istr_type); - if (PyModule_AddObject( - module, "istr", (PyObject*)&istr_type) < 0) - { + tmp = PyType_FromModuleAndSpec(mod, &cimultidict_spec, tpl); + if (tmp == NULL) { goto fail; } + state->CIMultiDictType = (PyTypeObject *)tmp; + Py_CLEAR(tpl); - Py_INCREF(&multidict_type); - if (PyModule_AddObject( - module, "MultiDict", (PyObject*)&multidict_type) < 0) - { + tmp = PyType_FromModuleAndSpec(mod, &multidict_proxy_spec, NULL); + if (tmp == NULL) { goto fail; } + state->MultiDictProxyType = (PyTypeObject *)tmp; - Py_INCREF(&cimultidict_type); - if (PyModule_AddObject( - module, "CIMultiDict", (PyObject*)&cimultidict_type) < 0) - { + tpl = PyTuple_Pack(1, (PyObject *)state->MultiDictProxyType); + if (tpl == NULL) { goto fail; } - - Py_INCREF(&multidict_proxy_type); - if (PyModule_AddObject( - module, "MultiDictProxy", (PyObject*)&multidict_proxy_type) < 0) - { + tmp = PyType_FromModuleAndSpec(mod, &cimultidict_proxy_spec, tpl); + if (tmp == NULL) { goto fail; } + state->CIMultiDictProxyType = (PyTypeObject *)tmp; + Py_CLEAR(tpl); - Py_INCREF(&cimultidict_proxy_type); - if (PyModule_AddObject( - module, "CIMultiDictProxy", (PyObject*)&cimultidict_proxy_type) < 0) - { + if (PyModule_AddType(mod, state->IStrType) < 0) { goto fail; } - - Py_INCREF(&multidict_keysview_type); - if (PyModule_AddObject( - module, "_KeysView", (PyObject*)&multidict_keysview_type) < 0) - { + if (PyModule_AddType(mod, state->MultiDictType) < 0) { goto fail; } - - Py_INCREF(&multidict_itemsview_type); - if (PyModule_AddObject( - module, "_ItemsView", (PyObject*)&multidict_itemsview_type) < 0) - { + if (PyModule_AddType(mod, state->CIMultiDictType) < 0) { goto fail; } - - Py_INCREF(&multidict_valuesview_type); - if (PyModule_AddObject( - module, "_ValuesView", (PyObject*)&multidict_valuesview_type) < 0) - { + if (PyModule_AddType(mod, state->MultiDictProxyType) < 0) { + goto fail; + } + if (PyModule_AddType(mod, state->CIMultiDictProxyType) < 0) { + goto fail; + } + if (PyModule_AddType(mod, state->ItemsViewType) < 0) { + goto fail; + } + if (PyModule_AddType(mod, state->KeysViewType) < 0) { + goto fail; + } + if (PyModule_AddType(mod, state->ValuesViewType) < 0) { goto fail; } - return module; - + return 0; fail: - Py_XDECREF(multidict_str_lower); - Py_XDECREF(multidict_str_canonical); + Py_CLEAR(tpl); + return -1; +} - return NULL; + +static struct PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, module_exec}, +#if PY_VERSION_HEX >= 0x030c00f0 + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, +#endif +#if PY_VERSION_HEX >= 0x030d00f0 + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL}, +}; + + +static PyModuleDef multidict_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_multidict", + .m_size = sizeof(mod_state), + .m_methods = module_methods, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = (freefunc)module_free, +}; + +PyMODINIT_FUNC +PyInit__multidict(void) +{ + return PyModuleDef_Init(&multidict_module); } diff --git a/contrib/python/multidict/multidict/_multidict_py.py b/contrib/python/multidict/multidict/_multidict_py.py index 7babdd5e6ef..3176861e787 100644 --- a/contrib/python/multidict/multidict/_multidict_py.py +++ b/contrib/python/multidict/multidict/_multidict_py.py @@ -1,4 +1,5 @@ import enum +import reprlib import sys from abc import abstractmethod from array import array @@ -55,7 +56,6 @@ class _Impl(Generic[_V]): self.incr_version() def incr_version(self) -> None: - global _version v = _version v[0] += 1 self._version = v[0] @@ -121,6 +121,7 @@ class _ItemsView(_ViewBase[_V], ItemsView[str, _V]): raise RuntimeError("Dictionary changed during iteration") yield self._keyfunc(k), v + @reprlib.recursive_repr() def __repr__(self) -> str: lst = [] for i, k, v in self._impl._items: @@ -254,7 +255,7 @@ class _ItemsView(_ViewBase[_V], ItemsView[str, _V]): except TypeError: return NotImplemented ret: set[Union[tuple[str, _V], _T]] = self - rgt - ret |= (rgt - self) + ret |= rgt - self return ret __rxor__ = __xor__ @@ -288,6 +289,7 @@ class _ValuesView(_ViewBase[_V], ValuesView[_V]): raise RuntimeError("Dictionary changed during iteration") yield v + @reprlib.recursive_repr() def __repr__(self) -> str: lst = [] for i, k, v in self._impl._items: @@ -425,7 +427,7 @@ class _KeysView(_ViewBase[_V], KeysView[str]): except TypeError: return NotImplemented ret: set[Union[str, _T]] = self - rgt # type: ignore[assignment] - ret |= (rgt - self) + ret |= rgt - self return ret __rxor__ = __xor__ @@ -453,6 +455,8 @@ class _CSMixin: class _CIMixin: + _ci: bool = True + def _key(self, key: str) -> str: if type(key) is istr: return key @@ -474,6 +478,7 @@ class _CIMixin: class _Base(MultiMapping[_V]): _impl: _Impl[_V] + _ci: bool = False @abstractmethod def _key(self, key: str) -> str: ... @@ -579,6 +584,7 @@ class _Base(MultiMapping[_V]): return True return False + @reprlib.recursive_repr() def __repr__(self) -> str: body = ", ".join(f"'{k}': {v!r}" for i, k, v in self._impl._items) return f"<{self.__class__.__name__}({body})>" @@ -628,7 +634,12 @@ class MultiDict(_CSMixin, _Base[_V], MutableMultiMapping[_V]): ) -> None: if arg: if isinstance(arg, (MultiDict, MultiDictProxy)): - items = arg._impl._items + if self._ci is not arg._ci: + items = [(self._title(k), k, v) for _, k, v in arg._impl._items] + else: + items = arg._impl._items + if kwargs: + items = items.copy() if kwargs: for key, value in kwargs.items(): items.append((self._title(key), key, value)) diff --git a/contrib/python/multidict/multidict/_multilib/defs.h b/contrib/python/multidict/multidict/_multilib/defs.h deleted file mode 100644 index 9e1cd724122..00000000000 --- a/contrib/python/multidict/multidict/_multilib/defs.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef _MULTIDICT_DEFS_H -#define _MULTIDICT_DEFS_H - -#ifdef __cplusplus -extern "C" { -#endif - -static PyObject *multidict_str_lower = NULL; -static PyObject *multidict_str_canonical = NULL; - -/* We link this module statically for convenience. If compiled as a shared - library instead, some compilers don't allow addresses of Python objects - defined in other libraries to be used in static initializers here. The - DEFERRED_ADDRESS macro is used to tag the slots where such addresses - appear; the module init function must fill in the tagged slots at runtime. - The argument is for documentation -- the macro ignores it. -*/ -#define DEFERRED_ADDRESS(ADDR) 0 - -#ifdef __cplusplus -} -#endif -#endif diff --git a/contrib/python/multidict/multidict/_multilib/dict.h b/contrib/python/multidict/multidict/_multilib/dict.h index 064101d47ea..fa07fdf4ac3 100644 --- a/contrib/python/multidict/multidict/_multilib/dict.h +++ b/contrib/python/multidict/multidict/_multilib/dict.h @@ -5,15 +5,27 @@ extern "C" { #endif +#include "pythoncapi_compat.h" +#include "pair_list.h" + +#if PY_VERSION_HEX >= 0x030c00f0 +#define MANAGED_WEAKREFS +#endif + + typedef struct { // 16 or 24 for GC prefix PyObject_HEAD // 16 +#ifndef MANAGED_WEAKREFS PyObject *weaklist; +#endif pair_list_t pairs; } MultiDictObject; typedef struct { PyObject_HEAD +#ifndef MANAGED_WEAKREFS PyObject *weaklist; +#endif MultiDictObject *md; } MultiDictProxyObject; diff --git a/contrib/python/multidict/multidict/_multilib/istr.h b/contrib/python/multidict/multidict/_multilib/istr.h index 68328054528..156b0dc04a3 100644 --- a/contrib/python/multidict/multidict/_multilib/istr.h +++ b/contrib/python/multidict/multidict/_multilib/istr.h @@ -5,14 +5,19 @@ extern "C" { #endif +#include "state.h" + typedef struct { PyUnicodeObject str; PyObject * canonical; + mod_state *state; } istrobject; -PyDoc_STRVAR(istr__doc__, "istr class implementation"); +#define IStr_CheckExact(state, obj) Py_IS_TYPE(obj, state->IStrType) +#define IStr_Check(state, obj) \ + (IStr_CheckExact(state, obj) || PyObject_TypeCheck(obj, state->IStrType)) -static PyTypeObject istr_type; +PyDoc_STRVAR(istr__doc__, "istr class implementation"); static inline void istr_dealloc(istrobject *self) @@ -24,26 +29,24 @@ istr_dealloc(istrobject *self) static inline PyObject * istr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { + PyObject *mod = PyType_GetModuleByDef(type, &multidict_module); + if (mod == NULL) { + return NULL; + } + mod_state *state = get_mod_state(mod); + PyObject *x = NULL; static char *kwlist[] = {"object", "encoding", "errors", 0}; PyObject *encoding = NULL; PyObject *errors = NULL; PyObject *canonical = NULL; - PyObject * ret = NULL; - if (kwds != NULL) { - int cmp = PyDict_Pop(kwds, multidict_str_canonical, &canonical); - if (cmp < 0) { - return NULL; - } else if (cmp > 0) { - Py_INCREF(canonical); - } - } + PyObject *ret = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO:str", kwlist, &x, &encoding, &errors)) { return NULL; } - if (x != NULL && Py_TYPE(x) == &istr_type) { + if (x != NULL && IStr_Check(state, x)) { Py_INCREF(x); return x; } @@ -51,29 +54,18 @@ istr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!ret) { goto fail; } - - if (canonical == NULL) { - canonical = PyObject_CallMethodNoArgs(ret, multidict_str_lower); - if (!canonical) { - goto fail; - } - } - if (!PyUnicode_CheckExact(canonical)) { - PyObject *tmp = PyUnicode_FromObject(canonical); - Py_CLEAR(canonical); - if (tmp == NULL) { - goto fail; - } - canonical = tmp; + canonical = PyObject_CallMethodNoArgs(ret, state->str_lower); + if (!canonical) { + goto fail; } ((istrobject*)ret)->canonical = canonical; + ((istrobject*)ret)->state = state; return ret; fail: Py_XDECREF(ret); return NULL; } - static inline PyObject * istr_reduce(PyObject *self) { @@ -102,62 +94,61 @@ static PyMethodDef istr_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyTypeObject istr_type = { - PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) - "multidict._multidict.istr", - sizeof(istrobject), - .tp_dealloc = (destructor)istr_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT +static PyType_Slot istr_slots[] = { + {Py_tp_dealloc, istr_dealloc}, + {Py_tp_doc, (void *)istr__doc__}, + {Py_tp_methods, istr_methods}, + {Py_tp_new, istr_new}, + {0, NULL}, +}; + +static PyType_Spec istr_spec = { + .name = "multidict._multidict.istr", + .basicsize = sizeof(istrobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_UNICODE_SUBCLASS, - .tp_doc = istr__doc__, - .tp_base = DEFERRED_ADDRESS(&PyUnicode_Type), - .tp_methods = istr_methods, - .tp_new = (newfunc)istr_new, +#if PY_VERSION_HEX >= 0x030a00f0 + | Py_TPFLAGS_IMMUTABLETYPE +#endif + | Py_TPFLAGS_UNICODE_SUBCLASS), + .slots = istr_slots, }; static inline PyObject * -IStr_New(PyObject *str, PyObject *canonical) +IStr_New(mod_state *state, PyObject *str, PyObject *canonical) { PyObject *args = NULL; - PyObject *kwds = NULL; PyObject *res = NULL; - args = PyTuple_Pack(1, str); if (args == NULL) { goto ret; } - - if (canonical != NULL) { - kwds = PyDict_New(); - if (kwds == NULL) { - goto ret; - } - if (!PyUnicode_CheckExact(canonical)) { - PyErr_SetString(PyExc_TypeError, - "'canonical' argument should be exactly str"); - goto ret; - } - if (PyDict_SetItem(kwds, multidict_str_canonical, canonical) < 0) { - goto ret; - } + res = PyUnicode_Type.tp_new(state->IStrType, args, NULL); + if (!res) { + goto ret; } - - res = istr_new(&istr_type, args, kwds); + Py_INCREF(canonical); + ((istrobject*)res)->canonical = canonical; + ((istrobject*)res)->state = state; ret: Py_CLEAR(args); - Py_CLEAR(kwds); return res; } static inline int -istr_init(void) +istr_init(PyObject *module, mod_state *state) { - istr_type.tp_base = &PyUnicode_Type; - if (PyType_Ready(&istr_type) < 0) { + PyObject *tpl = PyTuple_Pack(1, (PyObject *)&PyUnicode_Type); + if (tpl == NULL) { + return -1; + } + PyObject *tmp = PyType_FromModuleAndSpec(module, &istr_spec, tpl); + Py_DECREF(tpl); + if (tmp == NULL) { return -1; } + state->IStrType = (PyTypeObject *)tmp; return 0; } diff --git a/contrib/python/multidict/multidict/_multilib/iter.h b/contrib/python/multidict/multidict/_multilib/iter.h index 3fd34e2ca4b..2f3ca9de84c 100644 --- a/contrib/python/multidict/multidict/_multilib/iter.h +++ b/contrib/python/multidict/multidict/_multilib/iter.h @@ -5,9 +5,9 @@ extern "C" { #endif -static PyTypeObject multidict_items_iter_type; -static PyTypeObject multidict_values_iter_type; -static PyTypeObject multidict_keys_iter_type; +#include "dict.h" +#include "pair_list.h" +#include "state.h" typedef struct multidict_iter { PyObject_HEAD @@ -28,7 +28,7 @@ static inline PyObject * multidict_items_iter_new(MultiDictObject *md) { MultidictIter *it = PyObject_GC_New( - MultidictIter, &multidict_items_iter_type); + MultidictIter, md->pairs.state->ItemsIterType); if (it == NULL) { return NULL; } @@ -43,7 +43,7 @@ static inline PyObject * multidict_keys_iter_new(MultiDictObject *md) { MultidictIter *it = PyObject_GC_New( - MultidictIter, &multidict_keys_iter_type); + MultidictIter, md->pairs.state->KeysIterType); if (it == NULL) { return NULL; } @@ -58,7 +58,7 @@ static inline PyObject * multidict_values_iter_new(MultiDictObject *md) { MultidictIter *it = PyObject_GC_New( - MultidictIter, &multidict_values_iter_type); + MultidictIter, md->pairs.state->ValuesIterType); if (it == NULL) { return NULL; } @@ -180,53 +180,92 @@ static PyMethodDef multidict_iter_methods[] = { /***********************************************************************/ -static PyTypeObject multidict_items_iter_type = { - PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) - "multidict._multidict._itemsiter", /* tp_name */ - sizeof(MultidictIter), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_iter_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)multidict_iter_traverse, - .tp_clear = (inquiry)multidict_iter_clear, - .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)multidict_items_iter_iternext, - .tp_methods = multidict_iter_methods, +static PyType_Slot multidict_items_iter_slots[] = { + {Py_tp_dealloc, multidict_iter_dealloc}, + {Py_tp_methods, multidict_iter_methods}, + {Py_tp_traverse, multidict_iter_traverse}, + {Py_tp_clear, multidict_iter_clear}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, multidict_items_iter_iternext}, + {0, NULL}, }; -static PyTypeObject multidict_values_iter_type = { - PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) - "multidict._multidict._valuesiter", /* tp_name */ - sizeof(MultidictIter), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_iter_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)multidict_iter_traverse, - .tp_clear = (inquiry)multidict_iter_clear, - .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)multidict_values_iter_iternext, - .tp_methods = multidict_iter_methods, +static PyType_Spec multidict_items_iter_spec = { + .name = "multidict._multidict._itemsiter", + .basicsize = sizeof(MultidictIter), + .flags = (Py_TPFLAGS_DEFAULT +#if PY_VERSION_HEX >= 0x030a00f0 + | Py_TPFLAGS_IMMUTABLETYPE +#endif + | Py_TPFLAGS_HAVE_GC), + .slots = multidict_items_iter_slots, }; -static PyTypeObject multidict_keys_iter_type = { - PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) - "multidict._multidict._keysiter", /* tp_name */ - sizeof(MultidictIter), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_iter_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)multidict_iter_traverse, - .tp_clear = (inquiry)multidict_iter_clear, - .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)multidict_keys_iter_iternext, - .tp_methods = multidict_iter_methods, +static PyType_Slot multidict_values_iter_slots[] = { + {Py_tp_dealloc, multidict_iter_dealloc}, + {Py_tp_methods, multidict_iter_methods}, + {Py_tp_traverse, multidict_iter_traverse}, + {Py_tp_clear, multidict_iter_clear}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, multidict_values_iter_iternext}, + {0, NULL}, +}; + +static PyType_Spec multidict_values_iter_spec = { + .name = "multidict._multidict._valuesiter", + .basicsize = sizeof(MultidictIter), + .flags = (Py_TPFLAGS_DEFAULT +#if PY_VERSION_HEX >= 0x030a00f0 + | Py_TPFLAGS_IMMUTABLETYPE +#endif + | Py_TPFLAGS_HAVE_GC), + .slots = multidict_values_iter_slots, +}; + + +static PyType_Slot multidict_keys_iter_slots[] = { + {Py_tp_dealloc, multidict_iter_dealloc}, + {Py_tp_methods, multidict_iter_methods}, + {Py_tp_traverse, multidict_iter_traverse}, + {Py_tp_clear, multidict_iter_clear}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, multidict_keys_iter_iternext}, + {0, NULL}, +}; + +static PyType_Spec multidict_keys_iter_spec = { + .name = "multidict._multidict._keysiter", + .basicsize = sizeof(MultidictIter), + .flags = (Py_TPFLAGS_DEFAULT +#if PY_VERSION_HEX >= 0x030a00f0 + | Py_TPFLAGS_IMMUTABLETYPE +#endif + | Py_TPFLAGS_HAVE_GC), + .slots = multidict_keys_iter_slots, }; static inline int -multidict_iter_init(void) +multidict_iter_init(PyObject *module, mod_state *state) { - if (PyType_Ready(&multidict_items_iter_type) < 0 || - PyType_Ready(&multidict_values_iter_type) < 0 || - PyType_Ready(&multidict_keys_iter_type) < 0) { + PyObject * tmp; + tmp = PyType_FromModuleAndSpec(module, &multidict_items_iter_spec, NULL); + if (tmp == NULL) { + return -1; + } + state->ItemsIterType = (PyTypeObject *)tmp; + + tmp = PyType_FromModuleAndSpec(module, &multidict_values_iter_spec, NULL); + if (tmp == NULL) { return -1; } + state->ValuesIterType = (PyTypeObject *)tmp; + + tmp = PyType_FromModuleAndSpec(module, &multidict_keys_iter_spec, NULL); + if (tmp == NULL) { + return -1; + } + state->KeysIterType = (PyTypeObject *)tmp; + return 0; } diff --git a/contrib/python/multidict/multidict/_multilib/pair_list.h b/contrib/python/multidict/multidict/_multilib/pair_list.h index c5080743339..6c45673b73f 100644 --- a/contrib/python/multidict/multidict/_multilib/pair_list.h +++ b/contrib/python/multidict/multidict/_multilib/pair_list.h @@ -12,6 +12,9 @@ extern "C" { #include <stdint.h> #include <stdbool.h> +#include "istr.h" +#include "state.h" + /* Implementation note. identity always has exact PyUnicode_Type type, not a subclass. It guarantees that identity hashing and comparison never calls @@ -32,10 +35,7 @@ typedef struct pair { } pair_t; /* Note about the structure size -With 29 pairs the MultiDict object size is slightly less than 1KiB -(1000-1008 bytes depending on Python version, -plus extra 12 bytes for memory allocator internal structures). -As the result the max reserved size is 1020 bytes at most. +With 28 pairs the MultiDict object size is slightly less than 1KiB To fit into 512 bytes, the structure can contain only 13 pairs which is too small, e.g. https://www.python.org returns 16 headers @@ -45,9 +45,10 @@ The embedded buffer intention is to fit the vast majority of possible HTTP headers into the buffer without allocating an extra memory block. */ -#define EMBEDDED_CAPACITY 29 +#define EMBEDDED_CAPACITY 28 typedef struct pair_list { + mod_state *state; Py_ssize_t capacity; Py_ssize_t size; uint64_t version; @@ -92,10 +93,9 @@ str_cmp(PyObject *s1, PyObject *s2) static inline PyObject * -_key_to_ident(PyObject *key) +_key_to_ident(mod_state *state, PyObject *key) { - PyTypeObject *type = Py_TYPE(key); - if (type == &istr_type) { + if (IStr_Check(state, key)) { return Py_NewRef(((istrobject*)key)->canonical); } if (PyUnicode_CheckExact(key)) { @@ -112,14 +112,13 @@ _key_to_ident(PyObject *key) static inline PyObject * -_ci_key_to_ident(PyObject *key) +_ci_key_to_ident(mod_state *state, PyObject *key) { - PyTypeObject *type = Py_TYPE(key); - if (type == &istr_type) { + if (IStr_Check(state, key)) { return Py_NewRef(((istrobject*)key)->canonical); } if (PyUnicode_Check(key)) { - PyObject *ret = PyObject_CallMethodNoArgs(key, multidict_str_lower); + PyObject *ret = PyObject_CallMethodNoArgs(key, state->str_lower); if (!PyUnicode_CheckExact(ret)) { PyObject *tmp = PyUnicode_FromObject(ret); Py_CLEAR(ret); @@ -138,7 +137,7 @@ _ci_key_to_ident(PyObject *key) static inline PyObject * -_arg_to_key(PyObject *key, PyObject *ident) +_arg_to_key(mod_state *state, PyObject *key, PyObject *ident) { if (PyUnicode_Check(key)) { return Py_NewRef(key); @@ -151,14 +150,13 @@ _arg_to_key(PyObject *key, PyObject *ident) static inline PyObject * -_ci_arg_to_key(PyObject *key, PyObject *ident) +_ci_arg_to_key(mod_state *state, PyObject *key, PyObject *ident) { - PyTypeObject *type = Py_TYPE(key); - if (type == &istr_type) { + if (IStr_Check(state, key)) { return Py_NewRef(key); } if (PyUnicode_Check(key)) { - return IStr_New(key, ident); + return IStr_New(state, key, ident); } PyErr_SetString(PyExc_TypeError, "CIMultiDict keys should be either str " @@ -240,8 +238,10 @@ pair_list_shrink(pair_list_t *list) static inline int -_pair_list_init(pair_list_t *list, bool calc_ci_identity, Py_ssize_t preallocate) +_pair_list_init(pair_list_t *list, mod_state *state, + bool calc_ci_identity, Py_ssize_t preallocate) { + list->state = state; list->calc_ci_indentity = calc_ci_identity; Py_ssize_t capacity = EMBEDDED_CAPACITY; if (preallocate >= capacity) { @@ -257,16 +257,16 @@ _pair_list_init(pair_list_t *list, bool calc_ci_identity, Py_ssize_t preallocate } static inline int -pair_list_init(pair_list_t *list, Py_ssize_t size) +pair_list_init(pair_list_t *list, mod_state *state, Py_ssize_t size) { - return _pair_list_init(list, /* calc_ci_identity = */ false, size); + return _pair_list_init(list, state, /* calc_ci_identity = */ false, size); } static inline int -ci_pair_list_init(pair_list_t *list, Py_ssize_t size) +ci_pair_list_init(pair_list_t *list, mod_state *state, Py_ssize_t size) { - return _pair_list_init(list, /* calc_ci_identity = */ true, size); + return _pair_list_init(list, state, /* calc_ci_identity = */ true, size); } @@ -274,16 +274,16 @@ static inline PyObject * pair_list_calc_identity(pair_list_t *list, PyObject *key) { if (list->calc_ci_indentity) - return _ci_key_to_ident(key); - return _key_to_ident(key); + return _ci_key_to_ident(list->state, key); + return _key_to_ident(list->state, key); } static inline PyObject * pair_list_calc_key(pair_list_t *list, PyObject *key, PyObject *ident) { if (list->calc_ci_indentity) - return _ci_arg_to_key(key, ident); - return _arg_to_key(key, ident); + return _ci_arg_to_key(list->state, key, ident); + return _arg_to_key(list->state, key, ident); } static inline void @@ -363,9 +363,7 @@ _pair_list_add_with_hash(pair_list_t *list, static inline int -pair_list_add(pair_list_t *list, - PyObject *key, - PyObject *value) +pair_list_add(pair_list_t *list, PyObject *key, PyObject *value) { PyObject *identity = pair_list_calc_identity(list, key); if (identity == NULL) { @@ -1017,6 +1015,7 @@ _dict_set_number(PyObject *dict, PyObject *key, Py_ssize_t num) return -1; } + Py_DECREF(tmp); return 0; } @@ -1129,26 +1128,58 @@ _pair_list_update(pair_list_t *list, PyObject *key, static inline int -pair_list_update_from_pair_list(pair_list_t *list, PyObject* used, pair_list_t *other) +pair_list_update_from_pair_list(pair_list_t *list, + PyObject* used, pair_list_t *other) { Py_ssize_t pos; + Py_hash_t hash; + PyObject *identity = NULL; + PyObject *key = NULL; + bool recalc_identity = list->calc_ci_indentity != other->calc_ci_indentity; for (pos = 0; pos < other->size; pos++) { pair_t *pair = other->pairs + pos; + if (recalc_identity) { + identity = pair_list_calc_identity(list, pair->key); + if (identity == NULL) { + goto fail; + } + hash = PyObject_Hash(identity); + if (hash == -1) { + goto fail; + } + /* materialize key */ + key = pair_list_calc_key(other, pair->key, identity); + if (key == NULL) { + goto fail; + } + } else { + identity = pair->identity; + hash = pair->hash; + key = pair->key; + } if (used != NULL) { - if (_pair_list_update(list, pair->key, pair->value, used, - pair->identity, pair->hash) < 0) { + if (_pair_list_update(list, key, pair->value, used, + identity, hash) < 0) { goto fail; } } else { - if (_pair_list_add_with_hash(list, pair->identity, pair->key, - pair->value, pair->hash) < 0) { + if (_pair_list_add_with_hash(list, identity, key, + pair->value, hash) < 0) { goto fail; } } + if (recalc_identity) { + Py_CLEAR(identity); + Py_CLEAR(key); + } } return 0; fail: + if (recalc_identity) { + Py_CLEAR(identity); + Py_CLEAR(key); + } return -1; } @@ -1545,6 +1576,7 @@ fail: Py_CLEAR(key); Py_CLEAR(value); PyUnicodeWriter_Discard(writer); + return NULL; } diff --git a/contrib/python/multidict/multidict/_multilib/parser.h b/contrib/python/multidict/multidict/_multilib/parser.h index a804018cf1d..074f6fa7d9e 100644 --- a/contrib/python/multidict/multidict/_multilib/parser.h +++ b/contrib/python/multidict/multidict/_multilib/parser.h @@ -22,7 +22,6 @@ static int raise_missing_posarg(const char *fname, const char* argname) } - /* Parse FASTCALL|METH_KEYWORDS arguments as two args, the first arg is mandatory and the second one is optional. If the second arg is not passed it remains NULL pointer. diff --git a/contrib/python/multidict/multidict/_multilib/state.h b/contrib/python/multidict/multidict/_multilib/state.h new file mode 100644 index 00000000000..58110d973a7 --- /dev/null +++ b/contrib/python/multidict/multidict/_multilib/state.h @@ -0,0 +1,132 @@ +#ifndef _MULTIDICT_STATE_H +#define _MULTIDICT_STATE_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* State of the _multidict module */ +typedef struct { + PyTypeObject *IStrType; + + PyTypeObject *MultiDictType; + PyTypeObject *CIMultiDictType; + PyTypeObject *MultiDictProxyType; + PyTypeObject *CIMultiDictProxyType; + + PyTypeObject *KeysViewType; + PyTypeObject *ItemsViewType; + PyTypeObject *ValuesViewType; + + PyTypeObject *KeysIterType; + PyTypeObject *ItemsIterType; + PyTypeObject *ValuesIterType; + + PyObject *str_lower; + PyObject *str_canonical; +} mod_state; + +static inline mod_state * +get_mod_state(PyObject *mod) +{ + mod_state *state = (mod_state *)PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static inline mod_state * +get_mod_state_by_cls(PyTypeObject *cls) +{ + mod_state *state = (mod_state *)PyType_GetModuleState(cls); + assert(state != NULL); + return state; +} + + +#if PY_VERSION_HEX < 0x030b0000 +PyObject * +PyType_GetModuleByDef(PyTypeObject *tp, PyModuleDef *def) +{ + PyModuleDef * mod_def; + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + goto err; + } + PyObject *mod = NULL; + + mod = PyType_GetModule(tp); + if (mod == NULL) { + PyErr_Clear(); + } else { + mod_def = PyModule_GetDef(mod); + if (mod_def == def) { + return mod; + } + } + + PyObject *mro = tp->tp_mro; + assert(mro != NULL); + assert(PyTuple_Check(mro)); + assert(PyTuple_GET_SIZE(mro) >= 1); + assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)tp); + + Py_ssize_t n = PyTuple_GET_SIZE(mro); + for (Py_ssize_t i = 1; i < n; i++) { + PyObject *super = PyTuple_GET_ITEM(mro, i); + if (!PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { + continue; + } + mod = PyType_GetModule((PyTypeObject*)super); + if (mod == NULL) { + PyErr_Clear(); + } else { + mod_def = PyModule_GetDef(mod); + if (mod_def == def) { + return mod; + } + } + } + +err: + PyErr_Format( + PyExc_TypeError, + "PyType_GetModuleByDef: No superclass of '%s' has the given module", + tp->tp_name); + return NULL; + +} +#endif + +static PyModuleDef multidict_module; + +static inline int +get_mod_state_by_def_checked(PyObject *self, mod_state **ret) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject *mod = PyType_GetModuleByDef(tp, &multidict_module); + if (mod == NULL) { + *ret = NULL; + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + *ret = get_mod_state(mod); + return 1; +} + + +static inline mod_state * +get_mod_state_by_def(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject *mod = PyType_GetModuleByDef(tp, &multidict_module); + assert(mod != NULL); + return get_mod_state(mod); +} + + +#ifdef __cplusplus +} +#endif +#endif diff --git a/contrib/python/multidict/multidict/_multilib/views.h b/contrib/python/multidict/multidict/_multilib/views.h index 9cf002f803b..3e195f4d5b2 100644 --- a/contrib/python/multidict/multidict/_multilib/views.h +++ b/contrib/python/multidict/multidict/_multilib/views.h @@ -5,9 +5,9 @@ extern "C" { #endif -static PyTypeObject multidict_itemsview_type; -static PyTypeObject multidict_valuesview_type; -static PyTypeObject multidict_keysview_type; +#include "dict.h" +#include "pair_list.h" +#include "state.h" typedef struct { PyObject_HEAD @@ -15,6 +15,11 @@ typedef struct { } _Multidict_ViewObject; +#define Items_CheckExact(state, obj) Py_IS_TYPE(obj, state->ItemsViewType) +#define Keys_CheckExact(state, obj) Py_IS_TYPE(obj, state->KeysViewType) +#define Values_CheckExact(state, obj) Py_IS_TYPE(obj, state->ValuesViewType) + + /********** Base **********/ static inline void @@ -63,7 +68,7 @@ multidict_view_richcompare(PyObject *self, PyObject *other, int op) Py_ssize_t size = PyObject_Length(other); if (size < 0) { PyErr_Clear(); - Py_RETURN_NOTIMPLEMENTED;; + Py_RETURN_NOTIMPLEMENTED; } PyObject *iter = NULL; PyObject *item = NULL; @@ -147,7 +152,7 @@ static inline PyObject * multidict_itemsview_new(MultiDictObject *md) { _Multidict_ViewObject *mv = PyObject_GC_New( - _Multidict_ViewObject, &multidict_itemsview_type); + _Multidict_ViewObject, md->pairs.state->ItemsViewType); if (mv == NULL) { return NULL; } @@ -167,10 +172,20 @@ multidict_itemsview_iter(_Multidict_ViewObject *self) static inline PyObject * multidict_itemsview_repr(_Multidict_ViewObject *self) { + int tmp = Py_ReprEnter((PyObject *)self); + if (tmp < 0) { + return NULL; + } + if (tmp > 0) { + return PyUnicode_FromString("..."); + } PyObject *name = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "__name__"); - if (name == NULL) + if (name == NULL) { + Py_ReprLeave((PyObject *)self); return NULL; + } PyObject *ret = pair_list_repr(&self->md->pairs, name, true, true); + Py_ReprLeave((PyObject *)self); Py_CLEAR(name); return ret; } @@ -390,22 +405,25 @@ fail: static inline PyObject * multidict_itemsview_and(PyObject *lft, PyObject *rht) { - int tmp = PyObject_IsInstance(lft, (PyObject *)&multidict_itemsview_type); + mod_state * state; + int tmp = get_mod_state_by_def_checked(lft, &state); if (tmp < 0) { return NULL; - } - if (tmp > 0) { - return multidict_itemsview_and1((_Multidict_ViewObject *)lft, rht); - } else { - tmp = PyObject_IsInstance(rht, (PyObject *)&multidict_itemsview_type); + } else if (tmp == 0) { + tmp = get_mod_state_by_def_checked(rht, &state); if (tmp < 0) { return NULL; - } - if (tmp == 0) { + } else if (tmp == 0) { Py_RETURN_NOTIMPLEMENTED; } + } + assert(state != NULL); + if (Items_CheckExact(state, lft)) { + return multidict_itemsview_and1((_Multidict_ViewObject *)lft, rht); + } else if (Items_CheckExact(state, rht)) { return multidict_itemsview_and2((_Multidict_ViewObject *)rht, lft); } + Py_RETURN_NOTIMPLEMENTED; } static inline PyObject * @@ -579,22 +597,25 @@ fail: static inline PyObject * multidict_itemsview_or(PyObject *lft, PyObject *rht) { - int tmp = PyObject_IsInstance(lft, (PyObject *)&multidict_itemsview_type); + mod_state * state; + int tmp = get_mod_state_by_def_checked(lft, &state); if (tmp < 0) { return NULL; - } - if (tmp > 0) { - return multidict_itemsview_or1((_Multidict_ViewObject *)lft, rht); - } else { - tmp = PyObject_IsInstance(rht, (PyObject *)&multidict_itemsview_type); + } else if (tmp == 0) { + tmp = get_mod_state_by_def_checked(rht, &state); if (tmp < 0) { return NULL; - } - if (tmp == 0) { + } else if (tmp == 0) { Py_RETURN_NOTIMPLEMENTED; } + } + assert(state != NULL); + if (Items_CheckExact(state, lft)) { + return multidict_itemsview_or1((_Multidict_ViewObject *)lft, rht); + } else if (Items_CheckExact(state, rht)) { return multidict_itemsview_or2((_Multidict_ViewObject *)rht, lft); } + Py_RETURN_NOTIMPLEMENTED; } @@ -769,42 +790,50 @@ fail: static inline PyObject * multidict_itemsview_sub(PyObject *lft, PyObject *rht) { - int tmp = PyObject_IsInstance(lft, (PyObject *)&multidict_itemsview_type); + mod_state * state; + int tmp = get_mod_state_by_def_checked(lft, &state); if (tmp < 0) { return NULL; - } - if (tmp > 0) { - return multidict_itemsview_sub1((_Multidict_ViewObject *)lft, rht); - } else { - tmp = PyObject_IsInstance(rht, (PyObject *)&multidict_itemsview_type); + } else if (tmp == 0) { + tmp = get_mod_state_by_def_checked(rht, &state); if (tmp < 0) { return NULL; - } - if (tmp == 0) { + } else if (tmp == 0) { Py_RETURN_NOTIMPLEMENTED; } + } + assert(state != NULL); + if (Items_CheckExact(state, lft)) { + return multidict_itemsview_sub1((_Multidict_ViewObject *)lft, rht); + } else if (Items_CheckExact(state, rht)) { return multidict_itemsview_sub2((_Multidict_ViewObject *)rht, lft); } + Py_RETURN_NOTIMPLEMENTED; } static inline PyObject * multidict_itemsview_xor(_Multidict_ViewObject *self, PyObject *other) { - int tmp = PyObject_IsInstance((PyObject *)self, - (PyObject *)&multidict_itemsview_type); + mod_state * state; + int tmp = get_mod_state_by_def_checked((PyObject *)self, &state); if (tmp < 0) { - goto fail; - } - if (tmp == 0) { - tmp = PyObject_IsInstance(other, (PyObject *)&multidict_itemsview_type); + return NULL; + } else if (tmp == 0) { + tmp = get_mod_state_by_def_checked(other, &state); if (tmp < 0) { - goto fail; + return NULL; + } else if (tmp == 0) { + Py_RETURN_NOTIMPLEMENTED; } - if (tmp == 0) { + } + assert(state != NULL); + if (!Items_CheckExact(state, self)) { + if (Items_CheckExact(state, other)) { + return multidict_itemsview_xor((_Multidict_ViewObject *)other, + (PyObject *)self); + } else { Py_RETURN_NOTIMPLEMENTED; } - return multidict_itemsview_xor((_Multidict_ViewObject *)other, - (PyObject *)self); } PyObject *ret = NULL; @@ -842,13 +871,6 @@ fail: return NULL; } -static PyNumberMethods multidict_itemsview_as_number = { - .nb_subtract = (binaryfunc)multidict_itemsview_sub, - .nb_and = (binaryfunc)multidict_itemsview_and, - .nb_xor = (binaryfunc)multidict_itemsview_xor, - .nb_or = (binaryfunc)multidict_itemsview_or, -}; - static inline int multidict_itemsview_contains(_Multidict_ViewObject *self, PyObject *obj) { @@ -907,11 +929,6 @@ multidict_itemsview_contains(_Multidict_ViewObject *self, PyObject *obj) return 0; } -static PySequenceMethods multidict_itemsview_as_sequence = { - .sq_length = (lenfunc)multidict_view_len, - .sq_contains = (objobjproc)multidict_itemsview_contains, -}; - static inline PyObject * multidict_itemsview_isdisjoint(_Multidict_ViewObject *self, PyObject *other) { @@ -989,21 +1006,34 @@ static PyMethodDef multidict_itemsview_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyTypeObject multidict_itemsview_type = { - PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) - "multidict._multidict._ItemsView", /* tp_name */ - sizeof(_Multidict_ViewObject), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_view_dealloc, - .tp_repr = (reprfunc)multidict_itemsview_repr, - .tp_as_number = &multidict_itemsview_as_number, - .tp_as_sequence = &multidict_itemsview_as_sequence, - .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)multidict_view_traverse, - .tp_clear = (inquiry)multidict_view_clear, - .tp_richcompare = multidict_view_richcompare, - .tp_iter = (getiterfunc)multidict_itemsview_iter, - .tp_methods = multidict_itemsview_methods, +static PyType_Slot multidict_itemsview_slots[] = { + {Py_tp_dealloc, multidict_view_dealloc}, + {Py_tp_repr, multidict_itemsview_repr}, + + {Py_nb_subtract, multidict_itemsview_sub}, + {Py_nb_and, multidict_itemsview_and}, + {Py_nb_xor, multidict_itemsview_xor}, + {Py_nb_or, multidict_itemsview_or}, + {Py_sq_length, multidict_view_len}, + {Py_sq_contains, multidict_itemsview_contains}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_traverse, multidict_view_traverse}, + {Py_tp_clear, multidict_view_clear}, + {Py_tp_richcompare, multidict_view_richcompare}, + {Py_tp_iter, multidict_itemsview_iter}, + {Py_tp_methods, multidict_itemsview_methods}, + {0, NULL}, +}; + +static PyType_Spec multidict_itemsview_spec = { + .name = "multidict._multidict._ItemsView", + .basicsize = sizeof(_Multidict_ViewObject), + .flags = (Py_TPFLAGS_DEFAULT +#if PY_VERSION_HEX >= 0x030a0000 + | Py_TPFLAGS_IMMUTABLETYPE +#endif + | Py_TPFLAGS_HAVE_GC), + .slots = multidict_itemsview_slots, }; @@ -1013,7 +1043,7 @@ static inline PyObject * multidict_keysview_new(MultiDictObject *md) { _Multidict_ViewObject *mv = PyObject_GC_New( - _Multidict_ViewObject, &multidict_keysview_type); + _Multidict_ViewObject, md->pairs.state->KeysViewType); if (mv == NULL) { return NULL; } @@ -1034,8 +1064,9 @@ static inline PyObject * multidict_keysview_repr(_Multidict_ViewObject *self) { PyObject *name = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "__name__"); - if (name == NULL) + if (name == NULL) { return NULL; + } PyObject *ret = pair_list_repr(&self->md->pairs, name, true, false); Py_CLEAR(name); return ret; @@ -1137,22 +1168,25 @@ fail: static inline PyObject * multidict_keysview_and(PyObject *lft, PyObject *rht) { - int tmp = PyObject_IsInstance(lft, (PyObject *)&multidict_keysview_type); + mod_state * state; + int tmp = get_mod_state_by_def_checked(lft, &state); if (tmp < 0) { return NULL; - } - if (tmp > 0) { - return multidict_keysview_and1((_Multidict_ViewObject *)lft, rht); - } else { - tmp = PyObject_IsInstance(rht, (PyObject *)&multidict_keysview_type); + } else if (tmp == 0) { + tmp = get_mod_state_by_def_checked(rht, &state); if (tmp < 0) { return NULL; - } - if (tmp == 0) { + } else if (tmp == 0) { Py_RETURN_NOTIMPLEMENTED; } + } + assert(state != NULL); + if (Keys_CheckExact(state, lft)) { + return multidict_keysview_and1((_Multidict_ViewObject *)lft, rht); + } else if (Keys_CheckExact(state, rht)) { return multidict_keysview_and2((_Multidict_ViewObject *)rht, lft); } + Py_RETURN_NOTIMPLEMENTED; } static inline PyObject * @@ -1283,22 +1317,25 @@ fail: static inline PyObject * multidict_keysview_or(PyObject *lft, PyObject *rht) { - int tmp = PyObject_IsInstance(lft, (PyObject *)&multidict_keysview_type); + mod_state * state; + int tmp = get_mod_state_by_def_checked(lft, &state); if (tmp < 0) { return NULL; - } - if (tmp > 0) { - return multidict_keysview_or1((_Multidict_ViewObject *)lft, rht); - } else { - tmp = PyObject_IsInstance(rht, (PyObject *)&multidict_keysview_type); + } else if (tmp == 0) { + tmp = get_mod_state_by_def_checked(rht, &state); if (tmp < 0) { return NULL; - } - if (tmp == 0) { + } else if (tmp == 0) { Py_RETURN_NOTIMPLEMENTED; } + } + assert(state != NULL); + if (Keys_CheckExact(state, lft)) { + return multidict_keysview_or1((_Multidict_ViewObject *)lft, rht); + } else if (Keys_CheckExact(state, rht)) { return multidict_keysview_or2((_Multidict_ViewObject *)rht, lft); } + Py_RETURN_NOTIMPLEMENTED; } static inline PyObject * @@ -1399,42 +1436,50 @@ fail: static inline PyObject * multidict_keysview_sub(PyObject *lft, PyObject *rht) { - int tmp = PyObject_IsInstance(lft, (PyObject *)&multidict_keysview_type); + mod_state * state; + int tmp = get_mod_state_by_def_checked(lft, &state); if (tmp < 0) { return NULL; - } - if (tmp > 0) { - return multidict_keysview_sub1((_Multidict_ViewObject *)lft, rht); - } else { - tmp = PyObject_IsInstance(rht, (PyObject *)&multidict_keysview_type); + } else if (tmp == 0) { + tmp = get_mod_state_by_def_checked(rht, &state); if (tmp < 0) { return NULL; - } - if (tmp == 0) { + } else if (tmp == 0) { Py_RETURN_NOTIMPLEMENTED; } + } + assert(state != NULL); + if (Keys_CheckExact(state, lft)) { + return multidict_keysview_sub1((_Multidict_ViewObject *)lft, rht); + } else if (Keys_CheckExact(state, rht)) { return multidict_keysview_sub2((_Multidict_ViewObject *)rht, lft); } + Py_RETURN_NOTIMPLEMENTED; } static inline PyObject * multidict_keysview_xor(_Multidict_ViewObject *self, PyObject *other) { - int tmp = PyObject_IsInstance((PyObject *)self, - (PyObject *)&multidict_keysview_type); + mod_state * state; + int tmp = get_mod_state_by_def_checked((PyObject *)self, &state); if (tmp < 0) { - goto fail; - } - if (tmp == 0) { - tmp = PyObject_IsInstance(other, (PyObject *)&multidict_keysview_type); + return NULL; + } else if (tmp == 0) { + tmp = get_mod_state_by_def_checked(other, &state); if (tmp < 0) { - goto fail; + return NULL; + } else if (tmp == 0) { + Py_RETURN_NOTIMPLEMENTED; } - if (tmp == 0) { + } + assert(state != NULL); + if (!Keys_CheckExact(state, self)) { + if (Keys_CheckExact(state, other)) { + return multidict_keysview_xor((_Multidict_ViewObject *)other, + (PyObject *)self); + } else { Py_RETURN_NOTIMPLEMENTED; } - return multidict_keysview_xor((_Multidict_ViewObject *)other, - (PyObject *)self); } PyObject *ret = NULL; @@ -1472,24 +1517,12 @@ fail: return NULL; } -static PyNumberMethods multidict_keysview_as_number = { - .nb_subtract = (binaryfunc)multidict_keysview_sub, - .nb_and = (binaryfunc)multidict_keysview_and, - .nb_xor = (binaryfunc)multidict_keysview_xor, - .nb_or = (binaryfunc)multidict_keysview_or, -}; - static inline int multidict_keysview_contains(_Multidict_ViewObject *self, PyObject *key) { return pair_list_contains(&self->md->pairs, key, NULL); } -static PySequenceMethods multidict_keysview_as_sequence = { - .sq_length = (lenfunc)multidict_view_len, - .sq_contains = (objobjproc)multidict_keysview_contains, -}; - static inline PyObject * multidict_keysview_isdisjoint(_Multidict_ViewObject *self, PyObject *other) { @@ -1527,23 +1560,35 @@ static PyMethodDef multidict_keysview_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyTypeObject multidict_keysview_type = { - PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) - "multidict._multidict._KeysView", /* tp_name */ - sizeof(_Multidict_ViewObject), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_view_dealloc, - .tp_repr = (reprfunc)multidict_keysview_repr, - .tp_as_number = &multidict_keysview_as_number, - .tp_as_sequence = &multidict_keysview_as_sequence, - .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)multidict_view_traverse, - .tp_clear = (inquiry)multidict_view_clear, - .tp_richcompare = multidict_view_richcompare, - .tp_iter = (getiterfunc)multidict_keysview_iter, - .tp_methods = multidict_keysview_methods, +static PyType_Slot multidict_keysview_slots[] = { + {Py_tp_dealloc, multidict_view_dealloc}, + {Py_tp_repr, multidict_keysview_repr}, + + {Py_nb_subtract, multidict_keysview_sub}, + {Py_nb_and, multidict_keysview_and}, + {Py_nb_xor, multidict_keysview_xor}, + {Py_nb_or, multidict_keysview_or}, + {Py_sq_length, multidict_view_len}, + {Py_sq_contains, multidict_keysview_contains}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_traverse, multidict_view_traverse}, + {Py_tp_clear, multidict_view_clear}, + {Py_tp_richcompare, multidict_view_richcompare}, + {Py_tp_iter, multidict_keysview_iter}, + {Py_tp_methods, multidict_keysview_methods}, + {0, NULL}, }; +static PyType_Spec multidict_keysview_spec = { + .name = "multidict._multidict._KeysView", + .basicsize = sizeof(_Multidict_ViewObject), + .flags = (Py_TPFLAGS_DEFAULT +#if PY_VERSION_HEX >= 0x030a0000 + | Py_TPFLAGS_IMMUTABLETYPE +#endif + | Py_TPFLAGS_HAVE_GC), + .slots = multidict_keysview_slots, +}; /********** Values **********/ @@ -1551,7 +1596,7 @@ static inline PyObject * multidict_valuesview_new(MultiDictObject *md) { _Multidict_ViewObject *mv = PyObject_GC_New( - _Multidict_ViewObject, &multidict_valuesview_type); + _Multidict_ViewObject, md->pairs.state->ValuesViewType); if (mv == NULL) { return NULL; } @@ -1571,46 +1616,71 @@ multidict_valuesview_iter(_Multidict_ViewObject *self) static inline PyObject * multidict_valuesview_repr(_Multidict_ViewObject *self) { + int tmp = Py_ReprEnter((PyObject *)self); + if (tmp < 0) { + return NULL; + } + if (tmp > 0) { + return PyUnicode_FromString("..."); + } PyObject *name = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "__name__"); - if (name == NULL) + if (name == NULL) { + Py_ReprLeave((PyObject *)self); return NULL; + } PyObject *ret = pair_list_repr(&self->md->pairs, name, false, true); + Py_ReprLeave((PyObject *)self); Py_CLEAR(name); return ret; } -static PySequenceMethods multidict_valuesview_as_sequence = { - .sq_length = (lenfunc)multidict_view_len, +static PyType_Slot multidict_valuesview_slots[] = { + {Py_tp_dealloc, multidict_view_dealloc}, + {Py_tp_repr, multidict_valuesview_repr}, + + {Py_sq_length, multidict_view_len}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_traverse, multidict_view_traverse}, + {Py_tp_clear, multidict_view_clear}, + {Py_tp_iter, multidict_valuesview_iter}, + {0, NULL}, }; -static PyTypeObject multidict_valuesview_type = { - PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) - "multidict._multidict._ValuesView", /* tp_name */ - sizeof(_Multidict_ViewObject), /* tp_basicsize */ - .tp_dealloc = (destructor)multidict_view_dealloc, - .tp_repr = (reprfunc)multidict_valuesview_repr, - .tp_as_sequence = &multidict_valuesview_as_sequence, - .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)multidict_view_traverse, - .tp_clear = (inquiry)multidict_view_clear, - .tp_iter = (getiterfunc)multidict_valuesview_iter, +static PyType_Spec multidict_valuesview_spec = { + .name = "multidict._multidict._ValuesView", + .basicsize = sizeof(_Multidict_ViewObject), + .flags = (Py_TPFLAGS_DEFAULT +#if PY_VERSION_HEX >= 0x030a0000 + | Py_TPFLAGS_IMMUTABLETYPE +#endif + | Py_TPFLAGS_HAVE_GC), + .slots = multidict_valuesview_slots, }; static inline int -multidict_views_init(void) +multidict_views_init(PyObject *module, mod_state *state) { - if (PyType_Ready(&multidict_itemsview_type) < 0 || - PyType_Ready(&multidict_valuesview_type) < 0 || - PyType_Ready(&multidict_keysview_type) < 0) - { - goto fail; + PyObject * tmp; + tmp = PyType_FromModuleAndSpec(module, &multidict_itemsview_spec, NULL); + if (tmp == NULL) { + return -1; } + state->ItemsViewType = (PyTypeObject *)tmp; + + tmp = PyType_FromModuleAndSpec(module, &multidict_valuesview_spec, NULL); + if (tmp == NULL) { + return -1; + } + state->ValuesViewType = (PyTypeObject *)tmp; + + tmp = PyType_FromModuleAndSpec(module, &multidict_keysview_spec, NULL); + if (tmp == NULL) { + return -1; + } + state->KeysViewType = (PyTypeObject *)tmp; return 0; -fail: - return -1; } #ifdef __cplusplus diff --git a/contrib/python/multidict/tests/isolated/multidict_extend_dict.py b/contrib/python/multidict/tests/isolated/multidict_extend_dict.py new file mode 100644 index 00000000000..c7fc86d237f --- /dev/null +++ b/contrib/python/multidict/tests/isolated/multidict_extend_dict.py @@ -0,0 +1,27 @@ +import gc +import sys +from typing import Any + +import objgraph # type: ignore[import-untyped] + +from multidict import MultiDict + + +class NoLeakDict(dict[str, Any]): + """A subclassed dict to make it easier to test for leaks.""" + + +def _run_isolated_case() -> None: + md: MultiDict[str] = MultiDict() + for _ in range(100): + md.update(NoLeakDict()) + del md + gc.collect() + + leaked = len(objgraph.by_type("NoLeakDict")) + print(f"{leaked} instances of NoLeakDict not collected by GC") + sys.exit(1 if leaked else 0) + + +if __name__ == "__main__": + _run_isolated_case() diff --git a/contrib/python/multidict/tests/isolated/multidict_extend_multidict.py b/contrib/python/multidict/tests/isolated/multidict_extend_multidict.py new file mode 100644 index 00000000000..4c98972bf42 --- /dev/null +++ b/contrib/python/multidict/tests/isolated/multidict_extend_multidict.py @@ -0,0 +1,21 @@ +import gc +import sys + +import objgraph # type: ignore[import-untyped] + +from multidict import MultiDict + + +def _run_isolated_case() -> None: + md: MultiDict[str] = MultiDict() + for _ in range(100): + md.extend(MultiDict()) + del md + gc.collect() + leaked = len(objgraph.by_type("MultiDict")) + print(f"{leaked} instances of MultiDict not collected by GC") + sys.exit(1 if leaked else 0) + + +if __name__ == "__main__": + _run_isolated_case() diff --git a/contrib/python/multidict/tests/isolated/multidict_extend_tuple.py b/contrib/python/multidict/tests/isolated/multidict_extend_tuple.py new file mode 100644 index 00000000000..d96e922c316 --- /dev/null +++ b/contrib/python/multidict/tests/isolated/multidict_extend_tuple.py @@ -0,0 +1,27 @@ +import gc +import sys +from typing import Any + +import objgraph # type: ignore[import-untyped] + +from multidict import MultiDict + + +class NotLeakTuple(tuple[Any, ...]): + """A subclassed tuple to make it easier to test for leaks.""" + + +def _run_isolated_case() -> None: + md: MultiDict[str] = MultiDict() + for _ in range(100): + md.extend(NotLeakTuple()) + del md + gc.collect() + + leaked = len(objgraph.by_type("NotLeakTuple")) + print(f"{leaked} instances of NotLeakTuple not collected by GC") + sys.exit(1 if leaked else 0) + + +if __name__ == "__main__": + _run_isolated_case() diff --git a/contrib/python/multidict/tests/isolated/multidict_update_multidict.py b/contrib/python/multidict/tests/isolated/multidict_update_multidict.py new file mode 100644 index 00000000000..4c98972bf42 --- /dev/null +++ b/contrib/python/multidict/tests/isolated/multidict_update_multidict.py @@ -0,0 +1,21 @@ +import gc +import sys + +import objgraph # type: ignore[import-untyped] + +from multidict import MultiDict + + +def _run_isolated_case() -> None: + md: MultiDict[str] = MultiDict() + for _ in range(100): + md.extend(MultiDict()) + del md + gc.collect() + leaked = len(objgraph.by_type("MultiDict")) + print(f"{leaked} instances of MultiDict not collected by GC") + sys.exit(1 if leaked else 0) + + +if __name__ == "__main__": + _run_isolated_case() diff --git a/contrib/python/multidict/tests/test_leaks.py b/contrib/python/multidict/tests/test_leaks.py new file mode 100644 index 00000000000..ded7cf065b0 --- /dev/null +++ b/contrib/python/multidict/tests/test_leaks.py @@ -0,0 +1,31 @@ +import pathlib +import platform +import subprocess +import sys + +import pytest + +IS_PYPY = platform.python_implementation() == "PyPy" + + +@pytest.mark.parametrize( + ("script"), + ( + "multidict_extend_dict.py", + "multidict_extend_multidict.py", + "multidict_extend_tuple.py", + "multidict_update_multidict.py", + ), +) +@pytest.mark.leaks +@pytest.mark.skipif(IS_PYPY, reason="leak testing is not supported on PyPy") +def test_leak(script: str) -> None: + """Run isolated leak test script and check for leaks.""" + leak_test_script = pathlib.Path(__file__).parent.joinpath("isolated", script) + + subprocess.run( + [sys.executable, "-u", str(leak_test_script)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) diff --git a/contrib/python/multidict/tests/test_multidict.py b/contrib/python/multidict/tests/test_multidict.py index 48ad479deac..aa59d6adef4 100644 --- a/contrib/python/multidict/tests/test_multidict.py +++ b/contrib/python/multidict/tests/test_multidict.py @@ -2,6 +2,7 @@ from __future__ import annotations import gc import operator +import platform import sys import weakref from collections import deque @@ -18,9 +19,11 @@ from multidict import ( MultiDictProxy, MultiMapping, MutableMultiMapping, + istr, ) _T = TypeVar("_T") +IS_PYPY = platform.python_implementation() == "PyPy" def chained_callable( @@ -36,8 +39,6 @@ def chained_callable( *args: object, **kwargs: object, ) -> MultiMapping[int | str] | MutableMultiMapping[int | str]: - nonlocal callables - callable_chain = (getattr(module, name) for name in callables) first_callable = next(callable_chain) @@ -725,6 +726,17 @@ class TestMultiDict(BaseMultiDictTest): assert str(d) == "<%s('key': 'one', 'key': 'two')>" % _cls.__name__ + def test__repr___recursive( + self, any_multidict_class: type[MultiDict[object]] + ) -> None: + d = any_multidict_class() + _cls = type(d) + + d = any_multidict_class() + d["key"] = d + + assert str(d) == "<%s('key': ...)>" % _cls.__name__ + def test_getall(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") @@ -757,6 +769,14 @@ class TestMultiDict(BaseMultiDictTest): expected = "<_ItemsView('key': 'value1', 'key': 'value2')>" assert repr(d.items()) == expected + def test_items__repr__recursive( + self, any_multidict_class: type[MultiDict[object]] + ) -> None: + d = any_multidict_class() + d["key"] = d.items() + expected = "<_ItemsView('key': <_ItemsView('key': ...)>)>" + assert repr(d.items()) == expected + def test_keys__repr__(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") assert repr(d.keys()) == "<_KeysView('key', 'key')>" @@ -765,6 +785,13 @@ class TestMultiDict(BaseMultiDictTest): d = cls([("key", "value1")], key="value2") assert repr(d.values()) == "<_ValuesView('value1', 'value2')>" + def test_values__repr__recursive( + self, any_multidict_class: type[MultiDict[object]] + ) -> None: + d = any_multidict_class() + d["key"] = d.values() + assert repr(d.values()) == "<_ValuesView(<_ValuesView(...)>)>" + class TestCIMultiDict(BaseMultiDictTest): @pytest.fixture( @@ -1189,3 +1216,110 @@ class TestCIMultiDict(BaseMultiDictTest): ) -> None: d = cls([("KEY", "one")]) assert d.items().isdisjoint(arg) == expected + + +def test_create_multidict_from_existing_multidict_new_pairs() -> None: + """Test creating a MultiDict from an existing one does not mutate the original.""" + original = MultiDict([("h1", "header1"), ("h2", "header2"), ("h3", "header3")]) + new = MultiDict(original, h4="header4") + assert "h4" in new + assert "h4" not in original + + +def test_convert_multidict_to_cimultidict_and_back( + case_sensitive_multidict_class: type[MultiDict[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[istr], +) -> None: + """Test conversion from MultiDict to CIMultiDict.""" + start_as_md = case_sensitive_multidict_class( + [("KEY", "value1"), ("key2", "value2")] + ) + assert start_as_md.get("KEY") == "value1" + assert start_as_md["KEY"] == "value1" + assert start_as_md.get("key2") == "value2" + assert start_as_md["key2"] == "value2" + start_as_cimd = case_insensitive_multidict_class( + [("KEY", "value1"), ("key2", "value2")] + ) + assert start_as_cimd.get("key") == "value1" + assert start_as_cimd["key"] == "value1" + assert start_as_cimd.get("key2") == "value2" + assert start_as_cimd["key2"] == "value2" + converted_to_ci = case_insensitive_multidict_class(start_as_md) + assert converted_to_ci.get("key") == "value1" + assert converted_to_ci["key"] == "value1" + assert converted_to_ci.get("key2") == "value2" + assert converted_to_ci["key2"] == "value2" + converted_to_md = case_sensitive_multidict_class(converted_to_ci) + assert all(type(k) is case_insensitive_str_class for k in converted_to_ci.keys()) + assert converted_to_md.get("KEY") == "value1" + assert converted_to_md["KEY"] == "value1" + assert converted_to_md.get("key2") == "value2" + assert converted_to_md["key2"] == "value2" + + +def test_convert_multidict_to_cimultidict_eq( + case_sensitive_multidict_class: type[MultiDict[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], +) -> None: + """Test compare after conversion from MultiDict to CIMultiDict.""" + original = case_sensitive_multidict_class( + [("h1", "header1"), ("h2", "header2"), ("h3", "header3")] + ) + assert case_insensitive_multidict_class( + original + ) == case_insensitive_multidict_class( + [("H1", "header1"), ("H2", "header2"), ("H3", "header3")] + ) + + +@pytest.mark.skipif(IS_PYPY, reason="getrefcount is not supported on PyPy") +def test_extend_does_not_alter_refcount( + case_sensitive_multidict_class: type[MultiDict[str]], +) -> None: + """Test that extending a MultiDict with a MultiDict does not alter the refcount of the original.""" + original = case_sensitive_multidict_class([("h1", "header1")]) + new = case_sensitive_multidict_class([("h2", "header2")]) + original_refcount = sys.getrefcount(original) + new.extend(original) + assert sys.getrefcount(original) == original_refcount + + +@pytest.mark.skipif(IS_PYPY, reason="getrefcount is not supported on PyPy") +def test_update_does_not_alter_refcount( + case_sensitive_multidict_class: type[MultiDict[str]], +) -> None: + """Test that updating a MultiDict with a MultiDict does not alter the refcount of the original.""" + original = case_sensitive_multidict_class([("h1", "header1")]) + new = case_sensitive_multidict_class([("h2", "header2")]) + original_refcount = sys.getrefcount(original) + new.update(original) + assert sys.getrefcount(original) == original_refcount + + +@pytest.mark.skipif(IS_PYPY, reason="getrefcount is not supported on PyPy") +def test_init_does_not_alter_refcount( + case_sensitive_multidict_class: type[MultiDict[str]], +) -> None: + """Test that initializing a MultiDict with a MultiDict does not alter the refcount of the original.""" + original = case_sensitive_multidict_class([("h1", "header1")]) + original_refcount = sys.getrefcount(original) + case_sensitive_multidict_class(original) + assert sys.getrefcount(original) == original_refcount + + +def test_subclassed_multidict( + any_multidict_class: type[MultiDict[str]], +) -> None: + """Test that subclassed MultiDicts work as expected.""" + class SubclassedMultiDict(any_multidict_class): # type: ignore[valid-type, misc] + """Subclassed MultiDict.""" + + d1 = SubclassedMultiDict([("key", "value1")]) + d2 = SubclassedMultiDict([("key", "value2")]) + d3 = SubclassedMultiDict([("key", "value1")]) + assert d1 != d2 + assert d1 == d3 + assert d1 == SubclassedMultiDict([("key", "value1")]) + assert d1 != SubclassedMultiDict([("key", "value2")]) diff --git a/contrib/python/multidict/tests/test_multidict_benchmarks.py b/contrib/python/multidict/tests/test_multidict_benchmarks.py index bed9faa4038..a7a6a76e728 100644 --- a/contrib/python/multidict/tests/test_multidict_benchmarks.py +++ b/contrib/python/multidict/tests/test_multidict_benchmarks.py @@ -255,17 +255,19 @@ def test_cimultidict_delitem_istr( def test_multidict_getall_str_hit( benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] ) -> None: - md = any_multidict_class(("all", str(i)) for i in range(100)) + md = any_multidict_class((f"key{j}", str(f"{i}-{j}")) + for i in range(20) for j in range(5)) @benchmark def _run() -> None: - md.getall("all") + md.getall("key0") def test_multidict_getall_str_miss( benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] ) -> None: - md = any_multidict_class(("all", str(i)) for i in range(100)) + md = any_multidict_class((f"key{j}", str(f"{i}-{j}")) + for i in range(20) for j in range(5)) @benchmark def _run() -> None: @@ -276,8 +278,9 @@ def test_cimultidict_getall_istr_hit( benchmark: BenchmarkFixture, case_insensitive_multidict_class: Type[CIMultiDict[istr]], ) -> None: - all_istr = istr("all") - md = case_insensitive_multidict_class((all_istr, istr(i)) for i in range(100)) + all_istr = istr("key0") + md = case_insensitive_multidict_class((f"key{j}", istr(f"{i}-{j}")) + for i in range(20) for j in range(5)) @benchmark def _run() -> None: @@ -288,9 +291,9 @@ def test_cimultidict_getall_istr_miss( benchmark: BenchmarkFixture, case_insensitive_multidict_class: Type[CIMultiDict[istr]], ) -> None: - all_istr = istr("all") miss_istr = istr("miss") - md = case_insensitive_multidict_class((all_istr, istr(i)) for i in range(100)) + md = case_insensitive_multidict_class((istr(f"key{j}"), istr(f"{i}-{j}")) + for i in range(20) for j in range(5)) @benchmark def _run() -> None: @@ -565,6 +568,7 @@ def test_iterate_multidict( for _ in md: pass + def test_iterate_multidict_keys( benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] ) -> None: @@ -588,6 +592,7 @@ def test_iterate_multidict_values( for _ in md.values(): pass + def test_iterate_multidict_items( benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] ) -> None: diff --git a/contrib/python/multidict/tests/test_views_benchmarks.py b/contrib/python/multidict/tests/test_views_benchmarks.py index 6290b31f445..7f1b9fff995 100644 --- a/contrib/python/multidict/tests/test_views_benchmarks.py +++ b/contrib/python/multidict/tests/test_views_benchmarks.py @@ -7,7 +7,9 @@ from pytest_codspeed import BenchmarkFixture from multidict import MultiDict -def test_keys_view_equals(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_equals( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) @@ -16,7 +18,9 @@ def test_keys_view_equals(benchmark: BenchmarkFixture, any_multidict_class: Type assert md1.keys() == md2.keys() -def test_keys_view_not_equals(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_not_equals( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(20, 120)}) @@ -25,7 +29,9 @@ def test_keys_view_not_equals(benchmark: BenchmarkFixture, any_multidict_class: assert md1.keys() != md2.keys() -def test_keys_view_more(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_more( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) s = {str(i) for i in range(50)} @@ -34,7 +40,9 @@ def test_keys_view_more(benchmark: BenchmarkFixture, any_multidict_class: Type[M assert md.keys() > s -def test_keys_view_more_or_equal(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_more_or_equal( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) s = {str(i) for i in range(100)} @@ -43,7 +51,9 @@ def test_keys_view_more_or_equal(benchmark: BenchmarkFixture, any_multidict_clas assert md.keys() >= s -def test_keys_view_less(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_less( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) s = {str(i) for i in range(150)} @@ -52,7 +62,9 @@ def test_keys_view_less(benchmark: BenchmarkFixture, any_multidict_class: Type[M assert md.keys() < s -def test_keys_view_less_or_equal(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_less_or_equal( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) s = {str(i) for i in range(100)} @@ -61,7 +73,9 @@ def test_keys_view_less_or_equal(benchmark: BenchmarkFixture, any_multidict_clas assert md.keys() <= s -def test_keys_view_and(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_and( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(50, 150)}) @@ -70,7 +84,9 @@ def test_keys_view_and(benchmark: BenchmarkFixture, any_multidict_class: Type[Mu assert len(md1.keys() & md2.keys()) == 50 -def test_keys_view_or(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_or( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(50, 150)}) @@ -79,7 +95,9 @@ def test_keys_view_or(benchmark: BenchmarkFixture, any_multidict_class: Type[Mul assert len(md1.keys() | md2.keys()) == 150 -def test_keys_view_sub(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_sub( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(50, 150)}) @@ -88,7 +106,9 @@ def test_keys_view_sub(benchmark: BenchmarkFixture, any_multidict_class: Type[Mu assert len(md1.keys() - md2.keys()) == 50 -def test_keys_view_xor(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_xor( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(50, 150)}) @@ -97,7 +117,9 @@ def test_keys_view_xor(benchmark: BenchmarkFixture, any_multidict_class: Type[Mu assert len(md1.keys() ^ md2.keys()) == 100 -def test_keys_view_is_disjoint(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_is_disjoint( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100, 200)}) @@ -106,7 +128,9 @@ def test_keys_view_is_disjoint(benchmark: BenchmarkFixture, any_multidict_class: assert md1.keys().isdisjoint(md2.keys()) -def test_keys_view_repr(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_keys_view_repr( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) @benchmark @@ -114,7 +138,9 @@ def test_keys_view_repr(benchmark: BenchmarkFixture, any_multidict_class: Type[M repr(md.keys()) -def test_items_view_equals(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_equals( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) @@ -123,7 +149,9 @@ def test_items_view_equals(benchmark: BenchmarkFixture, any_multidict_class: Typ assert md1.items() == md2.items() -def test_items_view_not_equals(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_not_equals( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(20, 120)}) @@ -132,7 +160,9 @@ def test_items_view_not_equals(benchmark: BenchmarkFixture, any_multidict_class: assert md1.items() != md2.items() -def test_items_view_more(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_more( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) s = {(str(i), str(i)) for i in range(50)} @@ -141,7 +171,9 @@ def test_items_view_more(benchmark: BenchmarkFixture, any_multidict_class: Type[ assert md.items() > s -def test_items_view_more_or_equal(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_more_or_equal( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) s = {(str(i), str(i)) for i in range(100)} @@ -150,7 +182,9 @@ def test_items_view_more_or_equal(benchmark: BenchmarkFixture, any_multidict_cla assert md.items() >= s -def test_items_view_less(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_less( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) s = {(str(i), str(i)) for i in range(150)} @@ -159,7 +193,9 @@ def test_items_view_less(benchmark: BenchmarkFixture, any_multidict_class: Type[ assert md.items() < s -def test_items_view_less_or_equal(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_less_or_equal( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) s = {(str(i), str(i)) for i in range(100)} @@ -168,7 +204,9 @@ def test_items_view_less_or_equal(benchmark: BenchmarkFixture, any_multidict_cla assert md.items() <= s -def test_items_view_and(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_and( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(50, 150)}) @@ -177,7 +215,9 @@ def test_items_view_and(benchmark: BenchmarkFixture, any_multidict_class: Type[M assert len(md1.items() & md2.items()) == 50 -def test_items_view_or(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_or( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(50, 150)}) @@ -186,7 +226,9 @@ def test_items_view_or(benchmark: BenchmarkFixture, any_multidict_class: Type[Mu assert len(md1.items() | md2.items()) == 150 -def test_items_view_sub(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_sub( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(50, 150)}) @@ -195,7 +237,9 @@ def test_items_view_sub(benchmark: BenchmarkFixture, any_multidict_class: Type[M assert len(md1.items() - md2.items()) == 50 -def test_items_view_xor(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_xor( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(50, 150)}) @@ -204,7 +248,9 @@ def test_items_view_xor(benchmark: BenchmarkFixture, any_multidict_class: Type[M assert len(md1.items() ^ md2.items()) == 100 -def test_items_view_is_disjoint(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_is_disjoint( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md1: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) md2: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100, 200)}) @@ -213,7 +259,9 @@ def test_items_view_is_disjoint(benchmark: BenchmarkFixture, any_multidict_class assert md1.items().isdisjoint(md2.items()) -def test_items_view_repr(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_items_view_repr( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) @benchmark @@ -221,7 +269,9 @@ def test_items_view_repr(benchmark: BenchmarkFixture, any_multidict_class: Type[ repr(md.items()) -def test_values_view_repr(benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]]) -> None: +def test_values_view_repr( + benchmark: BenchmarkFixture, any_multidict_class: Type[MultiDict[str]] +) -> None: md: MultiDict[str] = any_multidict_class({str(i): str(i) for i in range(100)}) @benchmark diff --git a/contrib/python/multidict/ya.make b/contrib/python/multidict/ya.make index ed3eec6faaf..ad0d3d0777c 100644 --- a/contrib/python/multidict/ya.make +++ b/contrib/python/multidict/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.3.0) +VERSION(6.4.3) LICENSE(Apache-2.0) diff --git a/library/cpp/protobuf/json/ut/proto2json_ut.cpp b/library/cpp/protobuf/json/ut/proto2json_ut.cpp index f5bcfac49db..ce7d119be72 100644 --- a/library/cpp/protobuf/json/ut/proto2json_ut.cpp +++ b/library/cpp/protobuf/json/ut/proto2json_ut.cpp @@ -968,10 +968,10 @@ Y_UNIT_TEST(TestMapAsObject) { auto& items = *proto.MutableItems(); items["key1"] = "value1"; - items["key2"] = "value2"; + items[""] = "value2"; items["key3"] = "value3"; - TString modelStr(R"_({"Items":{"key3":"value3","key2":"value2","key1":"value1"}})_"); + TString modelStr(R"_({"Items":{"key3":"value3","":"value2","key1":"value1"}})_"); TStringStream jsonStr; TProto2JsonConfig config; diff --git a/library/cpp/yt/string/format-inl.h b/library/cpp/yt/string/format-inl.h index 01214399e8e..5b49fc5925c 100644 --- a/library/cpp/yt/string/format-inl.h +++ b/library/cpp/yt/string/format-inl.h @@ -316,7 +316,7 @@ typename TFormattableView<TRange, TFormatter>::TEnd TFormattableView<TRange, TFo } template <class TRange, class TFormatter> -TFormattableView<TRange, TFormatter> MakeFormattableView( +TFormattableView<TRange, std::decay_t<TFormatter>> MakeFormattableView( const TRange& range, TFormatter&& formatter) { @@ -324,7 +324,7 @@ TFormattableView<TRange, TFormatter> MakeFormattableView( } template <class TRange, class TFormatter> -TFormattableView<TRange, TFormatter> MakeShrunkFormattableView( +TFormattableView<TRange, std::decay_t<TFormatter>> MakeShrunkFormattableView( const TRange& range, TFormatter&& formatter, size_t limit) diff --git a/library/cpp/yt/string/format.h b/library/cpp/yt/string/format.h index d15127fae05..fa45f39e5ac 100644 --- a/library/cpp/yt/string/format.h +++ b/library/cpp/yt/string/format.h @@ -91,12 +91,12 @@ struct TFormattableView //! Annotates a given #range with #formatter to be applied to each item. template <class TRange, class TFormatter> -TFormattableView<TRange, TFormatter> MakeFormattableView( +TFormattableView<TRange, std::decay_t<TFormatter>> MakeFormattableView( const TRange& range, TFormatter&& formatter); template <class TRange, class TFormatter> -TFormattableView<TRange, TFormatter> MakeShrunkFormattableView( +TFormattableView<TRange, std::decay_t<TFormatter>> MakeShrunkFormattableView( const TRange& range, TFormatter&& formatter, size_t limit); diff --git a/library/cpp/yt/string/unittests/format_ut.cpp b/library/cpp/yt/string/unittests/format_ut.cpp index e2e23c737c2..a6c5ef6837f 100644 --- a/library/cpp/yt/string/unittests/format_ut.cpp +++ b/library/cpp/yt/string/unittests/format_ut.cpp @@ -267,6 +267,19 @@ TEST(TFormatTest, LazyMultiValueFormatter) EXPECT_EQ("int: 1, string: hello, range: [1, 2, 3]", Format("%v", lazyFormatter)); } +TEST(TFormatTest, ReusableLambdaFormatter) +{ + auto formatter = [&] (auto* builder, int value) { + builder->AppendFormat("%v", value); + }; + + std::vector<int> range1{1, 2, 3}; + EXPECT_EQ("[1, 2, 3]", Format("%v", MakeFormattableView(range1, formatter))); + + std::vector<int> range2{4, 5, 6}; + EXPECT_EQ("[4, 5, 6]", Format("%v", MakeFormattableView(range2, formatter))); +} + TEST(TFormatTest, VectorArg) { std::vector<TString> params = {"a", "b", "c"}; diff --git a/yql/essentials/core/common_opt/yql_co_flow2.cpp b/yql/essentials/core/common_opt/yql_co_flow2.cpp index 610bde15c60..7d521b2ac47 100644 --- a/yql/essentials/core/common_opt/yql_co_flow2.cpp +++ b/yql/essentials/core/common_opt/yql_co_flow2.cpp @@ -642,23 +642,100 @@ bool IsFlatmapSuitableForPullUpOverEqiuJoin(const TCoFlatMapBase& flatMap, return true; } -bool IsInputSuitableForPullingOverEquiJoin(const TCoEquiJoinInput& input, - const THashMap<TStringBuf, THashSet<TStringBuf>>& joinKeysByLabel, - THashMap<TStringBuf, TStringBuf>& renames, TOptimizeContext& optCtx) -{ - renames.clear(); - YQL_ENSURE(input.Scope().Ref().IsAtom()); - if (!optCtx.IsSingleUsage(input)) { +bool IsRenamingOrPassthroughFlatMapLabelList(const TCoFlatMapBase& flatMap, THashMap<TStringBuf, THashMap<TStringBuf, TStringBuf>>& renamesByLabel, + THashMap<TStringBuf, THashSet<TStringBuf>>& outputMembersByLabel, bool& isIdentity) { + isIdentity = false; + auto body = flatMap.Lambda().Body(); + auto arg = flatMap.Lambda().Args().Arg(0); + + if (!IsJustOrSingleAsList(body.Ref())) { return false; } - auto maybeFlatMap = TMaybeNode<TCoFlatMapBase>(input.List().Ptr()); - if (!maybeFlatMap) { + TExprBase outItem(body.Ref().ChildPtr(0)); + if (outItem.Raw() == arg.Raw()) { + isIdentity = true; + return true; + } + + if (auto maybeStruct = outItem.Maybe<TCoAsStruct>()) { + for (auto child : maybeStruct.Cast()) { + auto tuple = child.Cast<TCoNameValueTuple>(); + auto value = tuple.Value(); + auto outMemberName = tuple.Name().Value(); + YQL_ENSURE(outMemberName.find(".") != TString::npos); + TStringBuf tableName; + TStringBuf columnName; + SplitTableName(outMemberName, tableName, columnName); + YQL_ENSURE(outputMembersByLabel[tableName].insert(columnName).second); + + if (auto maybeMember = value.Maybe<TCoMember>()) { + auto member = maybeMember.Cast(); + if (member.Struct().Raw() == arg.Raw()) { + TStringBuf oldName = member.Name().Value(); + TStringBuf newName = tuple.Name().Value(); + YQL_ENSURE(oldName.find(".") != TString::npos && newName.find(".") != TString::npos); + + TStringBuf oldTableName; + TStringBuf oldColumnName; + SplitTableName(oldName, oldTableName, oldColumnName); + + TStringBuf newTableName; + TStringBuf newColumnName; + SplitTableName(newName, newTableName, newColumnName); + + YQL_ENSURE(oldTableName == newTableName); + renamesByLabel[oldTableName].insert({oldColumnName, newColumnName}); + } + } + } + return true; + } + + return false; +} + +bool IsFlatmapSuitableForPullUpOverEquiJoin(const TCoFlatMapBase& flatMap, TVector<TStringBuf>& labels, + THashMap<TStringBuf, THashMap<TStringBuf, TStringBuf>>& renamesByLabel, TOptimizeContext& optCtx) { + if ((flatMap.Lambda().Args().Arg(0).Ref().IsUsedInDependsOn()) || (!SilentGetSequenceItemType(flatMap.Input().Ref(), false)) || + (!optCtx.IsSingleUsage(flatMap)) || (IsTablePropsDependent(flatMap.Lambda().Body().Ref()))) { return false; } - const TStringBuf label = input.Scope().Ref().Content(); - return IsFlatmapSuitableForPullUpOverEqiuJoin(maybeFlatMap.Cast(), label, joinKeysByLabel, renames, optCtx); + bool isIdentity = false; + THashMap<TStringBuf, THashSet<TStringBuf>> outputMembersByLabel; + if (!IsRenamingOrPassthroughFlatMapLabelList(flatMap, renamesByLabel, outputMembersByLabel, isIdentity)) { + return false; + } + + if (isIdentity) { + return false; + } + + // Check if all renames are identical. + bool renamesAreIdentical = true; + for (const auto& label : labels) { + if (!renamesByLabel.contains(label)) { + continue; + } + const auto& renames = renamesByLabel[label]; + for (auto it = renames.begin(); it != renames.end(); ++it) { + if (it->first != it->second) { + renamesAreIdentical = false; + break; + } + } + } + + // If all renames are identical we can proceed futher, column projection semantics. + if (renamesAreIdentical) { + for (const auto& label : labels) { + renamesByLabel[label].clear(); + } + return true; + } + + return false; } TExprNode::TPtr ApplyRenames(const TExprNode::TPtr& input, const TMap<TStringBuf, TVector<TStringBuf>>& renames, @@ -763,175 +840,194 @@ const TTypeAnnotationNode* GetCanaryOutputType(const TStructExprType& outputType return outputType.GetItems()[*maybeIndex]->GetItemType(); } -TExprNode::TPtr BuildOutputFlattenMembersArg(const TCoEquiJoinInput& input, const TExprNode::TPtr& inputArg, +TVector<TExprNode::TPtr> BuildOutputFlattenMembersArg(const TCoEquiJoinInput& input, const TExprNode::TPtr& inputArg, const TString& canaryName, const TStructExprType& canaryResultTypeWithoutRenames, TExprContext& ctx) { - YQL_ENSURE(input.Scope().Ref().IsAtom()); - TStringBuf label = input.Scope().Ref().Content(); - auto flatMap = input.List().Cast<TCoFlatMapBase>(); auto lambda = flatMap.Lambda(); YQL_ENSURE(IsJustOrSingleAsList(lambda.Body().Ref())); auto strippedLambdaBody = lambda.Body().Ref().HeadPtr(); - const TString labelPrefix = TString::Join(label, "."); - const TString fullCanaryName = FullColumnName(label, canaryName); + if (input.Scope().Maybe<TCoAtomList>()) { + auto list = input.Scope().Cast<TCoAtomList>(); + TExprNode::TListType labelsPrefixList; + TVector<TStringBuf> labels; + labels.reserve(list.Size()); - const TTypeAnnotationNode* canaryOutType = GetCanaryOutputType(canaryResultTypeWithoutRenames, fullCanaryName); - if (!canaryOutType) { - // canary didn't survive join - return {}; - } + for (auto labelAtom : list) { + auto label = labelAtom.Value(); + labels.push_back(label); + TString prefix = TString::Join(label, "."); + labelsPrefixList.push_back(ctx.NewAtom(input.Pos(), prefix)); + } - auto flatMapInputItem = GetSequenceItemType(flatMap.Input(), false); + // Select members without stripping the prefixes. + auto myStruct = ctx.Builder(input.Pos()) + .Callable("SelectMembers") + .Add(0, inputArg) + .Add(1, ctx.NewList(input.Pos(), std::move(labelsPrefixList))) + .Seal() + .Build(); - auto myStruct = ctx.Builder(input.Pos()) - .Callable("DivePrefixMembers") - .Add(0, inputArg) - .List(1) - .Atom(0, labelPrefix) + auto lambdaResult = ctx.Builder(input.Pos()) + .ApplyPartial(lambda.Args().Ptr(), std::move(strippedLambdaBody)) + .With(0, std::move(myStruct)) .Seal() - .Seal() - .Build(); + .Build(); - if (canaryOutType->GetKind() == ETypeAnnotationKind::Data) { - YQL_ENSURE(canaryOutType->Cast<TDataExprType>()->GetSlot() == EDataSlot::Bool); - // our input passed as-is - return ctx.Builder(input.Pos()) - .List() - .Atom(0, labelPrefix) - .ApplyPartial(1, lambda.Args().Ptr(), std::move(strippedLambdaBody)) - .With(0, std::move(myStruct)) + TVector<TExprNode::TPtr> args; + for (ui32 i = 0; i < labels.size(); ++i) { + TString prefix = TString::Join(labels[i], "."); + auto arg = ctx.Builder(input.Pos()) + .List() + .Atom(0, prefix) + .Callable(1, "DivePrefixMembers") + .Add(0, lambdaResult) + .List(1) + .Atom(0, prefix) + .Seal() + .Seal() + .Seal() + .Build(); + args.push_back(arg); + } + return args; + } else { + TStringBuf label = input.Scope().Ref().Content(); + const TString labelPrefix = TString::Join(label, "."); + const TString fullCanaryName = FullColumnName(label, canaryName); + + const TTypeAnnotationNode* canaryOutType = GetCanaryOutputType(canaryResultTypeWithoutRenames, fullCanaryName); + if (!canaryOutType) { + // canary didn't survive join + return {}; + } + + auto flatMapInputItem = GetSequenceItemType(flatMap.Input(), false); + + auto myStruct = ctx.Builder(input.Pos()) + .Callable("DivePrefixMembers") + .Add(0, inputArg) + .List(1) + .Atom(0, labelPrefix) .Seal() .Seal() .Build(); - } - YQL_ENSURE(canaryOutType->GetKind() == ETypeAnnotationKind::Optional); + if (canaryOutType->GetKind() == ETypeAnnotationKind::Data) { + YQL_ENSURE(canaryOutType->Cast<TDataExprType>()->GetSlot() == EDataSlot::Bool); + // our input passed as-is + auto arg = ctx.Builder(input.Pos()) + .List() + .Atom(0, labelPrefix) + .ApplyPartial(1, lambda.Args().Ptr(), std::move(strippedLambdaBody)) + .With(0, std::move(myStruct)) + .Seal() + .Seal() + .Build(); + + return {arg}; + } - TExprNode::TListType membersForCheck; - auto flatMapInputItems = flatMapInputItem->Cast<TStructExprType>()->GetItems(); + YQL_ENSURE(canaryOutType->GetKind() == ETypeAnnotationKind::Optional); - flatMapInputItems.push_back(ctx.MakeType<TItemExprType>(canaryName, ctx.MakeType<TDataExprType>(EDataSlot::Bool))); - for (auto& item : flatMapInputItems) { - if (item->GetItemType()->GetKind() != ETypeAnnotationKind::Optional) { - membersForCheck.emplace_back(ctx.NewAtom(input.Pos(), item->GetName())); + TExprNode::TListType membersForCheck; + auto flatMapInputItems = flatMapInputItem->Cast<TStructExprType>()->GetItems(); + + flatMapInputItems.push_back(ctx.MakeType<TItemExprType>(canaryName, ctx.MakeType<TDataExprType>(EDataSlot::Bool))); + for (auto& item : flatMapInputItems) { + if (item->GetItemType()->GetKind() != ETypeAnnotationKind::Optional) { + membersForCheck.emplace_back(ctx.NewAtom(input.Pos(), item->GetName())); + } } - } - return ctx.Builder(input.Pos()) - .List() - .Atom(0, labelPrefix) - .Callable(1, "FlattenMembers") - .List(0) - .Atom(0, "") - .Callable(1, flatMap.CallableName()) - .Callable(0, "FilterNullMembers") - .Callable(0, "AssumeAllMembersNullableAtOnce") - .Callable(0, "Just") - .Add(0, std::move(myStruct)) + auto arg = ctx.Builder(input.Pos()) + .List() + .Atom(0, labelPrefix) + .Callable(1, "FlattenMembers") + .List(0) + .Atom(0, "") + .Callable(1, flatMap.CallableName()) + .Callable(0, "FilterNullMembers") + .Callable(0, "AssumeAllMembersNullableAtOnce") + .Callable(0, "Just") + .Add(0, std::move(myStruct)) + .Seal() + .Seal() + .List(1) + .Add(std::move(membersForCheck)) .Seal() .Seal() - .List(1) - .Add(std::move(membersForCheck)) - .Seal() - .Seal() - .Lambda(1) - .Param("canaryInput") - .Callable("Just") - .ApplyPartial(0, lambda.Args().Ptr(), std::move(strippedLambdaBody)) - .With(0) - .Callable("RemoveMember") - .Arg(0, "canaryInput") - .Atom(1, canaryName) - .Seal() - .Done() + .Lambda(1) + .Param("canaryInput") + .Callable("Just") + .ApplyPartial(0, lambda.Args().Ptr(), std::move(strippedLambdaBody)) + .With(0) + .Callable("RemoveMember") + .Arg(0, "canaryInput") + .Atom(1, canaryName) + .Seal() + .Done() + .Seal() .Seal() .Seal() .Seal() .Seal() .Seal() .Seal() - .Seal() - .Build(); + .Build(); + return {arg}; + } } -TExprNode::TPtr PullUpExtendOverEquiJoin(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.Types->PullUpFlatMapOverJoin || !AllowPullUpExtendOverEquiJoin(optCtx)) { - return node; - } +bool IsPullFlatMapOverJoinMultipleLabelsInputEnabled(TOptimizeContext &optCtx) { + static const char optName[] = "PullUpFlatMapOverJoinMultipleLabels"; + return IsOptimizerEnabled<optName>(*optCtx.Types); +} - YQL_ENSURE(node->ChildrenSize() >= 4); - auto inputsCount = ui32(node->ChildrenSize() - 2); +bool IsSuitableToPullUpFlatMapInputAssociatedWithLabelList(TCoEquiJoinInput& input, TOptimizeContext& optCtx) { + YQL_ENSURE(optCtx.Types); + if (input.List().Maybe<TCoFlatMapBase>()) { + auto flatMap = input.List().Cast<TCoFlatMapBase>(); + if (flatMap.Input().Maybe<TCoEquiJoin>()) { + return IsPullFlatMapOverJoinMultipleLabelsInputEnabled(optCtx); + } + } + return false; +} - auto joinTree = node->ChildPtr(inputsCount); - if (HasOnlyOneJoinType(*joinTree, "Cross")) { - return node; +bool IsInputSuitableForPullingOverEquiJoin(const TCoEquiJoinInput& input, + const THashMap<TStringBuf, THashSet<TStringBuf>>& joinKeysByLabel, + THashMap<TStringBuf, THashMap<TStringBuf, TStringBuf>>& renamesByLabel, TOptimizeContext& optCtx) +{ + if (!optCtx.IsSingleUsage(input)) { + return false; } - auto settings = node->ChildPtr(inputsCount + 1); - if (HasSetting(*settings, "flatten")) { - return node; + auto maybeFlatMap = TMaybeNode<TCoFlatMapBase>(input.List().Ptr()); + if (!maybeFlatMap) { + return false; } - const THashMap<TStringBuf, bool> additiveInputLabels = CollectAdditiveInputLabels(TCoEquiJoinTuple(joinTree)); - const THashMap<TStringBuf, THashSet<TStringBuf>> joinKeysByLabel = CollectEquiJoinKeyColumnsByLabel(*joinTree); - for (ui32 i = 0; i < inputsCount; ++i) { - TCoEquiJoinInput input(node->ChildPtr(i)); - if (!input.Scope().Ref().IsAtom()) { - return node; - } - const TStringBuf label = input.Scope().Ref().Content(); - auto addIt = additiveInputLabels.find(label); - YQL_ENSURE(addIt != additiveInputLabels.end()); - if (!addIt->second) { - continue; + if (input.Scope().Maybe<TCoAtomList>()) { + if (!IsPullFlatMapOverJoinMultipleLabelsInputEnabled(optCtx)) { + return false; } - auto maybeExtend = input.List().Maybe<TCoExtendBase>(); - if (!maybeExtend) { - continue; + TVector<TStringBuf> labels; + auto list = input.Scope().Cast<TCoAtomList>(); + for (auto labelAtom : list) { + auto label = labelAtom.Value(); + renamesByLabel[label].clear(); + labels.push_back(label); } - const TExprNodeList items = maybeExtend.Cast().Ref().ChildrenList(); - size_t pullableFlatmaps = 0; - size_t directReads = 0; - for (auto item : items) { - auto maybeFlatMap = TMaybeNode<TCoFlatMapBase>(item); - if (maybeFlatMap) { - THashMap<TStringBuf, TStringBuf> renames; - if (IsFlatmapSuitableForPullUpOverEqiuJoin(maybeFlatMap.Cast(), label, joinKeysByLabel, renames, optCtx)) { - ++pullableFlatmaps; - } - } else if (IsDirectRead(*item, optCtx)) { - ++directReads; - } - } - if (pullableFlatmaps > 0 && pullableFlatmaps + directReads == items.size()) { - YQL_CLOG(DEBUG, Core) << "Will pull up " << maybeExtend.Cast().CallableName() << " over EquiJoin input #" << i; - TExprNodeList newItems; - TExprNodeList reads; - auto processReads = [&]() { - if (!reads.empty()) { - auto newExtend = ctx.ChangeChildren(maybeExtend.Cast().Ref(), std::move(reads)); - auto newInput = ctx.ChangeChild(input.Ref(), TCoEquiJoinInput::idx_List, std::move(newExtend)); - newItems.push_back(ctx.ChangeChild(*node, i, std::move(newInput))); - } - }; - for (auto item : items) { - if (IsDirectRead(*item, optCtx)) { - reads.push_back(item); - continue; - } - processReads(); - auto newInput = ctx.ChangeChild(input.Ref(), TCoEquiJoinInput::idx_List, std::move(item)); - newItems.push_back(ctx.ChangeChild(*node, i, std::move(newInput))); - } - processReads(); - return ctx.ChangeChildren(maybeExtend.Cast().Ref(), std::move(newItems)); - } + return IsFlatmapSuitableForPullUpOverEquiJoin(maybeFlatMap.Cast(), labels, renamesByLabel, optCtx); } - return node; + + const TStringBuf label = input.Scope().Ref().Content(); + renamesByLabel[label].clear(); + return IsFlatmapSuitableForPullUpOverEqiuJoin(maybeFlatMap.Cast(), label, joinKeysByLabel, renamesByLabel[label], optCtx); } TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { @@ -964,7 +1060,7 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont for (ui32 i = 0; i < inputsCount; ++i) { TCoEquiJoinInput input(node->ChildPtr(i)); - if (!input.Scope().Ref().IsAtom()) { + if (input.Scope().Maybe<TCoAtomList>() && !IsSuitableToPullUpFlatMapInputAssociatedWithLabelList(input, optCtx)) { return node; } @@ -980,21 +1076,31 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont auto err = actualLabels.Add(ctx, *input.Scope().Ptr(), structType); YQL_ENSURE(!err); - auto label = input.Scope().Ref().Content(); - - - if (IsInputSuitableForPullingOverEquiJoin(input, joinKeysByLabel, inputJoinKeyRenamesByLabel[label], optCtx)) { + if (IsInputSuitableForPullingOverEquiJoin(input, joinKeysByLabel, inputJoinKeyRenamesByLabel, optCtx)) { auto flatMap = input.List().Cast<TCoFlatMapBase>(); auto flatMapInputItem = GetSequenceItemType(flatMap.Input(), false); auto structItems = flatMapInputItem->Cast<TStructExprType>()->GetItems(); - TString canaryName = TStringBuilder() << canaryBaseName << i; + + if (input.Scope().Maybe<TCoAtomList>()) { + auto list = input.Scope().Cast<TCoAtomList>(); + YQL_ENSURE(list.Size()); + // Take just a first label from list. + auto label = (*list.begin()).Value(); + // Canary name should have a label prefix when input is associated with labels list. + canaryName = FullColumnName(label, canaryName); + } + structItems.push_back(ctx.MakeType<TItemExprType>(canaryName, ctx.MakeType<TDataExprType>(EDataSlot::Bool))); structType = ctx.MakeType<TStructExprType>(structItems); YQL_CLOG(DEBUG, Core) << "Will pull up EquiJoin input #" << i; toPull.push_back(i); + } else { + // If cannot pull flat map with labels list stop processing. + if (!input.Scope().Ref().IsAtom()) + return node; } err = canaryLabels.Add(ctx, *input.Scope().Ptr(), structType); @@ -1026,7 +1132,6 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont } - TExprNode::TListType newEquiJoinArgs; newEquiJoinArgs.reserve(node->ChildrenSize()); @@ -1043,7 +1148,6 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont if (j < toPull.size() && i == toPull[j]) { j++; - const TString canaryName = TStringBuilder() << canaryBaseName << i; const TString fullCanaryName = FullColumnName(label, canaryName); @@ -1051,6 +1155,12 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont const TTypeAnnotationNode* canaryOutType = GetCanaryOutputType(*canaryResultType, fullCanaryName); if (canaryOutType && canaryOutType->GetKind() == ETypeAnnotationKind::Optional) { + // TODO: To support this, we have to implement support in `BuildOutputFlattenMemberArgs` for canary + // with optional type. + if (!input.Scope().Ref().IsAtom()) { + return node; + } + // remove leading flatmap from input and launch canary newEquiJoinArgs.push_back( ctx.Builder(input.Pos()) @@ -1086,9 +1196,9 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont ); } - auto flattenMembersArg = BuildOutputFlattenMembersArg(input, afterJoinArg, canaryName, *canaryResultType, ctx); - if (flattenMembersArg) { - flattenMembersArgs.push_back(flattenMembersArg); + auto flattenMembersArgsByInput = BuildOutputFlattenMembersArg(input, afterJoinArg, canaryName, *canaryResultType, ctx); + if (flattenMembersArgsByInput.size()) { + flattenMembersArgs.insert(flattenMembersArgs.end(), flattenMembersArgsByInput.begin(), flattenMembersArgsByInput.end()); } } else { flattenMembersArgs.push_back(ctx.Builder(input.Pos()) @@ -1125,6 +1235,84 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont return ctx.NewCallable(node->Pos(), "OrderedFlatMap", { newEquiJoin, newLambda }); } +TExprNode::TPtr PullUpExtendOverEquiJoin(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + if (!optCtx.Types->PullUpFlatMapOverJoin || !AllowPullUpExtendOverEquiJoin(optCtx)) { + return node; + } + + YQL_ENSURE(node->ChildrenSize() >= 4); + auto inputsCount = ui32(node->ChildrenSize() - 2); + + auto joinTree = node->ChildPtr(inputsCount); + if (HasOnlyOneJoinType(*joinTree, "Cross")) { + return node; + } + + auto settings = node->ChildPtr(inputsCount + 1); + if (HasSetting(*settings, "flatten")) { + return node; + } + + const THashMap<TStringBuf, bool> additiveInputLabels = CollectAdditiveInputLabels(TCoEquiJoinTuple(joinTree)); + const THashMap<TStringBuf, THashSet<TStringBuf>> joinKeysByLabel = CollectEquiJoinKeyColumnsByLabel(*joinTree); + for (ui32 i = 0; i < inputsCount; ++i) { + TCoEquiJoinInput input(node->ChildPtr(i)); + if (!input.Scope().Ref().IsAtom()) { + return node; + } + const TStringBuf label = input.Scope().Ref().Content(); + auto addIt = additiveInputLabels.find(label); + YQL_ENSURE(addIt != additiveInputLabels.end()); + if (!addIt->second) { + continue; + } + + auto maybeExtend = input.List().Maybe<TCoExtendBase>(); + if (!maybeExtend) { + continue; + } + + const TExprNodeList items = maybeExtend.Cast().Ref().ChildrenList(); + size_t pullableFlatmaps = 0; + size_t directReads = 0; + for (auto item : items) { + auto maybeFlatMap = TMaybeNode<TCoFlatMapBase>(item); + if (maybeFlatMap) { + THashMap<TStringBuf, TStringBuf> renames; + if (IsFlatmapSuitableForPullUpOverEqiuJoin(maybeFlatMap.Cast(), label, joinKeysByLabel, renames, optCtx)) { + ++pullableFlatmaps; + } + } else if (IsDirectRead(*item, optCtx)) { + ++directReads; + } + } + if (pullableFlatmaps > 0 && pullableFlatmaps + directReads == items.size()) { + YQL_CLOG(DEBUG, Core) << "Will pull up " << maybeExtend.Cast().CallableName() << " over EquiJoin input #" << i; + TExprNodeList newItems; + TExprNodeList reads; + auto processReads = [&]() { + if (!reads.empty()) { + auto newExtend = ctx.ChangeChildren(maybeExtend.Cast().Ref(), std::move(reads)); + auto newInput = ctx.ChangeChild(input.Ref(), TCoEquiJoinInput::idx_List, std::move(newExtend)); + newItems.push_back(ctx.ChangeChild(*node, i, std::move(newInput))); + } + }; + for (auto item : items) { + if (IsDirectRead(*item, optCtx)) { + reads.push_back(item); + continue; + } + processReads(); + auto newInput = ctx.ChangeChild(input.Ref(), TCoEquiJoinInput::idx_List, std::move(item)); + newItems.push_back(ctx.ChangeChild(*node, i, std::move(newInput))); + } + processReads(); + return ctx.ChangeChildren(maybeExtend.Cast().Ref(), std::move(newItems)); + } + } + return node; +} + TExprNode::TPtr OptimizeFromFlow(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { if (!optCtx.IsSingleUsage(node->Head())) { return node; diff --git a/yql/essentials/core/facade/ya.make b/yql/essentials/core/facade/ya.make index 5d1770241f2..ad37dfec239 100644 --- a/yql/essentials/core/facade/ya.make +++ b/yql/essentials/core/facade/ya.make @@ -30,6 +30,8 @@ PEERDIR( yql/essentials/utils/log yql/essentials/core yql/essentials/core/type_ann + yql/essentials/core/langver + yql/essentials/public/langver yql/essentials/providers/common/config yql/essentials/providers/common/proto yql/essentials/providers/common/provider diff --git a/yql/essentials/core/facade/yql_facade.cpp b/yql/essentials/core/facade/yql_facade.cpp index f58f7834664..b9c9fee537a 100644 --- a/yql/essentials/core/facade/yql_facade.cpp +++ b/yql/essentials/core/facade/yql_facade.cpp @@ -10,6 +10,7 @@ #include <yql/essentials/core/type_ann/type_ann_expr.h> #include <yql/essentials/core/services/yql_plan.h> #include <yql/essentials/core/services/yql_eval_params.h> +#include <yql/essentials/core/langver/yql_core_langver.h> #include <yql/essentials/sql/sql.h> #include <yql/essentials/sql/v1/sql.h> #include <yql/essentials/sql/v1/lexer/antlr4/lexer.h> @@ -39,6 +40,7 @@ #include <util/stream/file.h> #include <util/stream/null.h> +#include <util/string/cast.h> #include <util/string/join.h> #include <util/string/split.h> #include <util/generic/guid.h> @@ -56,6 +58,7 @@ const size_t DEFAULT_AST_BUF_SIZE = 1024; const size_t DEFAULT_PLAN_BUF_SIZE = 1024; const TString FacadeComponent = "Facade"; const TString SourceCodeLabel = "SourceCode"; +const TString LangVerLabel = "LangVer"; const TString GatewaysLabel = "Gateways"; const TString ParametersLabel = "Parameters"; const TString TranslationLabel = "Translation"; @@ -166,6 +169,14 @@ void TProgramFactory::EnableRangeComputeFor() { EnableRangeComputeFor_ = true; } +void TProgramFactory::SetLanguageVersion(TLangVersion version) { + LangVer_ = version; +} + +void TProgramFactory::SetMaxLanguageVersion(TLangVersion version) { + MaxLangVer_ = version; +} + void TProgramFactory::AddUserDataTable(const TUserDataTable& userDataTable) { for (const auto& p : userDataTable) { if (!UserDataTable_.emplace(p).second) { @@ -242,7 +253,7 @@ TProgramPtr TProgramFactory::Create( // make UserDataTable_ copy here return new TProgram(FunctionRegistry_, randomProvider, timeProvider, NextUniqueId_, DataProvidersInit_, - UserDataTable_, Credentials_, moduleResolver, urlListerManager, + LangVer_, MaxLangVer_, UserDataTable_, Credentials_, moduleResolver, urlListerManager, udfResolver, udfIndex, udfIndexPackageSet, FileStorage_, UrlPreprocessing_, GatewaysConfig_, filename, sourceCode, sessionId, Runner_, EnableRangeComputeFor_, ArrowResolver_, hiddenMode, qContext, gatewaysForMerge); @@ -257,6 +268,8 @@ TProgram::TProgram( const TIntrusivePtr<ITimeProvider> timeProvider, ui64 nextUniqueId, const TVector<TDataProviderInitializer>& dataProvidersInit, + TLangVersion langVer, + TLangVersion maxLangVer, const TUserDataTable& userDataTable, const TCredentials::TPtr& credentials, const IModuleResolver::TPtr& modules, @@ -284,6 +297,8 @@ TProgram::TProgram( , AstRoot_(nullptr) , Modules_(modules) , DataProvidersInit_(dataProvidersInit) + , LangVer_(langVer) + , MaxLangVer_(maxLangVer) , Credentials_(MakeIntrusive<NYql::TCredentials>(*credentials)) , UrlListerManager_(urlListerManager) , UdfResolver_(udfResolver) @@ -587,6 +602,14 @@ TString TProgram::GetSessionId() const { } } +void TProgram::SetLanguageVersion(TLangVersion version) { + LangVer_ = version; +} + +void TProgram::SetMaxLanguageVersion(TLangVersion version) { + MaxLangVer_ = version; +} + void TProgram::AddCredentials(const TVector<std::pair<TString, TCredential>>& credentials) { Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); @@ -740,8 +763,41 @@ bool TProgram::CheckParameters() { return true; } +bool TProgram::ValidateLangVersion() { + if (QContext_.CanRead()) { + auto loaded = QContext_.GetReader()->Get({FacadeComponent, LangVerLabel}).GetValueSync(); + if (loaded.Defined()) { + LangVer_ = FromString<TLangVersion>(loaded->Value); + } else { + LangVer_ = UnknownLangVersion; + } + + return true; + } + + if (QContext_.CanWrite()) { + QContext_.GetWriter()->Put({FacadeComponent, LangVerLabel}, ToString(LangVer_)).GetValueSync(); + } + + TMaybe<TIssue> issue; + auto ret = CheckLangVersion(LangVer_, MaxLangVer_, issue); + if (issue) { + if (!ExprCtx_) { + ExprCtx_.Reset(new TExprContext(NextUniqueId_)); + } + + ExprCtx_->AddError(*issue); + } + + return ret; +} + bool TProgram::ParseYql() { YQL_PROFILE_FUNC(TRACE); + if (!ValidateLangVersion()) { + return false; + } + if (!CheckParameters()) { return false; } @@ -769,6 +825,10 @@ bool TProgram::ParseSql() { bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings) { YQL_PROFILE_FUNC(TRACE); + if (!ValidateLangVersion()) { + return false; + } + if (!CheckParameters()) { return false; } @@ -788,6 +848,8 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings) } currentSettings->EmitReadsForExists = true; + currentSettings->LangVer = LangVer_; + NSQLTranslationV1::TLexers lexers; lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); @@ -1912,6 +1974,7 @@ TString TProgram::ResultsAsString() const { TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& username) { auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); + typeAnnotationContext->LangVer = LangVer_; typeAnnotationContext->UserDataStorage = UserDataStorage_; typeAnnotationContext->Credentials = Credentials_; typeAnnotationContext->Modules = Modules_; diff --git a/yql/essentials/core/facade/yql_facade.h b/yql/essentials/core/facade/yql_facade.h index 20b623d31cf..4c6532ab0a7 100644 --- a/yql/essentials/core/facade/yql_facade.h +++ b/yql/essentials/core/facade/yql_facade.h @@ -13,6 +13,7 @@ #include <yql/essentials/providers/result/provider/yql_result_provider.h> #include <yql/essentials/providers/common/proto/gateways_config.pb.h> #include <yql/essentials/public/issue/yql_issue.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/sql/sql.h> #include <library/cpp/random_provider/random_provider.h> @@ -50,6 +51,8 @@ public: const TVector<TDataProviderInitializer>& dataProvidersInit, const TString& runner); + void SetLanguageVersion(TLangVersion version); + void SetMaxLanguageVersion(TLangVersion version); void AddUserDataTable(const TUserDataTable& userDataTable); void SetCredentials(TCredentials::TPtr credentials); void SetGatewaysConfig(const TGatewaysConfig* gatewaysConfig); @@ -83,6 +86,8 @@ private: const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_; const ui64 NextUniqueId_; TVector<TDataProviderInitializer> DataProvidersInit_; + TLangVersion LangVer_ = UnknownLangVersion; + TLangVersion MaxLangVer_ = UnknownLangVersion; TUserDataTable UserDataTable_; TCredentials::TPtr Credentials_; const TGatewaysConfig* GatewaysConfig_; @@ -111,6 +116,9 @@ public: public: ~TProgram(); + void SetLanguageVersion(TLangVersion version); + void SetMaxLanguageVersion(TLangVersion version); + void AddCredentials(const TVector<std::pair<TString, TCredential>>& credentials); void ClearCredentials(); @@ -334,6 +342,8 @@ private: const TIntrusivePtr<ITimeProvider> timeProvider, ui64 nextUniqueId, const TVector<TDataProviderInitializer>& dataProvidersInit, + TLangVersion langVer, + TLangVersion maxLangVer, const TUserDataTable& userDataTable, const TCredentials::TPtr& credentials, const IModuleResolver::TPtr& modules, @@ -359,6 +369,7 @@ private: TTypeAnnotationContextPtr ProvideAnnotationContext(const TString& username); bool CollectUsedClusters(); bool CheckParameters(); + bool ValidateLangVersion(); NThreading::TFuture<void> OpenSession(const TString& username); @@ -395,6 +406,8 @@ private: const IModuleResolver::TPtr Modules_; TVector<TDataProviderInitializer> DataProvidersInit_; + TLangVersion LangVer_; + TLangVersion MaxLangVer_; TAdaptiveLock DataProvidersLock_; TVector<TDataProviderInfo> DataProviders_; TYqlOperationOptions OperationOptions_; diff --git a/yql/essentials/core/issue/protos/issue_id.proto b/yql/essentials/core/issue/protos/issue_id.proto index df65b98083e..a84eb6fac3a 100644 --- a/yql/essentials/core/issue/protos/issue_id.proto +++ b/yql/essentials/core/issue/protos/issue_id.proto @@ -38,6 +38,8 @@ message TIssuesIds { CORE_LEGACY_REGEX_ENGINE = 1110; CORE_ALIAS_SHADOWS_COLUMN = 1111; CORE_LINEAGE_INTERNAL_ERROR = 1112; + CORE_DEPRECATED_LANG_VER = 1113; + CORE_UNSUPPORTED_LANG_VER = 1114; // core informational CORE_TOP_UNSUPPORTED_BLOCK_TYPES = 1200; diff --git a/yql/essentials/core/issue/yql_issue.txt b/yql/essentials/core/issue/yql_issue.txt index 251d757b246..08e3cfb2e61 100644 --- a/yql/essentials/core/issue/yql_issue.txt +++ b/yql/essentials/core/issue/yql_issue.txt @@ -676,6 +676,14 @@ ids { severity: S_WARNING } ids { + code: CORE_DEPRECATED_LANG_VER + severity: S_WARNING +} +ids { + code: CORE_UNSUPPORTED_LANG_VER + severity: S_WARNING +} +ids { code: YT_SECURE_DATA_IN_COMMON_TMP severity: S_WARNING } diff --git a/yql/essentials/core/langver/ya.make b/yql/essentials/core/langver/ya.make new file mode 100644 index 00000000000..a3cc132af38 --- /dev/null +++ b/yql/essentials/core/langver/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + yql_core_langver.cpp +) + +PEERDIR( + yql/essentials/core/issue +) + +END() diff --git a/yql/essentials/core/langver/yql_core_langver.cpp b/yql/essentials/core/langver/yql_core_langver.cpp new file mode 100644 index 00000000000..e9d1e0881de --- /dev/null +++ b/yql/essentials/core/langver/yql_core_langver.cpp @@ -0,0 +1,64 @@ +#include "yql_core_langver.h" + +#include <yql/essentials/core/issue/yql_issue.h> +#include <util/string/builder.h> + +namespace NYql { + +namespace { + +void WriteVersion(TStringBuilder& builder, TLangVersion version) { + TLangVersionBuffer buffer; + TStringBuf str; + if (!FormatLangVersion(version, buffer, str)) { + builder << "unknown"; + } else { + builder << str; + } +} + +} + +bool CheckLangVersion(TLangVersion ver, TLangVersion max, TMaybe<TIssue>& issue) { + if (ver != UnknownLangVersion && !IsValidLangVersion(ver)) { + TStringBuilder builder; + builder << "Invalid YQL language version: '"; + WriteVersion(builder, ver); + builder << "'"; + issue = TIssue({}, builder); + return false; + } + + if (!IsAvalableLangVersion(ver, max)) { + TStringBuilder builder; + builder << "YQL language version '"; + WriteVersion(builder, ver); + builder << "' is not available, maximum version is '"; + WriteVersion(builder, max); + builder << "'"; + issue = TIssue({}, builder); + return false; + } + + if (IsDeprecatedLangVersion(ver, max)) { + TStringBuilder builder; + builder << "YQL language version '"; + WriteVersion(builder, ver); + builder << "' is deprecated, consider to upgrade"; + issue = YqlIssue({}, EYqlIssueCode::TIssuesIds_EIssueCode_CORE_DEPRECATED_LANG_VER, builder); + return true; + } + + if (IsUnsupportedLangVersion(ver, max)) { + TStringBuilder builder; + builder << "YQL language version '"; + WriteVersion(builder, ver); + builder << "' is unsupported, consider to upgrade. Queries may fail"; + issue = YqlIssue({}, EYqlIssueCode::TIssuesIds_EIssueCode_CORE_UNSUPPORTED_LANG_VER, builder); + return true; + } + + return true; +} + +} // namespace NYql diff --git a/yql/essentials/core/langver/yql_core_langver.h b/yql/essentials/core/langver/yql_core_langver.h new file mode 100644 index 00000000000..7003d9f4cdd --- /dev/null +++ b/yql/essentials/core/langver/yql_core_langver.h @@ -0,0 +1,9 @@ +#pragma once +#include <yql/essentials/public/issue/yql_issue.h> +#include <yql/essentials/public/langver/yql_langver.h> + +namespace NYql { + +bool CheckLangVersion(TLangVersion ver, TLangVersion max, TMaybe<TIssue>& issue); + +} // namespace NYql diff --git a/yql/essentials/core/ya.make b/yql/essentials/core/ya.make index 8626132b6f4..55d84a7ef4d 100644 --- a/yql/essentials/core/ya.make +++ b/yql/essentials/core/ya.make @@ -75,6 +75,7 @@ PEERDIR( yql/essentials/protos yql/essentials/public/udf yql/essentials/public/udf/tz + yql/essentials/public/langver yql/essentials/sql/settings yql/essentials/sql yql/essentials/utils @@ -103,6 +104,7 @@ RECURSE( dqs_expr_nodes file_storage issue + langver minsketch pg_ext pg_settings diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h index ab2db6e9fbd..bc09c963f4b 100644 --- a/yql/essentials/core/yql_type_annotation.h +++ b/yql/essentials/core/yql_type_annotation.h @@ -9,6 +9,7 @@ #include <yql/essentials/public/udf/udf_validate.h> #include <yql/essentials/public/udf/udf_log.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/core/credentials/yql_credentials.h> #include <yql/essentials/core/url_lister/interface/url_lister_manager.h> #include <yql/essentials/core/qplayer/storage/interface/yql_qstorage.h> @@ -375,6 +376,7 @@ inline TString GetRandomKey<TGUID>() { } struct TTypeAnnotationContext: public TThrRefBase { + TLangVersion LangVer = UnknownLangVersion; THashMap<TString, TIntrusivePtr<TOptimizerStatistics::TColumnStatMap>> ColumnStatisticsByTableName; THashMap<ui64, std::shared_ptr<TOptimizerStatistics>> StatisticsMap; TIntrusivePtr<ITimeProvider> TimeProvider; diff --git a/yql/essentials/public/langver/ut/ya.make b/yql/essentials/public/langver/ut/ya.make new file mode 100644 index 00000000000..ab179a9c34a --- /dev/null +++ b/yql/essentials/public/langver/ut/ya.make @@ -0,0 +1,10 @@ +UNITTEST_FOR(yql/essentials/public/langver) + +SRCS( + yql_langver_ut.cpp +) + +PEERDIR( +) + +END() diff --git a/yql/essentials/public/langver/ut/yql_langver_ut.cpp b/yql/essentials/public/langver/ut/yql_langver_ut.cpp new file mode 100644 index 00000000000..7467a7c5183 --- /dev/null +++ b/yql/essentials/public/langver/ut/yql_langver_ut.cpp @@ -0,0 +1,56 @@ +#include "yql_langver.h" + +#include <library/cpp/testing/unittest/registar.h> + +namespace NYql { + +Y_UNIT_TEST_SUITE(TLangVerTests) { + Y_UNIT_TEST(IsValidMin) { + UNIT_ASSERT(IsValidLangVersion(MinLangVersion)); + } + + Y_UNIT_TEST(Parse) { + TLangVersion v; + UNIT_ASSERT(!ParseLangVersion("",v)); + UNIT_ASSERT(!ParseLangVersion("2025.01X",v)); + UNIT_ASSERT(!ParseLangVersion("2025-01",v)); + UNIT_ASSERT(!ParseLangVersion("99999.99",v)); + UNIT_ASSERT(ParseLangVersion("2025.01",v)); + UNIT_ASSERT_VALUES_EQUAL(v, MakeLangVersion(2025,1)); + UNIT_ASSERT(ParseLangVersion("9999.99",v)); + UNIT_ASSERT_VALUES_EQUAL(v, MakeLangVersion(9999,99)); + } + + Y_UNIT_TEST(Format) { + TLangVersionBuffer b; + TStringBuf s; + UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(99999, 1), b, s)); + UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(999, 1), b, s)); + UNIT_ASSERT(FormatLangVersion(MakeLangVersion(2025, 1), b, s)); + UNIT_ASSERT_VALUES_EQUAL(s, "2025.01"); + UNIT_ASSERT_VALUES_EQUAL(b[s.Size()], 0); + UNIT_ASSERT(FormatLangVersion(MakeLangVersion(2025, 12), b, s)); + UNIT_ASSERT_VALUES_EQUAL(s, "2025.12"); + UNIT_ASSERT_VALUES_EQUAL(b[s.Size()], 0); + } + + Y_UNIT_TEST(Deprecated) { + UNIT_ASSERT(IsDeprecatedLangVersion(MakeLangVersion(2025,2),MakeLangVersion(2027,1))); + UNIT_ASSERT(!IsDeprecatedLangVersion(MakeLangVersion(2025,3),MakeLangVersion(2025,1))); + UNIT_ASSERT(!IsDeprecatedLangVersion(MakeLangVersion(2025,4),MakeLangVersion(2028,1))); + } + + Y_UNIT_TEST(Unsupported) { + UNIT_ASSERT(!IsUnsupportedLangVersion(MakeLangVersion(2025,2),MakeLangVersion(2025,1))); + UNIT_ASSERT(!IsUnsupportedLangVersion(MakeLangVersion(2025,3),MakeLangVersion(2027,1))); + UNIT_ASSERT(IsUnsupportedLangVersion(MakeLangVersion(2025,4),MakeLangVersion(2028,1))); + UNIT_ASSERT(IsUnsupportedLangVersion(MakeLangVersion(2025,5),MakeLangVersion(2029,1))); + } + + Y_UNIT_TEST(Available) { + UNIT_ASSERT(IsAvalableLangVersion(MakeLangVersion(2025,2),MakeLangVersion(2025,2))); + UNIT_ASSERT(!IsAvalableLangVersion(MakeLangVersion(2025,3),MakeLangVersion(2025,2))); + } +} + +} diff --git a/yql/essentials/public/langver/ya.make b/yql/essentials/public/langver/ya.make new file mode 100644 index 00000000000..ab1e88c5173 --- /dev/null +++ b/yql/essentials/public/langver/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + yql_langver.cpp +) + +PEERDIR( +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/yql/essentials/public/langver/yql_langver.cpp b/yql/essentials/public/langver/yql_langver.cpp new file mode 100644 index 00000000000..02584e81b6a --- /dev/null +++ b/yql/essentials/public/langver/yql_langver.cpp @@ -0,0 +1,70 @@ +#include "yql_langver.h" + +#include <util/string/cast.h> + +#include <vector> +#include <utility> + +namespace NYql { + +namespace { + +const std::pair<ui32,ui32> Versions[] = { +#include "yql_langver_list.inc" +}; + +} + +bool IsValidLangVersion(TLangVersion ver) { + for (size_t i = 0; i < Y_ARRAY_SIZE(Versions); ++i) { + if (ver == MakeLangVersion(Versions[i].first, Versions[i].second)) { + return true; + } + } + + return false; +} + +bool ParseLangVersion(TStringBuf str, TLangVersion& result) { + result = UnknownLangVersion; + if (str.size() != 7 || str[4] != '.') { + return false; + } + + ui32 year, minor; + if (!TryFromString(str.SubString(0, 4), year)) { + return false; + } + + if (!TryFromString(str.SubString(5, 2), minor)) { + return false; + } + + result = MakeLangVersion(year, minor); + return true; +} + +bool FormatLangVersion(TLangVersion ver, TLangVersionBuffer& buffer, TStringBuf& result) { + ui32 year = GetYearFromLangVersion(ver); + if (year > 9999) { + return false; + } + + ui32 minor = GetMinorFromLangVersion(ver); + Y_ASSERT(minor < 100); + if (ToString(year, buffer.data() + 0, 4) != 4) { + return false; + } + + buffer[4] = '.'; + if (ToString(minor, buffer.data() + 5, 2) == 1) { + buffer[6] = buffer[5]; + buffer[5] = '0'; + } + + buffer[7] = 0; + result = TStringBuf(buffer.data(), buffer.size() - 1); + return true; +} + +} diff --git a/yql/essentials/public/langver/yql_langver.h b/yql/essentials/public/langver/yql_langver.h new file mode 100644 index 00000000000..e14d5c91b12 --- /dev/null +++ b/yql/essentials/public/langver/yql_langver.h @@ -0,0 +1,58 @@ +#pragma once +#include <util/generic/strbuf.h> +#include <util/system/types.h> + +#include <array> + +namespace NYql { + +using TLangVersion = ui32; + +constexpr TLangVersion UnknownLangVersion = 0; + +constexpr inline TLangVersion MakeLangVersion(ui32 year, ui32 minor) { + return year * 100u + minor; +} + +constexpr inline ui32 GetYearFromLangVersion(TLangVersion ver) { + return ver / 100u; +} + +constexpr inline ui32 GetMinorFromLangVersion(TLangVersion ver) { + return ver % 100u; +} + +constexpr inline bool IsAvalableLangVersion(TLangVersion ver, TLangVersion max) { + if (ver == UnknownLangVersion || max == UnknownLangVersion) { + return true; + } + + return ver <= max; +} + +constexpr inline bool IsDeprecatedLangVersion(TLangVersion ver, TLangVersion max) { + if (ver == UnknownLangVersion || max == UnknownLangVersion) { + return false; + } + + return GetYearFromLangVersion(ver) == GetYearFromLangVersion(max) - 2; +} + +constexpr inline bool IsUnsupportedLangVersion(TLangVersion ver, TLangVersion max) { + if (ver == UnknownLangVersion || max == UnknownLangVersion) { + return false; + } + + return GetYearFromLangVersion(ver) <= GetYearFromLangVersion(max) - 3; +} + +constexpr TLangVersion MinLangVersion = MakeLangVersion(2025, 1); + +constexpr ui32 LangVersionBufferSize = 4 + 1 + 2 + 1; // year.minor\0 +using TLangVersionBuffer = std::array<char, LangVersionBufferSize>; + +bool IsValidLangVersion(TLangVersion ver); +bool ParseLangVersion(TStringBuf str, TLangVersion& result); +bool FormatLangVersion(TLangVersion ver, TLangVersionBuffer& buffer, TStringBuf& result); + +} diff --git a/yql/essentials/public/langver/yql_langver_list.inc b/yql/essentials/public/langver/yql_langver_list.inc new file mode 100644 index 00000000000..c2ef4658128 --- /dev/null +++ b/yql/essentials/public/langver/yql_langver_list.inc @@ -0,0 +1,2 @@ +{2025,1}, +{2025,2}, diff --git a/yql/essentials/public/ya.make b/yql/essentials/public/ya.make index cc6bef29309..d53fabdd192 100644 --- a/yql/essentials/public/ya.make +++ b/yql/essentials/public/ya.make @@ -2,6 +2,7 @@ RECURSE( decimal fastcheck issue + langver purecalc result_format sql_format diff --git a/yql/essentials/sql/settings/translation_settings.h b/yql/essentials/sql/settings/translation_settings.h index 98d537038e2..053e09acbec 100644 --- a/yql/essentials/sql/settings/translation_settings.h +++ b/yql/essentials/sql/settings/translation_settings.h @@ -1,6 +1,7 @@ #pragma once #include <yql/essentials/core/pg_settings/guc_settings.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <util/generic/hash.h> #include <util/generic/hash_set.h> @@ -76,6 +77,7 @@ namespace NSQLTranslation { TTranslationSettings(); google::protobuf::Arena* Arena = nullptr; + NYql::TLangVersion LangVer = NYql::UnknownLangVersion; THashMap<TString, TString> ClusterMapping; TString PathPrefix; // keys (cluster name) should be normalized diff --git a/yql/essentials/sql/settings/ya.make b/yql/essentials/sql/settings/ya.make index 784aa84d4f5..4e27ec7b400 100644 --- a/yql/essentials/sql/settings/ya.make +++ b/yql/essentials/sql/settings/ya.make @@ -10,6 +10,7 @@ PEERDIR( library/cpp/deprecated/split library/cpp/json yql/essentials/public/issue + yql/essentials/public/langver yql/essentials/core/issue yql/essentials/core/pg_settings yql/essentials/core/issue/protos diff --git a/yql/essentials/tests/common/udf_test/test.py b/yql/essentials/tests/common/udf_test/test.py index 218b05b4bde..95b8cf1e18a 100644 --- a/yql/essentials/tests/common/udf_test/test.py +++ b/yql/essentials/tests/common/udf_test/test.py @@ -67,9 +67,12 @@ def test(case): in_tables = yql_utils.get_input_tables(None, cfg, DATA_PATH, def_attr=yql_utils.KSV_ATTR) - udfs_dir = yql_utils.get_udfs_path([ - yatest.common.build_path(os.path.join(yatest.common.context.project_path, "..")) - ]) + udfs_list = [yatest.common.build_path(os.path.join(yatest.common.context.project_path, ".."))] + env_udfs_list = yql_utils.get_param("EXTRA_UDF_DIRS") + if env_udfs_list: + for udf_path in env_udfs_list.strip().split(":"): + udfs_list.append(yatest.common.build_path(udf_path)) + udfs_dir = yql_utils.get_udfs_path(udfs_list) xfail = yql_utils.is_xfail(cfg) if yql_utils.get_param('TARGET_PLATFORM') and xfail: diff --git a/yql/essentials/tests/s-expressions/minirun/part6/canondata/result.json b/yql/essentials/tests/s-expressions/minirun/part6/canondata/result.json index 181e08dbb8a..5f5370da933 100644 --- a/yql/essentials/tests/s-expressions/minirun/part6/canondata/result.json +++ b/yql/essentials/tests/s-expressions/minirun/part6/canondata/result.json @@ -111,6 +111,20 @@ "uri": "https://{canondata_backend}/1942525/03f327593648b37dc4a00dc0267d6412ab0962f1/resource.tar.gz#test.test_EquiJoin-JoinInMemAliasTwo12-default.txt-Results_/results.txt" } ], + "test.test[EquiJoin-PullUpFlatMapOverEquiJoins-default.txt-Debug]": [ + { + "checksum": "3df31dfd03165450b9e46b238654cfa2", + "size": 1356, + "uri": "https://{canondata_backend}/1900335/e6a961488cb1e9e4522ff434f8075f5ec48c274f/resource.tar.gz#test.test_EquiJoin-PullUpFlatMapOverEquiJoins-default.txt-Debug_/opt.yql" + } + ], + "test.test[EquiJoin-PullUpFlatMapOverEquiJoins-default.txt-Results]": [ + { + "checksum": "4fd94327acee5800e240ae30669086ad", + "size": 2357, + "uri": "https://{canondata_backend}/1777230/b1b0d9e9287141d50e299080d0e2e4404ac41ee1/resource.tar.gz#test.test_EquiJoin-PullUpFlatMapOverEquiJoins-default.txt-Results_/results.txt" + } + ], "test.test[File-ConfigureFileFromUrl-default.txt-Debug]": [ { "checksum": "06b6c365462b3f67076b04fa1fc55938", diff --git a/yql/essentials/tests/s-expressions/suites/EquiJoin/PullUpFlatMapOverEquiJoins.yqls b/yql/essentials/tests/s-expressions/suites/EquiJoin/PullUpFlatMapOverEquiJoins.yqls new file mode 100644 index 00000000000..7fcaf50c5f9 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/EquiJoin/PullUpFlatMapOverEquiJoins.yqls @@ -0,0 +1,33 @@ +( +(let flag (Configure! world (DataSource 'config) 'OptimizerFlags 'PullUpFlatMapOverJoinMultipleLabels)) +(let mr_source (DataSource 'yt 'plato)) +(let t1 (AsList + (AsStruct '('key1 (Int32 '1)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '2)) '('value1 (String 'B))) +)) + +(let t2 (AsList + (AsStruct '('key2 (Int32 '3)) '('value2 (String 'C))) + (AsStruct '('key2 (Int32 '4)) '('value2 (String 'D))) +)) + +(let t3 (AsList + (AsStruct '('key3 (Int32 '4)) '('value3 (String 'E))) + (AsStruct '('key3 (Int32 '5)) '('value3 (String 'F))) +)) + +(let t4 (AsList + (AsStruct '('key4 (Int32 '6)) '('value4 (String 'J))) + (AsStruct '('key4 (Int32 '7)) '('value4 (String 'H))) +)) + +(let ej1 (EquiJoin '(t1 'a) '((FlatMap t2 (lambda '(row) (OptionalIf (Coalesce (> (Member row 'key2) (Int32 '1)) (Bool 'false)) row))) 'b) '('Inner 'a 'b '('a 'key1) '('b 'key2) '()) '())) +(let ej2 (EquiJoin '((FlatMap ej1 (lambda '(row) (Just (AsStruct '('a.key1 (Member row 'a.key1)) '('a.value1 (Just (Member row 'a.value1))) '('b.value2 (Member row 'b.value2)))))) '('a 'b)) '(t3 'c) '('Inner 'a 'c '('a 'key1) '('c 'key3) '()) '())) +(let ej3 (EquiJoin '((FlatMap ej2 (lambda '(row) (Just (AsStruct '('a.key1 (Member row 'a.key1)) '('a.value1 (Just (Member row 'a.value1))) '('b.value2 (Just (Member row 'b.value2))) '('c.key3 (Member row 'c.key3)))))) '('a 'b 'c)) '(t4 'd) '('Inner 'c 'd '('c 'key3) '('d 'key4) '()) '())) + +(let res_sink (DataSink 'result)) +(let world (Write! flag res_sink (Key) ej3 '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tools/yql_facade_run/ya.make b/yql/essentials/tools/yql_facade_run/ya.make index 662276a8577..934d1e813c7 100644 --- a/yql/essentials/tools/yql_facade_run/ya.make +++ b/yql/essentials/tools/yql_facade_run/ya.make @@ -44,6 +44,8 @@ PEERDIR( yql/essentials/sql/v1/proto_parser/antlr4_ansi yql/essentials/sql/v1 yql/essentials/sql + yql/essentials/public/langver + yql/essentials/core/langver library/cpp/resource library/cpp/getopt diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp index 64fef291635..7a6a4f94abe 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp @@ -441,6 +441,20 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { opts.AddLongOption("validate-result-format", "Check that result-format can parse Result").NoArgument().SetFlag(&ValidateResultFormat); } + opts.AddLongOption("langver", "Set current language version").Optional().RequiredArgument("VER") + .Handler1T<TString>([this](const TString& str) { + if (!ParseLangVersion(str, LangVer)) { + throw yexception() << "Failed to parse language version: " << str; + } + }); + + opts.AddLongOption("max-langver", "Set maximum language version").Optional().RequiredArgument("VER") + .Handler1T<TString>([this](const TString& str) { + if (!ParseLangVersion(str, MaxLangVer)) { + throw yexception() << "Failed to parse language version: " << str; + } + }); + opts.SetFreeArgsMax(0); for (auto& ext: OptExtenders_) { @@ -697,6 +711,8 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) { int TFacadeRunner::DoRun(TProgramFactory& factory) { TProgramPtr program = factory.Create(RunOptions_.ProgramFile, RunOptions_.ProgramText, RunOptions_.OperationId, EHiddenMode::Disable, RunOptions_.QPlayerContext, RunOptions_.GatewaysPatch); + program->SetLanguageVersion(RunOptions_.LangVer); + program->SetMaxLanguageVersion(RunOptions_.MaxLangVer); if (RunOptions_.Params) { program->SetParametersYson(RunOptions_.Params); } diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.h b/yql/essentials/tools/yql_facade_run/yql_facade_run.h index e7169322924..2b4092c823b 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.h +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.h @@ -9,6 +9,7 @@ #include <yql/essentials/core/yql_user_data.h> #include <yql/essentials/core/facade/yql_facade.h> #include <yql/essentials/core/qplayer/storage/interface/yql_qstorage.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <library/cpp/getopt/last_getopt.h> #include <library/cpp/yson/public.h> @@ -69,6 +70,8 @@ public: ~TFacadeRunOptions(); EProgramType ProgramType = EProgramType::SExpr; + TLangVersion LangVer = UnknownLangVersion; + TLangVersion MaxLangVer = UnknownLangVersion; NYson::EYsonFormat ResultsFormat = NYson::EYsonFormat::Text; ERunMode Mode = ERunMode::Run; TString ProgramFile; diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp index 71fdc92c5cc..22e20173d6e 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp @@ -3624,6 +3624,7 @@ private: return execCtx->Session_->Queue_->Async([execCtx]() { return execCtx->LookupQueryCacheAsync().Apply([execCtx] (const auto& f) { YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_); + execCtx->SetNodeExecProgress("Preparing"); auto entry = execCtx->GetEntry(); bool cacheHit = f.GetValue(); TVector<TRichYPath> outYPaths = PrepareDestinations(execCtx->OutTables_, execCtx, entry, !cacheHit); @@ -3690,6 +3691,7 @@ private: return execCtx->Session_->Queue_->Async([forceTransform, combineChunks, limit, inputQueryExpr, execCtx]() { return execCtx->LookupQueryCacheAsync().Apply([forceTransform, combineChunks, limit, inputQueryExpr, execCtx] (const auto& f) { YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_); + execCtx->SetNodeExecProgress("Preparing"); auto entry = execCtx->GetEntry(); bool cacheHit = f.GetValue(); TVector<TRichYPath> outYPaths = PrepareDestinations(execCtx->OutTables_, execCtx, entry, !cacheHit); @@ -3762,6 +3764,7 @@ private: inputType, extraUsage, inputQueryExpr, execCtx, testRun] (const auto& f) mutable { YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_); + execCtx->SetNodeExecProgress("Preparing"); TTransactionCache::TEntry::TPtr entry; TVector<TRichYPath> outYPaths; if (testRun) { @@ -3990,6 +3993,7 @@ private: (const auto& f) mutable { YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_); + execCtx->SetNodeExecProgress("Preparing"); TTransactionCache::TEntry::TPtr entry; TVector<TRichYPath> outYPaths; if (testRun) { @@ -4208,6 +4212,7 @@ private: (const auto& f) mutable { YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_); + execCtx->SetNodeExecProgress("Preparing"); TTransactionCache::TEntry::TPtr entry; TVector<TRichYPath> outYPaths; @@ -4458,6 +4463,7 @@ private: (const auto& f) mutable { YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_); + execCtx->SetNodeExecProgress("Preparing"); TTransactionCache::TEntry::TPtr entry; TVector<TRichYPath> outYPaths; if (testRun) { @@ -4791,6 +4797,7 @@ private: TFuture<bool> ret = testRun ? MakeFuture<bool>(false) : execCtx->LookupQueryCacheAsync(); return ret.Apply([lambda, extraUsage, tmpTable, execCtx, testRun] (const auto& f) mutable { YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_); + execCtx->SetNodeExecProgress("Preparing"); TTransactionCache::TEntry::TPtr entry; TVector<TRichYPath> outYPaths; if (testRun) { @@ -5417,6 +5424,7 @@ private: bool localRun = execCtx->Config_->HasExecuteUdfLocallyIfPossible() ? execCtx->Config_->GetExecuteUdfLocallyIfPossible() : false; { + execCtx->SetNodeExecProgress("Preparing"); TUserJobSpec userJobSpec; TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); diff --git a/yt/yql/providers/yt/provider/yql_yt_horizontal_join.cpp b/yt/yql/providers/yt/provider/yql_yt_horizontal_join.cpp index 13d426e13dd..6b28a133e12 100644 --- a/yt/yql/providers/yt/provider/yql_yt_horizontal_join.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_horizontal_join.cpp @@ -603,8 +603,15 @@ TExprNode::TPtr THorizontalJoinOptimizer::HandleList(const TExprNode::TPtr& node size_t outNdx = JoinedMaps.size(); const TExprNode* columns = nullptr; const TExprNode* ranges = nullptr; + bool incompleteSectionSettings = false; if (sectionList) { - auto path = TYtSection(node->Child(sectionNum)).Paths().Item(pathNum); + const auto section = TYtSection(node->Child(sectionNum)); + const auto path = section.Paths().Item(pathNum); + + // Section cannot be changed right now. Keep all section inputs as is (add to ExclusiveOuts) + incompleteSectionSettings = NYql::HasAnySetting(section.Settings().Ref(), EYtSettingType::Take | EYtSettingType::Skip) + || HasNonEmptyKeyFilter(section); + columns = path.Columns().Raw(); ranges = path.Ranges().Raw(); } @@ -642,7 +649,7 @@ TExprNode::TPtr THorizontalJoinOptimizer::HandleList(const TExprNode::TPtr& node auto outRowSpec = TYtTableBaseInfo::GetRowSpec(map.Output().Item(0)); const bool sortedOut = outRowSpec && outRowSpec->IsSorted(); - if (sortedOut) { + if (sortedOut || incompleteSectionSettings) { if (ExclusiveOuts.find(outNdx) == ExclusiveOuts.end()) { // Sorted output cannot be joined with others outputCountIncrement = 1; @@ -694,7 +701,7 @@ TExprNode::TPtr THorizontalJoinOptimizer::HandleList(const TExprNode::TPtr& node Worlds.emplace(map.World().Ptr(), Worlds.size()); } - if (sortedOut) { + if (sortedOut || incompleteSectionSettings) { ExclusiveOuts[outNdx].emplace_back(sectionNum, pathNum); } else { GroupedOuts[std::make_tuple(sectionNum, columns, ranges)][outNdx] = pathNum; diff --git a/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp b/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp index 902668cf11d..a24102fd0c2 100644 --- a/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp @@ -2150,6 +2150,12 @@ private: // Used in unknown callables. Don't process exclusiveOuts.insert(outIndex); } + if (section && (NYql::HasAnySetting(*section->Child(TYtSection::idx_Settings), EYtSettingType::Take | EYtSettingType::Skip) + || HasNonEmptyKeyFilter(TYtSection(section)))) + { + exclusiveOuts.insert(outIndex); + } + // Section may be used multiple times in different operations // So, check only unique pair of operation + section if (!duplicateCheck[outIndex].insert(std::make_pair(op, section)).second) { diff --git a/yt/yt/core/ytree/public.h b/yt/yt/core/ytree/public.h index 368840528c1..deac48ae912 100644 --- a/yt/yt/core/ytree/public.h +++ b/yt/yt/core/ytree/public.h @@ -43,6 +43,7 @@ struct INodeFactory; struct ITransactionalNodeFactory; DECLARE_REFCOUNTED_STRUCT(IAttributeDictionary) +using IConstAttributeDictionaryPtr = TIntrusivePtr<const IAttributeDictionary>; struct IAttributeOwner; |