diff options
| author | AlexSm <alex@ydb.tech> | 2024-01-09 18:56:40 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-01-09 18:56:40 +0100 |
| commit | e95f266d2a3e48e62015220588a4fd73d5d5a5cb (patch) | |
| tree | a8a784b6931fe52ad5f511cfef85af14e5f63991 | |
| parent | 50a65e3b48a82d5b51f272664da389f2e0b0c99a (diff) | |
| download | ydb-e95f266d2a3e48e62015220588a4fd73d5d5a5cb.tar.gz | |
Library import 6 (#888)
256 files changed, 9815 insertions, 4193 deletions
diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 60e3cef573..0ffef27084 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -55,8 +55,16 @@ "4312064267": "https://devtools-registry.s3.yandex.net/4312064267", "4312063561": "https://devtools-registry.s3.yandex.net/4312063561", "2062930743": "https://devtools-registry.s3.yandex.net/2062930743", + "4111727026": "https://devtools-registry.s3.yandex.net/4111727026", + "4111727778": "https://devtools-registry.s3.yandex.net/4111727778", + "5599878022": "https://devtools-registry.s3.yandex.net/5599878022", + "5599877541": "https://devtools-registry.s3.yandex.net/5599877541", + "5599878769": "https://devtools-registry.s3.yandex.net/5599878769", + "5599877008": "https://devtools-registry.s3.yandex.net/5599877008", + "5599878473": "https://devtools-registry.s3.yandex.net/5599878473", "2842390994": "https://devtools-registry.s3.yandex.net/2842390994", "5310288728": "https://devtools-registry.s3.yandex.net/5310288728", + "5620327787": "https://devtools-registry.s3.yandex.net/5620327787", "2989597929": "https://devtools-registry.s3.yandex.net/2989597929", "2989596911": "https://devtools-registry.s3.yandex.net/2989596911", "2989598506": "https://devtools-registry.s3.yandex.net/2989598506", @@ -205,8 +213,16 @@ "4312064267": "black_linter for linux", "4312063561": "black_linter for linux-aarch64", "2062930743": "contrib/libs/clang11/pkg-windows-x86_64.json", + "4111727026": "contrib/libs/clang14/tools/clang-format/clang-format for darwin", + "4111727778": "contrib/libs/clang14/tools/clang-format/clang-format for linux", + "5599878022": "contrib/python/black/bin/black for darwin", + "5599877541": "contrib/python/black/bin/black for darwin-arm64", + "5599878769": "contrib/python/black/bin/black for linux", + "5599877008": "contrib/python/black/bin/black for linux-aarch64", + "5599878473": "contrib/python/black/bin/black for win32", "2842390994": "devtools jacoco agent 0.8.7 with shaded asm", "5310288728": "devtools jstyle runner 10.12.4", + "5620327787": "devtools jstyle runner 10.12.4", "2989597929": "devtools/huge_python/python for darwin", "2989596911": "devtools/huge_python/python for darwin-arm64", "2989598506": "devtools/huge_python/python for linux", diff --git a/build/platform/java/jstyle_lib/ya.make b/build/platform/java/jstyle_lib/ya.make index e83ebba6ca..bbfe6a1c48 100644 --- a/build/platform/java/jstyle_lib/ya.make +++ b/build/platform/java/jstyle_lib/ya.make @@ -5,7 +5,7 @@ OWNER(heretic) IF (USE_SYSTEM_JSTYLE_LIB) MESSAGE(WARNING System java codestyle library $USE_SYSTEM_JSTYLE_LIB will be used) ELSE() - DECLARE_EXTERNAL_RESOURCE(JSTYLE_LIB sbr:5310288728) + DECLARE_EXTERNAL_RESOURCE(JSTYLE_LIB sbr:5620327787) ENDIF() END() diff --git a/build/ya.conf.json b/build/ya.conf.json index 38ec090d58..d0be1fd1dd 100644 --- a/build/ya.conf.json +++ b/build/ya.conf.json @@ -15,6 +15,49 @@ } } }, + "black": { + "executable": { + "black": [ + "black" + ] + }, + "formula": { + "by_platform": { + "darwin-arm64": { + "uri": "sbr:5599877541" + }, + "darwin-x86_64": { + "uri": "sbr:5599878022" + }, + "linux-aarch64": { + "uri": "sbr:5599877008" + }, + "linux-x86_64": { + "uri": "sbr:5599878769" + }, + "win32-x86_64": { + "uri": "sbr:5599878473" + } + } + } + }, + "clang-format": { + "executable": { + "clang-format": [ + "clang-format" + ] + }, + "formula": { + "by_platform": { + "darwin-x86_64": { + "uri": "sbr:4111727026" + }, + "linux-x86_64": { + "uri": "sbr:4111727778" + } + } + } + }, "clang16": { "executable": { "c++": [ @@ -184,6 +227,92 @@ } } }, + "black": { + "platforms": [ + { + "default": true, + "host": { + "arch": "x86_64", + "os": "LINUX" + } + }, + { + "default": true, + "host": { + "arch": "aarch64", + "os": "LINUX" + } + }, + { + "default": true, + "host": { + "os": "WIN" + } + }, + { + "default": true, + "host": { + "os": "DARWIN" + } + }, + { + "default": true, + "host": { + "arch": "arm64", + "os": "DARWIN" + } + } + ], + "tools": { + "black": { + "bottle": "black", + "executable": "black" + } + } + }, + "clang-format": { + "platforms": [ + { + "default": true, + "host": { + "arch": "x86_64", + "os": "LINUX" + } + }, + { + "default": true, + "host": { + "arch": "aarch64", + "os": "LINUX" + } + }, + { + "default": true, + "host": { + "os": "WIN" + } + }, + { + "default": true, + "host": { + "os": "DARWIN" + } + }, + { + "default": true, + "host": { + "arch": "arm64", + "os": "DARWIN" + } + } + ], + "tools": { + "clang-format": { + "bottle": "clang-format", + "executable": "clang-format" + } + } + }, "clang16": { "env": { "CPATH": [ @@ -1186,6 +1315,9 @@ "atop": { "description": "Advanced System & Process Monitor" }, + "black": { + "description": "Python styler, Python 3 only" + }, "c++": { "description": "Run C++ compiler" }, @@ -1195,6 +1327,9 @@ "cc": { "description": "Run C compiler" }, + "clang-format": { + "description": "Run Clang-Format source code formatter" + }, "clang-rename": { "description": "Run Clang-Rename refactoring tool" }, diff --git a/contrib/libs/apache/arrow/cpp/src/arrow/array/diff.cc b/contrib/libs/apache/arrow/cpp/src/arrow/array/diff.cc index a94ca178a4..6867423522 100644 --- a/contrib/libs/apache/arrow/cpp/src/arrow/array/diff.cc +++ b/contrib/libs/apache/arrow/cpp/src/arrow/array/diff.cc @@ -629,42 +629,44 @@ class MakeFormatterImpl { auto fmt = fmt_str.c_str(); auto unit = checked_cast<const T&>(*array.type()).unit(); auto value = checked_cast<const NumericArray<T>&>(array).Value(index); - using arrow_vendored::date::format; + // Using unqualified `format` directly would produce ambiguous + // lookup because of `std::format` (ARROW-15520). + namespace avd = arrow_vendored::date; using std::chrono::nanoseconds; using std::chrono::microseconds; using std::chrono::milliseconds; using std::chrono::seconds; if (AddEpoch) { - static arrow_vendored::date::sys_days epoch{arrow_vendored::date::jan / 1 / 1970}; + static avd::sys_days epoch{avd::jan / 1 / 1970}; switch (unit) { case TimeUnit::NANO: - *os << format(fmt, static_cast<nanoseconds>(value) + epoch); + *os << avd::format(fmt, static_cast<nanoseconds>(value) + epoch); break; case TimeUnit::MICRO: - *os << format(fmt, static_cast<microseconds>(value) + epoch); + *os << avd::format(fmt, static_cast<microseconds>(value) + epoch); break; case TimeUnit::MILLI: - *os << format(fmt, static_cast<milliseconds>(value) + epoch); + *os << avd::format(fmt, static_cast<milliseconds>(value) + epoch); break; case TimeUnit::SECOND: - *os << format(fmt, static_cast<seconds>(value) + epoch); + *os << avd::format(fmt, static_cast<seconds>(value) + epoch); break; } return; } switch (unit) { case TimeUnit::NANO: - *os << format(fmt, static_cast<nanoseconds>(value)); + *os << avd::format(fmt, static_cast<nanoseconds>(value)); break; case TimeUnit::MICRO: - *os << format(fmt, static_cast<microseconds>(value)); + *os << avd::format(fmt, static_cast<microseconds>(value)); break; case TimeUnit::MILLI: - *os << format(fmt, static_cast<milliseconds>(value)); + *os << avd::format(fmt, static_cast<milliseconds>(value)); break; case TimeUnit::SECOND: - *os << format(fmt, static_cast<seconds>(value)); + *os << avd::format(fmt, static_cast<seconds>(value)); break; } }; diff --git a/contrib/libs/cctz/test/time_zone_format_test.cc b/contrib/libs/cctz/test/time_zone_format_test.cc index 0d07dad5c0..f9e598c000 100644 --- a/contrib/libs/cctz/test/time_zone_format_test.cc +++ b/contrib/libs/cctz/test/time_zone_format_test.cc @@ -56,10 +56,10 @@ const char RFC1123_no_wday[] = "%d %b %Y %H:%M:%S %z"; template <typename D> void TestFormatSpecifier(time_point<D> tp, time_zone tz, const std::string& fmt, const std::string& ans) { - EXPECT_EQ(ans, format(fmt, tp, tz)) << fmt; - EXPECT_EQ("xxx " + ans, format("xxx " + fmt, tp, tz)); - EXPECT_EQ(ans + " yyy", format(fmt + " yyy", tp, tz)); - EXPECT_EQ("xxx " + ans + " yyy", format("xxx " + fmt + " yyy", tp, tz)); + EXPECT_EQ(ans, cctz::format(fmt, tp, tz)) << fmt; + EXPECT_EQ("xxx " + ans, cctz::format("xxx " + fmt, tp, tz)); + EXPECT_EQ(ans + " yyy", cctz::format(fmt + " yyy", tp, tz)); + EXPECT_EQ("xxx " + ans + " yyy", cctz::format("xxx " + fmt + " yyy", tp, tz)); } } // namespace @@ -77,21 +77,21 @@ TEST(Format, TimePointResolution) { chrono::nanoseconds(789); EXPECT_EQ( "03:04:05.123456789", - format(kFmt, chrono::time_point_cast<chrono::nanoseconds>(t0), utc)); + cctz::format(kFmt, chrono::time_point_cast<chrono::nanoseconds>(t0), utc)); EXPECT_EQ( "03:04:05.123456", - format(kFmt, chrono::time_point_cast<chrono::microseconds>(t0), utc)); + cctz::format(kFmt, chrono::time_point_cast<chrono::microseconds>(t0), utc)); EXPECT_EQ( "03:04:05.123", - format(kFmt, chrono::time_point_cast<chrono::milliseconds>(t0), utc)); + cctz::format(kFmt, chrono::time_point_cast<chrono::milliseconds>(t0), utc)); EXPECT_EQ("03:04:05", - format(kFmt, chrono::time_point_cast<chrono::seconds>(t0), utc)); + cctz::format(kFmt, chrono::time_point_cast<chrono::seconds>(t0), utc)); EXPECT_EQ("03:04:05", - format(kFmt, chrono::time_point_cast<cctz::seconds>(t0), utc)); + cctz::format(kFmt, chrono::time_point_cast<cctz::seconds>(t0), utc)); EXPECT_EQ("03:04:00", - format(kFmt, chrono::time_point_cast<chrono::minutes>(t0), utc)); + cctz::format(kFmt, chrono::time_point_cast<chrono::minutes>(t0), utc)); EXPECT_EQ("03:00:00", - format(kFmt, chrono::time_point_cast<chrono::hours>(t0), utc)); + cctz::format(kFmt, chrono::time_point_cast<chrono::hours>(t0), utc)); } TEST(Format, TimePointExtendedResolution) { @@ -128,24 +128,24 @@ TEST(Format, Basics) { time_point<chrono::nanoseconds> tp = chrono::system_clock::from_time_t(0); // Starts with a couple basic edge cases. - EXPECT_EQ("", format("", tp, tz)); - EXPECT_EQ(" ", format(" ", tp, tz)); - EXPECT_EQ(" ", format(" ", tp, tz)); - EXPECT_EQ("xxx", format("xxx", tp, tz)); + EXPECT_EQ("", cctz::format("", tp, tz)); + EXPECT_EQ(" ", cctz::format(" ", tp, tz)); + EXPECT_EQ(" ", cctz::format(" ", tp, tz)); + EXPECT_EQ("xxx", cctz::format("xxx", tp, tz)); std::string big(128, 'x'); - EXPECT_EQ(big, format(big, tp, tz)); + EXPECT_EQ(big, cctz::format(big, tp, tz)); // Cause the 1024-byte buffer to grow. std::string bigger(100000, 'x'); - EXPECT_EQ(bigger, format(bigger, tp, tz)); + EXPECT_EQ(bigger, cctz::format(bigger, tp, tz)); tp += chrono::hours(13) + chrono::minutes(4) + chrono::seconds(5); tp += chrono::milliseconds(6) + chrono::microseconds(7) + chrono::nanoseconds(8); - EXPECT_EQ("1970-01-01", format("%Y-%m-%d", tp, tz)); - EXPECT_EQ("13:04:05", format("%H:%M:%S", tp, tz)); - EXPECT_EQ("13:04:05.006", format("%H:%M:%E3S", tp, tz)); - EXPECT_EQ("13:04:05.006007", format("%H:%M:%E6S", tp, tz)); - EXPECT_EQ("13:04:05.006007008", format("%H:%M:%E9S", tp, tz)); + EXPECT_EQ("1970-01-01", cctz::format("%Y-%m-%d", tp, tz)); + EXPECT_EQ("13:04:05", cctz::format("%H:%M:%S", tp, tz)); + EXPECT_EQ("13:04:05.006", cctz::format("%H:%M:%E3S", tp, tz)); + EXPECT_EQ("13:04:05.006007", cctz::format("%H:%M:%E6S", tp, tz)); + EXPECT_EQ("13:04:05.006007008", cctz::format("%H:%M:%E9S", tp, tz)); } TEST(Format, PosixConversions) { @@ -198,7 +198,7 @@ TEST(Format, LocaleSpecific) { TestFormatSpecifier(tp, tz, "%B", "January"); // %c should at least produce the numeric year and time-of-day. - const std::string s = format("%c", tp, utc_time_zone()); + const std::string s = cctz::format("%c", tp, utc_time_zone()); EXPECT_THAT(s, testing::HasSubstr("1970")); EXPECT_THAT(s, testing::HasSubstr("00:00:00")); @@ -262,49 +262,49 @@ TEST(Format, ExtendedSeconds) { // No subseconds. time_point<chrono::nanoseconds> tp = chrono::system_clock::from_time_t(0); tp += chrono::seconds(5); - EXPECT_EQ("05", format("%E*S", tp, tz)); - EXPECT_EQ("05", format("%E0S", tp, tz)); - EXPECT_EQ("05.0", format("%E1S", tp, tz)); - EXPECT_EQ("05.00", format("%E2S", tp, tz)); - EXPECT_EQ("05.000", format("%E3S", tp, tz)); - EXPECT_EQ("05.0000", format("%E4S", tp, tz)); - EXPECT_EQ("05.00000", format("%E5S", tp, tz)); - EXPECT_EQ("05.000000", format("%E6S", tp, tz)); - EXPECT_EQ("05.0000000", format("%E7S", tp, tz)); - EXPECT_EQ("05.00000000", format("%E8S", tp, tz)); - EXPECT_EQ("05.000000000", format("%E9S", tp, tz)); - EXPECT_EQ("05.0000000000", format("%E10S", tp, tz)); - EXPECT_EQ("05.00000000000", format("%E11S", tp, tz)); - EXPECT_EQ("05.000000000000", format("%E12S", tp, tz)); - EXPECT_EQ("05.0000000000000", format("%E13S", tp, tz)); - EXPECT_EQ("05.00000000000000", format("%E14S", tp, tz)); - EXPECT_EQ("05.000000000000000", format("%E15S", tp, tz)); + EXPECT_EQ("05", cctz::format("%E*S", tp, tz)); + EXPECT_EQ("05", cctz::format("%E0S", tp, tz)); + EXPECT_EQ("05.0", cctz::format("%E1S", tp, tz)); + EXPECT_EQ("05.00", cctz::format("%E2S", tp, tz)); + EXPECT_EQ("05.000", cctz::format("%E3S", tp, tz)); + EXPECT_EQ("05.0000", cctz::format("%E4S", tp, tz)); + EXPECT_EQ("05.00000", cctz::format("%E5S", tp, tz)); + EXPECT_EQ("05.000000", cctz::format("%E6S", tp, tz)); + EXPECT_EQ("05.0000000", cctz::format("%E7S", tp, tz)); + EXPECT_EQ("05.00000000", cctz::format("%E8S", tp, tz)); + EXPECT_EQ("05.000000000", cctz::format("%E9S", tp, tz)); + EXPECT_EQ("05.0000000000", cctz::format("%E10S", tp, tz)); + EXPECT_EQ("05.00000000000", cctz::format("%E11S", tp, tz)); + EXPECT_EQ("05.000000000000", cctz::format("%E12S", tp, tz)); + EXPECT_EQ("05.0000000000000", cctz::format("%E13S", tp, tz)); + EXPECT_EQ("05.00000000000000", cctz::format("%E14S", tp, tz)); + EXPECT_EQ("05.000000000000000", cctz::format("%E15S", tp, tz)); // With subseconds. tp += chrono::milliseconds(6) + chrono::microseconds(7) + chrono::nanoseconds(8); - EXPECT_EQ("05.006007008", format("%E*S", tp, tz)); - EXPECT_EQ("05", format("%E0S", tp, tz)); - EXPECT_EQ("05.0", format("%E1S", tp, tz)); - EXPECT_EQ("05.00", format("%E2S", tp, tz)); - EXPECT_EQ("05.006", format("%E3S", tp, tz)); - EXPECT_EQ("05.0060", format("%E4S", tp, tz)); - EXPECT_EQ("05.00600", format("%E5S", tp, tz)); - EXPECT_EQ("05.006007", format("%E6S", tp, tz)); - EXPECT_EQ("05.0060070", format("%E7S", tp, tz)); - EXPECT_EQ("05.00600700", format("%E8S", tp, tz)); - EXPECT_EQ("05.006007008", format("%E9S", tp, tz)); - EXPECT_EQ("05.0060070080", format("%E10S", tp, tz)); - EXPECT_EQ("05.00600700800", format("%E11S", tp, tz)); - EXPECT_EQ("05.006007008000", format("%E12S", tp, tz)); - EXPECT_EQ("05.0060070080000", format("%E13S", tp, tz)); - EXPECT_EQ("05.00600700800000", format("%E14S", tp, tz)); - EXPECT_EQ("05.006007008000000", format("%E15S", tp, tz)); + EXPECT_EQ("05.006007008", cctz::format("%E*S", tp, tz)); + EXPECT_EQ("05", cctz::format("%E0S", tp, tz)); + EXPECT_EQ("05.0", cctz::format("%E1S", tp, tz)); + EXPECT_EQ("05.00", cctz::format("%E2S", tp, tz)); + EXPECT_EQ("05.006", cctz::format("%E3S", tp, tz)); + EXPECT_EQ("05.0060", cctz::format("%E4S", tp, tz)); + EXPECT_EQ("05.00600", cctz::format("%E5S", tp, tz)); + EXPECT_EQ("05.006007", cctz::format("%E6S", tp, tz)); + EXPECT_EQ("05.0060070", cctz::format("%E7S", tp, tz)); + EXPECT_EQ("05.00600700", cctz::format("%E8S", tp, tz)); + EXPECT_EQ("05.006007008", cctz::format("%E9S", tp, tz)); + EXPECT_EQ("05.0060070080", cctz::format("%E10S", tp, tz)); + EXPECT_EQ("05.00600700800", cctz::format("%E11S", tp, tz)); + EXPECT_EQ("05.006007008000", cctz::format("%E12S", tp, tz)); + EXPECT_EQ("05.0060070080000", cctz::format("%E13S", tp, tz)); + EXPECT_EQ("05.00600700800000", cctz::format("%E14S", tp, tz)); + EXPECT_EQ("05.006007008000000", cctz::format("%E15S", tp, tz)); // Times before the Unix epoch. tp = chrono::system_clock::from_time_t(0) + chrono::microseconds(-1); EXPECT_EQ("1969-12-31 23:59:59.999999", - format("%Y-%m-%d %H:%M:%E*S", tp, tz)); + cctz::format("%Y-%m-%d %H:%M:%E*S", tp, tz)); // Here is a "%E*S" case we got wrong for a while. While the first // instant below is correctly rendered as "...:07.333304", the second @@ -312,10 +312,10 @@ TEST(Format, ExtendedSeconds) { tp = chrono::system_clock::from_time_t(0) + chrono::microseconds(1395024427333304); EXPECT_EQ("2014-03-17 02:47:07.333304", - format("%Y-%m-%d %H:%M:%E*S", tp, tz)); + cctz::format("%Y-%m-%d %H:%M:%E*S", tp, tz)); tp += chrono::microseconds(1); EXPECT_EQ("2014-03-17 02:47:07.333305", - format("%Y-%m-%d %H:%M:%E*S", tp, tz)); + cctz::format("%Y-%m-%d %H:%M:%E*S", tp, tz)); } TEST(Format, ExtendedSubeconds) { @@ -324,49 +324,49 @@ TEST(Format, ExtendedSubeconds) { // No subseconds. time_point<chrono::nanoseconds> tp = chrono::system_clock::from_time_t(0); tp += chrono::seconds(5); - EXPECT_EQ("0", format("%E*f", tp, tz)); - EXPECT_EQ("", format("%E0f", tp, tz)); - EXPECT_EQ("0", format("%E1f", tp, tz)); - EXPECT_EQ("00", format("%E2f", tp, tz)); - EXPECT_EQ("000", format("%E3f", tp, tz)); - EXPECT_EQ("0000", format("%E4f", tp, tz)); - EXPECT_EQ("00000", format("%E5f", tp, tz)); - EXPECT_EQ("000000", format("%E6f", tp, tz)); - EXPECT_EQ("0000000", format("%E7f", tp, tz)); - EXPECT_EQ("00000000", format("%E8f", tp, tz)); - EXPECT_EQ("000000000", format("%E9f", tp, tz)); - EXPECT_EQ("0000000000", format("%E10f", tp, tz)); - EXPECT_EQ("00000000000", format("%E11f", tp, tz)); - EXPECT_EQ("000000000000", format("%E12f", tp, tz)); - EXPECT_EQ("0000000000000", format("%E13f", tp, tz)); - EXPECT_EQ("00000000000000", format("%E14f", tp, tz)); - EXPECT_EQ("000000000000000", format("%E15f", tp, tz)); + EXPECT_EQ("0", cctz::format("%E*f", tp, tz)); + EXPECT_EQ("", cctz::format("%E0f", tp, tz)); + EXPECT_EQ("0", cctz::format("%E1f", tp, tz)); + EXPECT_EQ("00", cctz::format("%E2f", tp, tz)); + EXPECT_EQ("000", cctz::format("%E3f", tp, tz)); + EXPECT_EQ("0000", cctz::format("%E4f", tp, tz)); + EXPECT_EQ("00000", cctz::format("%E5f", tp, tz)); + EXPECT_EQ("000000", cctz::format("%E6f", tp, tz)); + EXPECT_EQ("0000000", cctz::format("%E7f", tp, tz)); + EXPECT_EQ("00000000", cctz::format("%E8f", tp, tz)); + EXPECT_EQ("000000000", cctz::format("%E9f", tp, tz)); + EXPECT_EQ("0000000000", cctz::format("%E10f", tp, tz)); + EXPECT_EQ("00000000000", cctz::format("%E11f", tp, tz)); + EXPECT_EQ("000000000000", cctz::format("%E12f", tp, tz)); + EXPECT_EQ("0000000000000", cctz::format("%E13f", tp, tz)); + EXPECT_EQ("00000000000000", cctz::format("%E14f", tp, tz)); + EXPECT_EQ("000000000000000", cctz::format("%E15f", tp, tz)); // With subseconds. tp += chrono::milliseconds(6) + chrono::microseconds(7) + chrono::nanoseconds(8); - EXPECT_EQ("006007008", format("%E*f", tp, tz)); - EXPECT_EQ("", format("%E0f", tp, tz)); - EXPECT_EQ("0", format("%E1f", tp, tz)); - EXPECT_EQ("00", format("%E2f", tp, tz)); - EXPECT_EQ("006", format("%E3f", tp, tz)); - EXPECT_EQ("0060", format("%E4f", tp, tz)); - EXPECT_EQ("00600", format("%E5f", tp, tz)); - EXPECT_EQ("006007", format("%E6f", tp, tz)); - EXPECT_EQ("0060070", format("%E7f", tp, tz)); - EXPECT_EQ("00600700", format("%E8f", tp, tz)); - EXPECT_EQ("006007008", format("%E9f", tp, tz)); - EXPECT_EQ("0060070080", format("%E10f", tp, tz)); - EXPECT_EQ("00600700800", format("%E11f", tp, tz)); - EXPECT_EQ("006007008000", format("%E12f", tp, tz)); - EXPECT_EQ("0060070080000", format("%E13f", tp, tz)); - EXPECT_EQ("00600700800000", format("%E14f", tp, tz)); - EXPECT_EQ("006007008000000", format("%E15f", tp, tz)); + EXPECT_EQ("006007008", cctz::format("%E*f", tp, tz)); + EXPECT_EQ("", cctz::format("%E0f", tp, tz)); + EXPECT_EQ("0", cctz::format("%E1f", tp, tz)); + EXPECT_EQ("00", cctz::format("%E2f", tp, tz)); + EXPECT_EQ("006", cctz::format("%E3f", tp, tz)); + EXPECT_EQ("0060", cctz::format("%E4f", tp, tz)); + EXPECT_EQ("00600", cctz::format("%E5f", tp, tz)); + EXPECT_EQ("006007", cctz::format("%E6f", tp, tz)); + EXPECT_EQ("0060070", cctz::format("%E7f", tp, tz)); + EXPECT_EQ("00600700", cctz::format("%E8f", tp, tz)); + EXPECT_EQ("006007008", cctz::format("%E9f", tp, tz)); + EXPECT_EQ("0060070080", cctz::format("%E10f", tp, tz)); + EXPECT_EQ("00600700800", cctz::format("%E11f", tp, tz)); + EXPECT_EQ("006007008000", cctz::format("%E12f", tp, tz)); + EXPECT_EQ("0060070080000", cctz::format("%E13f", tp, tz)); + EXPECT_EQ("00600700800000", cctz::format("%E14f", tp, tz)); + EXPECT_EQ("006007008000000", cctz::format("%E15f", tp, tz)); // Times before the Unix epoch. tp = chrono::system_clock::from_time_t(0) + chrono::microseconds(-1); EXPECT_EQ("1969-12-31 23:59:59.999999", - format("%Y-%m-%d %H:%M:%S.%E*f", tp, tz)); + cctz::format("%Y-%m-%d %H:%M:%S.%E*f", tp, tz)); // Here is a "%E*S" case we got wrong for a while. While the first // instant below is correctly rendered as "...:07.333304", the second @@ -374,10 +374,10 @@ TEST(Format, ExtendedSubeconds) { tp = chrono::system_clock::from_time_t(0) + chrono::microseconds(1395024427333304); EXPECT_EQ("2014-03-17 02:47:07.333304", - format("%Y-%m-%d %H:%M:%S.%E*f", tp, tz)); + cctz::format("%Y-%m-%d %H:%M:%S.%E*f", tp, tz)); tp += chrono::microseconds(1); EXPECT_EQ("2014-03-17 02:47:07.333305", - format("%Y-%m-%d %H:%M:%S.%E*f", tp, tz)); + cctz::format("%Y-%m-%d %H:%M:%S.%E*f", tp, tz)); } TEST(Format, CompareExtendSecondsVsSubseconds) { @@ -393,15 +393,15 @@ TEST(Format, CompareExtendSecondsVsSubseconds) { time_point<chrono::nanoseconds> tp = chrono::system_clock::from_time_t(0); tp += chrono::seconds(5); // ... %E*S and %S.%E*f are different. - EXPECT_EQ("05", format(fmt_A("*"), tp, tz)); - EXPECT_EQ("05.0", format(fmt_B("*"), tp, tz)); + EXPECT_EQ("05", cctz::format(fmt_A("*"), tp, tz)); + EXPECT_EQ("05.0", cctz::format(fmt_B("*"), tp, tz)); // ... %E0S and %S.%E0f are different. - EXPECT_EQ("05", format(fmt_A("0"), tp, tz)); - EXPECT_EQ("05.", format(fmt_B("0"), tp, tz)); + EXPECT_EQ("05", cctz::format(fmt_A("0"), tp, tz)); + EXPECT_EQ("05.", cctz::format(fmt_B("0"), tp, tz)); // ... %E<prec>S and %S.%E<prec>f are the same for prec in [1:15]. for (int prec = 1; prec <= 15; ++prec) { - const std::string a = format(fmt_A(std::to_string(prec)), tp, tz); - const std::string b = format(fmt_B(std::to_string(prec)), tp, tz); + const std::string a = cctz::format(fmt_A(std::to_string(prec)), tp, tz); + const std::string b = cctz::format(fmt_B(std::to_string(prec)), tp, tz); EXPECT_EQ(a, b) << "prec=" << prec; } @@ -409,15 +409,15 @@ TEST(Format, CompareExtendSecondsVsSubseconds) { // ... %E*S and %S.%E*f are the same. tp += chrono::milliseconds(6) + chrono::microseconds(7) + chrono::nanoseconds(8); - EXPECT_EQ("05.006007008", format(fmt_A("*"), tp, tz)); - EXPECT_EQ("05.006007008", format(fmt_B("*"), tp, tz)); + EXPECT_EQ("05.006007008", cctz::format(fmt_A("*"), tp, tz)); + EXPECT_EQ("05.006007008", cctz::format(fmt_B("*"), tp, tz)); // ... %E0S and %S.%E0f are different. - EXPECT_EQ("05", format(fmt_A("0"), tp, tz)); - EXPECT_EQ("05.", format(fmt_B("0"), tp, tz)); + EXPECT_EQ("05", cctz::format(fmt_A("0"), tp, tz)); + EXPECT_EQ("05.", cctz::format(fmt_B("0"), tp, tz)); // ... %E<prec>S and %S.%E<prec>f are the same for prec in [1:15]. for (int prec = 1; prec <= 15; ++prec) { - const std::string a = format(fmt_A(std::to_string(prec)), tp, tz); - const std::string b = format(fmt_B(std::to_string(prec)), tp, tz); + const std::string a = cctz::format(fmt_A(std::to_string(prec)), tp, tz); + const std::string b = cctz::format(fmt_B(std::to_string(prec)), tp, tz); EXPECT_EQ(a, b) << "prec=" << prec; } } @@ -590,31 +590,31 @@ TEST(Format, ExtendedYears) { // %E4Y zero-pads the year to produce at least 4 chars, including the sign. auto tp = convert(civil_second(-999, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("-9991127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("-9991127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(-99, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("-0991127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("-0991127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(-9, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("-0091127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("-0091127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(-1, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("-0011127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("-0011127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(0, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("00001127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("00001127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(1, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("00011127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("00011127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(9, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("00091127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("00091127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(99, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("00991127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("00991127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(999, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("09991127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("09991127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(9999, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("99991127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("99991127", cctz::format(e4y_fmt, tp, utc)); // When the year is outside [-999:9999], more than 4 chars are produced. tp = convert(civil_second(-1000, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("-10001127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("-10001127", cctz::format(e4y_fmt, tp, utc)); tp = convert(civil_second(10000, 11, 27, 0, 0, 0), utc); - EXPECT_EQ("100001127", format(e4y_fmt, tp, utc)); + EXPECT_EQ("100001127", cctz::format(e4y_fmt, tp, utc)); } TEST(Format, RFC3339Format) { @@ -623,45 +623,45 @@ TEST(Format, RFC3339Format) { time_point<chrono::nanoseconds> tp = convert(civil_second(1977, 6, 28, 9, 8, 7), tz); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); tp += chrono::milliseconds(100); - EXPECT_EQ("1977-06-28T09:08:07.1-07:00", format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07.1-07:00", cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); tp += chrono::milliseconds(20); - EXPECT_EQ("1977-06-28T09:08:07.12-07:00", format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07.12-07:00", cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); tp += chrono::milliseconds(3); - EXPECT_EQ("1977-06-28T09:08:07.123-07:00", format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07.123-07:00", cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); tp += chrono::microseconds(400); - EXPECT_EQ("1977-06-28T09:08:07.1234-07:00", format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07.1234-07:00", cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); tp += chrono::microseconds(50); - EXPECT_EQ("1977-06-28T09:08:07.12345-07:00", format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07.12345-07:00", cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); tp += chrono::microseconds(6); - EXPECT_EQ("1977-06-28T09:08:07.123456-07:00", format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07.123456-07:00", cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); tp += chrono::nanoseconds(700); - EXPECT_EQ("1977-06-28T09:08:07.1234567-07:00", format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07.1234567-07:00", cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); tp += chrono::nanoseconds(80); - EXPECT_EQ("1977-06-28T09:08:07.12345678-07:00", format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07.12345678-07:00", cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); tp += chrono::nanoseconds(9); EXPECT_EQ("1977-06-28T09:08:07.123456789-07:00", - format(RFC3339_full, tp, tz)); - EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + cctz::format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", cctz::format(RFC3339_sec, tp, tz)); } TEST(Format, RFC1123Format) { // locale specific @@ -669,36 +669,36 @@ TEST(Format, RFC1123Format) { // locale specific EXPECT_TRUE(load_time_zone("America/Los_Angeles", &tz)); auto tp = convert(civil_second(1977, 6, 28, 9, 8, 7), tz); - EXPECT_EQ("Tue, 28 Jun 1977 09:08:07 -0700", format(RFC1123_full, tp, tz)); - EXPECT_EQ("28 Jun 1977 09:08:07 -0700", format(RFC1123_no_wday, tp, tz)); + EXPECT_EQ("Tue, 28 Jun 1977 09:08:07 -0700", cctz::format(RFC1123_full, tp, tz)); + EXPECT_EQ("28 Jun 1977 09:08:07 -0700", cctz::format(RFC1123_no_wday, tp, tz)); } TEST(Format, Week) { const time_zone utc = utc_time_zone(); auto tp = convert(civil_second(2017, 1, 1, 0, 0, 0), utc); - EXPECT_EQ("2017-01-7", format("%Y-%U-%u", tp, utc)); - EXPECT_EQ("2017-00-0", format("%Y-%W-%w", tp, utc)); + EXPECT_EQ("2017-01-7", cctz::format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2017-00-0", cctz::format("%Y-%W-%w", tp, utc)); tp = convert(civil_second(2017, 12, 31, 0, 0, 0), utc); - EXPECT_EQ("2017-53-7", format("%Y-%U-%u", tp, utc)); - EXPECT_EQ("2017-52-0", format("%Y-%W-%w", tp, utc)); + EXPECT_EQ("2017-53-7", cctz::format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2017-52-0", cctz::format("%Y-%W-%w", tp, utc)); tp = convert(civil_second(2018, 1, 1, 0, 0, 0), utc); - EXPECT_EQ("2018-00-1", format("%Y-%U-%u", tp, utc)); - EXPECT_EQ("2018-01-1", format("%Y-%W-%w", tp, utc)); + EXPECT_EQ("2018-00-1", cctz::format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2018-01-1", cctz::format("%Y-%W-%w", tp, utc)); tp = convert(civil_second(2018, 12, 31, 0, 0, 0), utc); - EXPECT_EQ("2018-52-1", format("%Y-%U-%u", tp, utc)); - EXPECT_EQ("2018-53-1", format("%Y-%W-%w", tp, utc)); + EXPECT_EQ("2018-52-1", cctz::format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2018-53-1", cctz::format("%Y-%W-%w", tp, utc)); tp = convert(civil_second(2019, 1, 1, 0, 0, 0), utc); - EXPECT_EQ("2019-00-2", format("%Y-%U-%u", tp, utc)); - EXPECT_EQ("2019-00-2", format("%Y-%W-%w", tp, utc)); + EXPECT_EQ("2019-00-2", cctz::format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2019-00-2", cctz::format("%Y-%W-%w", tp, utc)); tp = convert(civil_second(2019, 12, 31, 0, 0, 0), utc); - EXPECT_EQ("2019-52-2", format("%Y-%U-%u", tp, utc)); - EXPECT_EQ("2019-52-2", format("%Y-%W-%w", tp, utc)); + EXPECT_EQ("2019-52-2", cctz::format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2019-52-2", cctz::format("%Y-%W-%w", tp, utc)); } // @@ -711,39 +711,39 @@ TEST(Parse, TimePointResolution) { time_point<chrono::nanoseconds> tp_ns; EXPECT_TRUE(parse(kFmt, "03:04:05.123456789", utc, &tp_ns)); - EXPECT_EQ("03:04:05.123456789", format(kFmt, tp_ns, utc)); + EXPECT_EQ("03:04:05.123456789", cctz::format(kFmt, tp_ns, utc)); EXPECT_TRUE(parse(kFmt, "03:04:05.123456", utc, &tp_ns)); - EXPECT_EQ("03:04:05.123456", format(kFmt, tp_ns, utc)); + EXPECT_EQ("03:04:05.123456", cctz::format(kFmt, tp_ns, utc)); time_point<chrono::microseconds> tp_us; EXPECT_TRUE(parse(kFmt, "03:04:05.123456789", utc, &tp_us)); - EXPECT_EQ("03:04:05.123456", format(kFmt, tp_us, utc)); + EXPECT_EQ("03:04:05.123456", cctz::format(kFmt, tp_us, utc)); EXPECT_TRUE(parse(kFmt, "03:04:05.123456", utc, &tp_us)); - EXPECT_EQ("03:04:05.123456", format(kFmt, tp_us, utc)); + EXPECT_EQ("03:04:05.123456", cctz::format(kFmt, tp_us, utc)); EXPECT_TRUE(parse(kFmt, "03:04:05.123", utc, &tp_us)); - EXPECT_EQ("03:04:05.123", format(kFmt, tp_us, utc)); + EXPECT_EQ("03:04:05.123", cctz::format(kFmt, tp_us, utc)); time_point<chrono::milliseconds> tp_ms; EXPECT_TRUE(parse(kFmt, "03:04:05.123456", utc, &tp_ms)); - EXPECT_EQ("03:04:05.123", format(kFmt, tp_ms, utc)); + EXPECT_EQ("03:04:05.123", cctz::format(kFmt, tp_ms, utc)); EXPECT_TRUE(parse(kFmt, "03:04:05.123", utc, &tp_ms)); - EXPECT_EQ("03:04:05.123", format(kFmt, tp_ms, utc)); + EXPECT_EQ("03:04:05.123", cctz::format(kFmt, tp_ms, utc)); EXPECT_TRUE(parse(kFmt, "03:04:05", utc, &tp_ms)); - EXPECT_EQ("03:04:05", format(kFmt, tp_ms, utc)); + EXPECT_EQ("03:04:05", cctz::format(kFmt, tp_ms, utc)); time_point<chrono::seconds> tp_s; EXPECT_TRUE(parse(kFmt, "03:04:05.123", utc, &tp_s)); - EXPECT_EQ("03:04:05", format(kFmt, tp_s, utc)); + EXPECT_EQ("03:04:05", cctz::format(kFmt, tp_s, utc)); EXPECT_TRUE(parse(kFmt, "03:04:05", utc, &tp_s)); - EXPECT_EQ("03:04:05", format(kFmt, tp_s, utc)); + EXPECT_EQ("03:04:05", cctz::format(kFmt, tp_s, utc)); time_point<chrono::minutes> tp_m; EXPECT_TRUE(parse(kFmt, "03:04:05", utc, &tp_m)); - EXPECT_EQ("03:04:00", format(kFmt, tp_m, utc)); + EXPECT_EQ("03:04:00", cctz::format(kFmt, tp_m, utc)); time_point<chrono::hours> tp_h; EXPECT_TRUE(parse(kFmt, "03:04:05", utc, &tp_h)); - EXPECT_EQ("03:00:00", format(kFmt, tp_h, utc)); + EXPECT_EQ("03:00:00", cctz::format(kFmt, tp_h, utc)); } TEST(Parse, TimePointExtendedResolution) { @@ -1538,7 +1538,7 @@ TEST(FormatParse, RoundTrip) { // RFC3339, which renders subseconds. { time_point<chrono::nanoseconds> out; - const std::string s = format(RFC3339_full, in + subseconds, lax); + const std::string s = cctz::format(RFC3339_full, in + subseconds, lax); EXPECT_TRUE(parse(RFC3339_full, s, lax, &out)) << s; EXPECT_EQ(in + subseconds, out); // RFC3339_full includes %Ez } @@ -1546,7 +1546,7 @@ TEST(FormatParse, RoundTrip) { // RFC1123, which only does whole seconds. { time_point<chrono::nanoseconds> out; - const std::string s = format(RFC1123_full, in, lax); + const std::string s = cctz::format(RFC1123_full, in, lax); EXPECT_TRUE(parse(RFC1123_full, s, lax, &out)) << s; EXPECT_EQ(in, out); // RFC1123_full includes %z } @@ -1560,7 +1560,7 @@ TEST(FormatParse, RoundTrip) { { time_point<chrono::nanoseconds> out; time_zone utc = utc_time_zone(); - const std::string s = format("%c", in, utc); + const std::string s = cctz::format("%c", in, utc); EXPECT_TRUE(parse("%c", s, utc, &out)) << s; EXPECT_EQ(in, out); } @@ -1570,7 +1570,7 @@ TEST(FormatParse, RoundTrip) { TEST(FormatParse, RoundTripDistantFuture) { const time_zone utc = utc_time_zone(); const time_point<cctz::seconds> in = time_point<cctz::seconds>::max(); - const std::string s = format(RFC3339_full, in, utc); + const std::string s = cctz::format(RFC3339_full, in, utc); time_point<cctz::seconds> out; EXPECT_TRUE(parse(RFC3339_full, s, utc, &out)) << s; EXPECT_EQ(in, out); @@ -1579,7 +1579,7 @@ TEST(FormatParse, RoundTripDistantFuture) { TEST(FormatParse, RoundTripDistantPast) { const time_zone utc = utc_time_zone(); const time_point<cctz::seconds> in = time_point<cctz::seconds>::min(); - const std::string s = format(RFC3339_full, in, utc); + const std::string s = cctz::format(RFC3339_full, in, utc); time_point<cctz::seconds> out; EXPECT_TRUE(parse(RFC3339_full, s, utc, &out)) << s; EXPECT_EQ(in, out); diff --git a/contrib/libs/cctz/test/time_zone_lookup_test.cc b/contrib/libs/cctz/test/time_zone_lookup_test.cc index 056e076771..5bc04b7947 100644 --- a/contrib/libs/cctz/test/time_zone_lookup_test.cc +++ b/contrib/libs/cctz/test/time_zone_lookup_test.cc @@ -899,19 +899,19 @@ TEST(MakeTime, TimePointResolution) { const time_zone utc = utc_time_zone(); const time_point<chrono::nanoseconds> tp_ns = convert(civil_second(2015, 1, 2, 3, 4, 5), utc); - EXPECT_EQ("04:05", format("%M:%E*S", tp_ns, utc)); + EXPECT_EQ("04:05", cctz::format("%M:%E*S", tp_ns, utc)); const time_point<chrono::microseconds> tp_us = convert(civil_second(2015, 1, 2, 3, 4, 5), utc); - EXPECT_EQ("04:05", format("%M:%E*S", tp_us, utc)); + EXPECT_EQ("04:05", cctz::format("%M:%E*S", tp_us, utc)); const time_point<chrono::milliseconds> tp_ms = convert(civil_second(2015, 1, 2, 3, 4, 5), utc); - EXPECT_EQ("04:05", format("%M:%E*S", tp_ms, utc)); + EXPECT_EQ("04:05", cctz::format("%M:%E*S", tp_ms, utc)); const time_point<chrono::seconds> tp_s = convert(civil_second(2015, 1, 2, 3, 4, 5), utc); - EXPECT_EQ("04:05", format("%M:%E*S", tp_s, utc)); + EXPECT_EQ("04:05", cctz::format("%M:%E*S", tp_s, utc)); const time_point<cctz::seconds> tp_s64 = convert(civil_second(2015, 1, 2, 3, 4, 5), utc); - EXPECT_EQ("04:05", format("%M:%E*S", tp_s64, utc)); + EXPECT_EQ("04:05", cctz::format("%M:%E*S", tp_s64, utc)); // These next two require chrono::time_point_cast because the conversion // from a resolution of seconds (the return value of convert()) to a @@ -919,11 +919,11 @@ TEST(MakeTime, TimePointResolution) { const time_point<chrono::minutes> tp_m = chrono::time_point_cast<chrono::minutes>( convert(civil_second(2015, 1, 2, 3, 4, 5), utc)); - EXPECT_EQ("04:00", format("%M:%E*S", tp_m, utc)); + EXPECT_EQ("04:00", cctz::format("%M:%E*S", tp_m, utc)); const time_point<chrono::hours> tp_h = chrono::time_point_cast<chrono::hours>( convert(civil_second(2015, 1, 2, 3, 4, 5), utc)); - EXPECT_EQ("00:00", format("%M:%E*S", tp_h, utc)); + EXPECT_EQ("00:00", cctz::format("%M:%E*S", tp_h, utc)); } TEST(MakeTime, Normalization) { @@ -949,9 +949,9 @@ TEST(MakeTime, SysSecondsLimits) { // Approach the maximal time_point<cctz::seconds> value from below. tp = convert(civil_second(292277026596, 12, 4, 15, 30, 6), utc); - EXPECT_EQ("292277026596-12-04T15:30:06+00:00", format(RFC3339, tp, utc)); + EXPECT_EQ("292277026596-12-04T15:30:06+00:00", cctz::format(RFC3339, tp, utc)); tp = convert(civil_second(292277026596, 12, 4, 15, 30, 7), utc); - EXPECT_EQ("292277026596-12-04T15:30:07+00:00", format(RFC3339, tp, utc)); + EXPECT_EQ("292277026596-12-04T15:30:07+00:00", cctz::format(RFC3339, tp, utc)); EXPECT_EQ(time_point<cctz::seconds>::max(), tp); tp = convert(civil_second(292277026596, 12, 4, 15, 30, 8), utc); EXPECT_EQ(time_point<cctz::seconds>::max(), tp); @@ -960,7 +960,7 @@ TEST(MakeTime, SysSecondsLimits) { // Checks that we can also get the maximal value for a far-east zone. tp = convert(civil_second(292277026596, 12, 5, 5, 30, 7), east); - EXPECT_EQ("292277026596-12-05T05:30:07+14:00", format(RFC3339, tp, east)); + EXPECT_EQ("292277026596-12-05T05:30:07+14:00", cctz::format(RFC3339, tp, east)); EXPECT_EQ(time_point<cctz::seconds>::max(), tp); tp = convert(civil_second(292277026596, 12, 5, 5, 30, 8), east); EXPECT_EQ(time_point<cctz::seconds>::max(), tp); @@ -969,7 +969,7 @@ TEST(MakeTime, SysSecondsLimits) { // Checks that we can also get the maximal value for a far-west zone. tp = convert(civil_second(292277026596, 12, 4, 1, 30, 7), west); - EXPECT_EQ("292277026596-12-04T01:30:07-14:00", format(RFC3339, tp, west)); + EXPECT_EQ("292277026596-12-04T01:30:07-14:00", cctz::format(RFC3339, tp, west)); EXPECT_EQ(time_point<cctz::seconds>::max(), tp); tp = convert(civil_second(292277026596, 12, 4, 7, 30, 8), west); EXPECT_EQ(time_point<cctz::seconds>::max(), tp); @@ -978,9 +978,9 @@ TEST(MakeTime, SysSecondsLimits) { // Approach the minimal time_point<cctz::seconds> value from above. tp = convert(civil_second(-292277022657, 1, 27, 8, 29, 53), utc); - EXPECT_EQ("-292277022657-01-27T08:29:53+00:00", format(RFC3339, tp, utc)); + EXPECT_EQ("-292277022657-01-27T08:29:53+00:00", cctz::format(RFC3339, tp, utc)); tp = convert(civil_second(-292277022657, 1, 27, 8, 29, 52), utc); - EXPECT_EQ("-292277022657-01-27T08:29:52+00:00", format(RFC3339, tp, utc)); + EXPECT_EQ("-292277022657-01-27T08:29:52+00:00", cctz::format(RFC3339, tp, utc)); EXPECT_EQ(time_point<cctz::seconds>::min(), tp); tp = convert(civil_second(-292277022657, 1, 27, 8, 29, 51), utc); EXPECT_EQ(time_point<cctz::seconds>::min(), tp); @@ -989,7 +989,7 @@ TEST(MakeTime, SysSecondsLimits) { // Checks that we can also get the minimal value for a far-east zone. tp = convert(civil_second(-292277022657, 1, 27, 22, 29, 52), east); - EXPECT_EQ("-292277022657-01-27T22:29:52+14:00", format(RFC3339, tp, east)); + EXPECT_EQ("-292277022657-01-27T22:29:52+14:00", cctz::format(RFC3339, tp, east)); EXPECT_EQ(time_point<cctz::seconds>::min(), tp); tp = convert(civil_second(-292277022657, 1, 27, 22, 29, 51), east); EXPECT_EQ(time_point<cctz::seconds>::min(), tp); @@ -998,7 +998,7 @@ TEST(MakeTime, SysSecondsLimits) { // Checks that we can also get the minimal value for a far-west zone. tp = convert(civil_second(-292277022657, 1, 26, 18, 29, 52), west); - EXPECT_EQ("-292277022657-01-26T18:29:52-14:00", format(RFC3339, tp, west)); + EXPECT_EQ("-292277022657-01-26T18:29:52-14:00", cctz::format(RFC3339, tp, west)); EXPECT_EQ(time_point<cctz::seconds>::min(), tp); tp = convert(civil_second(-292277022657, 1, 26, 18, 29, 51), west); EXPECT_EQ(time_point<cctz::seconds>::min(), tp); @@ -1018,11 +1018,11 @@ TEST(MakeTime, SysSecondsLimits) { #if defined(__FreeBSD__) || defined(__OpenBSD__) // The BSD gmtime_r() fails on extreme positive tm_year values. #else - EXPECT_EQ("2147485547-12-31T23:59:59+00:00", format(RFC3339, tp, cut)); + EXPECT_EQ("2147485547-12-31T23:59:59+00:00", cctz::format(RFC3339, tp, cut)); #endif const year_t min_tm_year = year_t{std::numeric_limits<int>::min()} + 1900; tp = convert(civil_second(min_tm_year, 1, 1, 0, 0, 0), cut); - EXPECT_EQ("-2147481748-01-01T00:00:00+00:00", format(RFC3339, tp, cut)); + EXPECT_EQ("-2147481748-01-01T00:00:00+00:00", cctz::format(RFC3339, tp, cut)); #endif } } diff --git a/contrib/libs/llvm12/utils/TableGen/GlobalISel/GIMatchDag.cpp b/contrib/libs/llvm12/utils/TableGen/GlobalISel/GIMatchDag.cpp index 7e037dd03b..8be32d2eff 100644 --- a/contrib/libs/llvm12/utils/TableGen/GlobalISel/GIMatchDag.cpp +++ b/contrib/libs/llvm12/utils/TableGen/GlobalISel/GIMatchDag.cpp @@ -48,7 +48,7 @@ void GIMatchDag::writeDOTGraph(raw_ostream &OS, StringRef ID) const { << Assignment.first << ")"; Separator = ", "; } - OS << format("|%p|", &N); + OS << llvm::format("|%p|", &N); writePorts("d", N->getOperandInfo()); OS << "}\""; if (N->isMatchRoot()) @@ -82,7 +82,7 @@ void GIMatchDag::writeDOTGraph(raw_ostream &OS, StringRef ID) const { writePorts("s", N->getOperandInfo()); OS << "|" << N->getName() << "|"; N->printDescription(OS); - OS << format("|%p|", &N); + OS << llvm::format("|%p|", &N); writePorts("d", N->getOperandInfo()); OS << "}\",style=dotted]\n"; } diff --git a/contrib/libs/lz4/ya.make b/contrib/libs/lz4/ya.make index fa735a4a6d..78477250e2 100644 --- a/contrib/libs/lz4/ya.make +++ b/contrib/libs/lz4/ya.make @@ -1,17 +1,14 @@ -# Generated by devtools/yamaker from nixpkgs 22.05. +# Generated by devtools/yamaker from nixpkgs 22.11. LIBRARY() -VERSION(1.9.4) +LICENSE(BSD-2-Clause) -ORIGINAL_SOURCE(https://github.com/lz4/lz4/archive/v1.9.4.tar.gz) +LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -LICENSE( - BSD-2-Clause AND - BSD-3-Clause -) +VERSION(1.9.4) -LICENSE_TEXTS(.yandex_meta/licenses.list.txt) +ORIGINAL_SOURCE(https://github.com/lz4/lz4/archive/v1.9.4.tar.gz) PEERDIR( contrib/libs/xxhash diff --git a/contrib/libs/pybind11/README.rst b/contrib/libs/pybind11/README.rst index 45c4af5a60..80213a4062 100644 --- a/contrib/libs/pybind11/README.rst +++ b/contrib/libs/pybind11/README.rst @@ -32,9 +32,9 @@ this heavy machinery has become an excessively large and unnecessary dependency. Think of this library as a tiny self-contained version of Boost.Python -with everything stripped away that isn’t relevant for binding +with everything stripped away that isn't relevant for binding generation. Without comments, the core header files only require ~4K -lines of code and depend on Python (2.7 or 3.5+, or PyPy) and the C++ +lines of code and depend on Python (3.6+, or PyPy) and the C++ standard library. This compact implementation was possible thanks to some of the new C++11 language features (specifically: tuples, lambda functions and variadic templates). Since its creation, this library has @@ -78,8 +78,8 @@ Goodies In addition to the core functionality, pybind11 provides some extra goodies: -- Python 2.7, 3.5+, and PyPy/PyPy3 7.3 are supported with an - implementation-agnostic interface. +- Python 3.6+, and PyPy3 7.3 are supported with an implementation-agnostic + interface (pybind11 2.9 was the last version to support Python 2 and 3.5). - It is possible to bind C++11 lambda functions with captured variables. The lambda capture data is stored inside the resulting @@ -88,8 +88,8 @@ goodies: - pybind11 uses C++11 move constructors and move assignment operators whenever possible to efficiently transfer custom data types. -- It’s easy to expose the internal storage of custom data types through - Pythons’ buffer protocols. This is handy e.g. for fast conversion +- It's easy to expose the internal storage of custom data types through + Pythons' buffer protocols. This is handy e.g. for fast conversion between C++ matrix classes like Eigen and NumPy without expensive copy operations. @@ -119,10 +119,10 @@ goodies: Supported compilers ------------------- -1. Clang/LLVM 3.3 or newer (for Apple Xcode’s clang, this is 5.0.0 or +1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or newer) 2. GCC 4.8 or newer -3. Microsoft Visual Studio 2015 Update 3 or newer +3. Microsoft Visual Studio 2017 or newer 4. Intel classic C++ compiler 18 or newer (ICC 20.2 tested in CI) 5. Cygwin/GCC (previously tested on 2.5.1) 6. NVCC (CUDA 11.0 tested in CI) @@ -135,7 +135,7 @@ This project was created by `Wenzel Jakob <http://rgl.epfl.ch/people/wjakob>`_. Significant features and/or improvements to the code were contributed by Jonas Adler, Lori A. Burns, Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel -Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz Miąsko, +Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov, Johan Mabille, Tomasz Miąsko, Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. diff --git a/contrib/libs/pybind11/SECURITY.md b/contrib/libs/pybind11/SECURITY.md new file mode 100644 index 0000000000..3d74611f2d --- /dev/null +++ b/contrib/libs/pybind11/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +Security updates are applied only to the latest release. + +## Reporting a Vulnerability + +If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. + +Please disclose it at [security advisory](https://github.com/pybind/pybind11/security/advisories/new). + +This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure. diff --git a/contrib/libs/pybind11/include/pybind11/attr.h b/contrib/libs/pybind11/include/pybind11/attr.h index de95f89d0f..1044db94d9 100644 --- a/contrib/libs/pybind11/include/pybind11/attr.h +++ b/contrib/libs/pybind11/include/pybind11/attr.h @@ -10,6 +10,7 @@ #pragma once +#include "detail/common.h" #include "cast.h" #include <functional> @@ -25,6 +26,9 @@ struct is_method { explicit is_method(const handle &c) : class_(c) {} }; +/// Annotation for setters +struct is_setter {}; + /// Annotation for operators struct is_operator {}; @@ -61,7 +65,7 @@ struct base { PYBIND11_DEPRECATED( "base<T>() was deprecated in favor of specifying 'T' as a template argument to class_") - base() {} // NOLINT(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute + base() = default; }; /// Keep patient alive while nurse lives @@ -82,8 +86,7 @@ struct metaclass { handle value; PYBIND11_DEPRECATED("py::metaclass() is no longer required. It's turned on by default now.") - // NOLINTNEXTLINE(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute - metaclass() {} + metaclass() = default; /// Override pybind11's default metaclass explicit metaclass(handle value) : value(value) {} @@ -188,8 +191,8 @@ struct argument_record { struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), has_args(false), has_kwargs(false), - prepend(false) {} + is_operator(false), is_method(false), is_setter(false), has_args(false), + has_kwargs(false), prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -230,6 +233,9 @@ struct function_record { /// True if this is a method bool is_method : 1; + /// True if this is a setter + bool is_setter : 1; + /// True if the function has a '*args' argument bool has_args : 1; @@ -345,9 +351,11 @@ struct type_record { bases.append((PyObject *) base_info->type); - if (base_info->type->tp_dictoffset != 0) { - dynamic_attr = true; - } +#if PY_VERSION_HEX < 0x030B0000 + dynamic_attr |= base_info->type->tp_dictoffset != 0; +#else + dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; +#endif if (caster) { base_info->implicit_casts.emplace_back(type, caster); @@ -397,7 +405,7 @@ struct process_attribute<doc> : process_attribute_default<doc> { template <> struct process_attribute<const char *> : process_attribute_default<const char *> { static void init(const char *d, function_record *r) { r->doc = const_cast<char *>(d); } - static void init(const char *d, type_record *r) { r->doc = const_cast<char *>(d); } + static void init(const char *d, type_record *r) { r->doc = d; } }; template <> struct process_attribute<char *> : process_attribute<const char *> {}; @@ -424,6 +432,12 @@ struct process_attribute<is_method> : process_attribute_default<is_method> { } }; +/// Process an attribute which indicates that this function is a setter +template <> +struct process_attribute<is_setter> : process_attribute_default<is_setter> { + static void init(const is_setter &, function_record *r) { r->is_setter = true; } +}; + /// Process an attribute which indicates the parent scope of a method template <> struct process_attribute<scope> : process_attribute_default<scope> { @@ -478,7 +492,7 @@ struct process_attribute<arg_v> : process_attribute_default<arg_v> { } if (!a.value) { -#if !defined(NDEBUG) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) std::string descr("'"); if (a.name) { descr += std::string(a.name) + ": "; @@ -499,7 +513,8 @@ struct process_attribute<arg_v> : process_attribute_default<arg_v> { #else pybind11_fail("arg(): could not convert default argument " "into a Python object (type not registered yet?). " - "Compile in debug mode for more information."); + "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "more information."); #endif } r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert, a.flag_none); diff --git a/contrib/libs/pybind11/include/pybind11/buffer_info.h b/contrib/libs/pybind11/include/pybind11/buffer_info.h index 06120d5563..b99ee8bef4 100644 --- a/contrib/libs/pybind11/include/pybind11/buffer_info.h +++ b/contrib/libs/pybind11/include/pybind11/buffer_info.h @@ -37,6 +37,9 @@ inline std::vector<ssize_t> f_strides(const std::vector<ssize_t> &shape, ssize_t return strides; } +template <typename T, typename SFINAE = void> +struct compare_buffer_info; + PYBIND11_NAMESPACE_END(detail) /// Information record describing a Python buffer object @@ -150,6 +153,17 @@ struct buffer_info { Py_buffer *view() const { return m_view; } Py_buffer *&view() { return m_view; } + /* True if the buffer item type is equivalent to `T`. */ + // To define "equivalent" by example: + // `buffer_info::item_type_is_equivalent_to<int>(b)` and + // `buffer_info::item_type_is_equivalent_to<long>(b)` may both be true + // on some platforms, but `int` and `unsigned` will never be equivalent. + // For the ground truth, please inspect `detail::compare_buffer_info<>`. + template <typename T> + bool item_type_is_equivalent_to() const { + return detail::compare_buffer_info<T>::compare(*this); + } + private: struct private_ctr_tag {}; @@ -170,9 +184,10 @@ private: PYBIND11_NAMESPACE_BEGIN(detail) -template <typename T, typename SFINAE = void> +template <typename T, typename SFINAE> struct compare_buffer_info { static bool compare(const buffer_info &b) { + // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *` return b.format == format_descriptor<T>::format() && b.itemsize == (ssize_t) sizeof(T); } }; diff --git a/contrib/libs/pybind11/include/pybind11/cast.h b/contrib/libs/pybind11/include/pybind11/cast.h index 92f65119bf..aa7dd494ad 100644 --- a/contrib/libs/pybind11/include/pybind11/cast.h +++ b/contrib/libs/pybind11/include/pybind11/cast.h @@ -30,6 +30,9 @@ #include <util/generic/string.h> PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template <typename type, typename SFINAE = void> @@ -89,7 +92,8 @@ public: template <typename T_, \ ::pybind11::detail::enable_if_t< \ std::is_same<type, ::pybind11::detail::remove_cv_t<T_>>::value, \ - int> = 0> \ + int> \ + = 0> \ static ::pybind11::handle cast( \ T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) { \ if (!src) \ @@ -249,7 +253,7 @@ public: return false; } static handle cast(T, return_value_policy /* policy */, handle /* parent */) { - return none().inc_ref(); + return none().release(); } PYBIND11_TYPE_CASTER(T, const_name("None")); }; @@ -292,7 +296,7 @@ public: if (ptr) { return capsule(ptr).release(); } - return none().inc_ref(); + return none().release(); } template <typename T> @@ -330,7 +334,7 @@ public: res = 0; // None is implicitly converted to False } #if defined(PYPY_VERSION) - // On PyPy, check that "__bool__" (or "__nonzero__" on Python 2.7) attr exists + // On PyPy, check that "__bool__" attr exists else if (hasattr(src, PYBIND11_BOOL_ATTR)) { res = PyObject_IsTrue(src.ptr()); } @@ -389,10 +393,10 @@ struct string_caster { } if (!PyUnicode_Check(load_src.ptr())) { #if PY_MAJOR_VERSION >= 3 - return load_bytes(load_src); + return load_raw(load_src); #else if (std::is_same<CharT, char>::value) { - return load_bytes(load_src); + return load_raw(load_src); } // The below is a guaranteed failure in Python 3 when PyUnicode_Check returns false @@ -409,9 +413,9 @@ struct string_caster { } #if PY_VERSION_HEX >= 0x03030000 - // On Python >= 3.3, for UTF-8 we avoid the need for a temporary `bytes` - // object by using `PyUnicode_AsUTF8AndSize`. - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N == 8)) { + // For UTF-8 we avoid the need for a temporary `bytes` object by using + // `PyUnicode_AsUTF8AndSize`. + if (UTF_N == 8) { Py_ssize_t size = -1; const auto *buffer = reinterpret_cast<const CharT *>(PyUnicode_AsUTF8AndSize(load_src.ptr(), &size)); @@ -439,7 +443,7 @@ struct string_caster { = reinterpret_cast<const CharT *>(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); // Skip BOM for UTF-16/32 - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N > 8)) { + if (UTF_N > 8) { buffer++; length--; } @@ -485,26 +489,37 @@ private: #endif } - // When loading into a std::string or char*, accept a bytes object as-is (i.e. + // When loading into a std::string or char*, accept a bytes/bytearray object as-is (i.e. // without any encoding/decoding attempt). For other C++ char sizes this is a no-op. // which supports loading a unicode from a str, doesn't take this path. template <typename C = CharT> - bool load_bytes(enable_if_t<std::is_same<C, char>::value, handle> src) { + bool load_raw(enable_if_t<std::is_same<C, char>::value, handle> src) { if (PYBIND11_BYTES_CHECK(src.ptr())) { - // We were passed a Python 3 raw bytes; accept it into a std::string or char* + // We were passed raw bytes; accept it into a std::string or char* // without any encoding attempt. const char *bytes = PYBIND11_BYTES_AS_STRING(src.ptr()); - if (bytes) { - value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); - return true; + if (!bytes) { + pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure."); + } + value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); + return true; + } + if (PyByteArray_Check(src.ptr())) { + // We were passed a bytearray; accept it into a std::string or char* + // without any encoding attempt. + const char *bytearray = PyByteArray_AsString(src.ptr()); + if (!bytearray) { + pybind11_fail("Unexpected PyByteArray_AsString() failure."); } + value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); + return true; } return false; } template <typename C = CharT> - bool load_bytes(enable_if_t<!std::is_same<C, char>::value, handle>) { + bool load_raw(enable_if_t<!std::is_same<C, char>::value, handle>) { return false; } }; @@ -530,7 +545,7 @@ struct type_caster<std::basic_string_view<CharT, Traits>, template <typename CharT> struct type_caster<CharT, enable_if_t<is_std_char_type<CharT>::value>> { using StringType = std::basic_string<CharT>; - using StringCaster = type_caster<StringType>; + using StringCaster = make_caster<StringType>; StringCaster str_caster; bool none = false; CharT one_char = 0; @@ -553,7 +568,7 @@ public: static handle cast(const CharT *src, return_value_policy policy, handle parent) { if (src == nullptr) { - return pybind11::none().inc_ref(); + return pybind11::none().release(); } return StringCaster::cast(StringType(src), policy, parent); } @@ -588,7 +603,7 @@ public: // figure out how long the first encoded character is in bytes to distinguish between these // two errors. We also allow want to allow unicode characters U+0080 through U+00FF, as // those can fit into a single char value. - if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 8) && str_len > 1 && str_len <= 4) { + if (StringCaster::UTF_N == 8 && str_len > 1 && str_len <= 4) { auto v0 = static_cast<unsigned char>(value[0]); // low bits only: 0-127 // 0b110xxxxx - start of 2-byte sequence @@ -614,7 +629,7 @@ public: // UTF-16 is much easier: we can only have a surrogate pair for values above U+FFFF, thus a // surrogate pair with total length 2 instantly indicates a range error (but not a "your // string was too long" error). - else if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 16) && str_len == 2) { + else if (StringCaster::UTF_N == 16 && str_len == 2) { one_char = static_cast<CharT>(value[0]); if (one_char >= 0xD800 && one_char < 0xE000) { throw value_error("Character code point not in range(0x10000)"); @@ -793,8 +808,9 @@ protected: return true; } throw cast_error("Unable to cast from non-held to held instance (T& to Holder<T>) " -#if defined(NDEBUG) - "(compile in debug mode for type information)"); +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "type information)"); #else "of type '" + type_id<holder_type>() + "''"); @@ -861,7 +877,7 @@ struct always_construct_holder { /// Create a specialization for custom holder types (silently ignores std::shared_ptr) #define PYBIND11_DECLARE_HOLDER_TYPE(type, holder_type, ...) \ - namespace pybind11 { \ + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) \ namespace detail { \ template <typename type> \ struct always_construct_holder<holder_type> : always_construct_holder<void, ##__VA_ARGS__> { \ @@ -870,7 +886,7 @@ struct always_construct_holder { class type_caster<holder_type, enable_if_t<!is_shared_ptr<holder_type>::value>> \ : public type_caster_holder<type, holder_type> {}; \ } \ - } + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) // PYBIND11_DECLARE_HOLDER_TYPE holder types: template <typename base, typename holder> @@ -924,6 +940,14 @@ struct handle_type_name<kwargs> { template <typename type> struct pyobject_caster { template <typename T = type, enable_if_t<std::is_same<T, handle>::value, int> = 0> + pyobject_caster() : value() {} + + // `type` may not be default constructible (e.g. frozenset, anyset). Initializing `value` + // to a nil handle is safe since it will only be accessed if `load` succeeds. + template <typename T = type, enable_if_t<std::is_base_of<object, T>::value, int> = 0> + pyobject_caster() : value(reinterpret_steal<type>(handle())) {} + + template <typename T = type, enable_if_t<std::is_same<T, handle>::value, int> = 0> bool load(handle src, bool /* convert */) { value = src; return static_cast<bool>(value); @@ -935,7 +959,7 @@ struct pyobject_caster { // For Python 2, without this implicit conversion, Python code would // need to be cluttered with six.ensure_text() or similar, only to be // un-cluttered later after Python 2 support is dropped. - if (PYBIND11_SILENCE_MSVC_C4127(std::is_same<T, str>::value) && isinstance<bytes>(src)) { + if ((std::is_same<T, str>::value) && isinstance<bytes>(src)) { PyObject *str_from_bytes = PyUnicode_FromEncodedObject(src.ptr(), "utf-8", nullptr); if (!str_from_bytes) throw error_already_set(); @@ -979,7 +1003,7 @@ struct move_always< enable_if_t< all_of<move_is_plain_type<T>, negation<is_copy_constructible<T>>, - std::is_move_constructible<T>, + is_move_constructible<T>, std::is_same<decltype(std::declval<make_caster<T>>().operator T &()), T &>>::value>> : std::true_type {}; template <typename T, typename SFINAE = void> @@ -990,7 +1014,7 @@ struct move_if_unreferenced< enable_if_t< all_of<move_is_plain_type<T>, negation<move_always<T>>, - std::is_move_constructible<T>, + is_move_constructible<T>, std::is_same<decltype(std::declval<make_caster<T>>().operator T &()), T &>>::value>> : std::true_type {}; template <typename T> @@ -1028,13 +1052,18 @@ struct return_value_policy_override< // Basic python -> C++ casting; throws if casting fails template <typename T, typename SFINAE> type_caster<T, SFINAE> &load_type(type_caster<T, SFINAE> &conv, const handle &handle) { + static_assert(!detail::is_pyobject<T>::value, + "Internal error: type_caster should only be used for C++ types"); if (!conv.load(handle, true)) { -#if defined(NDEBUG) +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) throw cast_error( - "Unable to cast Python instance to C++ type (compile in debug mode for details)"); + "Unable to cast Python instance of type " + + str(type::handle_of(handle)).cast<std::string>() + + " to C++ type '?' (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else throw cast_error("Unable to cast Python instance of type " - + (std::string) str(type::handle_of(handle)) + " to C++ type '" + + str(type::handle_of(handle)).cast<std::string>() + " to C++ type '" + type_id<T>() + "'"); #endif } @@ -1051,7 +1080,11 @@ make_caster<T> load_type(const handle &handle) { PYBIND11_NAMESPACE_END(detail) // pytype -> C++ type -template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0> +template <typename T, + detail::enable_if_t<!detail::is_pyobject<T>::value + && !detail::is_same_ignoring_cvref<T, PyObject *>::value, + int> + = 0> T cast(const handle &handle) { using namespace detail; static_assert(!cast_is_temporary_value_reference<T>::value, @@ -1065,6 +1098,34 @@ T cast(const handle &handle) { return T(reinterpret_borrow<object>(handle)); } +// Note that `cast<PyObject *>(obj)` increments the reference count of `obj`. +// This is necessary for the case that `obj` is a temporary, and could +// not possibly be different, given +// 1. the established convention that the passed `handle` is borrowed, and +// 2. we don't want to force all generic code using `cast<T>()` to special-case +// handling of `T` = `PyObject *` (to increment the reference count there). +// It is the responsibility of the caller to ensure that the reference count +// is decremented. +template <typename T, + typename Handle, + detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value + && detail::is_same_ignoring_cvref<Handle, handle>::value, + int> + = 0> +T cast(Handle &&handle) { + return handle.inc_ref().ptr(); +} +// To optimize way an inc_ref/dec_ref cycle: +template <typename T, + typename Object, + detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value + && detail::is_same_ignoring_cvref<Object, object>::value, + int> + = 0> +T cast(Object &&obj) { + return obj.release().ptr(); +} + // C++ type -> py::object template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0> object cast(T &&value, @@ -1096,14 +1157,15 @@ inline void handle::cast() const { template <typename T> detail::enable_if_t<!detail::move_never<T>::value, T> move(object &&obj) { if (obj.ref_count() > 1) { -#if defined(NDEBUG) +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) throw cast_error( - "Unable to cast Python instance to C++ rvalue: instance has multiple references" - " (compile in debug mode for details)"); + "Unable to cast Python " + str(type::handle_of(obj)).cast<std::string>() + + " instance to C++ rvalue: instance has multiple references" + " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else - throw cast_error("Unable to move from Python " + (std::string) str(type::handle_of(obj)) - + " instance to C++ " + type_id<T>() - + " instance: instance has multiple references"); + throw cast_error("Unable to move from Python " + + str(type::handle_of(obj)).cast<std::string>() + " instance to C++ " + + type_id<T>() + " instance: instance has multiple references"); #endif } @@ -1118,21 +1180,30 @@ detail::enable_if_t<!detail::move_never<T>::value, T> move(object &&obj) { // - If both movable and copyable, check ref count: if 1, move; otherwise copy // - Otherwise (not movable), copy. template <typename T> -detail::enable_if_t<detail::move_always<T>::value, T> cast(object &&object) { +detail::enable_if_t<!detail::is_pyobject<T>::value && detail::move_always<T>::value, T> +cast(object &&object) { return move<T>(std::move(object)); } template <typename T> -detail::enable_if_t<detail::move_if_unreferenced<T>::value, T> cast(object &&object) { +detail::enable_if_t<!detail::is_pyobject<T>::value && detail::move_if_unreferenced<T>::value, T> +cast(object &&object) { if (object.ref_count() > 1) { return cast<T>(object); } return move<T>(std::move(object)); } template <typename T> -detail::enable_if_t<detail::move_never<T>::value, T> cast(object &&object) { +detail::enable_if_t<!detail::is_pyobject<T>::value && detail::move_never<T>::value, T> +cast(object &&object) { return cast<T>(object); } +// pytype rvalue -> pytype (calls converting constructor) +template <typename T> +detail::enable_if_t<detail::is_pyobject<T>::value, T> cast(object &&object) { + return T(std::move(object)); +} + template <typename T> T object::cast() const & { return pybind11::cast<T>(*this); @@ -1183,24 +1254,26 @@ enable_if_t<!cast_is_temporary_value_reference<T>::value, T> cast_ref(object &&, // static_assert, even though if it's in dead code, so we provide a "trampoline" to pybind11::cast // that only does anything in cases where pybind11::cast is valid. template <typename T> -enable_if_t<!cast_is_temporary_value_reference<T>::value, T> cast_safe(object &&o) { - return pybind11::cast<T>(std::move(o)); -} -template <typename T> enable_if_t<cast_is_temporary_value_reference<T>::value, T> cast_safe(object &&) { pybind11_fail("Internal error: cast_safe fallback invoked"); } -template <> -inline void cast_safe<void>(object &&) {} +template <typename T> +enable_if_t<std::is_void<T>::value, void> cast_safe(object &&) {} +template <typename T> +enable_if_t<detail::none_of<cast_is_temporary_value_reference<T>, std::is_void<T>>::value, T> +cast_safe(object &&o) { + return pybind11::cast<T>(std::move(o)); +} PYBIND11_NAMESPACE_END(detail) // The overloads could coexist, i.e. the #if is not strictly speaking needed, // but it is an easy minor optimization. -#if defined(NDEBUG) -inline cast_error cast_error_unable_to_convert_call_arg() { - return cast_error( - "Unable to convert call argument to Python object (compile in debug mode for details)"); +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) +inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name) { + return cast_error("Unable to convert call argument '" + name + + "' to Python object (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } #else inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name, @@ -1222,8 +1295,8 @@ tuple make_tuple(Args &&...args_) { detail::make_caster<Args>::cast(std::forward<Args>(args_), policy, nullptr))...}}; for (size_t i = 0; i < args.size(); i++) { if (!args[i]) { -#if defined(NDEBUG) - throw cast_error_unable_to_convert_call_arg(); +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + throw cast_error_unable_to_convert_call_arg(std::to_string(i)); #else std::array<std::string, size> argtypes{{type_id<Args>()...}}; throw cast_error_unable_to_convert_call_arg(std::to_string(i), argtypes[i]); @@ -1271,10 +1344,10 @@ struct arg_v : arg { private: template <typename T> arg_v(arg &&base, T &&x, const char *descr = nullptr) - : arg(base), value(reinterpret_steal<object>( - detail::make_caster<T>::cast(x, return_value_policy::automatic, {}))), + : arg(base), value(reinterpret_steal<object>(detail::make_caster<T>::cast( + std::forward<T>(x), return_value_policy::automatic, {}))), descr(descr) -#if !defined(NDEBUG) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) , type(type_id<T>()) #endif @@ -1314,7 +1387,7 @@ public: object value; /// The (optional) description of the default value const char *descr; -#if !defined(NDEBUG) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) /// The C++ type name of the default value (only available when compiled in debug mode) std::string type; #endif @@ -1322,7 +1395,7 @@ public: /// \ingroup annotations /// Annotation indicating that all following arguments are keyword-only; the is the equivalent of -/// an unnamed '*' argument (in Python 3) +/// an unnamed '*' argument struct kw_only {}; /// \ingroup annotations @@ -1512,14 +1585,14 @@ private: auto o = reinterpret_steal<object>( detail::make_caster<T>::cast(std::forward<T>(x), policy, {})); if (!o) { -#if defined(NDEBUG) - throw cast_error_unable_to_convert_call_arg(); +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size())); #else throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()), type_id<T>()); #endif } - args_list.append(o); + args_list.append(std::move(o)); } void process(list &args_list, detail::args_proxy ap) { @@ -1530,27 +1603,27 @@ private: void process(list & /*args_list*/, arg_v a) { if (!a.name) { -#if defined(NDEBUG) +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) nameless_argument_error(); #else nameless_argument_error(a.type); #endif } if (m_kwargs.contains(a.name)) { -#if defined(NDEBUG) +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) multiple_values_error(); #else multiple_values_error(a.name); #endif } if (!a.value) { -#if defined(NDEBUG) - throw cast_error_unable_to_convert_call_arg(); +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + throw cast_error_unable_to_convert_call_arg(a.name); #else throw cast_error_unable_to_convert_call_arg(a.name, a.type); #endif } - m_kwargs[a.name] = a.value; + m_kwargs[a.name] = std::move(a.value); } void process(list & /*args_list*/, detail::kwargs_proxy kp) { @@ -1559,7 +1632,7 @@ private: } for (auto k : reinterpret_borrow<dict>(kp)) { if (m_kwargs.contains(k.first)) { -#if defined(NDEBUG) +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) multiple_values_error(); #else multiple_values_error(str(k.first)); @@ -1570,9 +1643,10 @@ private: } [[noreturn]] static void nameless_argument_error() { - throw type_error("Got kwargs without a name; only named arguments " - "may be passed via py::arg() to a python function call. " - "(compile in debug mode for details)"); + throw type_error( + "Got kwargs without a name; only named arguments " + "may be passed via py::arg() to a python function call. " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } [[noreturn]] static void nameless_argument_error(const std::string &type) { throw type_error("Got kwargs without a name of type '" + type @@ -1580,8 +1654,9 @@ private: "arguments may be passed via py::arg() to a python function call. "); } [[noreturn]] static void multiple_values_error() { - throw type_error("Got multiple values for keyword argument " - "(compile in debug mode for details)"); + throw type_error( + "Got multiple values for keyword argument " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } [[noreturn]] static void multiple_values_error(const std::string &name) { @@ -1653,12 +1728,12 @@ handle type::handle_of() { } #define PYBIND11_MAKE_OPAQUE(...) \ - namespace pybind11 { \ + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) \ namespace detail { \ template <> \ class type_caster<__VA_ARGS__> : public type_caster_base<__VA_ARGS__> {}; \ } \ - } + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) /// Lets you pass a type containing a `,` through a macro parameter without needing a separate /// typedef, e.g.: diff --git a/contrib/libs/pybind11/include/pybind11/detail/class.h b/contrib/libs/pybind11/include/pybind11/detail/class.h index 0e06b94f10..fd192530b5 100644 --- a/contrib/libs/pybind11/include/pybind11/detail/class.h +++ b/contrib/libs/pybind11/include/pybind11/detail/class.h @@ -19,8 +19,8 @@ PYBIND11_NAMESPACE_BEGIN(detail) # define PYBIND11_BUILTIN_QUALNAME # define PYBIND11_SET_OLDPY_QUALNAME(obj, nameobj) #else -// In pre-3.3 Python, we still set __qualname__ so that we can produce reliable function type -// signatures; in 3.3+ this macro expands to nothing: +// In PyPy, we still set __qualname__ so that we can produce reliable function type +// signatures; in CPython this macro expands to nothing: # define PYBIND11_SET_OLDPY_QUALNAME(obj, nameobj) \ setattr((PyObject *) obj, "__qualname__", nameobj) #endif @@ -55,6 +55,9 @@ extern "C" inline int pybind11_static_set(PyObject *self, PyObject *obj, PyObjec return PyProperty_Type.tp_descr_set(self, cls, value); } +// Forward declaration to use in `make_static_property_type()` +inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type); + /** A `static_property` is the same as a `property` but the `__get__()` and `__set__()` methods are modified to always use the object type instead of a concrete instance. Return value: New reference. */ @@ -87,6 +90,13 @@ inline PyTypeObject *make_static_property_type() { pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); } +# if PY_VERSION_HEX >= 0x030C0000 + // PRE 3.12 FEATURE FREEZE. PLEASE REVIEW AFTER FREEZE. + // Since Python-3.12 property-derived types are required to + // have dynamic attributes (to set `__doc__`) + enable_dynamic_attributes(heap_type); +# endif + setattr((PyObject *) type, "__module__", str("pybind11_builtins")); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); @@ -439,9 +449,17 @@ inline void clear_instance(PyObject *self) { /// Instance destructor function for all pybind11 types. It calls `type_info.dealloc` /// to destroy the C++ object itself, while the rest is Python bookkeeping. extern "C" inline void pybind11_object_dealloc(PyObject *self) { + auto *type = Py_TYPE(self); + + // If this is a GC tracked object, untrack it first + // Note that the track call is implicitly done by the + // default tp_alloc, which we never override. + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) != 0) { + PyObject_GC_UnTrack(self); + } + clear_instance(self); - auto *type = Py_TYPE(self); type->tp_free(self); #if PY_VERSION_HEX < 0x03080000 @@ -459,6 +477,8 @@ extern "C" inline void pybind11_object_dealloc(PyObject *self) { #endif } +std::string error_string(); + /** Create the type which can be used as a common base for all classes. This is needed in order to satisfy Python's requirements for multiple inheritance. Return value: New reference. */ @@ -494,7 +514,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { type->tp_weaklistoffset = offsetof(instance, weakrefs); if (PyType_Ready(type) < 0) { - pybind11_fail("PyType_Ready failed in make_object_base_type():" + error_string()); + pybind11_fail("PyType_Ready failed in make_object_base_type(): " + error_string()); } setattr((PyObject *) type, "__module__", str("pybind11_builtins")); @@ -504,6 +524,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { return (PyObject *) heap_type; } +#if PY_MAJOR_VERSION < 3 /// dynamic_attr: Support for `d = instance.__dict__`. extern "C" inline PyObject *pybind11_get_dict(PyObject *self, void *) { PyObject *&dict = *_PyObject_GetDictPtr(self); @@ -528,11 +549,16 @@ extern "C" inline int pybind11_set_dict(PyObject *self, PyObject *new_dict, void dict = new_dict; return 0; } +#endif /// dynamic_attr: Allow the garbage collector to traverse the internal instance `__dict__`. extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { PyObject *&dict = *_PyObject_GetDictPtr(self); Py_VISIT(dict); +// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse +#if PY_VERSION_HEX >= 0x03090000 + Py_VISIT(Py_TYPE(self)); +#endif return 0; } @@ -547,14 +573,31 @@ extern "C" inline int pybind11_clear(PyObject *self) { inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; +#if PY_VERSION_HEX < 0x030B0000 type->tp_dictoffset = type->tp_basicsize; // place dict at the end type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it +#else + type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; +#endif type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; - static PyGetSetDef getset[] = { - {const_cast<char *>("__dict__"), pybind11_get_dict, pybind11_set_dict, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr}}; + static PyGetSetDef getset[] = {{ +#if PY_VERSION_HEX < 0x03070000 + const_cast<char *>("__dict__"), +#else + "__dict__", +#endif +#if PY_MAJOR_VERSION < 3 + pybind11_get_dict, + pybind11_set_dict, +#else + PyObject_GenericGetDict, + PyObject_GenericSetDict, +#endif + nullptr, + nullptr}, + {nullptr, nullptr, nullptr, nullptr, nullptr}}; type->tp_getset = getset; } @@ -723,7 +766,7 @@ inline PyObject *make_new_python_type(const type_record &rec) { } if (PyType_Ready(type) < 0) { - pybind11_fail(std::string(rec.name) + ": PyType_Ready failed (" + error_string() + ")!"); + pybind11_fail(std::string(rec.name) + ": PyType_Ready failed: " + error_string()); } assert(!rec.dynamic_attr || PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)); diff --git a/contrib/libs/pybind11/include/pybind11/detail/common.h b/contrib/libs/pybind11/include/pybind11/detail/common.h index 6e8cfd4922..601408153a 100644 --- a/contrib/libs/pybind11/include/pybind11/detail/common.h +++ b/contrib/libs/pybind11/include/pybind11/detail/common.h @@ -10,15 +10,76 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 9 -#define PYBIND11_VERSION_PATCH 2 +#define PYBIND11_VERSION_MINOR 11 +#define PYBIND11_VERSION_PATCH 1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x02090200 +#define PYBIND11_VERSION_HEX 0x020B0100 + +// Define some generic pybind11 helper macros for warning management. +// +// Note that compiler-specific push/pop pairs are baked into the +// PYBIND11_NAMESPACE_BEGIN/PYBIND11_NAMESPACE_END pair of macros. Therefore manual +// PYBIND11_WARNING_PUSH/PYBIND11_WARNING_POP are usually only needed in `#include` sections. +// +// If you find you need to suppress a warning, please try to make the suppression as local as +// possible using these macros. Please also be sure to push/pop with the pybind11 macros. Please +// only use compiler specifics if you need to check specific versions, e.g. Apple Clang vs. vanilla +// Clang. +#if defined(_MSC_VER) +# define PYBIND11_COMPILER_MSVC +# define PYBIND11_PRAGMA(...) __pragma(__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning(push)) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning(pop)) +#elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_INTEL +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning pop) +#elif defined(__clang__) +# define PYBIND11_COMPILER_CLANG +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push) +#elif defined(__GNUC__) +# define PYBIND11_COMPILER_GCC +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(GCC diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(GCC diagnostic pop) +#endif + +#ifdef PYBIND11_COMPILER_MSVC +# define PYBIND11_WARNING_DISABLE_MSVC(name) PYBIND11_PRAGMA(warning(disable : name)) +#else +# define PYBIND11_WARNING_DISABLE_MSVC(name) +#endif + +#ifdef PYBIND11_COMPILER_CLANG +# define PYBIND11_WARNING_DISABLE_CLANG(name) PYBIND11_PRAGMA(clang diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_CLANG(name) +#endif + +#ifdef PYBIND11_COMPILER_GCC +# define PYBIND11_WARNING_DISABLE_GCC(name) PYBIND11_PRAGMA(GCC diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_GCC(name) +#endif + +#ifdef PYBIND11_COMPILER_INTEL +# define PYBIND11_WARNING_DISABLE_INTEL(name) PYBIND11_PRAGMA(warning disable name) +#else +# define PYBIND11_WARNING_DISABLE_INTEL(name) +#endif + +#define PYBIND11_NAMESPACE_BEGIN(name) \ + namespace name { \ + PYBIND11_WARNING_PUSH -#define PYBIND11_NAMESPACE_BEGIN(name) namespace name { -#define PYBIND11_NAMESPACE_END(name) } +#define PYBIND11_NAMESPACE_END(name) \ + PYBIND11_WARNING_POP \ + } // Robust support for some features and loading modules compiled against different pybind versions // requires forcing hidden visibility on pybind code, so we enforce this by setting the attribute @@ -38,6 +99,7 @@ # define PYBIND11_CPP17 # if __cplusplus >= 202002L # define PYBIND11_CPP20 +// Please update tests/pybind11_tests.cpp `cpp_std()` when adding a macro here. # endif # endif # endif @@ -47,7 +109,7 @@ // or newer. # if _MSVC_LANG >= 201402L # define PYBIND11_CPP14 -# if _MSVC_LANG > 201402L && _MSC_VER >= 1910 +# if _MSVC_LANG > 201402L # define PYBIND11_CPP17 # if _MSVC_LANG >= 202002L # define PYBIND11_CPP20 @@ -81,10 +143,8 @@ # error pybind11 requires gcc 4.8 or newer # endif #elif defined(_MSC_VER) -// Pybind hits various compiler bugs in 2015u2 and earlier, and also makes use of some stl features -// (e.g. std::negation) added in 2015u3: -# if _MSC_FULL_VER < 190024210 -# error pybind11 requires MSVC 2015 update 3 or newer +# if _MSC_VER < 1910 +# error pybind11 2.10+ requires MSVC 2017 or newer # endif #endif @@ -97,13 +157,10 @@ #endif #if !defined(PYBIND11_EXPORT_EXCEPTION) -# ifdef __MINGW32__ -// workaround for: -// error: 'dllexport' implies default visibility, but xxx has already been declared with a -// different visibility -# define PYBIND11_EXPORT_EXCEPTION -# else +# if defined(__apple_build_version__) # define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT +# else +# define PYBIND11_EXPORT_EXCEPTION # endif #endif @@ -149,15 +206,15 @@ /* Don't let Python.h #define (v)snprintf as macro because they are implemented properly in Visual Studio since 2015. */ -#if defined(_MSC_VER) && _MSC_VER >= 1900 +#if defined(_MSC_VER) # define HAVE_SNPRINTF 1 #endif /// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode #if defined(_MSC_VER) -# pragma warning(push) +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4505) // C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) -# pragma warning(disable : 4505) # if defined(_DEBUG) && !defined(Py_DEBUG) // Workaround for a VS 2022 issue. // NOTE: This workaround knowingly violates the Python.h include order requirement: @@ -206,11 +263,8 @@ # endif #endif -#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L -# define PYBIND11_HAS_U8STRING -#endif - #include <Python.h> +// Reminder: WITH_THREAD is always defined if PY_VERSION_HEX >= 0x03070000 #include <frameobject.h> #include <pythread.h> @@ -231,12 +285,16 @@ # undef copysign #endif +#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# define PYBIND11_SIMPLE_GIL_MANAGEMENT +#endif + #if defined(_MSC_VER) # if defined(PYBIND11_DEBUG_MARKER) # define _DEBUG # undef PYBIND11_DEBUG_MARKER # endif -# pragma warning(pop) +PYBIND11_WARNING_POP #endif #include <cstddef> @@ -257,6 +315,20 @@ # endif #endif +// Must be after including <version> or one of the other headers specified by the standard +#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L +# define PYBIND11_HAS_U8STRING +#endif + +// See description of PR #4246: +#if PY_MAJOR_VERSION < 3 +# define PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF +#endif +#if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG) \ + && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +# define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF +#endif + // #define PYBIND11_STR_LEGACY_PERMISSIVE // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject // (probably surprising and never documented, but this was the @@ -266,7 +338,7 @@ // If UNDEFINED, pybind11::str can only hold PyUnicodeObject, and // pybind11::isinstance<str>() is true only for pybind11::str. // However, for Python 2 only (!), the pybind11::str caster -// implicitly decodes bytes to PyUnicodeObject. This is to ease +// implicitly decoded bytes to PyUnicodeObject. This was to ease // the transition from the legacy behavior to the non-permissive // behavior. @@ -330,15 +402,6 @@ PyObject *pybind11_init_wrapper() #endif -#if PY_VERSION_HEX >= 0x03050000 && PY_VERSION_HEX < 0x03050200 -extern "C" { -struct _Py_atomic_address { - void *value; -}; -PyAPI_DATA(_Py_atomic_address) _PyThreadState_Current; -} -#endif - #define PYBIND11_TRY_NEXT_OVERLOAD ((PyObject *) 1) // special failure return code #define PYBIND11_STRINGIFY(x) #x #define PYBIND11_TOSTRING(x) PYBIND11_STRINGIFY(x) @@ -418,7 +481,7 @@ PyAPI_DATA(_Py_atomic_address) _PyThreadState_Current; /** \rst This macro creates the entry point that will be invoked when the Python interpreter - imports an extension module. The module name is given as the fist argument and it + imports an extension module. The module name is given as the first argument and it should not be in quotes. The second macro argument defines a variable of type `py::module_` which can be used to initialize the module. @@ -483,7 +546,7 @@ enum class return_value_policy : uint8_t { /** Reference an existing object (i.e. do not create a new copy) and take ownership. Python will call the destructor and delete operator when the - object’s reference count reaches zero. Undefined behavior ensues when + object's reference count reaches zero. Undefined behavior ensues when the C++ side does the same.. */ take_ownership, @@ -499,7 +562,7 @@ enum class return_value_policy : uint8_t { move, /** Reference an existing object, but do not take ownership. The C++ side - is responsible for managing the object’s lifetime and deallocating it + is responsible for managing the object's lifetime and deallocating it when it is no longer used. Warning: undefined behavior will ensue when the C++ side deletes an object that is still referenced and used by Python. */ @@ -508,7 +571,7 @@ enum class return_value_policy : uint8_t { /** This policy only applies to methods and properties. It references the object without taking ownership similar to the above return_value_policy::reference policy. In contrast to that policy, the - function or property’s implicit this argument (called the parent) is + function or property's implicit this argument (called the parent) is considered to be the the owner of the return value (the child). pybind11 then couples the lifetime of the parent to the child via a reference relationship that ensures that the parent cannot be garbage @@ -615,7 +678,7 @@ static_assert(std::is_standard_layout<instance>::value, "Internal error: `pybind11::detail::instance` is not standard layout!"); /// from __cpp_future__ import (convenient aliases from C++14/17) -#if defined(PYBIND11_CPP14) && (!defined(_MSC_VER) || _MSC_VER >= 1910) +#if defined(PYBIND11_CPP14) using std::conditional_t; using std::enable_if_t; using std::remove_cv_t; @@ -643,6 +706,10 @@ template <class T> using remove_cvref_t = typename remove_cvref<T>::type; #endif +/// Example usage: is_same_ignoring_cvref<T, PyObject *>::value +template <typename T, typename U> +using is_same_ignoring_cvref = std::is_same<detail::remove_cvref_t<T>, U>; + /// Index sequences #if defined(PYBIND11_CPP14) using std::index_sequence; @@ -736,7 +803,16 @@ template <typename C, typename R, typename... A> struct remove_class<R (C::*)(A...) const> { using type = R(A...); }; - +#ifdef __cpp_noexcept_function_type +template <typename C, typename R, typename... A> +struct remove_class<R (C::*)(A...) noexcept> { + using type = R(A...); +}; +template <typename C, typename R, typename... A> +struct remove_class<R (C::*)(A...) const noexcept> { + using type = R(A...); +}; +#endif /// Helper template to strip away type modifiers template <typename T> struct intrinsic_type { @@ -878,10 +954,12 @@ struct is_template_base_of_impl { /// Check if a template is the base of a type. For example: /// `is_template_base_of<Base, T>` is true if `struct T : Base<U> {}` where U can be anything template <template <typename...> class Base, typename T> +// Sadly, all MSVC versions incl. 2022 need the workaround, even in C++20 mode. +// See also: https://github.com/pybind/pybind11/pull/3741 #if !defined(_MSC_VER) using is_template_base_of = decltype(is_template_base_of_impl<Base>::check((intrinsic_t<T> *) nullptr)); -#else // MSVC2015 has trouble with decltype in template aliases +#else struct is_template_base_of : decltype(is_template_base_of_impl<Base>::check((intrinsic_t<T> *) nullptr)) { }; @@ -951,12 +1029,6 @@ using expand_side_effects = bool[]; PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif /// C++ bindings of builtin Python exceptions class PYBIND11_EXPORT_EXCEPTION builtin_exception : public std::runtime_error { public: @@ -964,9 +1036,6 @@ public: /// Set the error using the Python C API virtual void set_error() const = 0; }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif #define PYBIND11_RUNTIME_EXCEPTION(name, type) \ class PYBIND11_EXPORT_EXCEPTION name : public builtin_exception { \ @@ -990,15 +1059,26 @@ PYBIND11_RUNTIME_EXCEPTION(cast_error, PyExc_RuntimeError) /// Thrown when pybin PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used internally [[noreturn]] PYBIND11_NOINLINE void pybind11_fail(const char *reason) { + assert(!PyErr_Occurred()); throw std::runtime_error(reason); } [[noreturn]] PYBIND11_NOINLINE void pybind11_fail(const std::string &reason) { + assert(!PyErr_Occurred()); throw std::runtime_error(reason); } template <typename T, typename SFINAE = void> struct format_descriptor {}; +template <typename T> +struct format_descriptor< + T, + detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value>> { + static constexpr const char c = 'O'; + static constexpr const char value[2] = {c, '\0'}; + static std::string format() { return std::string(1, c); } +}; + PYBIND11_NAMESPACE_BEGIN(detail) // Returns the index of the given type in the type char array below, and in the list in numpy.h // The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double; @@ -1044,6 +1124,8 @@ constexpr const char struct error_scope { PyObject *type, *value, *trace; error_scope() { PyErr_Fetch(&type, &value, &trace); } + error_scope(const error_scope &) = delete; + error_scope &operator=(const error_scope &) = delete; ~error_scope() { PyErr_Restore(type, value, trace); } }; @@ -1056,9 +1138,6 @@ struct nodelete { PYBIND11_NAMESPACE_BEGIN(detail) template <typename... Args> struct overload_cast_impl { - // NOLINTNEXTLINE(modernize-use-equals-default): MSVC 2015 needs this - constexpr overload_cast_impl() {} - template <typename Return> constexpr auto operator()(Return (*pf)(Args...)) const noexcept -> decltype(pf) { return pf; @@ -1085,8 +1164,7 @@ PYBIND11_NAMESPACE_END(detail) /// - regular: static_cast<Return (Class::*)(Arg0, Arg1, Arg2)>(&Class::func) /// - sweet: overload_cast<Arg0, Arg1, Arg2>(&Class::func) template <typename... Args> -static constexpr detail::overload_cast_impl<Args...> overload_cast = {}; -// MSVC 2015 only accepts this particular initialization syntax for this variable template. +static constexpr detail::overload_cast_impl<Args...> overload_cast{}; #endif /// Const member function selector for overload_cast @@ -1172,7 +1250,7 @@ try_get_shared_from_this(std::enable_shared_from_this<T> *holder_value_ptr) { // For silencing "unused" compiler warnings in special situations. template <typename... Args> -#if defined(_MSC_VER) && _MSC_VER >= 1910 && _MSC_VER < 1920 // MSVC 2017 +#if defined(_MSC_VER) && _MSC_VER < 1920 // MSVC 2017 constexpr #endif inline void @@ -1195,15 +1273,30 @@ constexpr # define PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(...) #endif -#if defined(_MSC_VER) // All versions (as of July 2021). - -// warning C4127: Conditional expression is constant -constexpr inline bool silence_msvc_c4127(bool cond) { return cond; } - -# define PYBIND11_SILENCE_MSVC_C4127(...) ::pybind11::detail::silence_msvc_c4127(__VA_ARGS__) +#if defined(__clang__) \ + && (defined(__apple_build_version__) /* AppleClang 13.0.0.13000029 was the only data point \ + available. */ \ + || (__clang_major__ >= 7 \ + && __clang_major__ <= 12) /* Clang 3, 5, 13, 14, 15 do not generate the warning. */ \ + ) +# define PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING +// Example: +// tests/test_kwargs_and_defaults.cpp:46:68: error: local variable 'args' will be copied despite +// being returned by name [-Werror,-Wreturn-std-move] +// m.def("args_function", [](py::args args) -> py::tuple { return args; }); +// ^~~~ +// test_kwargs_and_defaults.cpp:46:68: note: call 'std::move' explicitly to avoid copying +// m.def("args_function", [](py::args args) -> py::tuple { return args; }); +// ^~~~ +// std::move(args) +#endif -#else -# define PYBIND11_SILENCE_MSVC_C4127(...) __VA_ARGS__ +// Pybind offers detailed error messages by default for all builts that are debug (through the +// negation of NDEBUG). This can also be manually enabled by users, for any builds, through +// defining PYBIND11_DETAILED_ERROR_MESSAGES. This information is primarily useful for those +// who are writing (as opposed to merely using) libraries that use pybind11. +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(NDEBUG) +# define PYBIND11_DETAILED_ERROR_MESSAGES #endif PYBIND11_NAMESPACE_END(detail) diff --git a/contrib/libs/pybind11/include/pybind11/detail/descr.h b/contrib/libs/pybind11/include/pybind11/detail/descr.h index e7a5e2c145..635614b0d6 100644 --- a/contrib/libs/pybind11/include/pybind11/detail/descr.h +++ b/contrib/libs/pybind11/include/pybind11/detail/descr.h @@ -143,11 +143,24 @@ constexpr descr<N, Ts...> concat(const descr<N, Ts...> &descr) { return descr; } +#ifdef __cpp_fold_expressions +template <size_t N1, size_t N2, typename... Ts1, typename... Ts2> +constexpr descr<N1 + N2 + 2, Ts1..., Ts2...> operator,(const descr<N1, Ts1...> &a, + const descr<N2, Ts2...> &b) { + return a + const_name(", ") + b; +} + +template <size_t N, typename... Ts, typename... Args> +constexpr auto concat(const descr<N, Ts...> &d, const Args &...args) { + return (d, ..., args); +} +#else template <size_t N, typename... Ts, typename... Args> constexpr auto concat(const descr<N, Ts...> &d, const Args &...args) -> decltype(std::declval<descr<N + 2, Ts...>>() + concat(args...)) { return d + const_name(", ") + concat(args...); } +#endif template <size_t N, typename... Ts> constexpr descr<N + 2, Ts...> type_descr(const descr<N, Ts...> &descr) { diff --git a/contrib/libs/pybind11/include/pybind11/detail/init.h b/contrib/libs/pybind11/include/pybind11/detail/init.h index 3807d6d5a3..ca5a5178c8 100644 --- a/contrib/libs/pybind11/include/pybind11/detail/init.h +++ b/contrib/libs/pybind11/include/pybind11/detail/init.h @@ -12,6 +12,9 @@ #include "class.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template <> @@ -115,7 +118,7 @@ template <typename Class> void construct(value_and_holder &v_h, Cpp<Class> *ptr, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); no_nullptr(ptr); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias<Class>(ptr)) { + if (Class::has_alias && need_alias && !is_alias<Class>(ptr)) { // We're going to try to construct an alias by moving the cpp type. Whether or not // that succeeds, we still need to destroy the original cpp pointer (either the // moved away leftover, if the alias construction works, or the value itself if we @@ -156,7 +159,7 @@ void construct(value_and_holder &v_h, Holder<Class> holder, bool need_alias) { auto *ptr = holder_helper<Holder<Class>>::get(holder); no_nullptr(ptr); // If we need an alias, check that the held pointer is actually an alias instance - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias<Class>(ptr)) { + if (Class::has_alias && need_alias && !is_alias<Class>(ptr)) { throw type_error("pybind11::init(): construction failed: returned holder-wrapped instance " "is not an alias instance"); } @@ -172,9 +175,9 @@ void construct(value_and_holder &v_h, Holder<Class> holder, bool need_alias) { template <typename Class> void construct(value_and_holder &v_h, Cpp<Class> &&result, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); - static_assert(std::is_move_constructible<Cpp<Class>>::value, + static_assert(is_move_constructible<Cpp<Class>>::value, "pybind11::init() return-by-value factory function requires a movable class"); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias) { + if (Class::has_alias && need_alias) { construct_alias_from_cpp<Class>(is_alias_constructible<Class>{}, v_h, std::move(result)); } else { v_h.value_ptr() = new Cpp<Class>(std::move(result)); @@ -187,7 +190,7 @@ void construct(value_and_holder &v_h, Cpp<Class> &&result, bool need_alias) { template <typename Class> void construct(value_and_holder &v_h, Alias<Class> &&result, bool) { static_assert( - std::is_move_constructible<Alias<Class>>::value, + is_move_constructible<Alias<Class>>::value, "pybind11::init() return-by-alias-value factory function requires a movable alias class"); v_h.value_ptr() = new Alias<Class>(std::move(result)); } @@ -206,10 +209,11 @@ struct constructor { extra...); } - template <typename Class, - typename... Extra, - enable_if_t<Class::has_alias && std::is_constructible<Cpp<Class>, Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t<Class::has_alias && std::is_constructible<Cpp<Class>, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -226,10 +230,11 @@ struct constructor { extra...); } - template <typename Class, - typename... Extra, - enable_if_t<Class::has_alias && !std::is_constructible<Cpp<Class>, Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t<Class::has_alias && !std::is_constructible<Cpp<Class>, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -245,10 +250,11 @@ struct constructor { // Implementing class for py::init_alias<...>() template <typename... Args> struct alias_constructor { - template <typename Class, - typename... Extra, - enable_if_t<Class::has_alias && std::is_constructible<Alias<Class>, Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t<Class::has_alias && std::is_constructible<Alias<Class>, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -459,4 +465,4 @@ struct pickle_factory<Get, Set, RetState(Self), NewInstance(ArgState)> { PYBIND11_NAMESPACE_END(initimpl) PYBIND11_NAMESPACE_END(detail) -PYBIND11_NAMESPACE_END(pybind11) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/contrib/libs/pybind11/include/pybind11/detail/internals.h b/contrib/libs/pybind11/include/pybind11/detail/internals.h index eaf8998b4a..5f4ba6959a 100644 --- a/contrib/libs/pybind11/include/pybind11/detail/internals.h +++ b/contrib/libs/pybind11/include/pybind11/detail/internals.h @@ -9,6 +9,12 @@ #pragma once +#include "common.h" + +#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "../gil.h" +#endif + #include "../pytypes.h" #include <exception> @@ -30,15 +36,26 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 4 +# if PY_VERSION_HEX >= 0x030C0000 +// Version bump for Python 3.12+, before first 3.12 beta release. +# define PYBIND11_INTERNALS_VERSION 5 +# else +# define PYBIND11_INTERNALS_VERSION 4 +# endif #endif +// This requirement is mainly to reduce the support burden (see PR #4570). +static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5, + "pybind11 ABI version 5 is the minimum for Python 3.12+"); + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) using ExceptionTranslator = void (*)(std::exception_ptr); PYBIND11_NAMESPACE_BEGIN(detail) +constexpr const char *internals_function_record_capsule_name = "pybind11_function_record_capsule"; + // Forward declarations inline PyTypeObject *make_static_property_type(); inline PyTypeObject *make_default_metaclass(); @@ -51,7 +68,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); // `Py_LIMITED_API` anyway. # if PYBIND11_INTERNALS_VERSION > 4 # define PYBIND11_TLS_KEY_REF Py_tss_t & -# ifdef __GNUC__ +# if defined(__GNUC__) && !defined(__INTEL_COMPILER) // Clang on macOS warns due to `Py_tss_NEEDS_INIT` not specifying an initializer // for every field. # define PYBIND11_TLS_KEY_INIT(var) \ @@ -108,7 +125,8 @@ inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, // which works. If not under a known-good stl, provide our own name-based hash and equality // functions that use the type name. -#if defined(__GLIBCXX__) +#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__)) \ + || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION)) inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } using type_hash = std::hash<std::type_index>; using type_equal_to = std::equal_to<std::type_index>; @@ -171,11 +189,23 @@ struct internals { PyTypeObject *default_metaclass; PyObject *instance_base; #if defined(WITH_THREAD) + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PYBIND11_TLS_KEY_INIT(tstate) # if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) # endif // PYBIND11_INTERNALS_VERSION > 4 + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; + +# if PYBIND11_INTERNALS_VERSION > 4 + // Note that we have to use a std::string to allocate memory to ensure a unique address + // We want unique addresses since we use pointer equality to compare function records + std::string function_record_capsule_name = internals_function_record_capsule_name; +# endif + + internals() = default; + internals(const internals &other) = delete; + internals &operator=(const internals &other) = delete; ~internals() { # if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_FREE(loader_life_support_tls_key); @@ -415,6 +445,46 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif +inline object get_python_state_dict() { + object state_dict; +#if PYBIND11_INTERNALS_VERSION <= 4 || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) + state_dict = reinterpret_borrow<object>(PyEval_GetBuiltins()); +#else +# if PY_VERSION_HEX < 0x03090000 + PyInterpreterState *istate = _PyInterpreterState_Get(); +# else + PyInterpreterState *istate = PyInterpreterState_Get(); +# endif + if (istate) { + state_dict = reinterpret_borrow<object>(PyInterpreterState_GetDict(istate)); + } +#endif + if (!state_dict) { +#if PY_VERSION_HEX >= 0x03030000 + raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); +#else + PyErr_SetString(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); +#endif + } + return state_dict; +} + +inline object get_internals_obj_from_state_dict(handle state_dict) { + return reinterpret_borrow<object>(dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID)); +} + +inline internals **get_internals_pp_from_capsule(handle obj) { + void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr); + if (raw_ptr == nullptr) { +#if PY_VERSION_HEX >= 0x03030000 + raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED"); +#else + PyErr_SetString(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED"); +#endif + } + return static_cast<internals **>(raw_ptr); +} + /// Return a reference to the current `internals` data PYBIND11_NOINLINE internals &get_internals() { auto **&internals_pp = get_internals_pp(); @@ -422,20 +492,29 @@ PYBIND11_NOINLINE internals &get_internals() { return **internals_pp; } +#if defined(WITH_THREAD) +# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + gil_scoped_acquire gil; +# else // Ensure that the GIL is held since we will need to make Python calls. // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. struct gil_scoped_acquire_local { gil_scoped_acquire_local() : state(PyGILState_Ensure()) {} + gil_scoped_acquire_local(const gil_scoped_acquire_local &) = delete; + gil_scoped_acquire_local &operator=(const gil_scoped_acquire_local &) = delete; ~gil_scoped_acquire_local() { PyGILState_Release(state); } const PyGILState_STATE state; } gil; +# endif +#endif + error_scope err_scope; - PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); - auto builtins = handle(PyEval_GetBuiltins()); - if (builtins.contains(id) && isinstance<capsule>(builtins[id])) { - internals_pp = static_cast<internals **>(capsule(builtins[id])); - - // We loaded builtins through python's builtins, which means that our `error_already_set` + dict state_dict = get_python_state_dict(); + if (object internals_obj = get_internals_obj_from_state_dict(state_dict)) { + internals_pp = get_internals_pp_from_capsule(internals_obj); + } + if (internals_pp && *internals_pp) { + // We loaded the internals through `state_dict`, which means that our `error_already_set` // and `builtin_exception` may be different local classes than the ones set up in the // initial exception translator, below, so add another for our local exception classes. // @@ -453,16 +532,15 @@ PYBIND11_NOINLINE internals &get_internals() { internals_ptr = new internals(); #if defined(WITH_THREAD) -# if PY_VERSION_HEX < 0x03090000 - PyEval_InitThreads(); -# endif PyThreadState *tstate = PyThreadState_Get(); + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->tstate)) { pybind11_fail("get_internals: could not successfully initialize the tstate TSS key!"); } PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate); # if PYBIND11_INTERNALS_VERSION > 4 + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) { pybind11_fail("get_internals: could not successfully initialize the " "loader_life_support TSS key!"); @@ -470,7 +548,7 @@ PYBIND11_NOINLINE internals &get_internals() { # endif internals_ptr->istate = tstate->interp; #endif - builtins[id] = capsule(internals_pp); + state_dict[PYBIND11_INTERNALS_ID] = capsule(internals_pp); internals_ptr->registered_exception_translators.push_front(&translate_exception); internals_ptr->static_property_type = make_static_property_type(); internals_ptr->default_metaclass = make_default_metaclass(); @@ -502,6 +580,7 @@ struct local_internals { struct shared_loader_life_support_data { PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) shared_loader_life_support_data() { + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) { pybind11_fail("local_internals: could not successfully initialize the " "loader_life_support TLS key!"); @@ -525,8 +604,13 @@ struct local_internals { /// Works like `get_internals`, but for things which are locally registered. inline local_internals &get_local_internals() { - static local_internals locals; - return locals; + // Current static can be created in the interpreter finalization routine. If the later will be + // destroyed in another static variable destructor, creation of this static there will cause + // static deinitialization fiasco. In order to avoid it we avoid destruction of the + // local_internals static. One can read more about the problem and current solution here: + // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables + static auto *locals = new local_internals(); + return *locals; } /// Constructs a std::string with the given arguments, stores it in `internals`, and returns its @@ -540,6 +624,25 @@ const char *c_str(Args &&...args) { return strings.front().c_str(); } +inline const char *get_function_record_capsule_name() { +#if PYBIND11_INTERNALS_VERSION > 4 + return get_internals().function_record_capsule_name.c_str(); +#else + return nullptr; +#endif +} + +// Determine whether or not the following capsule contains a pybind11 function record. +// Note that we use `internals` to make sure that only ABI compatible records are touched. +// +// This check is currently used in two places: +// - An important optimization in functional.h to avoid overhead in C++ -> Python -> C++ +// - The sibling feature of cpp_function to allow overloads +inline bool is_function_record_capsule(const capsule &cap) { + // Pointer equality as we rely on internals() to ensure unique pointers + return cap.name() == get_function_record_capsule_name(); +} + PYBIND11_NAMESPACE_END(detail) /// Returns a named pointer that is shared among all extension modules (using the same diff --git a/contrib/libs/pybind11/include/pybind11/detail/type_caster_base.h b/contrib/libs/pybind11/include/pybind11/detail/type_caster_base.h index 3c32aa07c0..245bf99d2d 100644 --- a/contrib/libs/pybind11/include/pybind11/detail/type_caster_base.h +++ b/contrib/libs/pybind11/include/pybind11/detail/type_caster_base.h @@ -226,8 +226,8 @@ PYBIND11_NOINLINE detail::type_info *get_type_info(const std::type_index &tp, if (throw_if_missing) { std::string tname = tp.name(); detail::clean_type_id(tname); - pybind11_fail("pybind11::detail::get_type_info: unable to find type info for \"" + tname - + "\""); + pybind11_fail("pybind11::detail::get_type_info: unable to find type info for \"" + + std::move(tname) + '"'); } return nullptr; } @@ -259,9 +259,9 @@ struct value_and_holder { // Main constructor for a found value/holder: value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) - : inst{i}, index{index}, type{type}, vh{inst->simple_layout - ? inst->simple_value_holder - : &inst->nonsimple.values_and_holders[vpos]} {} + : inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder + : &inst->nonsimple.values_and_holders[vpos]} {} // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) value_and_holder() = default; @@ -395,15 +395,16 @@ instance::get_value_and_holder(const type_info *find_type /*= nullptr default in return value_and_holder(); } -#if defined(NDEBUG) - pybind11_fail("pybind11::detail::instance::get_value_and_holder: " - "type is not a pybind11 base of the given instance " - "(compile in debug mode for type details)"); -#else +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" + get_fully_qualified_tp_name(find_type->type) + "' is not a pybind11 base of the given `" + get_fully_qualified_tp_name(Py_TYPE(this)) + "' instance"); +#else + pybind11_fail( + "pybind11::detail::instance::get_value_and_holder: " + "type is not a pybind11 base of the given instance " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for type details)"); #endif } @@ -440,10 +441,11 @@ PYBIND11_NOINLINE void instance::allocate_layout() { // instance_registered) // Allocate space for flags, values, and holders, and initialize it to 0 (flags and values, - // in particular, need to be 0). Use Python's memory allocation functions: in Python 3.6 - // they default to using pymalloc, which is designed to be efficient for small allocations - // like the one we're doing here; in earlier versions (and for larger allocations) they are - // just wrappers around malloc. + // in particular, need to be 0). Use Python's memory allocation + // functions: Python is using pymalloc, which is designed to be + // efficient for small allocations like the one we're doing here; + // for larger allocations they are just wrappers around malloc. + // TODO: is this still true for pure Python 3.6? #if PY_VERSION_HEX >= 0x03050000 nonsimple.values_and_holders = (void **) PyMem_Calloc(space, sizeof(void *)); if (!nonsimple.values_and_holders) { @@ -476,70 +478,6 @@ PYBIND11_NOINLINE bool isinstance_generic(handle obj, const std::type_info &tp) return isinstance(obj, type); } -PYBIND11_NOINLINE std::string error_string() { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_RuntimeError, "Unknown internal error occurred"); - return "Unknown internal error occurred"; - } - - error_scope scope; // Preserve error state - - std::string errorString; - if (scope.type) { - errorString += handle(scope.type).attr("__name__").cast<std::string>(); - errorString += ": "; - } - if (scope.value) { - errorString += (std::string) str(scope.value); - } - - PyErr_NormalizeException(&scope.type, &scope.value, &scope.trace); - -#if PY_MAJOR_VERSION >= 3 - if (scope.trace != nullptr) { - PyException_SetTraceback(scope.value, scope.trace); - } -#endif - -#if !defined(PYPY_VERSION) - if (scope.trace) { - auto *trace = (PyTracebackObject *) scope.trace; - - /* Get the deepest trace possible */ - while (trace->tb_next) { - trace = trace->tb_next; - } - - PyFrameObject *frame = trace->tb_frame; - Py_XINCREF(frame); - errorString += "\n\nAt:\n"; - while (frame) { -# if PY_VERSION_HEX >= 0x030900B1 - PyCodeObject *f_code = PyFrame_GetCode(frame); -# else - PyCodeObject *f_code = frame->f_code; - Py_INCREF(f_code); -# endif - int lineno = PyFrame_GetLineNumber(frame); - errorString += " " + handle(f_code->co_filename).cast<std::string>() + "(" - + std::to_string(lineno) - + "): " + handle(f_code->co_name).cast<std::string>() + "\n"; - Py_DECREF(f_code); -# if PY_VERSION_HEX >= 0x030900B1 - auto *b_frame = PyFrame_GetBack(frame); -# else - auto *b_frame = frame->f_back; - Py_XINCREF(b_frame); -# endif - Py_DECREF(frame); - frame = b_frame; - } - } -#endif - - return errorString; -} - PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_info *type) { auto &instances = get_internals().registered_instances; auto range = instances.equal_range(ptr); @@ -558,10 +496,6 @@ inline PyThreadState *get_thread_state_unchecked() { return PyThreadState_GET(); #elif PY_VERSION_HEX < 0x03000000 return _PyThreadState_Current; -#elif PY_VERSION_HEX < 0x03050000 - return (PyThreadState *) _Py_atomic_load_relaxed(&_PyThreadState_Current); -#elif PY_VERSION_HEX < 0x03050200 - return (PyThreadState *) _PyThreadState_Current.value; #else return _PyThreadState_UncheckedGet(); #endif @@ -623,14 +557,15 @@ public: if (copy_constructor) { valueptr = copy_constructor(src); } else { -#if defined(NDEBUG) - throw cast_error("return_value_policy = copy, but type is " - "non-copyable! (compile in debug mode for details)"); -#else +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) std::string type_name(tinfo->cpptype->name()); detail::clean_type_id(type_name); throw cast_error("return_value_policy = copy, but type " + type_name + " is non-copyable!"); +#else + throw cast_error("return_value_policy = copy, but type is " + "non-copyable! (#define PYBIND11_DETAILED_ERROR_MESSAGES or " + "compile in debug mode for details)"); #endif } wrapper->owned = true; @@ -642,15 +577,16 @@ public: } else if (copy_constructor) { valueptr = copy_constructor(src); } else { -#if defined(NDEBUG) - throw cast_error("return_value_policy = move, but type is neither " - "movable nor copyable! " - "(compile in debug mode for details)"); -#else +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) std::string type_name(tinfo->cpptype->name()); detail::clean_type_id(type_name); throw cast_error("return_value_policy = move, but type " + type_name + " is neither movable nor copyable!"); +#else + throw cast_error("return_value_policy = move, but type is neither " + "movable nor copyable! " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in " + "debug mode for details)"); #endif } wrapper->owned = true; @@ -896,23 +832,179 @@ using movable_cast_op_type typename std::add_rvalue_reference<intrinsic_t<T>>::type, typename std::add_lvalue_reference<intrinsic_t<T>>::type>>; -// std::is_copy_constructible isn't quite enough: it lets std::vector<T> (and similar) through when -// T is non-copyable, but code containing such a copy constructor fails to actually compile. -template <typename T, typename SFINAE = void> -struct is_copy_constructible : std::is_copy_constructible<T> {}; +// Does the container have a mapped type and is it recursive? +// Implemented by specializations below. +template <typename Container, typename SFINAE = void> +struct container_mapped_type_traits { + static constexpr bool has_mapped_type = false; + static constexpr bool has_recursive_mapped_type = false; +}; -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. template <typename Container> -struct is_copy_constructible< +struct container_mapped_type_traits< Container, - enable_if_t< - all_of<std::is_copy_constructible<Container>, - std::is_same<typename Container::value_type &, typename Container::reference>, - // Avoid infinite recursion - negation<std::is_same<Container, typename Container::value_type>>>::value>> - : is_copy_constructible<typename Container::value_type> {}; + typename std::enable_if< + std::is_same<typename Container::mapped_type, Container>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = true; +}; + +template <typename Container> +struct container_mapped_type_traits< + Container, + typename std::enable_if< + negation<std::is_same<typename Container::mapped_type, Container>>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = false; +}; + +// Does the container have a value type and is it recursive? +// Implemented by specializations below. +template <typename Container, typename SFINAE = void> +struct container_value_type_traits : std::false_type { + static constexpr bool has_value_type = false; + static constexpr bool has_recursive_value_type = false; +}; + +template <typename Container> +struct container_value_type_traits< + Container, + typename std::enable_if< + std::is_same<typename Container::value_type, Container>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = true; +}; + +template <typename Container> +struct container_value_type_traits< + Container, + typename std::enable_if< + negation<std::is_same<typename Container::value_type, Container>>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = false; +}; + +/* + * Tag to be used for representing the bottom of recursively defined types. + * Define this tag so we don't have to use void. + */ +struct recursive_bottom {}; + +/* + * Implementation detail of `recursive_container_traits` below. + * `T` is the `value_type` of the container, which might need to be modified to + * avoid recursive types and const types. + */ +template <typename T, bool is_this_a_map> +struct impl_type_to_check_recursively { + /* + * If the container is recursive, then no further recursion should be done. + */ + using if_recursive = recursive_bottom; + /* + * Otherwise yield `T` unchanged. + */ + using if_not_recursive = T; +}; + +/* + * For pairs - only as value type of a map -, the first type should remove the `const`. + * Also, if the map is recursive, then the recursive checking should consider + * the first type only. + */ +template <typename A, typename B> +struct impl_type_to_check_recursively<std::pair<A, B>, /* is_this_a_map = */ true> { + using if_recursive = typename std::remove_const<A>::type; + using if_not_recursive = std::pair<typename std::remove_const<A>::type, B>; +}; + +/* + * Implementation of `recursive_container_traits` below. + */ +template <typename Container, typename SFINAE = void> +struct impl_recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; + +template <typename Container> +struct impl_recursive_container_traits< + Container, + typename std::enable_if<container_value_type_traits<Container>::has_value_type>::type> { + static constexpr bool is_recursive + = container_mapped_type_traits<Container>::has_recursive_mapped_type + || container_value_type_traits<Container>::has_recursive_value_type; + /* + * This member dictates which type Pybind11 should check recursively in traits + * such as `is_move_constructible`, `is_copy_constructible`, `is_move_assignable`, ... + * Direct access to `value_type` should be avoided: + * 1. `value_type` might recursively contain the type again + * 2. `value_type` of STL map types is `std::pair<A const, B>`, the `const` + * should be removed. + * + */ + using type_to_check_recursively = typename std::conditional< + is_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits<Container>::has_mapped_type>::if_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits<Container>::has_mapped_type>::if_not_recursive>::type; +}; + +/* + * This trait defines the `type_to_check_recursively` which is needed to properly + * handle recursively defined traits such as `is_move_constructible` without going + * into an infinite recursion. + * Should be used instead of directly accessing the `value_type`. + * It cancels the recursion by returning the `recursive_bottom` tag. + * + * The default definition of `type_to_check_recursively` is as follows: + * + * 1. By default, it is `recursive_bottom`, so that the recursion is canceled. + * 2. If the type is non-recursive and defines a `value_type`, then the `value_type` is used. + * If the `value_type` is a pair and a `mapped_type` is defined, + * then the `const` is removed from the first type. + * 3. If the type is recursive and `value_type` is not a pair, then `recursive_bottom` is returned. + * 4. If the type is recursive and `value_type` is a pair and a `mapped_type` is defined, + * then `const` is removed from the first type and the first type is returned. + * + * This behavior can be extended by the user as seen in test_stl_binders.cpp. + * + * This struct is exactly the same as impl_recursive_container_traits. + * The duplication achieves that user-defined specializations don't compete + * with internal specializations, but take precedence. + */ +template <typename Container, typename SFINAE = void> +struct recursive_container_traits : impl_recursive_container_traits<Container> {}; + +template <typename T> +struct is_move_constructible + : all_of<std::is_move_constructible<T>, + is_move_constructible< + typename recursive_container_traits<T>::type_to_check_recursively>> {}; + +template <> +struct is_move_constructible<recursive_bottom> : std::true_type {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the move constructor not exist when the two types aren't +// themselves move constructible, but this can not be relied upon when T1 or T2 are themselves +// containers). +template <typename T1, typename T2> +struct is_move_constructible<std::pair<T1, T2>> + : all_of<is_move_constructible<T1>, is_move_constructible<T2>> {}; + +// std::is_copy_constructible isn't quite enough: it lets std::vector<T> (and similar) through when +// T is non-copyable, but code containing such a copy constructor fails to actually compile. +template <typename T> +struct is_copy_constructible + : all_of<std::is_copy_constructible<T>, + is_copy_constructible< + typename recursive_container_traits<T>::type_to_check_recursively>> {}; + +template <> +struct is_copy_constructible<recursive_bottom> : std::true_type {}; // Likewise for std::pair // (after C++17 it is mandatory that the copy constructor not exist when the two types aren't @@ -923,14 +1015,16 @@ struct is_copy_constructible<std::pair<T1, T2>> : all_of<is_copy_constructible<T1>, is_copy_constructible<T2>> {}; // The same problems arise with std::is_copy_assignable, so we use the same workaround. -template <typename T, typename SFINAE = void> -struct is_copy_assignable : std::is_copy_assignable<T> {}; -template <typename Container> -struct is_copy_assignable<Container, - enable_if_t<all_of<std::is_copy_assignable<Container>, - std::is_same<typename Container::value_type &, - typename Container::reference>>::value>> - : is_copy_assignable<typename Container::value_type> {}; +template <typename T> +struct is_copy_assignable + : all_of< + std::is_copy_assignable<T>, + is_copy_assignable<typename recursive_container_traits<T>::type_to_check_recursively>> { +}; + +template <> +struct is_copy_assignable<recursive_bottom> : std::true_type {}; + template <typename T1, typename T2> struct is_copy_assignable<std::pair<T1, T2>> : all_of<is_copy_assignable<T1>, is_copy_assignable<T2>> {}; @@ -1068,7 +1162,7 @@ protected: return [](const void *arg) -> void * { return new T(*reinterpret_cast<const T *>(arg)); }; } - template <typename T, typename = enable_if_t<std::is_move_constructible<T>::value>> + template <typename T, typename = enable_if_t<is_move_constructible<T>::value>> static auto make_move_constructor(const T *) -> decltype(new T(std::declval<T &&>()), Constructor{}) { return [](const void *arg) -> void * { @@ -1080,5 +1174,14 @@ protected: static Constructor make_move_constructor(...) { return nullptr; } }; +PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) { + if (auto *type_data = get_type_info(ti)) { + handle th((PyObject *) type_data->type); + return th.attr("__module__").cast<std::string>() + '.' + + th.attr("__qualname__").cast<std::string>(); + } + return clean_type_id(ti.name()); +} + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/contrib/libs/pybind11/include/pybind11/detail/typeid.h b/contrib/libs/pybind11/include/pybind11/detail/typeid.h index 8d99fc0286..a67b52135b 100644 --- a/contrib/libs/pybind11/include/pybind11/detail/typeid.h +++ b/contrib/libs/pybind11/include/pybind11/detail/typeid.h @@ -20,6 +20,7 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) + /// Erase all occurrences of a substring inline void erase_all(std::string &string, const std::string &search) { for (size_t pos = 0;;) { @@ -46,14 +47,19 @@ PYBIND11_NOINLINE void clean_type_id(std::string &name) { #endif detail::erase_all(name, "pybind11::"); } + +inline std::string clean_type_id(const char *typeid_name) { + std::string name(typeid_name); + detail::clean_type_id(name); + return name; +} + PYBIND11_NAMESPACE_END(detail) /// Return a string representation of a C++ type template <typename T> static std::string type_id() { - std::string name(typeid(T).name()); - detail::clean_type_id(name); - return name; + return detail::clean_type_id(typeid(T).name()); } PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/contrib/libs/pybind11/include/pybind11/gil.h b/contrib/libs/pybind11/include/pybind11/gil.h index 446f4620b2..570a5581d5 100644 --- a/contrib/libs/pybind11/include/pybind11/gil.h +++ b/contrib/libs/pybind11/include/pybind11/gil.h @@ -10,7 +10,10 @@ #pragma once #include "detail/common.h" -#include "detail/internals.h" + +#if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "detail/internals.h" +#endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -21,7 +24,9 @@ PyThreadState *get_thread_state_unchecked(); PYBIND11_NAMESPACE_END(detail) -#if defined(WITH_THREAD) && !defined(PYPY_VERSION) +#if defined(WITH_THREAD) + +# if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) /* The functions below essentially reproduce the PyGILState_* API using a RAII * pattern, but there are a few important differences: @@ -62,11 +67,11 @@ public: if (!tstate) { tstate = PyThreadState_New(internals.istate); -# if !defined(NDEBUG) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!tstate) { pybind11_fail("scoped_acquire: could not create thread state!"); } -# endif +# endif tstate->gilstate_counter = 0; PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); } else { @@ -80,24 +85,27 @@ public: inc_ref(); } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; + void inc_ref() { ++tstate->gilstate_counter; } PYBIND11_NOINLINE void dec_ref() { --tstate->gilstate_counter; -# if !defined(NDEBUG) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (detail::get_thread_state_unchecked() != tstate) { pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); } if (tstate->gilstate_counter < 0) { pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); } -# endif +# endif if (tstate->gilstate_counter == 0) { -# if !defined(NDEBUG) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!release) { pybind11_fail("scoped_acquire::dec_ref(): internal error!"); } -# endif +# endif PyThreadState_Clear(tstate); if (active) { PyThreadState_DeleteCurrent(); @@ -144,6 +152,9 @@ public: } } + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; + /// This method will disable the PyThreadState_DeleteCurrent call and the /// GIL won't be acquired. This method should be used if the interpreter /// could be shutting down when this is called, as thread deletion is not @@ -172,12 +183,16 @@ private: bool disassoc; bool active = true; }; -#elif defined(PYPY_VERSION) + +# else // PYBIND11_SIMPLE_GIL_MANAGEMENT + class gil_scoped_acquire { PyGILState_STATE state; public: - gil_scoped_acquire() { state = PyGILState_Ensure(); } + gil_scoped_acquire() : state{PyGILState_Ensure()} {} + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_acquire() { PyGILState_Release(state); } void disarm() {} }; @@ -186,17 +201,39 @@ class gil_scoped_release { PyThreadState *state; public: - gil_scoped_release() { state = PyEval_SaveThread(); } + gil_scoped_release() : state{PyEval_SaveThread()} {} + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; -#else + +# endif // PYBIND11_SIMPLE_GIL_MANAGEMENT + +#else // WITH_THREAD + class gil_scoped_acquire { +public: + gil_scoped_acquire() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; void disarm() {} }; + class gil_scoped_release { +public: + gil_scoped_release() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; void disarm() {} }; -#endif + +#endif // WITH_THREAD PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/contrib/libs/pybind11/include/pybind11/numpy.h b/contrib/libs/pybind11/include/pybind11/numpy.h index 9f4d516f5f..8162908edd 100644 --- a/contrib/libs/pybind11/include/pybind11/numpy.h +++ b/contrib/libs/pybind11/include/pybind11/numpy.h @@ -36,6 +36,8 @@ static_assert(std::is_signed<Py_intptr_t>::value, "Py_intptr_t must be signed"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + class array; // Forward declaration PYBIND11_NAMESPACE_BEGIN(detail) @@ -264,7 +266,7 @@ private: module_ m = module_::import("numpy.core.multiarray"); auto c = m.attr("_ARRAY_API"); #if PY_MAJOR_VERSION >= 3 - void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), NULL); + void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); #else void **api_ptr = (void **) PyCObject_AsVoidPtr(c.ptr()); #endif @@ -541,21 +543,21 @@ PYBIND11_NAMESPACE_END(detail) class dtype : public object { public: - PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); + PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_) explicit dtype(const buffer_info &info) { - dtype descr(_dtype_from_pep3118()(PYBIND11_STR_TYPE(info.format))); + dtype descr(_dtype_from_pep3118()(pybind11::str(info.format))); // If info.itemsize == 0, use the value calculated from the format string m_ptr = descr.strip_padding(info.itemsize != 0 ? info.itemsize : descr.itemsize()) .release() .ptr(); } - explicit dtype(const std::string &format) { - m_ptr = from_args(pybind11::str(format)).release().ptr(); - } + explicit dtype(const pybind11::str &format) : dtype(from_args(format)) {} - explicit dtype(const char *format) : dtype(std::string(format)) {} + explicit dtype(const std::string &format) : dtype(pybind11::str(format)) {} + + explicit dtype(const char *format) : dtype(pybind11::str(format)) {} dtype(list names, list formats, list offsets, ssize_t itemsize) { dict args; @@ -563,11 +565,20 @@ public: args["formats"] = std::move(formats); args["offsets"] = std::move(offsets); args["itemsize"] = pybind11::int_(itemsize); - m_ptr = from_args(std::move(args)).release().ptr(); + m_ptr = from_args(args).release().ptr(); + } + + /// Return dtype for the given typenum (one of the NPY_TYPES). + /// https://numpy.org/devdocs/reference/c-api/array.html#c.PyArray_DescrFromType + explicit dtype(int typenum) + : object(detail::npy_api::get().PyArray_DescrFromType_(typenum), stolen_t{}) { + if (m_ptr == nullptr) { + throw error_already_set(); + } } /// This is essentially the same as calling numpy.dtype(args) in Python. - static dtype from_args(object args) { + static dtype from_args(const object &args) { PyObject *ptr = nullptr; if ((detail::npy_api::get().PyArray_DescrConverter_(args.ptr(), &ptr) == 0) || !ptr) { throw error_already_set(); @@ -600,6 +611,23 @@ public: return detail::array_descriptor_proxy(m_ptr)->type; } + /// type number of dtype. + int num() const { + // Note: The signature, `dtype::num` follows the naming of NumPy's public + // Python API (i.e., ``dtype.num``), rather than its internal + // C API (``PyArray_Descr::type_num``). + return detail::array_descriptor_proxy(m_ptr)->type_num; + } + + /// Single character for byteorder + char byteorder() const { return detail::array_descriptor_proxy(m_ptr)->byteorder; } + + /// Alignment of the data type + int alignment() const { return detail::array_descriptor_proxy(m_ptr)->alignment; } + + /// Flags for the array descriptor + char flags() const { return detail::array_descriptor_proxy(m_ptr)->flags; } + private: static object _dtype_from_pep3118() { static PyObject *obj = module_::import("numpy.core._internal") @@ -618,22 +646,27 @@ private: } struct field_descr { - PYBIND11_STR_TYPE name; + pybind11::str name; object format; pybind11::int_ offset; + field_descr(pybind11::str &&name, object &&format, pybind11::int_ &&offset) + : name{std::move(name)}, format{std::move(format)}, offset{std::move(offset)} {}; }; + auto field_dict = attr("fields").cast<dict>(); std::vector<field_descr> field_descriptors; + field_descriptors.reserve(field_dict.size()); - for (auto field : attr("fields").attr("items")()) { + for (auto field : field_dict.attr("items")()) { auto spec = field.cast<tuple>(); auto name = spec[0].cast<pybind11::str>(); - auto format = spec[1].cast<tuple>()[0].cast<dtype>(); - auto offset = spec[1].cast<tuple>()[1].cast<pybind11::int_>(); + auto spec_fo = spec[1].cast<tuple>(); + auto format = spec_fo[0].cast<dtype>(); + auto offset = spec_fo[1].cast<pybind11::int_>(); if ((len(name) == 0u) && format.kind() == 'V') { continue; } - field_descriptors.push_back( - {(PYBIND11_STR_TYPE) name, format.strip_padding(format.itemsize()), offset}); + field_descriptors.emplace_back( + std::move(name), format.strip_padding(format.itemsize()), std::move(offset)); } std::sort(field_descriptors.begin(), @@ -644,9 +677,9 @@ private: list names, formats, offsets; for (auto &descr : field_descriptors) { - names.append(descr.name); - formats.append(descr.format); - offsets.append(descr.offset); + names.append(std::move(descr.name)); + formats.append(std::move(descr.format)); + offsets.append(std::move(descr.offset)); } return dtype(std::move(names), std::move(formats), std::move(offsets), itemsize); } @@ -850,7 +883,7 @@ public: */ template <typename T, ssize_t Dims = -1> detail::unchecked_mutable_reference<T, Dims> mutable_unchecked() & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -868,7 +901,7 @@ public: */ template <typename T, ssize_t Dims = -1> detail::unchecked_reference<T, Dims> unchecked() const & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -944,7 +977,7 @@ protected: void fail_dim_check(ssize_t dim, const std::string &msg) const { throw index_error(msg + ": " + std::to_string(dim) + " (ndim = " + std::to_string(ndim()) - + ")"); + + ')'); } template <typename... Ix> @@ -1094,10 +1127,10 @@ public: /** * Returns a proxy object that provides const access to the array's data without bounds or - * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying - * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped - * for the duration of the returned object, and the caller must take care not to access invalid - * dimensions or dimension indices. + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed + * or reshaped for the duration of the returned object, and the caller must take care not to + * access invalid dimensions or dimension indices. */ template <ssize_t Dims = -1> detail::unchecked_reference<T, Dims> unchecked() const & { @@ -1148,11 +1181,11 @@ struct format_descriptor<T, detail::enable_if_t<detail::is_pod_struct<T>::value> template <size_t N> struct format_descriptor<char[N]> { - static std::string format() { return std::to_string(N) + "s"; } + static std::string format() { return std::to_string(N) + 's'; } }; template <size_t N> struct format_descriptor<std::array<char, N>> { - static std::string format() { return std::to_string(N) + "s"; } + static std::string format() { return std::to_string(N) + 's'; } }; template <typename T> @@ -1256,12 +1289,16 @@ private: public: static constexpr int value = values[detail::is_fmt_numeric<T>::index]; - static pybind11::dtype dtype() { - if (auto *ptr = npy_api::get().PyArray_DescrFromType_(value)) { - return reinterpret_steal<pybind11::dtype>(ptr); - } - pybind11_fail("Unsupported buffer format!"); - } + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } +}; + +template <typename T> +struct npy_format_descriptor<T, enable_if_t<is_same_ignoring_cvref<T, PyObject *>::value>> { + static constexpr auto name = const_name("object"); + + static constexpr int value = npy_api::NPY_OBJECT_; + + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } }; #define PYBIND11_DECL_CHAR_FMT \ @@ -1292,7 +1329,8 @@ public: static pybind11::dtype dtype() { list shape; array_info<T>::append_extents(shape); - return pybind11::dtype::from_args(pybind11::make_tuple(base_descr::dtype(), shape)); + return pybind11::dtype::from_args( + pybind11::make_tuple(base_descr::dtype(), std::move(shape))); } }; @@ -1338,7 +1376,7 @@ PYBIND11_NOINLINE void register_structured_dtype(any_container<field_descriptor> pybind11_fail(std::string("NumPy: unsupported field dtype: `") + field.name + "` @ " + tinfo.name()); } - names.append(PYBIND11_STR_TYPE(field.name)); + names.append(pybind11::str(field.name)); formats.append(field.descr); offsets.append(pybind11::int_(field.offset)); } @@ -1375,7 +1413,7 @@ PYBIND11_NOINLINE void register_structured_dtype(any_container<field_descriptor> oss << '}'; auto format_str = oss.str(); - // Sanity check: verify that NumPy properly parses our buffer format string + // Smoke test: verify that NumPy properly parses our buffer format string auto &api = npy_api::get(); auto arr = array(buffer_info(nullptr, itemsize, format_str, 1)); if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) { @@ -1383,7 +1421,7 @@ PYBIND11_NOINLINE void register_structured_dtype(any_container<field_descriptor> } auto tindex = std::type_index(tinfo); - numpy_internals.registered_dtypes[tindex] = {dtype_ptr, format_str}; + numpy_internals.registered_dtypes[tindex] = {dtype_ptr, std::move(format_str)}; get_internals().direct_conversions[tindex].push_back(direct_converter); } @@ -1443,7 +1481,7 @@ private: } // Extract name, offset and format descriptor for a struct field -# define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, # Field) +# define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, #Field) // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz @@ -1530,7 +1568,7 @@ public: void *data() const { return p_ptr; } private: - char *p_ptr{0}; + char *p_ptr{nullptr}; container_type m_strides; }; @@ -1840,8 +1878,13 @@ private: auto result = returned_array::create(trivial, shape); + PYBIND11_WARNING_PUSH +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") +#endif + if (size == 0) { - return std::move(result); + return result; } /* Call the function */ @@ -1852,7 +1895,8 @@ private: apply_trivial(buffers, params, mutable_data, size, i_seq, vi_seq, bi_seq); } - return std::move(result); + return result; + PYBIND11_WARNING_POP } template <size_t... Index, size_t... VIndex, size_t... BIndex> diff --git a/contrib/libs/pybind11/include/pybind11/options.h b/contrib/libs/pybind11/include/pybind11/options.h index 1e493bdcc4..1b2122522d 100644 --- a/contrib/libs/pybind11/include/pybind11/options.h +++ b/contrib/libs/pybind11/include/pybind11/options.h @@ -47,6 +47,16 @@ public: return *this; } + options &disable_enum_members_docstring() & { + global_state().show_enum_members_docstring = false; + return *this; + } + + options &enable_enum_members_docstring() & { + global_state().show_enum_members_docstring = true; + return *this; + } + // Getter methods (return the global state): static bool show_user_defined_docstrings() { @@ -55,6 +65,10 @@ public: static bool show_function_signatures() { return global_state().show_function_signatures; } + static bool show_enum_members_docstring() { + return global_state().show_enum_members_docstring; + } + // This type is not meant to be allocated on the heap. void *operator new(size_t) = delete; @@ -63,6 +77,8 @@ private: bool show_user_defined_docstrings = true; //< Include user-supplied texts in docstrings. bool show_function_signatures = true; //< Include auto-generated function signatures // in docstrings. + bool show_enum_members_docstring = true; //< Include auto-generated member list in enum + // docstrings. }; static state &global_state() { diff --git a/contrib/libs/pybind11/include/pybind11/pybind11.h b/contrib/libs/pybind11/include/pybind11/pybind11.h index e349bfdc96..cd30063303 100644 --- a/contrib/libs/pybind11/include/pybind11/pybind11.h +++ b/contrib/libs/pybind11/include/pybind11/pybind11.h @@ -35,6 +35,8 @@ # include <cxxabi.h> #endif +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + /* https://stackoverflow.com/questions/46798456/handling-gccs-noexcept-type-warning This warning is about ABI compatibility, not code health. It is only actually needed in a couple places, but apparently GCC 7 "generates this warning if @@ -43,11 +45,10 @@ No other GCC version generates this warning. */ #if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wnoexcept-type" +PYBIND11_WARNING_DISABLE_GCC("-Wnoexcept-type") #endif -PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) @@ -83,6 +84,7 @@ public: cpp_function() = default; // NOLINTNEXTLINE(google-explicit-constructor) cpp_function(std::nullptr_t) {} + cpp_function(std::nullptr_t, const is_setter &) {} /// Construct a cpp_function from a vanilla function pointer template <typename Return, typename... Args, typename... Extra> @@ -177,22 +179,22 @@ protected: auto *rec = unique_rec.get(); /* Store the capture object directly in the function record if there is enough space */ - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(capture) <= sizeof(rec->data))) { + if (sizeof(capture) <= sizeof(rec->data)) { /* Without these pragmas, GCC warns that there might not be enough space to use the placement new operator. However, the 'if' statement above ensures that this is the case. */ -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wplacement-new" + PYBIND11_WARNING_PUSH + +#if defined(__GNUG__) && __GNUC__ >= 6 + PYBIND11_WARNING_DISABLE_GCC("-Wplacement-new") #endif + new ((capture *) &rec->data) capture{std::forward<Func>(f)}; -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wstrict-aliasing" + +#if !PYBIND11_HAS_STD_LAUNDER + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") #endif + // UB without std::launder, but without breaking ABI and/or // a significant refactoring it's "impossible" to solve. if (!std::is_trivially_destructible<capture>::value) { @@ -202,9 +204,7 @@ protected: data->~capture(); }; } -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif + PYBIND11_WARNING_POP } else { rec->data[0] = new capture{std::forward<Func>(f)}; rec->free_data = [](function_record *r) { delete ((capture *) r->data[0]); }; @@ -245,10 +245,16 @@ protected: using Guard = extract_guard_t<Extra...>; /* Perform the function call */ - handle result - = cast_out::cast(std::move(args_converter).template call<Return, Guard>(cap->f), - policy, - call.parent); + handle result; + if (call.func.is_setter) { + (void) std::move(args_converter).template call<Return, Guard>(cap->f); + result = none().release(); + } else { + result = cast_out::cast( + std::move(args_converter).template call<Return, Guard>(cap->f), + policy, + call.parent); + } /* Invoke call policy post-call hook */ process_attributes<Extra...>::postcall(call, result); @@ -312,6 +318,10 @@ protected: // along the way. class strdup_guard { public: + strdup_guard() = default; + strdup_guard(const strdup_guard &) = delete; + strdup_guard &operator=(const strdup_guard &) = delete; + ~strdup_guard() { for (auto *s : strings) { std::free(s); @@ -366,7 +376,7 @@ protected: rec->is_constructor = (std::strcmp(rec->name, "__init__") == 0) || (std::strcmp(rec->name, "__setstate__") == 0); -#if !defined(NDEBUG) && !defined(PYBIND11_DISABLE_NEW_STYLE_INIT_WARNING) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(PYBIND11_DISABLE_NEW_STYLE_INIT_WARNING) if (rec->is_constructor && !rec->is_new_style_constructor) { const auto class_name = detail::get_fully_qualified_tp_name((PyTypeObject *) rec->scope.ptr()); @@ -431,9 +441,8 @@ protected: } if (auto *tinfo = detail::get_type_info(*t)) { handle th((PyObject *) tinfo->type); - signature += th.attr("__module__").cast<std::string>() + "." + - // Python 3.3+, but we backport it to earlier versions - th.attr("__qualname__").cast<std::string>(); + signature += th.attr("__module__").cast<std::string>() + "." + + th.attr("__qualname__").cast<std::string>(); } else if (rec->is_new_style_constructor && arg_index == 0) { // A new-style `__init__` takes `self` as `value_and_holder`. // Rewrite it to the proper class type. @@ -474,13 +483,20 @@ protected: if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { auto *self = PyCFunction_GET_SELF(rec->sibling.ptr()); - capsule rec_capsule = isinstance<capsule>(self) ? reinterpret_borrow<capsule>(self) - : capsule(self); - chain = (detail::function_record *) rec_capsule; - /* Never append a method to an overload chain of a parent class; - instead, hide the parent's overloads in this case */ - if (!chain->scope.is(rec->scope)) { + if (!isinstance<capsule>(self)) { chain = nullptr; + } else { + auto rec_capsule = reinterpret_borrow<capsule>(self); + if (detail::is_function_record_capsule(rec_capsule)) { + chain = rec_capsule.get_pointer<detail::function_record>(); + /* Never append a method to an overload chain of a parent class; + instead, hide the parent's overloads in this case */ + if (!chain->scope.is(rec->scope)) { + chain = nullptr; + } + } else { + chain = nullptr; + } } } // Don't trigger for things like the default __init__, which are wrapper_descriptors @@ -501,6 +517,7 @@ protected: rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; capsule rec_capsule(unique_rec.release(), + detail::get_function_record_capsule_name(), [](void *ptr) { destruct((detail::function_record *) ptr); }); guarded_strdup.release(); @@ -524,8 +541,9 @@ protected: if (chain->is_method != rec->is_method) { pybind11_fail( "overloading a method with both static and instance methods is not supported; " -#if defined(NDEBUG) - "compile in debug mode for more details" +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for more " + "details" #else "error while attempting to bind " + std::string(rec->is_method ? "instance" : "static") + " method " @@ -571,14 +589,14 @@ protected: for (auto *it = chain_start; it != nullptr; it = it->next) { if (options::show_function_signatures()) { if (index > 0) { - signatures += "\n"; + signatures += '\n'; } if (chain) { signatures += std::to_string(++index) + ". "; } signatures += rec->name; signatures += it->signature; - signatures += "\n"; + signatures += '\n'; } if (it->doc && it->doc[0] != '\0' && options::show_user_defined_docstrings()) { // If we're appending another docstring, and aren't printing function signatures, @@ -587,15 +605,15 @@ protected: if (first_user_def) { first_user_def = false; } else { - signatures += "\n"; + signatures += '\n'; } } if (options::show_function_signatures()) { - signatures += "\n"; + signatures += '\n'; } signatures += it->doc; if (options::show_function_signatures()) { - signatures += "\n"; + signatures += '\n'; } } } @@ -666,10 +684,13 @@ protected: /// Main dispatch logic for calls to functions bound using pybind11 static PyObject *dispatcher(PyObject *self, PyObject *args_in, PyObject *kwargs_in) { using namespace detail; + assert(isinstance<capsule>(self)); /* Iterator over the list of potentially admissible overloads */ - const function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr), + const function_record *overloads = reinterpret_cast<function_record *>( + PyCapsule_GetPointer(self, get_function_record_capsule_name())), *it = overloads; + assert(overloads != nullptr); /* Need to know how many arguments + keyword arguments there are to pick the right overload */ @@ -912,7 +933,7 @@ protected: // 5. Put everything in a vector. Not technically step 5, we've been building it // in `call.args` all along. -#if !defined(NDEBUG) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (call.args.size() != func.nargs || call.args_convert.size() != func.nargs) { pybind11_fail("Internal error: function call dispatcher inserted wrong number " "of arguments!"); @@ -1065,7 +1086,7 @@ protected: msg += it2->signature; } - msg += "\n"; + msg += '\n'; } msg += "\nInvoked with: "; auto args_ = reinterpret_borrow<tuple>(args_in); @@ -1186,9 +1207,16 @@ public: py::module_ m3 = m2.def_submodule("subsub", "A submodule of 'example.sub'"); \endrst */ module_ def_submodule(const char *name, const char *doc = nullptr) { - std::string full_name - = std::string(PyModule_GetName(m_ptr)) + std::string(".") + std::string(name); - auto result = reinterpret_borrow<module_>(PyImport_AddModule(full_name.c_str())); + const char *this_name = PyModule_GetName(m_ptr); + if (this_name == nullptr) { + throw error_already_set(); + } + std::string full_name = std::string(this_name) + '.' + name; + handle submodule = PyImport_AddModule(full_name.c_str()); + if (!submodule) { + throw error_already_set(); + } + auto result = reinterpret_borrow<module_>(submodule); if (doc && options::show_user_defined_docstrings()) { result.attr("__doc__") = pybind11::str(doc); } @@ -1232,7 +1260,7 @@ public: } #if PY_MAJOR_VERSION >= 3 - using module_def = PyModuleDef; + using module_def = PyModuleDef; // TODO: Can this be removed (it was needed only for Python 2)? #else struct module_def {}; #endif @@ -1271,8 +1299,8 @@ public: pybind11_fail("Internal error in module_::create_extension_module()"); } // TODO: Should be reinterpret_steal for Python 3, but Python also steals it again when - // returned from PyInit_... - // For Python 2, reinterpret_borrow is correct. + // returned from PyInit_... + // For Python 2, reinterpret_borrow was correct. return reinterpret_borrow<module_>(m); } }; @@ -1436,9 +1464,9 @@ template <typename T, enable_if_t<has_operator_delete<T>::value, int> = 0> void call_operator_delete(T *p, size_t, size_t) { T::operator delete(p); } -template < - typename T, - enable_if_t<!has_operator_delete<T>::value && has_operator_delete_size<T>::value, int> = 0> +template <typename T, + enable_if_t<!has_operator_delete<T>::value && has_operator_delete_size<T>::value, int> + = 0> void call_operator_delete(T *p, size_t s, size_t) { T::operator delete(p, s); } @@ -1593,18 +1621,19 @@ public: scope(*this), sibling(getattr(*this, name_, none())), extra...); - attr(cf.name()) = staticmethod(cf); + auto cf_name = cf.name(); + attr(std::move(cf_name)) = staticmethod(std::move(cf)); return *this; } - template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra> - class_ &def(const detail::op_<id, ot, L, R> &op, const Extra &...extra) { + template <typename T, typename... Extra, detail::enable_if_t<T::op_enable_if_hook, int> = 0> + class_ &def(const T &op, const Extra &...extra) { op.execute(*this, extra...); return *this; } - template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra> - class_ &def_cast(const detail::op_<id, ot, L, R> &op, const Extra &...extra) { + template <typename T, typename... Extra, detail::enable_if_t<T::op_enable_if_hook, int> = 0> + class_ &def_cast(const T &op, const Extra &...extra) { op.execute_cast(*this, extra...); return *this; } @@ -1647,7 +1676,7 @@ public: if (!caster.load(obj, false)) { return nullptr; } - return new buffer_info(((capture *) ptr)->func(caster)); + return new buffer_info(((capture *) ptr)->func(std::move(caster))); }, ptr); weakref(m_ptr, cpp_function([ptr](handle wr) { @@ -1738,7 +1767,8 @@ public: template <typename Getter, typename Setter, typename... Extra> class_ & def_property(const char *name, const Getter &fget, const Setter &fset, const Extra &...extra) { - return def_property(name, fget, cpp_function(method_adaptor<type>(fset)), extra...); + return def_property( + name, fget, cpp_function(method_adaptor<type>(fset), is_setter()), extra...); } template <typename Getter, typename... Extra> class_ &def_property(const char *name, @@ -1849,7 +1879,7 @@ private: if (holder_ptr) { init_holder_from_existing(v_h, holder_ptr, std::is_copy_constructible<holder_type>()); v_h.set_holder_constructed(); - } else if (inst->owned || detail::always_construct_holder<holder_type>::value) { + } else if (detail::always_construct_holder<holder_type>::value || inst->owned) { new (std::addressof(v_h.holder<holder_type>())) holder_type(v_h.value_ptr<type>()); v_h.set_holder_constructed(); } @@ -1889,9 +1919,22 @@ private: static detail::function_record *get_function_record(handle h) { h = detail::get_function(h); - return h ? (detail::function_record *) reinterpret_borrow<capsule>( - PyCFunction_GET_SELF(h.ptr())) - : nullptr; + if (!h) { + return nullptr; + } + + handle func_self = PyCFunction_GET_SELF(h.ptr()); + if (!func_self) { + throw error_already_set(); + } + if (!isinstance<capsule>(func_self)) { + return nullptr; + } + auto cap = reinterpret_borrow<capsule>(func_self); + if (!detail::is_function_record_capsule(cap)) { + return nullptr; + } + return cap.get_pointer<detail::function_record>(); } }; @@ -1952,7 +1995,8 @@ struct enum_base { [](const object &arg) -> str { handle type = type::handle_of(arg); object type_name = type.attr("__name__"); - return pybind11::str("<{}.{}: {}>").format(type_name, enum_name(arg), int_(arg)); + return pybind11::str("<{}.{}: {}>") + .format(std::move(type_name), enum_name(arg), int_(arg)); }, name("__repr__"), is_method(m_base)); @@ -1962,34 +2006,40 @@ struct enum_base { m_base.attr("__str__") = cpp_function( [](handle arg) -> str { object type_name = type::handle_of(arg).attr("__name__"); - return pybind11::str("{}.{}").format(type_name, enum_name(arg)); + return pybind11::str("{}.{}").format(std::move(type_name), enum_name(arg)); }, name("name"), is_method(m_base)); - m_base.attr("__doc__") = static_property( - cpp_function( - [](handle arg) -> std::string { - std::string docstring; - dict entries = arg.attr("__entries"); - if (((PyTypeObject *) arg.ptr())->tp_doc) { - docstring += std::string(((PyTypeObject *) arg.ptr())->tp_doc) + "\n\n"; - } - docstring += "Members:"; - for (auto kv : entries) { - auto key = std::string(pybind11::str(kv.first)); - auto comment = kv.second[int_(1)]; - docstring += "\n\n " + key; - if (!comment.is_none()) { - docstring += " : " + (std::string) pybind11::str(comment); + if (options::show_enum_members_docstring()) { + m_base.attr("__doc__") = static_property( + cpp_function( + [](handle arg) -> std::string { + std::string docstring; + dict entries = arg.attr("__entries"); + if (((PyTypeObject *) arg.ptr())->tp_doc) { + docstring += std::string( + reinterpret_cast<PyTypeObject *>(arg.ptr())->tp_doc); + docstring += "\n\n"; } - } - return docstring; - }, - name("__doc__")), - none(), - none(), - ""); + docstring += "Members:"; + for (auto kv : entries) { + auto key = std::string(pybind11::str(kv.first)); + auto comment = kv.second[int_(1)]; + docstring += "\n\n "; + docstring += key; + if (!comment.is_none()) { + docstring += " : "; + docstring += pybind11::str(comment).cast<std::string>(); + } + } + return docstring; + }, + name("__doc__")), + none(), + none(), + ""); + } m_base.attr("__members__") = static_property(cpp_function( [](handle arg) -> dict { @@ -2086,12 +2136,12 @@ struct enum_base { str name(name_); if (entries.contains(name)) { std::string type_name = (std::string) str(m_base.attr("__name__")); - throw value_error(type_name + ": element \"" + std::string(name_) + throw value_error(std::move(type_name) + ": element \"" + std::string(name_) + "\" already exists!"); } - entries[name] = std::make_pair(value, doc); - m_base.attr(name) = value; + entries[name] = pybind11::make_tuple(value, doc); + m_base.attr(std::move(name)) = std::move(value); } PYBIND11_NOINLINE void export_values() { @@ -2437,7 +2487,8 @@ template <return_value_policy Policy = return_value_policy::reference_internal, typename Type, typename... Extra> iterator make_iterator(Type &value, Extra &&...extra) { - return make_iterator<Policy>(std::begin(value), std::end(value), extra...); + return make_iterator<Policy>( + std::begin(value), std::end(value), std::forward<Extra>(extra)...); } /// Makes an iterator over the keys (`.first`) of a stl map-like container supporting @@ -2446,7 +2497,8 @@ template <return_value_policy Policy = return_value_policy::reference_internal, typename Type, typename... Extra> iterator make_key_iterator(Type &value, Extra &&...extra) { - return make_key_iterator<Policy>(std::begin(value), std::end(value), extra...); + return make_key_iterator<Policy>( + std::begin(value), std::end(value), std::forward<Extra>(extra)...); } /// Makes an iterator over the values (`.second`) of a stl map-like container supporting @@ -2455,7 +2507,8 @@ template <return_value_policy Policy = return_value_policy::reference_internal, typename Type, typename... Extra> iterator make_value_iterator(Type &value, Extra &&...extra) { - return make_value_iterator<Policy>(std::begin(value), std::end(value), extra...); + return make_value_iterator<Policy>( + std::begin(value), std::end(value), std::forward<Extra>(extra)...); } template <typename InputType, typename OutputType> @@ -2484,7 +2537,7 @@ void implicitly_convertible() { }; if (auto *tinfo = detail::get_type_info(typeid(OutputType))) { - tinfo->implicit_conversions.push_back(implicit_caster); + tinfo->implicit_conversions.emplace_back(std::move(implicit_caster)); } else { pybind11_fail("implicitly_convertible: Unable to find type " + type_id<OutputType>()); } @@ -2520,7 +2573,7 @@ public: exception(handle scope, const char *name, handle base = PyExc_Exception) { std::string full_name = scope.attr("__name__").cast<std::string>() + std::string(".") + name; - m_ptr = PyErr_NewException(const_cast<char *>(full_name.c_str()), base.ptr(), NULL); + m_ptr = PyErr_NewException(const_cast<char *>(full_name.c_str()), base.ptr(), nullptr); if (hasattr(scope, "__dict__") && scope.attr("__dict__").contains(name)) { pybind11_fail("Error during initialization: multiple incompatible " "definitions with name \"" @@ -2602,8 +2655,8 @@ PYBIND11_NOINLINE void print(const tuple &args, const dict &kwargs) { for (size_t i = 0; i < args.size(); ++i) { strings[i] = str(args[i]); } - auto sep = kwargs.contains("sep") ? kwargs["sep"] : cast(" "); - auto line = sep.attr("join")(strings); + auto sep = kwargs.contains("sep") ? kwargs["sep"] : str(" "); + auto line = sep.attr("join")(std::move(strings)); object file; if (kwargs.contains("file")) { @@ -2621,8 +2674,8 @@ PYBIND11_NOINLINE void print(const tuple &args, const dict &kwargs) { } auto write = file.attr("write"); - write(line); - write(kwargs.contains("end") ? kwargs["end"] : cast("\n")); + write(std::move(line)); + write(kwargs.contains("end") ? kwargs["end"] : str("\n")); if (kwargs.contains("flush") && kwargs["flush"].cast<bool>()) { file.attr("flush")(); @@ -2636,17 +2689,21 @@ void print(Args &&...args) { detail::print(c.args(), c.kwargs()); } -error_already_set::~error_already_set() { - if (m_type) { - gil_scoped_acquire gil; - error_scope scope; - m_type.release().dec_ref(); - m_value.release().dec_ref(); - m_trace.release().dec_ref(); - } +inline void +error_already_set::m_fetched_error_deleter(detail::error_fetch_and_normalize *raw_ptr) { + gil_scoped_acquire gil; + error_scope scope; + delete raw_ptr; +} + +inline const char *error_already_set::what() const noexcept { + gil_scoped_acquire gil; + error_scope scope; + return m_fetched_error->error_string().c_str(); } PYBIND11_NAMESPACE_BEGIN(detail) + inline function get_type_override(const void *this_ptr, const type_info *this_type, const char *name) { handle self = get_object_handle(this_ptr, this_type); @@ -2665,7 +2722,7 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char * function override = getattr(self, name, function()); if (override.is_cpp_function()) { - cache.insert(key); + cache.insert(std::move(key)); return function(); } @@ -2865,7 +2922,3 @@ inline function get_overload(const T *this_ptr, const char *name) { PYBIND11_OVERRIDE_PURE(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), fn, __VA_ARGS__); PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic pop // -Wnoexcept-type -#endif diff --git a/contrib/libs/pybind11/include/pybind11/pytypes.h b/contrib/libs/pybind11/include/pybind11/pytypes.h index 75ea7de88e..fcad4c609e 100644 --- a/contrib/libs/pybind11/include/pybind11/pytypes.h +++ b/contrib/libs/pybind11/include/pybind11/pytypes.h @@ -12,7 +12,15 @@ #include "detail/common.h" #include "buffer_info.h" +#include <assert.h> +#include <cstddef> +#include <exception> +#include <frameobject.h> +#include <iterator> +#include <memory> +#include <string> #include <type_traits> +#include <typeinfo> #include <utility> #if defined(PYBIND11_HAS_OPTIONAL) @@ -25,6 +33,8 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + /* A few forward declarations */ class handle; class object; @@ -85,7 +95,9 @@ public: or `object` subclass causes a call to ``__setitem__``. \endrst */ item_accessor operator[](handle key) const; - /// See above (the only difference is that they key is provided as a string literal) + /// See above (the only difference is that the key's reference is stolen) + item_accessor operator[](object &&key) const; + /// See above (the only difference is that the key is provided as a string literal) item_accessor operator[](const char *key) const; /** \rst @@ -95,7 +107,9 @@ public: or `object` subclass causes a call to ``setattr``. \endrst */ obj_attr_accessor attr(handle key) const; - /// See above (the only difference is that they key is provided as a string literal) + /// See above (the only difference is that the key's reference is stolen) + obj_attr_accessor attr(object &&key) const; + /// See above (the only difference is that the key is provided as a string literal) str_attr_accessor attr(const char *key) const; /** \rst @@ -143,23 +157,23 @@ public: object operator-() const; object operator~() const; object operator+(object_api const &other) const; - object operator+=(object_api const &other) const; + object operator+=(object_api const &other); object operator-(object_api const &other) const; - object operator-=(object_api const &other) const; + object operator-=(object_api const &other); object operator*(object_api const &other) const; - object operator*=(object_api const &other) const; + object operator*=(object_api const &other); object operator/(object_api const &other) const; - object operator/=(object_api const &other) const; + object operator/=(object_api const &other); object operator|(object_api const &other) const; - object operator|=(object_api const &other) const; + object operator|=(object_api const &other); object operator&(object_api const &other) const; - object operator&=(object_api const &other) const; + object operator&=(object_api const &other); object operator^(object_api const &other) const; - object operator^=(object_api const &other) const; + object operator^=(object_api const &other); object operator<<(object_api const &other) const; - object operator<<=(object_api const &other) const; + object operator<<=(object_api const &other); object operator>>(object_api const &other) const; - object operator>>=(object_api const &other) const; + object operator>>=(object_api const &other); PYBIND11_DEPRECATED("Use py::str(obj) instead") pybind11::str str() const; @@ -178,8 +192,17 @@ private: bool rich_compare(object_api const &other, int value) const; }; +template <typename T> +using is_pyobj_ptr_or_nullptr_t = detail::any_of<std::is_same<T, PyObject *>, + std::is_same<T, PyObject *const>, + std::is_same<T, std::nullptr_t>>; + PYBIND11_NAMESPACE_END(detail) +#if !defined(PYBIND11_HANDLE_REF_DEBUG) && !defined(NDEBUG) +# define PYBIND11_HANDLE_REF_DEBUG +#endif + /** \rst Holds a reference to a Python object (no reference counting) @@ -195,9 +218,24 @@ class handle : public detail::object_api<handle> { public: /// The default constructor creates a handle with a ``nullptr``-valued pointer handle() = default; - /// Creates a ``handle`` from the given raw Python object pointer + + /// Enable implicit conversion from ``PyObject *`` and ``nullptr``. + /// Not using ``handle(PyObject *ptr)`` to avoid implicit conversion from ``0``. + template <typename T, + detail::enable_if_t<detail::is_pyobj_ptr_or_nullptr_t<T>::value, int> = 0> + // NOLINTNEXTLINE(google-explicit-constructor) + handle(T ptr) : m_ptr(ptr) {} + + /// Enable implicit conversion through ``T::operator PyObject *()``. + template < + typename T, + detail::enable_if_t<detail::all_of<detail::none_of<std::is_base_of<handle, T>, + detail::is_pyobj_ptr_or_nullptr_t<T>>, + std::is_convertible<T, PyObject *>>::value, + int> + = 0> // NOLINTNEXTLINE(google-explicit-constructor) - handle(PyObject *ptr) : m_ptr(ptr) {} // Allow implicit conversion from PyObject* + handle(T &obj) : m_ptr(obj) {} /// Return the underlying ``PyObject *`` pointer PyObject *ptr() const { return m_ptr; } @@ -209,6 +247,14 @@ public: this function automatically. Returns a reference to itself. \endrst */ const handle &inc_ref() const & { +#ifdef PYBIND11_HANDLE_REF_DEBUG + inc_ref_counter(1); +#endif +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + if (m_ptr != nullptr && !PyGILState_Check()) { + throw_gilstate_error("pybind11::handle::inc_ref()"); + } +#endif Py_XINCREF(m_ptr); return *this; } @@ -219,6 +265,11 @@ public: this function automatically. Returns a reference to itself. \endrst */ const handle &dec_ref() const & { +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + if (m_ptr != nullptr && !PyGILState_Check()) { + throw_gilstate_error("pybind11::handle::dec_ref()"); + } +#endif Py_XDECREF(m_ptr); return *this; } @@ -244,6 +295,43 @@ public: protected: PyObject *m_ptr = nullptr; + +private: +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + void throw_gilstate_error(const std::string &function_name) const { + fprintf( + stderr, + "%s is being called while the GIL is either not held or invalid. Please see " + "https://pybind11.readthedocs.io/en/stable/advanced/" + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n" + "If you are convinced there is no bug in your code, you can #define " + "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF" + "to disable this check. In that case you have to ensure this #define is consistently " + "used for all translation units linked into a given pybind11 extension, otherwise " + "there will be ODR violations.", + function_name.c_str()); + fflush(stderr); + if (Py_TYPE(m_ptr)->tp_name != nullptr) { + fprintf(stderr, + "The failing %s call was triggered on a %s object.\n", + function_name.c_str(), + Py_TYPE(m_ptr)->tp_name); + fflush(stderr); + } + throw std::runtime_error(function_name + " PyGILState_Check() failure."); + } +#endif + +#ifdef PYBIND11_HANDLE_REF_DEBUG + static std::size_t inc_ref_counter(std::size_t add) { + thread_local std::size_t counter = 0; + counter += add; + return counter; + } + +public: + static std::size_t inc_ref_counter() { return inc_ref_counter(0); } +#endif }; /** \rst @@ -268,10 +356,7 @@ public: /// Copy constructor; always increases the reference count object(const object &o) : handle(o) { inc_ref(); } /// Move constructor; steals the object from ``other`` and preserves its reference count - object(object &&other) noexcept { - m_ptr = other.m_ptr; - other.m_ptr = nullptr; - } + object(object &&other) noexcept : handle(other) { other.m_ptr = nullptr; } /// Destructor; automatically calls `handle::dec_ref()` ~object() { dec_ref(); } @@ -287,12 +372,15 @@ public: } object &operator=(const object &other) { - other.inc_ref(); - // Use temporary variable to ensure `*this` remains valid while - // `Py_XDECREF` executes, in case `*this` is accessible from Python. - handle temp(m_ptr); - m_ptr = other.m_ptr; - temp.dec_ref(); + // Skip inc_ref and dec_ref if both objects are the same + if (!this->is(other)) { + other.inc_ref(); + // Use temporary variable to ensure `*this` remains valid while + // `Py_XDECREF` executes, in case `*this` is accessible from Python. + handle temp(m_ptr); + m_ptr = other.m_ptr; + temp.dec_ref(); + } return *this; } @@ -306,6 +394,20 @@ public: return *this; } +#define PYBIND11_INPLACE_OP(iop) \ + object iop(object_api const &other) { return operator=(handle::iop(other)); } + + PYBIND11_INPLACE_OP(operator+=) + PYBIND11_INPLACE_OP(operator-=) + PYBIND11_INPLACE_OP(operator*=) + PYBIND11_INPLACE_OP(operator/=) + PYBIND11_INPLACE_OP(operator|=) + PYBIND11_INPLACE_OP(operator&=) + PYBIND11_INPLACE_OP(operator^=) + PYBIND11_INPLACE_OP(operator<<=) + PYBIND11_INPLACE_OP(operator>>=) +#undef PYBIND11_INPLACE_OP + // Calling cast() on an object lvalue just copies (via handle::cast) template <typename T> T cast() const &; @@ -363,51 +465,291 @@ T reinterpret_steal(handle h) { } PYBIND11_NAMESPACE_BEGIN(detail) + +// Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). +inline const char *obj_class_name(PyObject *obj) { + if (PyType_Check(obj)) { + return reinterpret_cast<PyTypeObject *>(obj)->tp_name; + } + return Py_TYPE(obj)->tp_name; +} + std::string error_string(); -PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275 4251) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif -/// Fetch and hold an error which was already set in Python. An instance of this is typically -/// thrown to propagate python-side errors back through C++ which can either be caught manually or -/// else falls back to the function dispatcher (which then raises the captured error back to -/// python). -class PYBIND11_EXPORT_EXCEPTION error_already_set : public std::runtime_error { -public: - /// Constructs a new exception from the current Python error indicator, if any. The current - /// Python error indicator will be cleared. - error_already_set() : std::runtime_error(detail::error_string()) { +// The code in this struct is very unusual, to minimize the chances of +// masking bugs (elsewhere) by errors during the error handling (here). +// This is meant to be a lifeline for troubleshooting long-running processes +// that crash under conditions that are virtually impossible to reproduce. +// Low-level implementation alternatives are preferred to higher-level ones +// that might raise cascading exceptions. Last-ditch-kind-of attempts are made +// to report as much of the original error as possible, even if there are +// secondary issues obtaining some of the details. +struct error_fetch_and_normalize { + // This comment only applies to Python <= 3.11: + // Immediate normalization is long-established behavior (starting with + // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 + // from Sep 2016) and safest. Normalization could be deferred, but this could mask + // errors elsewhere, the performance gain is very minor in typical situations + // (usually the dominant bottleneck is EH unwinding), and the implementation here + // would be more complex. + // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately. + // Any errors during normalization are tracked under __notes__. + explicit error_fetch_and_normalize(const char *called) { PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); + if (!m_type) { + pybind11_fail("Internal error: " + std::string(called) + + " called while " + "Python error indicator not set."); + } + const char *exc_type_name_orig = detail::obj_class_name(m_type.ptr()); + if (exc_type_name_orig == nullptr) { + pybind11_fail("Internal error: " + std::string(called) + + " failed to obtain the name " + "of the original active exception type."); + } + m_lazy_error_string = exc_type_name_orig; +#if PY_VERSION_HEX >= 0x030C0000 + // The presence of __notes__ is likely due to exception normalization + // errors, although that is not necessarily true, therefore insert a + // hint only: + if (PyObject_HasAttrString(m_value.ptr(), "__notes__")) { + m_lazy_error_string += "[WITH __notes__]"; + } +#else + // PyErr_NormalizeException() may change the exception type if there are cascading + // failures. This can potentially be extremely confusing. + PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); + if (m_type.ptr() == nullptr) { + pybind11_fail("Internal error: " + std::string(called) + + " failed to normalize the " + "active exception."); + } + const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); + if (exc_type_name_norm == nullptr) { + pybind11_fail("Internal error: " + std::string(called) + + " failed to obtain the name " + "of the normalized active exception type."); + } +# if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 + // This behavior runs the risk of masking errors in the error handling, but avoids a + // conflict with PyPy, which relies on the normalization here to change OSError to + // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). + m_lazy_error_string = exc_type_name_norm; +# else + if (exc_type_name_norm != m_lazy_error_string) { + std::string msg = std::string(called) + + ": MISMATCH of original and normalized " + "active exception types: "; + msg += "ORIGINAL "; + msg += m_lazy_error_string; + msg += " REPLACED BY "; + msg += exc_type_name_norm; + msg += ": " + format_value_and_trace(); + pybind11_fail(msg); + } +# endif +#endif } - error_already_set(const error_already_set &) = default; - error_already_set(error_already_set &&) = default; + error_fetch_and_normalize(const error_fetch_and_normalize &) = delete; + error_fetch_and_normalize(error_fetch_and_normalize &&) = delete; + + std::string format_value_and_trace() const { + std::string result; + std::string message_error_string; + if (m_value) { + auto value_str = reinterpret_steal<object>(PyObject_Str(m_value.ptr())); + constexpr const char *message_unavailable_exc + = "<MESSAGE UNAVAILABLE DUE TO ANOTHER EXCEPTION>"; + if (!value_str) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + // Not using `value_str.cast<std::string>()`, to not potentially throw a secondary + // error_already_set that will then result in process termination (#4288). + auto value_bytes = reinterpret_steal<object>( + PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); + if (!value_bytes) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + result = std::string(buffer, static_cast<std::size_t>(length)); + } + } + } +#if PY_VERSION_HEX >= 0x030B0000 + auto notes + = reinterpret_steal<object>(PyObject_GetAttrString(m_value.ptr(), "__notes__")); + if (!notes) { + PyErr_Clear(); // No notes is good news. + } else { + auto len_notes = PyList_Size(notes.ptr()); + if (len_notes < 0) { + result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); + } else { + result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; + for (ssize_t i = 0; i < len_notes; i++) { + PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + auto note_bytes = reinterpret_steal<object>( + PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); + if (!note_bytes) { + result += "\nFAILURE obtaining __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) + == -1) { + result += "\nFAILURE formatting __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + result += '\n'; + result += std::string(buffer, static_cast<std::size_t>(length)); + } + } + } + } + } +#endif + } else { + result = "<MESSAGE UNAVAILABLE>"; + } + if (result.empty()) { + result = "<EMPTY MESSAGE>"; + } + + bool have_trace = false; + if (m_trace) { +#if !defined(PYPY_VERSION) + auto *tb = reinterpret_cast<PyTracebackObject *>(m_trace.ptr()); + + // Get the deepest trace possible. + while (tb->tb_next) { + tb = tb->tb_next; + } - inline ~error_already_set() override; + PyFrameObject *frame = tb->tb_frame; + Py_XINCREF(frame); + result += "\n\nAt:\n"; + while (frame) { +# if PY_VERSION_HEX >= 0x030900B1 + PyCodeObject *f_code = PyFrame_GetCode(frame); +# else + PyCodeObject *f_code = frame->f_code; + Py_INCREF(f_code); +# endif + int lineno = PyFrame_GetLineNumber(frame); + result += " "; + result += handle(f_code->co_filename).cast<std::string>(); + result += '('; + result += std::to_string(lineno); + result += "): "; + result += handle(f_code->co_name).cast<std::string>(); + result += '\n'; + Py_DECREF(f_code); +# if PY_VERSION_HEX >= 0x030900B1 + auto *b_frame = PyFrame_GetBack(frame); +# else + auto *b_frame = frame->f_back; + Py_XINCREF(b_frame); +# endif + Py_DECREF(frame); + frame = b_frame; + } + + have_trace = true; +#endif //! defined(PYPY_VERSION) + } + + if (!message_error_string.empty()) { + if (!have_trace) { + result += '\n'; + } + result += "\nMESSAGE UNAVAILABLE DUE TO EXCEPTION: " + message_error_string; + } + + return result; + } + + std::string const &error_string() const { + if (!m_lazy_error_string_completed) { + m_lazy_error_string += ": " + format_value_and_trace(); + m_lazy_error_string_completed = true; + } + return m_lazy_error_string; + } - /// Give the currently-held error back to Python, if any. If there is currently a Python error - /// already set it is cleared first. After this call, the current object no longer stores the - /// error variables (but the `.what()` string is still available). void restore() { - PyErr_Restore(m_type.release().ptr(), m_value.release().ptr(), m_trace.release().ptr()); + if (m_restore_called) { + pybind11_fail("Internal error: pybind11::detail::error_fetch_and_normalize::restore() " + "called a second time. ORIGINAL ERROR: " + + error_string()); + } + PyErr_Restore(m_type.inc_ref().ptr(), m_value.inc_ref().ptr(), m_trace.inc_ref().ptr()); + m_restore_called = true; } + bool matches(handle exc) const { + return (PyErr_GivenExceptionMatches(m_type.ptr(), exc.ptr()) != 0); + } + + // Not protecting these for simplicity. + object m_type, m_value, m_trace; + +private: + // Only protecting invariants. + mutable std::string m_lazy_error_string; + mutable bool m_lazy_error_string_completed = false; + mutable bool m_restore_called = false; +}; + +inline std::string error_string() { + return error_fetch_and_normalize("pybind11::detail::error_string").error_string(); +} + +PYBIND11_NAMESPACE_END(detail) + +/// Fetch and hold an error which was already set in Python. An instance of this is typically +/// thrown to propagate python-side errors back through C++ which can either be caught manually or +/// else falls back to the function dispatcher (which then raises the captured error back to +/// python). +class PYBIND11_EXPORT_EXCEPTION error_already_set : public std::exception { +public: + /// Fetches the current Python exception (using PyErr_Fetch()), which will clear the + /// current Python error indicator. + error_already_set() + : m_fetched_error{new detail::error_fetch_and_normalize("pybind11::error_already_set"), + m_fetched_error_deleter} {} + + /// The what() result is built lazily on demand. + /// WARNING: This member function needs to acquire the Python GIL. This can lead to + /// crashes (undefined behavior) if the Python interpreter is finalizing. + const char *what() const noexcept override; + + /// Restores the currently-held Python error (which will clear the Python error indicator first + /// if already set). + /// NOTE: This member function will always restore the normalized exception, which may or may + /// not be the original Python exception. + /// WARNING: The GIL must be held when this member function is called! + void restore() { m_fetched_error->restore(); } + /// If it is impossible to raise the currently-held error, such as in a destructor, we can /// write it out using Python's unraisable hook (`sys.unraisablehook`). The error context /// should be some object whose `repr()` helps identify the location of the error. Python - /// already knows the type and value of the error, so there is no need to repeat that. After - /// this call, the current object no longer stores the error variables, and neither does - /// Python. + /// already knows the type and value of the error, so there is no need to repeat that. void discard_as_unraisable(object err_context) { restore(); PyErr_WriteUnraisable(err_context.ptr()); } /// An alternate version of `discard_as_unraisable()`, where a string provides information on /// the location of the error. For example, `__func__` could be helpful. + /// WARNING: The GIL must be held when this member function is called! void discard_as_unraisable(const char *err_context) { discard_as_unraisable(reinterpret_steal<object>(PYBIND11_FROM_STRING(err_context))); } @@ -419,20 +761,19 @@ public: /// Check if the currently trapped error type matches the given Python exception class (or a /// subclass thereof). May also be passed a tuple to search for any exception class matches in /// the given tuple. - bool matches(handle exc) const { - return (PyErr_GivenExceptionMatches(m_type.ptr(), exc.ptr()) != 0); - } + bool matches(handle exc) const { return m_fetched_error->matches(exc); } - const object &type() const { return m_type; } - const object &value() const { return m_value; } - const object &trace() const { return m_trace; } + const object &type() const { return m_fetched_error->m_type; } + const object &value() const { return m_fetched_error->m_value; } + const object &trace() const { return m_fetched_error->m_trace; } private: - object m_type, m_value, m_trace; + std::shared_ptr<detail::error_fetch_and_normalize> m_fetched_error; + + /// WARNING: This custom deleter needs to acquire the Python GIL. This can lead to + /// crashes (undefined behavior) if the Python interpreter is finalizing. + static void m_fetched_error_deleter(detail::error_fetch_and_normalize *raw_ptr); }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif #if PY_VERSION_HEX >= 0x03030000 @@ -466,8 +807,7 @@ inline void raise_from(PyObject *type, const char *message) { /// Sets the current Python error indicator with the chosen error, performing a 'raise from' /// from the error contained in error_already_set to indicate that the chosen error was -/// caused by the original error. After this function is called error_already_set will -/// no longer contain an error. +/// caused by the original error. inline void raise_from(error_already_set &err, PyObject *type, const char *message) { err.restore(); raise_from(type, message); @@ -611,13 +951,13 @@ inline PyObject *dict_getitemstring(PyObject *v, const char *key) { #if PY_MAJOR_VERSION >= 3 PyObject *kv = nullptr, *rv = nullptr; kv = PyUnicode_FromString(key); - if (kv == NULL) { + if (kv == nullptr) { throw error_already_set(); } rv = PyDict_GetItemWithError(v, kv); Py_DECREF(kv); - if (rv == NULL && PyErr_Occurred()) { + if (rv == nullptr && PyErr_Occurred()) { throw error_already_set(); } return rv; @@ -629,7 +969,7 @@ inline PyObject *dict_getitemstring(PyObject *v, const char *key) { inline PyObject *dict_getitem(PyObject *v, PyObject *key) { #if PY_MAJOR_VERSION >= 3 PyObject *rv = PyDict_GetItemWithError(v, key); - if (rv == NULL && PyErr_Occurred()) { + if (rv == nullptr && PyErr_Occurred()) { throw error_already_set(); } return rv; @@ -651,10 +991,8 @@ object object_or_cast(T &&o); // Match a PyObject*, which we want to convert directly to handle via its converting constructor inline handle object_or_cast(PyObject *ptr) { return ptr; } -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(push) -# pragma warning(disable : 4522) // warning C4522: multiple assignment operators specified -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4522) // warning C4522: multiple assignment operators specified template <typename Policy> class accessor : public object_api<accessor<Policy>> { using key_type = typename Policy::key_type; @@ -675,7 +1013,7 @@ public: } template <typename T> void operator=(T &&value) & { - get_cache() = reinterpret_borrow<object>(object_or_cast(std::forward<T>(value))); + get_cache() = ensure_object(object_or_cast(std::forward<T>(value))); } template <typename T = Policy> @@ -703,6 +1041,9 @@ public: } private: + static object ensure_object(object &&o) { return std::move(o); } + static object ensure_object(handle h) { return reinterpret_borrow<object>(h); } + object &get_cache() const { if (!cache) { cache = Policy::get(obj, key); @@ -715,9 +1056,7 @@ private: key_type key; mutable object cache; }; -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(pop) -#endif +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(accessor_policies) struct obj_attr { @@ -1053,17 +1392,17 @@ public: Name(const object &o) \ : Parent(check_(o) ? o.inc_ref().ptr() : ConvertFun(o.ptr()), stolen_t{}) { \ if (!m_ptr) \ - throw error_already_set(); \ + throw ::pybind11::error_already_set(); \ } \ /* NOLINTNEXTLINE(google-explicit-constructor) */ \ Name(object &&o) : Parent(check_(o) ? o.release().ptr() : ConvertFun(o.ptr()), stolen_t{}) { \ if (!m_ptr) \ - throw error_already_set(); \ + throw ::pybind11::error_already_set(); \ } #define PYBIND11_OBJECT_CVT_DEFAULT(Name, Parent, CheckFun, ConvertFun) \ PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun) \ - Name() : Parent() {} + Name() = default; #define PYBIND11_OBJECT_CHECK_FAILED(Name, o_ptr) \ ::pybind11::type_error("Object of type '" \ @@ -1086,7 +1425,7 @@ public: #define PYBIND11_OBJECT_DEFAULT(Name, Parent, CheckFun) \ PYBIND11_OBJECT(Name, Parent, CheckFun) \ - Name() : Parent() {} + Name() = default; /// \addtogroup pytypes /// @{ @@ -1155,7 +1494,7 @@ public: private: void advance() { value = reinterpret_steal<object>(PyIter_Next(m_ptr)); - if (PyErr_Occurred()) { + if (value.ptr() == nullptr && PyErr_Occurred()) { throw error_already_set(); } } @@ -1205,6 +1544,9 @@ public: str(const char *c, const SzType &n) : object(PyUnicode_FromStringAndSize(c, ssize_t_cast(n)), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1214,6 +1556,9 @@ public: // NOLINTNEXTLINE(google-explicit-constructor) str(const char *c = "") : object(PyUnicode_FromString(c), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1258,8 +1603,8 @@ public: } char *buffer = nullptr; ssize_t length = 0; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length)) { - pybind11_fail("Unable to extract string contents! (invalid type)"); + if (PyBytes_AsStringAndSize(temp.ptr(), &buffer, &length) != 0) { + throw error_already_set(); } return std::string(buffer, (size_t) length); } @@ -1321,14 +1666,7 @@ public: explicit bytes(const pybind11::str &s); // NOLINTNEXTLINE(google-explicit-constructor) - operator std::string() const { - char *buffer = nullptr; - ssize_t length = 0; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length)) { - pybind11_fail("Unable to extract bytes contents!"); - } - return std::string(buffer, (size_t) length); - } + operator std::string() const { return string_op<std::string>(); } #ifdef PYBIND11_HAS_STRING_VIEW // enable_if is needed to avoid "ambiguous conversion" errors (see PR #3521). @@ -1340,15 +1678,18 @@ public: // valid so long as the `bytes` instance remains alive and so generally should not outlive the // lifetime of the `bytes` instance. // NOLINTNEXTLINE(google-explicit-constructor) - operator std::string_view() const { + operator std::string_view() const { return string_op<std::string_view>(); } +#endif +private: + template <typename T> + T string_op() const { char *buffer = nullptr; ssize_t length = 0; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length)) { - pybind11_fail("Unable to extract bytes contents!"); + if (PyBytes_AsStringAndSize(m_ptr, &buffer, &length) != 0) { + throw error_already_set(); } return {buffer, static_cast<size_t>(length)}; } -#endif }; // Note: breathe >= 4.17.0 will fail to build docs if the below two constructors // are included in the doxygen group; close here and reopen after as a workaround @@ -1359,13 +1700,13 @@ inline bytes::bytes(const pybind11::str &s) { if (PyUnicode_Check(s.ptr())) { temp = reinterpret_steal<object>(PyUnicode_AsUTF8String(s.ptr())); if (!temp) { - pybind11_fail("Unable to extract string contents! (encoding issue)"); + throw error_already_set(); } } char *buffer = nullptr; ssize_t length = 0; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length)) { - pybind11_fail("Unable to extract string contents! (invalid type)"); + if (PyBytes_AsStringAndSize(temp.ptr(), &buffer, &length) != 0) { + throw error_already_set(); } auto obj = reinterpret_steal<object>(PYBIND11_BYTES_FROM_STRING_AND_SIZE(buffer, length)); if (!obj) { @@ -1377,11 +1718,14 @@ inline bytes::bytes(const pybind11::str &s) { inline str::str(const bytes &b) { char *buffer = nullptr; ssize_t length = 0; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(b.ptr(), &buffer, &length)) { - pybind11_fail("Unable to extract bytes contents!"); + if (PyBytes_AsStringAndSize(b.ptr(), &buffer, &length) != 0) { + throw error_already_set(); } auto obj = reinterpret_steal<object>(PyUnicode_FromStringAndSize(buffer, length)); if (!obj) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } m_ptr = obj.release().ptr(); @@ -1459,7 +1803,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // unsigned type: (A)-1 != (B)-1 when A and B are unsigned types of different sizes). template <typename Unsigned> Unsigned as_unsigned(PyObject *o) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(Unsigned) <= sizeof(unsigned long)) + if (sizeof(Unsigned) <= sizeof(unsigned long) #if PY_VERSION_HEX < 0x03000000 || PyInt_Check(o) #endif @@ -1480,7 +1824,7 @@ public: template <typename T, detail::enable_if_t<std::is_integral<T>::value, int> = 0> // NOLINTNEXTLINE(google-explicit-constructor) int_(T value) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(T) <= sizeof(long))) { + if (sizeof(T) <= sizeof(long)) { if (std::is_signed<T>::value) { m_ptr = PyLong_FromLong((long) value); } else { @@ -1535,6 +1879,9 @@ public: explicit weakref(handle obj, handle callback = {}) : object(PyWeakref_NewRef(obj.ptr(), callback.ptr()), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate weak reference!"); } } @@ -1546,8 +1893,8 @@ private: class slice : public object { public: PYBIND11_OBJECT_DEFAULT(slice, object, PySlice_Check) - slice(handle start, handle stop, handle step) { - m_ptr = PySlice_New(start.ptr(), stop.ptr(), step.ptr()); + slice(handle start, handle stop, handle step) + : object(PySlice_New(start.ptr(), stop.ptr(), step.ptr()), stolen_t{}) { if (!m_ptr) { pybind11_fail("Could not allocate slice object!"); } @@ -1594,45 +1941,42 @@ public: explicit capsule(const void *value, const char *name = nullptr, - void (*destructor)(PyObject *) = nullptr) + PyCapsule_Destructor destructor = nullptr) : object(PyCapsule_New(const_cast<void *>(value), name, destructor), stolen_t{}) { if (!m_ptr) { - pybind11_fail("Could not allocate capsule object!"); + throw error_already_set(); } } - PYBIND11_DEPRECATED("Please pass a destructor that takes a void pointer as input") - capsule(const void *value, void (*destruct)(PyObject *)) - : object(PyCapsule_New(const_cast<void *>(value), nullptr, destruct), stolen_t{}) { + PYBIND11_DEPRECATED("Please use the ctor with value, name, destructor args") + capsule(const void *value, PyCapsule_Destructor destructor) + : object(PyCapsule_New(const_cast<void *>(value), nullptr, destructor), stolen_t{}) { if (!m_ptr) { - pybind11_fail("Could not allocate capsule object!"); + throw error_already_set(); } } + /// Capsule name is nullptr. capsule(const void *value, void (*destructor)(void *)) { - m_ptr = PyCapsule_New(const_cast<void *>(value), nullptr, [](PyObject *o) { - auto destructor = reinterpret_cast<void (*)(void *)>(PyCapsule_GetContext(o)); - void *ptr = PyCapsule_GetPointer(o, nullptr); - destructor(ptr); - }); - - if (!m_ptr) { - pybind11_fail("Could not allocate capsule object!"); - } + initialize_with_void_ptr_destructor(value, nullptr, destructor); + } - if (PyCapsule_SetContext(m_ptr, (void *) destructor) != 0) { - pybind11_fail("Could not set capsule context!"); - } + capsule(const void *value, const char *name, void (*destructor)(void *)) { + initialize_with_void_ptr_destructor(value, name, destructor); } explicit capsule(void (*destructor)()) { m_ptr = PyCapsule_New(reinterpret_cast<void *>(destructor), nullptr, [](PyObject *o) { - auto destructor = reinterpret_cast<void (*)()>(PyCapsule_GetPointer(o, nullptr)); + const char *name = get_name_in_error_scope(o); + auto destructor = reinterpret_cast<void (*)()>(PyCapsule_GetPointer(o, name)); + if (destructor == nullptr) { + throw error_already_set(); + } destructor(); }); if (!m_ptr) { - pybind11_fail("Could not allocate capsule object!"); + throw error_already_set(); } } @@ -1647,8 +1991,7 @@ public: const auto *name = this->name(); T *result = static_cast<T *>(PyCapsule_GetPointer(m_ptr, name)); if (!result) { - PyErr_Clear(); - pybind11_fail("Unable to extract capsule contents!"); + throw error_already_set(); } return result; } @@ -1656,12 +1999,63 @@ public: /// Replaces a capsule's pointer *without* calling the destructor on the existing one. void set_pointer(const void *value) { if (PyCapsule_SetPointer(m_ptr, const_cast<void *>(value)) != 0) { - PyErr_Clear(); - pybind11_fail("Could not set capsule pointer"); + throw error_already_set(); + } + } + + const char *name() const { + const char *name = PyCapsule_GetName(m_ptr); + if ((name == nullptr) && PyErr_Occurred()) { + throw error_already_set(); + } + return name; + } + + /// Replaces a capsule's name *without* calling the destructor on the existing one. + void set_name(const char *new_name) { + if (PyCapsule_SetName(m_ptr, new_name) != 0) { + throw error_already_set(); + } + } + +private: + static const char *get_name_in_error_scope(PyObject *o) { + error_scope error_guard; + + const char *name = PyCapsule_GetName(o); + if ((name == nullptr) && PyErr_Occurred()) { + // write out and consume error raised by call to PyCapsule_GetName + PyErr_WriteUnraisable(o); } + + return name; } - const char *name() const { return PyCapsule_GetName(m_ptr); } + void initialize_with_void_ptr_destructor(const void *value, + const char *name, + void (*destructor)(void *)) { + m_ptr = PyCapsule_New(const_cast<void *>(value), name, [](PyObject *o) { + // guard if destructor called while err indicator is set + error_scope error_guard; + auto destructor = reinterpret_cast<void (*)(void *)>(PyCapsule_GetContext(o)); + if (destructor == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + const char *name = get_name_in_error_scope(o); + void *ptr = PyCapsule_GetPointer(o, name); + if (ptr == nullptr) { + throw error_already_set(); + } + + if (destructor != nullptr) { + destructor(ptr); + } + }); + + if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast<void *>(destructor)) != 0) { + throw error_already_set(); + } + } }; class tuple : public object { @@ -1678,7 +2072,10 @@ public: size_t size() const { return (size_t) PyTuple_Size(m_ptr); } bool empty() const { return size() == 0; } detail::tuple_accessor operator[](size_t index) const { return {*this, index}; } - detail::item_accessor operator[](handle h) const { return object::operator[](h); } + template <typename T, detail::enable_if_t<detail::is_pyobject<T>::value, int> = 0> + detail::item_accessor operator[](T &&o) const { + return object::operator[](std::forward<T>(o)); + } detail::tuple_iterator begin() const { return {*this, 0}; } detail::tuple_iterator end() const { return {*this, PyTuple_GET_SIZE(m_ptr)}; } }; @@ -1713,7 +2110,11 @@ public: void clear() /* py-non-const */ { PyDict_Clear(ptr()); } template <typename T> bool contains(T &&key) const { - return PyDict_Contains(m_ptr, detail::object_or_cast(std::forward<T>(key)).ptr()) == 1; + auto result = PyDict_Contains(m_ptr, detail::object_or_cast(std::forward<T>(key)).ptr()); + if (result == -1) { + throw error_already_set(); + } + return result == 1; } private: @@ -1738,7 +2139,10 @@ public: } bool empty() const { return size() == 0; } detail::sequence_accessor operator[](size_t index) const { return {*this, index}; } - detail::item_accessor operator[](handle h) const { return object::operator[](h); } + template <typename T, detail::enable_if_t<detail::is_pyobject<T>::value, int> = 0> + detail::item_accessor operator[](T &&o) const { + return object::operator[](std::forward<T>(o)); + } detail::sequence_iterator begin() const { return {*this, 0}; } detail::sequence_iterator end() const { return {*this, PySequence_Size(m_ptr)}; } }; @@ -1757,19 +2161,28 @@ public: size_t size() const { return (size_t) PyList_Size(m_ptr); } bool empty() const { return size() == 0; } detail::list_accessor operator[](size_t index) const { return {*this, index}; } - detail::item_accessor operator[](handle h) const { return object::operator[](h); } + template <typename T, detail::enable_if_t<detail::is_pyobject<T>::value, int> = 0> + detail::item_accessor operator[](T &&o) const { + return object::operator[](std::forward<T>(o)); + } detail::list_iterator begin() const { return {*this, 0}; } detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; } template <typename T> void append(T &&val) /* py-non-const */ { - PyList_Append(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()); + if (PyList_Append(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) != 0) { + throw error_already_set(); + } } template <typename IdxType, typename ValType, detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0> void insert(const IdxType &index, ValType &&val) /* py-non-const */ { - PyList_Insert( - m_ptr, ssize_t_cast(index), detail::object_or_cast(std::forward<ValType>(val)).ptr()); + if (PyList_Insert(m_ptr, + ssize_t_cast(index), + detail::object_or_cast(std::forward<ValType>(val)).ptr()) + != 0) { + throw error_already_set(); + } } }; @@ -1780,25 +2193,39 @@ class kwargs : public dict { PYBIND11_OBJECT_DEFAULT(kwargs, dict, PyDict_Check) }; -class set : public object { +class anyset : public object { public: - PYBIND11_OBJECT_CVT(set, object, PySet_Check, PySet_New) - set() : object(PySet_New(nullptr), stolen_t{}) { + PYBIND11_OBJECT(anyset, object, PyAnySet_Check) + size_t size() const { return static_cast<size_t>(PySet_Size(m_ptr)); } + bool empty() const { return size() == 0; } + template <typename T> + bool contains(T &&val) const { + auto result = PySet_Contains(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()); + if (result == -1) { + throw error_already_set(); + } + return result == 1; + } +}; + +class set : public anyset { +public: + PYBIND11_OBJECT_CVT(set, anyset, PySet_Check, PySet_New) + set() : anyset(PySet_New(nullptr), stolen_t{}) { if (!m_ptr) { pybind11_fail("Could not allocate set object!"); } } - size_t size() const { return (size_t) PySet_Size(m_ptr); } - bool empty() const { return size() == 0; } template <typename T> bool add(T &&val) /* py-non-const */ { return PySet_Add(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) == 0; } void clear() /* py-non-const */ { PySet_Clear(m_ptr); } - template <typename T> - bool contains(T &&val) const { - return PySet_Contains(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) == 1; - } +}; + +class frozenset : public anyset { +public: + PYBIND11_OBJECT_CVT(frozenset, anyset, PyFrozenSet_Check, PyFrozenSet_New) }; class function : public object { @@ -1910,8 +2337,8 @@ public: return memoryview::from_buffer(reinterpret_cast<void *>(ptr), sizeof(T), format_descriptor<T>::value, - shape, - strides, + std::move(shape), + std::move(strides), readonly); } @@ -1919,7 +2346,8 @@ public: static memoryview from_buffer(const T *ptr, detail::any_container<ssize_t> shape, detail::any_container<ssize_t> strides) { - return memoryview::from_buffer(const_cast<T *>(ptr), shape, strides, true); + return memoryview::from_buffer( + const_cast<T *>(ptr), std::move(shape), std::move(strides), true); } #if PY_MAJOR_VERSION >= 3 @@ -1930,8 +2358,6 @@ public: managed by Python. The caller is responsible for managing the lifetime of ``mem``, which MUST outlive the memoryview constructed here. - This method is not available in Python 2. - See also: Python C API documentation for `PyMemoryView_FromBuffer`_. .. _PyMemoryView_FromMemory: @@ -2062,6 +2488,10 @@ item_accessor object_api<D>::operator[](handle key) const { return {derived(), reinterpret_borrow<object>(key)}; } template <typename D> +item_accessor object_api<D>::operator[](object &&key) const { + return {derived(), std::move(key)}; +} +template <typename D> item_accessor object_api<D>::operator[](const char *key) const { return {derived(), pybind11::str(key)}; } @@ -2070,6 +2500,10 @@ obj_attr_accessor object_api<D>::attr(handle key) const { return {derived(), reinterpret_borrow<object>(key)}; } template <typename D> +obj_attr_accessor object_api<D>::attr(object &&key) const { + return {derived(), std::move(key)}; +} +template <typename D> str_attr_accessor object_api<D>::attr(const char *key) const { return {derived(), key}; } @@ -2125,29 +2559,39 @@ bool object_api<D>::rich_compare(object_api const &other, int value) const { return result; \ } +#define PYBIND11_MATH_OPERATOR_BINARY_INPLACE(iop, fn) \ + template <typename D> \ + object object_api<D>::iop(object_api const &other) { \ + object result = reinterpret_steal<object>(fn(derived().ptr(), other.derived().ptr())); \ + if (!result.ptr()) \ + throw error_already_set(); \ + return result; \ + } + PYBIND11_MATH_OPERATOR_UNARY(operator~, PyNumber_Invert) PYBIND11_MATH_OPERATOR_UNARY(operator-, PyNumber_Negative) PYBIND11_MATH_OPERATOR_BINARY(operator+, PyNumber_Add) -PYBIND11_MATH_OPERATOR_BINARY(operator+=, PyNumber_InPlaceAdd) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator+=, PyNumber_InPlaceAdd) PYBIND11_MATH_OPERATOR_BINARY(operator-, PyNumber_Subtract) -PYBIND11_MATH_OPERATOR_BINARY(operator-=, PyNumber_InPlaceSubtract) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator-=, PyNumber_InPlaceSubtract) PYBIND11_MATH_OPERATOR_BINARY(operator*, PyNumber_Multiply) -PYBIND11_MATH_OPERATOR_BINARY(operator*=, PyNumber_InPlaceMultiply) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator*=, PyNumber_InPlaceMultiply) PYBIND11_MATH_OPERATOR_BINARY(operator/, PyNumber_TrueDivide) -PYBIND11_MATH_OPERATOR_BINARY(operator/=, PyNumber_InPlaceTrueDivide) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator/=, PyNumber_InPlaceTrueDivide) PYBIND11_MATH_OPERATOR_BINARY(operator|, PyNumber_Or) -PYBIND11_MATH_OPERATOR_BINARY(operator|=, PyNumber_InPlaceOr) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator|=, PyNumber_InPlaceOr) PYBIND11_MATH_OPERATOR_BINARY(operator&, PyNumber_And) -PYBIND11_MATH_OPERATOR_BINARY(operator&=, PyNumber_InPlaceAnd) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator&=, PyNumber_InPlaceAnd) PYBIND11_MATH_OPERATOR_BINARY(operator^, PyNumber_Xor) -PYBIND11_MATH_OPERATOR_BINARY(operator^=, PyNumber_InPlaceXor) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator^=, PyNumber_InPlaceXor) PYBIND11_MATH_OPERATOR_BINARY(operator<<, PyNumber_Lshift) -PYBIND11_MATH_OPERATOR_BINARY(operator<<=, PyNumber_InPlaceLshift) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator<<=, PyNumber_InPlaceLshift) PYBIND11_MATH_OPERATOR_BINARY(operator>>, PyNumber_Rshift) -PYBIND11_MATH_OPERATOR_BINARY(operator>>=, PyNumber_InPlaceRshift) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_UNARY #undef PYBIND11_MATH_OPERATOR_BINARY +#undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/contrib/libs/pybind11/ya.make b/contrib/libs/pybind11/ya.make index d8f9d60781..95a7f21ea3 100644 --- a/contrib/libs/pybind11/ya.make +++ b/contrib/libs/pybind11/ya.make @@ -1,4 +1,4 @@ -# Generated by devtools/yamaker from nixpkgs 22.05. +# Generated by devtools/yamaker from nixpkgs 22.11. LIBRARY() @@ -6,9 +6,9 @@ LICENSE(BSD-3-Clause) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(2.9.2) +VERSION(2.11.1) -ORIGINAL_SOURCE(https://github.com/pybind/pybind11/archive/v2.9.2.tar.gz) +ORIGINAL_SOURCE(https://github.com/pybind/pybind11/archive/v2.11.1.tar.gz) ADDINCL( GLOBAL contrib/libs/pybind11/include diff --git a/contrib/python/PyJWT/py3/.dist-info/METADATA b/contrib/python/PyJWT/py3/.dist-info/METADATA index f5fbdf64a5..65b6141fa1 100644 --- a/contrib/python/PyJWT/py3/.dist-info/METADATA +++ b/contrib/python/PyJWT/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: PyJWT -Version: 2.3.0 +Version: 2.4.0 Summary: JSON Web Token implementation in Python Home-page: https://github.com/jpadilla/pyjwt Author: Jose Padilla @@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 Classifier: Topic :: Utilities Requires-Python: >=3.6 Description-Content-Type: text/x-rst diff --git a/contrib/python/PyJWT/py3/LICENSE b/contrib/python/PyJWT/py3/LICENSE index bdc7819ea1..fd0ecbc889 100644 --- a/contrib/python/PyJWT/py3/LICENSE +++ b/contrib/python/PyJWT/py3/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 José Padilla +Copyright (c) 2015-2022 José Padilla Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/contrib/python/PyJWT/py3/jwt/__init__.py b/contrib/python/PyJWT/py3/jwt/__init__.py index 3208c39f39..6b3f8ab160 100644 --- a/contrib/python/PyJWT/py3/jwt/__init__.py +++ b/contrib/python/PyJWT/py3/jwt/__init__.py @@ -25,19 +25,19 @@ from .exceptions import ( ) from .jwks_client import PyJWKClient -__version__ = "2.3.0" +__version__ = "2.4.0" __title__ = "PyJWT" __description__ = "JSON Web Token implementation in Python" __url__ = "https://pyjwt.readthedocs.io" __uri__ = __url__ -__doc__ = __description__ + " <" + __uri__ + ">" +__doc__ = f"{__description__} <{__uri__}>" __author__ = "José Padilla" __email__ = "hello@jpadilla.com" __license__ = "MIT" -__copyright__ = "Copyright 2015-2020 José Padilla" +__copyright__ = "Copyright 2015-2022 José Padilla" __all__ = [ diff --git a/contrib/python/PyJWT/py3/jwt/algorithms.py b/contrib/python/PyJWT/py3/jwt/algorithms.py index 1f8865afbd..46a1a532e7 100644 --- a/contrib/python/PyJWT/py3/jwt/algorithms.py +++ b/contrib/python/PyJWT/py3/jwt/algorithms.py @@ -9,6 +9,8 @@ from .utils import ( der_to_raw_signature, force_bytes, from_base64url_uint, + is_pem_format, + is_ssh_key, raw_to_der_signature, to_base64url_uint, ) @@ -183,14 +185,7 @@ class HMACAlgorithm(Algorithm): def prepare_key(self, key): key = force_bytes(key) - invalid_strings = [ - b"-----BEGIN PUBLIC KEY-----", - b"-----BEGIN CERTIFICATE-----", - b"-----BEGIN RSA PUBLIC KEY-----", - b"ssh-rsa", - ] - - if any(string_value in key for string_value in invalid_strings): + if is_pem_format(key) or is_ssh_key(key): raise InvalidKeyError( "The specified key is an asymmetric key or x509 certificate and" " should not be used as an HMAC secret." @@ -417,6 +412,12 @@ if has_crypto: except ValueError: key = load_pem_private_key(key, password=None) + # Explicit check the key to prevent confusing errors from cryptography + if not isinstance(key, (EllipticCurvePrivateKey, EllipticCurvePublicKey)): + raise InvalidKeyError( + "Expecting a EllipticCurvePrivateKey/EllipticCurvePublicKey. Wrong key provided for ECDSA algorithms" + ) + return key def sign(self, msg, key): @@ -545,26 +546,28 @@ if has_crypto: pass def prepare_key(self, key): - - if isinstance( - key, - (Ed25519PrivateKey, Ed25519PublicKey, Ed448PrivateKey, Ed448PublicKey), - ): - return key - if isinstance(key, (bytes, str)): if isinstance(key, str): key = key.encode("utf-8") str_key = key.decode("utf-8") if "-----BEGIN PUBLIC" in str_key: - return load_pem_public_key(key) - if "-----BEGIN PRIVATE" in str_key: - return load_pem_private_key(key, password=None) - if str_key[0:4] == "ssh-": - return load_ssh_public_key(key) + key = load_pem_public_key(key) + elif "-----BEGIN PRIVATE" in str_key: + key = load_pem_private_key(key, password=None) + elif str_key[0:4] == "ssh-": + key = load_ssh_public_key(key) + + # Explicit check the key to prevent confusing errors from cryptography + if not isinstance( + key, + (Ed25519PrivateKey, Ed25519PublicKey, Ed448PrivateKey, Ed448PublicKey), + ): + raise InvalidKeyError( + "Expecting a EllipticCurvePrivateKey/EllipticCurvePublicKey. Wrong key provided for EdDSA algorithms" + ) - raise TypeError("Expecting a PEM-formatted or OpenSSH key.") + return key def sign(self, msg, key): """ diff --git a/contrib/python/PyJWT/py3/jwt/api_jwk.py b/contrib/python/PyJWT/py3/jwt/api_jwk.py index a0f6364da0..31250d57f2 100644 --- a/contrib/python/PyJWT/py3/jwt/api_jwk.py +++ b/contrib/python/PyJWT/py3/jwt/api_jwk.py @@ -11,7 +11,7 @@ class PyJWK: kty = self._jwk_data.get("kty", None) if not kty: - raise InvalidKeyError("kty is not found: %s" % self._jwk_data) + raise InvalidKeyError(f"kty is not found: {self._jwk_data}") if not algorithm and isinstance(self._jwk_data, dict): algorithm = self._jwk_data.get("alg", None) @@ -29,25 +29,25 @@ class PyJWK: elif crv == "secp256k1": algorithm = "ES256K" else: - raise InvalidKeyError("Unsupported crv: %s" % crv) + raise InvalidKeyError(f"Unsupported crv: {crv}") elif kty == "RSA": algorithm = "RS256" elif kty == "oct": algorithm = "HS256" elif kty == "OKP": if not crv: - raise InvalidKeyError("crv is not found: %s" % self._jwk_data) + raise InvalidKeyError(f"crv is not found: {self._jwk_data}") if crv == "Ed25519": algorithm = "EdDSA" else: - raise InvalidKeyError("Unsupported crv: %s" % crv) + raise InvalidKeyError(f"Unsupported crv: {crv}") else: - raise InvalidKeyError("Unsupported kty: %s" % kty) + raise InvalidKeyError(f"Unsupported kty: {kty}") self.Algorithm = self._algorithms.get(algorithm) if not self.Algorithm: - raise PyJWKError("Unable to find a algorithm for key: %s" % self._jwk_data) + raise PyJWKError(f"Unable to find a algorithm for key: {self._jwk_data}") self.key = self.Algorithm.from_jwk(self._jwk_data) @@ -95,3 +95,9 @@ class PyJWKSet: def from_json(data): obj = json.loads(data) return PyJWKSet.from_dict(obj) + + def __getitem__(self, kid): + for key in self.keys: + if key.key_id == kid: + return key + raise KeyError(f"keyset has no key for kid: {kid}") diff --git a/contrib/python/PyJWT/py3/jwt/api_jws.py b/contrib/python/PyJWT/py3/jwt/api_jws.py index f85072e05e..cbf4f6f5b8 100644 --- a/contrib/python/PyJWT/py3/jwt/api_jws.py +++ b/contrib/python/PyJWT/py3/jwt/api_jws.py @@ -80,34 +80,54 @@ class PyJWS: algorithm: Optional[str] = "HS256", headers: Optional[Dict] = None, json_encoder: Optional[Type[json.JSONEncoder]] = None, + is_payload_detached: bool = False, ) -> str: segments = [] if algorithm is None: algorithm = "none" - # Prefer headers["alg"] if present to algorithm parameter. - if headers and "alg" in headers and headers["alg"]: - algorithm = headers["alg"] + # Prefer headers values if present to function parameters. + if headers: + headers_alg = headers.get("alg") + if headers_alg: + algorithm = headers["alg"] + + headers_b64 = headers.get("b64") + if headers_b64 is False: + is_payload_detached = True # Header - header = {"typ": self.header_typ, "alg": algorithm} + header = {"typ": self.header_typ, "alg": algorithm} # type: Dict[str, Any] if headers: self._validate_headers(headers) header.update(headers) - if not header["typ"]: - del header["typ"] + + if not header["typ"]: + del header["typ"] + + if is_payload_detached: + header["b64"] = False + elif "b64" in header: + # True is the standard value for b64, so no need for it + del header["b64"] json_header = json.dumps( header, separators=(",", ":"), cls=json_encoder ).encode() segments.append(base64url_encode(json_header)) - segments.append(base64url_encode(payload)) + + if is_payload_detached: + msg_payload = payload + else: + msg_payload = base64url_encode(payload) + segments.append(msg_payload) # Segments signing_input = b".".join(segments) + try: alg_obj = self._algorithms[algorithm] key = alg_obj.prepare_key(key) @@ -116,14 +136,15 @@ class PyJWS: except KeyError as e: if not has_crypto and algorithm in requires_cryptography: raise NotImplementedError( - "Algorithm '%s' could not be found. Do you have cryptography " - "installed?" % algorithm + f"Algorithm '{algorithm}' could not be found. Do you have cryptography installed?" ) from e - else: - raise NotImplementedError("Algorithm not supported") from e + raise NotImplementedError("Algorithm not supported") from e segments.append(base64url_encode(signature)) + # Don't put the payload content inside the encoded token when detached + if is_payload_detached: + segments[1] = b"" encoded_string = b".".join(segments) return encoded_string.decode("utf-8") @@ -132,8 +153,9 @@ class PyJWS: self, jwt: str, key: str = "", - algorithms: List[str] = None, - options: Dict = None, + algorithms: Optional[List[str]] = None, + options: Optional[Dict] = None, + detached_payload: Optional[bytes] = None, **kwargs, ) -> Dict[str, Any]: if options is None: @@ -148,6 +170,14 @@ class PyJWS: payload, signing_input, header, signature = self._load(jwt) + if header.get("b64", True) is False: + if detached_payload is None: + raise DecodeError( + 'It is required that you pass in a value for the "detached_payload" argument to decode a message having the b64 header set to false.' + ) + payload = detached_payload + signing_input = b".".join([signing_input.rsplit(b".", 1)[0], payload]) + if verify_signature: self._verify_signature(signing_input, header, signature, key, algorithms) @@ -161,8 +191,8 @@ class PyJWS: self, jwt: str, key: str = "", - algorithms: List[str] = None, - options: Dict = None, + algorithms: Optional[List[str]] = None, + options: Optional[Dict] = None, **kwargs, ) -> str: decoded = self.decode_complete(jwt, key, algorithms, options, **kwargs) @@ -200,7 +230,7 @@ class PyJWS: try: header = json.loads(header_data) except ValueError as e: - raise DecodeError("Invalid header string: %s" % e) from e + raise DecodeError(f"Invalid header string: {e}") from e if not isinstance(header, Mapping): raise DecodeError("Invalid header string: must be a json object") diff --git a/contrib/python/PyJWT/py3/jwt/api_jwt.py b/contrib/python/PyJWT/py3/jwt/api_jwt.py index f3b55d360e..7d2177bf53 100644 --- a/contrib/python/PyJWT/py3/jwt/api_jwt.py +++ b/contrib/python/PyJWT/py3/jwt/api_jwt.py @@ -1,4 +1,5 @@ import json +import warnings from calendar import timegm from collections.abc import Iterable, Mapping from datetime import datetime, timedelta, timezone @@ -66,14 +67,23 @@ class PyJWT: self, jwt: str, key: str = "", - algorithms: List[str] = None, - options: Dict = None, + algorithms: Optional[List[str]] = None, + options: Optional[Dict] = None, **kwargs, ) -> Dict[str, Any]: - if options is None: - options = {"verify_signature": True} - else: - options.setdefault("verify_signature", True) + options = dict(options or {}) # shallow-copy or initialize an empty dict + options.setdefault("verify_signature", True) + + # If the user has set the legacy `verify` argument, and it doesn't match + # what the relevant `options` entry for the argument is, inform the user + # that they're likely making a mistake. + if "verify" in kwargs and kwargs["verify"] != options["verify_signature"]: + warnings.warn( + "The `verify` argument to `decode` does nothing in PyJWT 2.0 and newer. " + "The equivalent is setting `verify_signature` to False in the `options` dictionary. " + "This invocation has a mismatch between the kwarg and the option entry.", + category=DeprecationWarning, + ) if not options["verify_signature"]: options.setdefault("verify_exp", False) @@ -98,7 +108,7 @@ class PyJWT: try: payload = json.loads(decoded["payload"]) except ValueError as e: - raise DecodeError("Invalid payload string: %s" % e) + raise DecodeError(f"Invalid payload string: {e}") if not isinstance(payload, dict): raise DecodeError("Invalid payload string: must be a json object") @@ -112,8 +122,8 @@ class PyJWT: self, jwt: str, key: str = "", - algorithms: List[str] = None, - options: Dict = None, + algorithms: Optional[List[str]] = None, + options: Optional[Dict] = None, **kwargs, ) -> Dict[str, Any]: decoded = self.decode_complete(jwt, key, algorithms, options, **kwargs) diff --git a/contrib/python/PyJWT/py3/jwt/exceptions.py b/contrib/python/PyJWT/py3/jwt/exceptions.py index 308899aa6a..ee201add1a 100644 --- a/contrib/python/PyJWT/py3/jwt/exceptions.py +++ b/contrib/python/PyJWT/py3/jwt/exceptions.py @@ -51,7 +51,7 @@ class MissingRequiredClaimError(InvalidTokenError): self.claim = claim def __str__(self): - return 'Token is missing the "%s" claim' % self.claim + return f'Token is missing the "{self.claim}" claim' class PyJWKError(PyJWTError): diff --git a/contrib/python/PyJWT/py3/jwt/help.py b/contrib/python/PyJWT/py3/jwt/help.py index d8f2302421..d5c3ebbfff 100644 --- a/contrib/python/PyJWT/py3/jwt/help.py +++ b/contrib/python/PyJWT/py3/jwt/help.py @@ -28,10 +28,10 @@ def info(): if implementation == "CPython": implementation_version = platform.python_version() elif implementation == "PyPy": - implementation_version = "{}.{}.{}".format( - sys.pypy_version_info.major, - sys.pypy_version_info.minor, - sys.pypy_version_info.micro, + implementation_version = ( + f"{sys.pypy_version_info.major}." + f"{sys.pypy_version_info.minor}." + f"{sys.pypy_version_info.micro}" ) if sys.pypy_version_info.releaselevel != "final": implementation_version = "".join( diff --git a/contrib/python/PyJWT/py3/jwt/utils.py b/contrib/python/PyJWT/py3/jwt/utils.py index 9dde10cf8e..8ab73b42d3 100644 --- a/contrib/python/PyJWT/py3/jwt/utils.py +++ b/contrib/python/PyJWT/py3/jwt/utils.py @@ -1,5 +1,6 @@ import base64 import binascii +import re from typing import Any, Union try: @@ -97,3 +98,63 @@ def raw_to_der_signature(raw_sig: bytes, curve: EllipticCurve) -> bytes: s = bytes_to_number(raw_sig[num_bytes:]) return encode_dss_signature(r, s) + + +# Based on https://github.com/hynek/pem/blob/7ad94db26b0bc21d10953f5dbad3acfdfacf57aa/src/pem/_core.py#L224-L252 +_PEMS = { + b"CERTIFICATE", + b"TRUSTED CERTIFICATE", + b"PRIVATE KEY", + b"PUBLIC KEY", + b"ENCRYPTED PRIVATE KEY", + b"OPENSSH PRIVATE KEY", + b"DSA PRIVATE KEY", + b"RSA PRIVATE KEY", + b"RSA PUBLIC KEY", + b"EC PRIVATE KEY", + b"DH PARAMETERS", + b"NEW CERTIFICATE REQUEST", + b"CERTIFICATE REQUEST", + b"SSH2 PUBLIC KEY", + b"SSH2 ENCRYPTED PRIVATE KEY", + b"X509 CRL", +} + +_PEM_RE = re.compile( + b"----[- ]BEGIN (" + + b"|".join(_PEMS) + + b""")[- ]----\r? +.+?\r? +----[- ]END \\1[- ]----\r?\n?""", + re.DOTALL, +) + + +def is_pem_format(key: bytes) -> bool: + return bool(_PEM_RE.search(key)) + + +# Based on https://github.com/pyca/cryptography/blob/bcb70852d577b3f490f015378c75cba74986297b/src/cryptography/hazmat/primitives/serialization/ssh.py#L40-L46 +_CERT_SUFFIX = b"-cert-v01@openssh.com" +_SSH_PUBKEY_RC = re.compile(br"\A(\S+)[ \t]+(\S+)") +_SSH_KEY_FORMATS = [ + b"ssh-ed25519", + b"ssh-rsa", + b"ssh-dss", + b"ecdsa-sha2-nistp256", + b"ecdsa-sha2-nistp384", + b"ecdsa-sha2-nistp521", +] + + +def is_ssh_key(key: bytes) -> bool: + if any(string_value in key for string_value in _SSH_KEY_FORMATS): + return True + + ssh_pubkey_match = _SSH_PUBKEY_RC.match(key) + if ssh_pubkey_match: + key_type = ssh_pubkey_match.group(1) + if _CERT_SUFFIX == key_type[-len(_CERT_SUFFIX) :]: + return True + + return False diff --git a/contrib/python/PyJWT/py3/ya.make b/contrib/python/PyJWT/py3/ya.make index 0cbee2bb2e..3be98b643e 100644 --- a/contrib/python/PyJWT/py3/ya.make +++ b/contrib/python/PyJWT/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.3.0) +VERSION(2.4.0) LICENSE(MIT) diff --git a/contrib/python/contextlib2/py3/contextlib2/__init__.py b/contrib/python/contextlib2/py3/contextlib2/__init__.py index d6c0c4ac4a..63ecced65a 100644 --- a/contrib/python/contextlib2/py3/contextlib2/__init__.py +++ b/contrib/python/contextlib2/py3/contextlib2/__init__.py @@ -177,8 +177,10 @@ class _GeneratorContextManager(_GeneratorContextManagerBase, # Need to force instantiation so we can reliably # tell if we get the same exception back value = type() + if traceback is not None: + value = value.with_traceback(traceback) try: - self.gen.throw(type, value, traceback) + self.gen.throw(value) except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration diff --git a/contrib/python/fonttools/.dist-info/METADATA b/contrib/python/fonttools/.dist-info/METADATA index 9026fc11f8..f156c42697 100644 --- a/contrib/python/fonttools/.dist-info/METADATA +++ b/contrib/python/fonttools/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: fonttools -Version: 4.46.0 +Version: 4.47.0 Summary: Tools to manipulate font files Home-page: http://github.com/fonttools/fonttools Author: Just van Rossum @@ -34,6 +34,7 @@ Requires-Dist: fs <3,>=2.2.0 ; extra == 'all' Requires-Dist: lxml <5,>=4.0 ; extra == 'all' Requires-Dist: zopfli >=0.1.4 ; extra == 'all' Requires-Dist: lz4 >=1.7.4.2 ; extra == 'all' +Requires-Dist: pycairo ; extra == 'all' Requires-Dist: matplotlib ; extra == 'all' Requires-Dist: sympy ; extra == 'all' Requires-Dist: skia-pathops >=0.5.0 ; extra == 'all' @@ -47,6 +48,7 @@ Requires-Dist: xattr ; (sys_platform == "darwin") and extra == 'all' Provides-Extra: graphite Requires-Dist: lz4 >=1.7.4.2 ; extra == 'graphite' Provides-Extra: interpolatable +Requires-Dist: pycairo ; extra == 'interpolatable' Requires-Dist: scipy ; (platform_python_implementation != "PyPy") and extra == 'interpolatable' Requires-Dist: munkres ; (platform_python_implementation == "PyPy") and extra == 'interpolatable' Provides-Extra: lxml @@ -210,6 +212,13 @@ are required to unlock the extra features named "ufo", etc. * `munkres <https://pypi.python.org/pypi/munkres>`__: a pure-Python module that implements the Hungarian or Kuhn-Munkres algorithm. + To plot the results to a PDF or HTML format, you also need to install: + + * `pycairo <https://pypi.org/project/pycairo/>`__: Python bindings for the + Cairo graphics library. Note that wheels are currently only available for + Windows, for other platforms see pycairo's `installation instructions + <https://pycairo.readthedocs.io/en/latest/getting_started.html>`__. + *Extra:* ``interpolatable`` - ``Lib/fontTools/varLib/plot.py`` @@ -366,6 +375,16 @@ Have fun! Changelog ~~~~~~~~~ +4.47.0 (released 2023-12-18) +---------------------------- + +- [varLib.models] New API for VariationModel: ``getMasterScalars`` and + ``interpolateFromValuesAndScalars``. +- [varLib.interpolatable] Various bugfixes and rendering improvements. In particular, + add a Summary page in the front, and an Index and Table-of-Contents in the back. + Change the page size to Letter. +- [Docs/designspaceLib] Defined a new ``public.fontInfo`` lib key, not used anywhere yet (#3358). + 4.46.0 (released 2023-12-02) ---------------------------- diff --git a/contrib/python/fonttools/README.rst b/contrib/python/fonttools/README.rst index bcb7f0d47f..d84282fc76 100644 --- a/contrib/python/fonttools/README.rst +++ b/contrib/python/fonttools/README.rst @@ -138,6 +138,13 @@ are required to unlock the extra features named "ufo", etc. * `munkres <https://pypi.python.org/pypi/munkres>`__: a pure-Python module that implements the Hungarian or Kuhn-Munkres algorithm. + To plot the results to a PDF or HTML format, you also need to install: + + * `pycairo <https://pypi.org/project/pycairo/>`__: Python bindings for the + Cairo graphics library. Note that wheels are currently only available for + Windows, for other platforms see pycairo's `installation instructions + <https://pycairo.readthedocs.io/en/latest/getting_started.html>`__. + *Extra:* ``interpolatable`` - ``Lib/fontTools/varLib/plot.py`` diff --git a/contrib/python/fonttools/fontTools/__init__.py b/contrib/python/fonttools/fontTools/__init__.py index dfe589402f..6c00e567a4 100644 --- a/contrib/python/fonttools/fontTools/__init__.py +++ b/contrib/python/fonttools/fontTools/__init__.py @@ -3,6 +3,6 @@ from fontTools.misc.loggingTools import configLogger log = logging.getLogger(__name__) -version = __version__ = "4.46.0" +version = __version__ = "4.47.0" __all__ = ["version", "log", "configLogger"] diff --git a/contrib/python/fonttools/fontTools/afmLib.py b/contrib/python/fonttools/fontTools/afmLib.py index 935a1e8e0d..e89646951c 100644 --- a/contrib/python/fonttools/fontTools/afmLib.py +++ b/contrib/python/fonttools/fontTools/afmLib.py @@ -82,7 +82,10 @@ kernRE = re.compile( # regular expressions to parse composite info lines of the form: # Aacute 2 ; PCC A 0 0 ; PCC acute 182 211 ; compositeRE = re.compile( - r"([.A-Za-z0-9_]+)" r"\s+" r"(\d+)" r"\s*;\s*" # char name # number of parts + r"([.A-Za-z0-9_]+)" # char name + r"\s+" + r"(\d+)" # number of parts + r"\s*;\s*" ) componentRE = re.compile( r"PCC\s+" # PPC diff --git a/contrib/python/fonttools/fontTools/designspaceLib/__init__.py b/contrib/python/fonttools/fontTools/designspaceLib/__init__.py index 1c71fd002e..69d4912c09 100644 --- a/contrib/python/fonttools/fontTools/designspaceLib/__init__.py +++ b/contrib/python/fonttools/fontTools/designspaceLib/__init__.py @@ -312,7 +312,7 @@ class SourceDescriptor(SimpleDescriptor): return self.designLocation @location.setter - def location(self, location: Optional[AnisotropicLocationDict]): + def location(self, location: Optional[SimpleLocationDict]): self.designLocation = location or {} def setFamilyName(self, familyName, languageCode="en"): @@ -329,15 +329,13 @@ class SourceDescriptor(SimpleDescriptor): """ return self.localisedFamilyName.get(languageCode) - def getFullDesignLocation( - self, doc: "DesignSpaceDocument" - ) -> AnisotropicLocationDict: + def getFullDesignLocation(self, doc: "DesignSpaceDocument") -> SimpleLocationDict: """Get the complete design location of this source, from its :attr:`designLocation` and the document's axis defaults. .. versionadded:: 5.0 """ - result: AnisotropicLocationDict = {} + result: SimpleLocationDict = {} for axis in doc.axes: if axis.name in self.designLocation: result[axis.name] = self.designLocation[axis.name] diff --git a/contrib/python/fonttools/fontTools/pens/recordingPen.py b/contrib/python/fonttools/fontTools/pens/recordingPen.py index 2ed8d32ec7..e24b65265a 100644 --- a/contrib/python/fonttools/fontTools/pens/recordingPen.py +++ b/contrib/python/fonttools/fontTools/pens/recordingPen.py @@ -8,6 +8,7 @@ __all__ = [ "RecordingPen", "DecomposingRecordingPen", "RecordingPointPen", + "lerpRecordings", ] @@ -172,6 +173,34 @@ class RecordingPointPen(AbstractPointPen): drawPoints = replay +def lerpRecordings(recording1, recording2, factor=0.5): + """Linearly interpolate between two recordings. The recordings + must be decomposed, i.e. they must not contain any components. + + Factor is typically between 0 and 1. 0 means the first recording, + 1 means the second recording, and 0.5 means the average of the + two recordings. Other values are possible, and can be useful to + extrapolate. Defaults to 0.5. + + Returns a generator with the new recording. + """ + if len(recording1) != len(recording2): + raise ValueError( + "Mismatched lengths: %d and %d" % (len(recording1), len(recording2)) + ) + for (op1, args1), (op2, args2) in zip(recording1, recording2): + if op1 != op2: + raise ValueError("Mismatched operations: %s, %s" % (op1, op2)) + if op1 == "addComponent": + raise ValueError("Cannot interpolate components") + else: + mid_args = [ + (x1 + (x2 - x1) * factor, y1 + (y2 - y1) * factor) + for (x1, y1), (x2, y2) in zip(args1, args2) + ] + yield (op1, mid_args) + + if __name__ == "__main__": pen = RecordingPen() pen.moveTo((0, 0)) diff --git a/contrib/python/fonttools/fontTools/ttLib/ttGlyphSet.py b/contrib/python/fonttools/fontTools/ttLib/ttGlyphSet.py index 349cc2c73f..5d188d6a10 100644 --- a/contrib/python/fonttools/fontTools/ttLib/ttGlyphSet.py +++ b/contrib/python/fonttools/fontTools/ttLib/ttGlyphSet.py @@ -9,6 +9,11 @@ from fontTools.misc.fixedTools import otRound from fontTools.misc.loggingTools import deprecateFunction from fontTools.misc.transform import Transform from fontTools.pens.transformPen import TransformPen, TransformPointPen +from fontTools.pens.recordingPen import ( + DecomposingRecordingPen, + lerpRecordings, + replayRecording, +) class _TTGlyphSet(Mapping): @@ -321,3 +326,52 @@ def _setCoordinates(glyph, coord, glyfTable, *, recalcBounds=True): verticalAdvanceWidth, topSideBearing, ) + + +class LerpGlyphSet(Mapping): + """A glyphset that interpolates between two other glyphsets. + + Factor is typically between 0 and 1. 0 means the first glyphset, + 1 means the second glyphset, and 0.5 means the average of the + two glyphsets. Other values are possible, and can be useful to + extrapolate. Defaults to 0.5. + """ + + def __init__(self, glyphset1, glyphset2, factor=0.5): + self.glyphset1 = glyphset1 + self.glyphset2 = glyphset2 + self.factor = factor + + def __getitem__(self, glyphname): + if glyphname in self.glyphset1 and glyphname in self.glyphset2: + return LerpGlyph(glyphname, self) + raise KeyError(glyphname) + + def __contains__(self, glyphname): + return glyphname in self.glyphset1 and glyphname in self.glyphset2 + + def __iter__(self): + set1 = set(self.glyphset1) + set2 = set(self.glyphset2) + return iter(set1.intersection(set2)) + + def __len__(self): + set1 = set(self.glyphset1) + set2 = set(self.glyphset2) + return len(set1.intersection(set2)) + + +class LerpGlyph: + def __init__(self, glyphname, glyphset): + self.glyphset = glyphset + self.glyphname = glyphname + + def draw(self, pen): + recording1 = DecomposingRecordingPen(self.glyphset.glyphset1) + self.glyphset.glyphset1[self.glyphname].draw(recording1) + recording2 = DecomposingRecordingPen(self.glyphset.glyphset2) + self.glyphset.glyphset2[self.glyphname].draw(recording2) + + factor = self.glyphset.factor + + replayRecording(lerpRecordings(recording1.value, recording2.value, factor), pen) diff --git a/contrib/python/fonttools/fontTools/varLib/instancer/__init__.py b/contrib/python/fonttools/fontTools/varLib/instancer/__init__.py index a887e5d38f..d1cde0df7a 100644 --- a/contrib/python/fonttools/fontTools/varLib/instancer/__init__.py +++ b/contrib/python/fonttools/fontTools/varLib/instancer/__init__.py @@ -1433,7 +1433,7 @@ def parseArgs(args): nargs="*", help="List of space separated locations. A location consists of " "the tag of a variation axis, followed by '=' and the literal, " - "string 'drop', or comma-separate list of one to three values, " + "string 'drop', or colon-separated list of one to three values, " "each of which is the empty string, or a number. " "E.g.: wdth=100 or wght=75.0:125.0 or wght=100:400:700 or wght=:500: " "or wght=drop", diff --git a/contrib/python/fonttools/fontTools/varLib/instancer/solver.py b/contrib/python/fonttools/fontTools/varLib/instancer/solver.py index 9c568fe9a5..ba5231b796 100644 --- a/contrib/python/fonttools/fontTools/varLib/instancer/solver.py +++ b/contrib/python/fonttools/fontTools/varLib/instancer/solver.py @@ -178,7 +178,9 @@ def _solve(tent, axisLimit, negative=False): # newUpper = peak + (1 - gain) * (upper - peak) assert axisMax <= newUpper # Because outGain > gain - if newUpper <= axisDef + (axisMax - axisDef) * 2: + # Disabled because ots doesn't like us: + # https://github.com/fonttools/fonttools/issues/3350 + if False and newUpper <= axisDef + (axisMax - axisDef) * 2: upper = newUpper if not negative and axisDef + (axisMax - axisDef) * MAX_F2DOT14 < upper: # we clamp +2.0 to the max F2Dot14 (~1.99994) for convenience diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatable.py b/contrib/python/fonttools/fontTools/varLib/interpolatable.py index f03e946207..0a9bbebc41 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatable.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatable.py @@ -9,7 +9,11 @@ $ fonttools varLib.interpolatable font1 font2 ... from .interpolatableHelpers import * from .interpolatableTestContourOrder import test_contour_order from .interpolatableTestStartingPoint import test_starting_point -from fontTools.pens.recordingPen import RecordingPen, DecomposingRecordingPen +from fontTools.pens.recordingPen import ( + RecordingPen, + DecomposingRecordingPen, + lerpRecordings, +) from fontTools.pens.transformPen import TransformPen from fontTools.pens.statisticsPen import StatisticsPen, StatisticsControlPen from fontTools.pens.momentsPen import OpenContourError @@ -22,6 +26,7 @@ from functools import wraps from pprint import pformat from math import sqrt, atan2, pi import logging +import os log = logging.getLogger("fontTools.varLib.interpolatable") @@ -34,11 +39,9 @@ DEFAULT_UPEM = 1000 class Glyph: ITEMS = ( "recordings", - "recordingsNormalized", "greenStats", "controlStats", "greenVectors", - "greenVectorsNormalized", "controlVectors", "nodeTypes", "isomorphisms", @@ -90,21 +93,6 @@ class Glyph: self.greenVectors.append(contour_vector_from_stats(greenStats)) self.controlVectors.append(contour_vector_from_stats(controlStats)) - # Save a "normalized" version of the outlines - try: - rpen = DecomposingRecordingPen(glyphset) - tpen = TransformPen( - rpen, transform_from_stats(greenStats, inverse=True) - ) - contour.replay(tpen) - self.recordingsNormalized.append(rpen) - except ZeroDivisionError: - self.recordingsNormalized.append(None) - - greenStats = StatisticsPen(glyphset=glyphset) - rpen.replay(greenStats) - self.greenVectorsNormalized.append(contour_vector_from_stats(greenStats)) - # Check starting point if nodeTypes[0] == "addComponent": self._fill_in(ix) @@ -186,7 +174,11 @@ def test_gen( if not ignore_missing: yield ( glyph_name, - {"type": "missing", "master": name, "master_idx": master_idx}, + { + "type": InterpolatableProblem.MISSING, + "master": name, + "master_idx": master_idx, + }, ) continue @@ -198,10 +190,10 @@ def test_gen( yield ( glyph_name, { + "type": InterpolatableProblem.OPEN_PATH, "master": name, "master_idx": master_idx, "contour": ix, - "type": "open_path", }, ) if has_open: @@ -230,7 +222,7 @@ def test_gen( yield ( glyph_name, { - "type": "path_count", + "type": InterpolatableProblem.PATH_COUNT, "master_1": names[m0idx], "master_2": names[m1idx], "master_1_idx": m0idx, @@ -249,7 +241,7 @@ def test_gen( yield ( glyph_name, { - "type": "node_count", + "type": InterpolatableProblem.NODE_COUNT, "path": pathIx, "master_1": names[m0idx], "master_2": names[m1idx], @@ -265,7 +257,7 @@ def test_gen( yield ( glyph_name, { - "type": "node_incompatibility", + "type": InterpolatableProblem.NODE_INCOMPATIBILITY, "path": pathIx, "node": nodeIx, "master_1": names[m0idx], @@ -279,21 +271,15 @@ def test_gen( continue # - # "contour_order" check + # InterpolatableProblem.CONTOUR_ORDER check # - matching, matching_cost, identity_cost = test_contour_order(glyph0, glyph1) - if matching_cost < identity_cost * tolerance: - log.debug( - "matching_ratio %g", - matching_cost / identity_cost, - ) - this_tolerance = matching_cost / identity_cost - log.debug("tolerance: %g", this_tolerance) + this_tolerance, matching = test_contour_order(glyph0, glyph1) + if this_tolerance < tolerance: yield ( glyph_name, { - "type": "contour_order", + "type": InterpolatableProblem.CONTOUR_ORDER, "master_1": names[m0idx], "master_2": names[m1idx], "master_1_idx": m0idx, @@ -306,19 +292,15 @@ def test_gen( matchings[m1idx] = matching # - # "wrong_start_point" / weight check + # wrong-start-point / weight check # m0Isomorphisms = glyph0.isomorphisms m1Isomorphisms = glyph1.isomorphisms m0Vectors = glyph0.greenVectors m1Vectors = glyph1.greenVectors - m0VectorsNormalized = glyph0.greenVectorsNormalized - m1VectorsNormalized = glyph1.greenVectorsNormalized recording0 = glyph0.recordings recording1 = glyph1.recordings - recording0Normalized = glyph0.recordingsNormalized - recording1Normalized = glyph1.recordingsNormalized # If contour-order is wrong, adjust it matching = matchings[m1idx] @@ -327,14 +309,14 @@ def test_gen( ): # m1 is empty for composite glyphs m1Isomorphisms = [m1Isomorphisms[i] for i in matching] m1Vectors = [m1Vectors[i] for i in matching] - m1VectorsNormalized = [m1VectorsNormalized[i] for i in matching] recording1 = [recording1[i] for i in matching] - recording1Normalized = [recording1Normalized[i] for i in matching] midRecording = [] for c0, c1 in zip(recording0, recording1): try: - midRecording.append(lerp_recordings(c0, c1)) + r = RecordingPen() + r.value = list(lerpRecordings(c0.value, c1.value)) + midRecording.append(r) except ValueError: # Mismatch because of the reordering above midRecording.append(None) @@ -352,118 +334,100 @@ def test_gen( # after reordering above. continue - proposed_point, reverse, min_cost, first_cost = test_starting_point( + this_tolerance, proposed_point, reverse = test_starting_point( glyph0, glyph1, ix, tolerance, matching ) - if proposed_point or reverse: - this_tolerance = min_cost / first_cost - log.debug("tolerance: %g", this_tolerance) - if min_cost < first_cost * tolerance: - yield ( - glyph_name, - { - "type": "wrong_start_point", - "contour": ix, - "master_1": names[m0idx], - "master_2": names[m1idx], - "master_1_idx": m0idx, - "master_2_idx": m1idx, - "value_1": 0, - "value_2": proposed_point, - "reversed": reverse, - "tolerance": this_tolerance, - }, - ) - else: - # Weight check. - # - # If contour could be mid-interpolated, and the two - # contours have the same area sign, proceeed. - # - # The sign difference can happen if it's a werido - # self-intersecting contour; ignore it. - contour = midRecording[ix] - - normalized = False - if contour and (m0Vectors[ix][0] < 0) == (m1Vectors[ix][0] < 0): - if normalized: - midStats = StatisticsPen(glyphset=None) - tpen = TransformPen( - midStats, transform_from_stats(midStats, inverse=True) - ) - contour.replay(tpen) - else: - midStats = StatisticsPen(glyphset=None) - contour.replay(midStats) - - midVector = contour_vector_from_stats(midStats) + if this_tolerance < tolerance: + yield ( + glyph_name, + { + "type": InterpolatableProblem.WRONG_START_POINT, + "contour": ix, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "value_1": 0, + "value_2": proposed_point, + "reversed": reverse, + "tolerance": this_tolerance, + }, + ) - m0Vec = ( - m0Vectors[ix] if not normalized else m0VectorsNormalized[ix] + # Weight check. + # + # If contour could be mid-interpolated, and the two + # contours have the same area sign, proceeed. + # + # The sign difference can happen if it's a weirdo + # self-intersecting contour; ignore it. + contour = midRecording[ix] + + if contour and (m0Vectors[ix][0] < 0) == (m1Vectors[ix][0] < 0): + midStats = StatisticsPen(glyphset=None) + contour.replay(midStats) + + midVector = contour_vector_from_stats(midStats) + + m0Vec = m0Vectors[ix] + m1Vec = m1Vectors[ix] + size0 = m0Vec[0] * m0Vec[0] + size1 = m1Vec[0] * m1Vec[0] + midSize = midVector[0] * midVector[0] + + power = 1 + t = tolerance**power + + for overweight, problem_type in enumerate( + ( + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, ) - m1Vec = ( - m1Vectors[ix] if not normalized else m1VectorsNormalized[ix] + ): + if overweight: + expectedSize = sqrt(size0 * size1) + expectedSize = (size0 + size1) - expectedSize + continue + else: + expectedSize = sqrt(size0 * size1) + + log.debug( + "%s: actual size %g; threshold size %g, master sizes: %g, %g", + problem_type, + midSize, + expectedSize, + size0, + size1, ) - size0 = m0Vec[0] * m0Vec[0] - size1 = m1Vec[0] * m1Vec[0] - midSize = midVector[0] * midVector[0] - - power = 1 - t = tolerance**power - - for overweight, problem_type in enumerate( - ("underweight", "overweight") - ): - if overweight: - expectedSize = sqrt(size0 * size1) - expectedSize = (size0 + size1) - expectedSize - expectedSize = size1 + (midSize - size1) - continue - else: - expectedSize = sqrt(size0 * size1) - - log.debug( - "%s: actual size %g; threshold size %g, master sizes: %g, %g", - problem_type, - midSize, - expectedSize, - size0, - size1, - ) - size0, size1 = sorted((size0, size1)) - - if ( - not overweight - and expectedSize * tolerance > midSize + 1e-5 - ) or ( - overweight and 1e-5 + expectedSize / tolerance < midSize - ): - try: - if overweight: - this_tolerance = (expectedSize / midSize) ** ( - 1 / power - ) - else: - this_tolerance = (midSize / expectedSize) ** ( - 1 / power - ) - except ZeroDivisionError: - this_tolerance = 0 - log.debug("tolerance %g", this_tolerance) - yield ( - glyph_name, - { - "type": problem_type, - "contour": ix, - "master_1": names[m0idx], - "master_2": names[m1idx], - "master_1_idx": m0idx, - "master_2_idx": m1idx, - "tolerance": this_tolerance, - }, - ) + if ( + not overweight and expectedSize * tolerance > midSize + 1e-5 + ) or (overweight and 1e-5 + expectedSize / tolerance < midSize): + try: + if overweight: + this_tolerance = (expectedSize / midSize) ** ( + 1 / power + ) + else: + this_tolerance = (midSize / expectedSize) ** ( + 1 / power + ) + except ZeroDivisionError: + this_tolerance = 0 + log.debug("tolerance %g", this_tolerance) + yield ( + glyph_name, + { + "type": problem_type, + "contour": ix, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "tolerance": this_tolerance, + }, + ) # # "kink" detector @@ -585,7 +549,7 @@ def test_gen( this_tolerance = t / (abs(sin_mid) * kinkiness) log.debug( - "deviation %g; deviation_ratio %g; sin_mid %g; r_diff %g", + "kink: deviation %g; deviation_ratio %g; sin_mid %g; r_diff %g", deviation, deviation_ratio, sin_mid, @@ -595,7 +559,7 @@ def test_gen( yield ( glyph_name, { - "type": "kink", + "type": InterpolatableProblem.KINK, "contour": ix, "master_1": names[m0idx], "master_2": names[m1idx], @@ -614,7 +578,7 @@ def test_gen( yield ( glyph_name, { - "type": "nothing", + "type": InterpolatableProblem.NOTHING, "master_1": names[m0idx], "master_2": names[m1idx], "master_1_idx": m0idx, @@ -640,6 +604,13 @@ def recursivelyAddGlyph(glyphname, glyphset, ttGlyphSet, glyf): recursivelyAddGlyph(component.glyphName, glyphset, ttGlyphSet, glyf) +def ensure_parent_dir(path): + dirname = os.path.dirname(path) + if dirname: + os.makedirs(dirname, exist_ok=True) + return path + + def main(args=None): """Test for interpolatability issues between fonts""" import argparse @@ -759,7 +730,7 @@ def main(args=None): for k, vv in axis_triples.items() } - elif args.inputs[0].endswith(".glyphs"): + elif args.inputs[0].endswith((".glyphs", ".glyphspackage")): from glyphsLib import GSFont, to_designspace gsfont = GSFont(args.inputs[0]) @@ -929,7 +900,11 @@ def main(args=None): ) problems = defaultdict(list) - f = sys.stdout if args.output is None else open(args.output, "w") + f = ( + sys.stdout + if args.output is None + else open(ensure_parent_dir(args.output), "w") + ) if not args.quiet: if args.json: @@ -963,16 +938,16 @@ def main(args=None): print(f" Masters: %s:" % ", ".join(master_names), file=f) last_master_idxs = master_idxs - if p["type"] == "missing": + if p["type"] == InterpolatableProblem.MISSING: print( " Glyph was missing in master %s" % p["master"], file=f ) - elif p["type"] == "open_path": + elif p["type"] == InterpolatableProblem.OPEN_PATH: print( " Glyph has an open path in master %s" % p["master"], file=f, ) - elif p["type"] == "path_count": + elif p["type"] == InterpolatableProblem.PATH_COUNT: print( " Path count differs: %i in %s, %i in %s" % ( @@ -983,7 +958,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "node_count": + elif p["type"] == InterpolatableProblem.NODE_COUNT: print( " Node count differs in path %i: %i in %s, %i in %s" % ( @@ -995,7 +970,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "node_incompatibility": + elif p["type"] == InterpolatableProblem.NODE_INCOMPATIBILITY: print( " Node %o incompatible in path %i: %s in %s, %s in %s" % ( @@ -1008,7 +983,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "contour_order": + elif p["type"] == InterpolatableProblem.CONTOUR_ORDER: print( " Contour order differs: %s in %s, %s in %s" % ( @@ -1019,7 +994,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "wrong_start_point": + elif p["type"] == InterpolatableProblem.WRONG_START_POINT: print( " Contour %d start point differs: %s in %s, %s in %s; reversed: %s" % ( @@ -1032,7 +1007,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "underweight": + elif p["type"] == InterpolatableProblem.UNDERWEIGHT: print( " Contour %d interpolation is underweight: %s, %s" % ( @@ -1042,7 +1017,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "overweight": + elif p["type"] == InterpolatableProblem.OVERWEIGHT: print( " Contour %d interpolation is overweight: %s, %s" % ( @@ -1052,7 +1027,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "kink": + elif p["type"] == InterpolatableProblem.KINK: print( " Contour %d has a kink at %s: %s, %s" % ( @@ -1063,7 +1038,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "nothing": + elif p["type"] == InterpolatableProblem.NOTHING: print( " Showing %s and %s" % ( @@ -1076,29 +1051,31 @@ def main(args=None): for glyphname, problem in problems_gen: problems[glyphname].append(problem) - if args.pdf: - log.info("Writing PDF to %s", args.pdf) - from .interpolatablePlot import InterpolatablePDF + problems = sort_problems(problems) - with InterpolatablePDF(args.pdf, glyphsets=glyphsets, names=names) as pdf: - pdf.add_title_page( - original_args_inputs, tolerance=tolerance, kinkiness=kinkiness - ) - pdf.add_problems(problems) - if not problems and not args.quiet: - pdf.draw_cupcake() + for p in "ps", "pdf": + arg = getattr(args, p) + if arg is None: + continue + log.info("Writing %s to %s", p.upper(), arg) + from .interpolatablePlot import InterpolatablePS, InterpolatablePDF - if args.ps: - log.info("Writing PS to %s", args.pdf) - from .interpolatablePlot import InterpolatablePS + PlotterClass = InterpolatablePS if p == "ps" else InterpolatablePDF - with InterpolatablePS(args.ps, glyphsets=glyphsets, names=names) as ps: - ps.add_title_page( + with PlotterClass( + ensure_parent_dir(arg), glyphsets=glyphsets, names=names + ) as doc: + doc.add_title_page( original_args_inputs, tolerance=tolerance, kinkiness=kinkiness ) - ps.add_problems(problems) + if problems: + doc.add_summary(problems) + doc.add_problems(problems) if not problems and not args.quiet: - ps.draw_cupcake() + doc.draw_cupcake() + if problems: + doc.add_index() + doc.add_table_of_contents() if args.html: log.info("Writing HTML to %s", args.html) @@ -1125,7 +1102,7 @@ def main(args=None): import base64 - with open(args.html, "wb") as f: + with open(ensure_parent_dir(args.html), "wb") as f: f.write(b"<!DOCTYPE html>\n") f.write( b'<html><body align="center" style="font-family: sans-serif; text-color: #222">\n' diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatableHelpers.py b/contrib/python/fonttools/fontTools/varLib/interpolatableHelpers.py index 513e5f7409..2a3540fff2 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatableHelpers.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatableHelpers.py @@ -1,9 +1,11 @@ +from fontTools.ttLib.ttGlyphSet import LerpGlyphSet from fontTools.pens.basePen import AbstractPen, BasePen, DecomposingPen from fontTools.pens.pointPen import AbstractPointPen, SegmentToPointPen from fontTools.pens.recordingPen import RecordingPen, DecomposingRecordingPen from fontTools.misc.transform import Transform from collections import defaultdict, deque from math import sqrt, copysign, atan2, pi +from enum import Enum import itertools import logging @@ -11,6 +13,50 @@ import logging log = logging.getLogger("fontTools.varLib.interpolatable") +class InterpolatableProblem: + NOTHING = "nothing" + MISSING = "missing" + OPEN_PATH = "open_path" + PATH_COUNT = "path_count" + NODE_COUNT = "node_count" + NODE_INCOMPATIBILITY = "node_incompatibility" + CONTOUR_ORDER = "contour_order" + WRONG_START_POINT = "wrong_start_point" + KINK = "kink" + UNDERWEIGHT = "underweight" + OVERWEIGHT = "overweight" + + severity = { + MISSING: 1, + OPEN_PATH: 2, + PATH_COUNT: 3, + NODE_COUNT: 4, + NODE_INCOMPATIBILITY: 5, + CONTOUR_ORDER: 6, + WRONG_START_POINT: 7, + KINK: 8, + UNDERWEIGHT: 9, + OVERWEIGHT: 10, + NOTHING: 11, + } + + +def sort_problems(problems): + """Sort problems by severity, then by glyph name, then by problem message.""" + return dict( + sorted( + problems.items(), + key=lambda _: -min( + ( + (InterpolatableProblem.severity[p["type"]] + p.get("tolerance", 0)) + for p in _[1] + ), + ), + reverse=True, + ) + ) + + def rot_list(l, k): """Rotate list by k items forward. Ie. item at position 0 will be at position k in returned list. Negative k is allowed.""" @@ -332,52 +378,3 @@ def transform_from_stats(stats, inverse=False): trans = trans.translate(stats.meanX, stats.meanY) return trans - - -class LerpGlyphSet: - def __init__(self, glyphset1, glyphset2, factor=0.5): - self.glyphset1 = glyphset1 - self.glyphset2 = glyphset2 - self.factor = factor - - def __getitem__(self, glyphname): - return LerpGlyph(glyphname, self) - - -class LerpGlyph: - def __init__(self, glyphname, glyphset): - self.glyphset = glyphset - self.glyphname = glyphname - - def draw(self, pen): - recording1 = DecomposingRecordingPen(self.glyphset.glyphset1) - self.glyphset.glyphset1[self.glyphname].draw(recording1) - recording2 = DecomposingRecordingPen(self.glyphset.glyphset2) - self.glyphset.glyphset2[self.glyphname].draw(recording2) - - factor = self.glyphset.factor - for (op1, args1), (op2, args2) in zip(recording1.value, recording2.value): - if op1 != op2: - raise ValueError("Mismatching operations: %s, %s" % (op1, op2)) - mid_args = [ - (x1 + (x2 - x1) * factor, y1 + (y2 - y1) * factor) - for (x1, y1), (x2, y2) in zip(args1, args2) - ] - getattr(pen, op1)(*mid_args) - - -def lerp_recordings(recording1, recording2, factor=0.5): - pen = RecordingPen() - value = pen.value - for (op1, args1), (op2, args2) in zip(recording1.value, recording2.value): - if op1 != op2: - raise ValueError("Mismatched operations: %s, %s" % (op1, op2)) - if op1 == "addComponent": - mid_args = args1 # XXX Interpolate transformation? - else: - mid_args = [ - (x1 + (x2 - x1) * factor, y1 + (y2 - y1) * factor) - for (x1, y1), (x2, y2) in zip(args1, args2) - ] - value.append((op1, mid_args)) - return pen diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatablePlot.py b/contrib/python/fonttools/fontTools/varLib/interpolatablePlot.py index eef4a47160..3c206c6ee2 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatablePlot.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatablePlot.py @@ -1,4 +1,6 @@ +from .interpolatableHelpers import * from fontTools.ttLib import TTFont +from fontTools.ttLib.ttGlyphSet import LerpGlyphSet from fontTools.pens.recordingPen import ( RecordingPen, DecomposingRecordingPen, @@ -11,10 +13,9 @@ from fontTools.pens.pointPen import ( PointToSegmentPen, ReverseContourPointPen, ) -from fontTools.varLib.interpolatable import ( +from fontTools.varLib.interpolatableHelpers import ( PerContourOrComponentPen, SimpleRecordingPointPen, - LerpGlyphSet, ) from itertools import cycle from functools import wraps @@ -36,33 +37,34 @@ class OverridingDict(dict): class InterpolatablePlot: - width = 640 - height = 480 - pad = 16 - line_height = 36 + width = 8.5 * 72 + height = 11 * 72 + pad = 0.1 * 72 + title_font_size = 24 + font_size = 16 page_number = 1 head_color = (0.3, 0.3, 0.3) label_color = (0.2, 0.2, 0.2) border_color = (0.9, 0.9, 0.9) - border_width = 1 + border_width = 0.5 fill_color = (0.8, 0.8, 0.8) stroke_color = (0.1, 0.1, 0.1) - stroke_width = 2 + stroke_width = 1 oncurve_node_color = (0, 0.8, 0, 0.7) - oncurve_node_diameter = 10 + oncurve_node_diameter = 6 offcurve_node_color = (0, 0.5, 0, 0.7) - offcurve_node_diameter = 8 + offcurve_node_diameter = 4 handle_color = (0, 0.5, 0, 0.7) - handle_width = 1 + handle_width = 0.5 corrected_start_point_color = (0, 0.9, 0, 0.7) - corrected_start_point_size = 15 + corrected_start_point_size = 7 wrong_start_point_color = (1, 0, 0, 0.7) start_point_color = (0, 0, 1, 0.7) - start_arrow_length = 20 - kink_point_size = 10 + start_arrow_length = 9 + kink_point_size = 7 kink_point_color = (1, 0, 1, 0.7) - kink_circle_size = 25 - kink_circle_stroke_width = 1.5 + kink_circle_size = 15 + kink_circle_stroke_width = 1 kink_circle_color = (1, 0, 1, 0.7) contour_colors = ((1, 0, 0), (0, 0, 1), (0, 1, 0), (1, 1, 0), (1, 0, 1), (0, 1, 1)) contour_alpha = 0.5 @@ -113,62 +115,59 @@ class InterpolatablePlot: self.out = out self.glyphsets = glyphsets self.names = names or [repr(g) for g in glyphsets] + self.toc = {} for k, v in kwargs.items(): if not hasattr(self, k): raise TypeError("Unknown keyword argument: %s" % k) setattr(self, k, v) + self.panel_width = self.width / 2 - self.pad * 3 + self.panel_height = ( + self.height / 2 - self.pad * 6 - self.font_size * 2 - self.title_font_size + ) + def __enter__(self): return self def __exit__(self, type, value, traceback): pass - def set_size(self, width, height): - raise NotImplementedError - def show_page(self): self.page_number += 1 - def total_width(self): - return self.width * 2 + self.pad * 3 - - def total_height(self): - return ( - self.pad - + self.line_height - + self.pad - + self.line_height - + self.pad - + 2 * (self.height + self.pad * 2 + self.line_height) - + self.pad - ) - def add_title_page( self, files, *, show_tolerance=True, tolerance=None, kinkiness=None ): - self.set_size(self.total_width(), self.total_height()) - pad = self.pad - width = self.total_width() - 3 * self.pad - height = self.total_height() - 2 * self.pad + width = self.width - 3 * self.pad + height = self.height - 2 * self.pad x = y = pad - self.draw_label("Problem report for:", x=x, y=y, bold=True, width=width) - y += self.line_height + self.draw_label( + "Problem report for:", + x=x, + y=y, + bold=True, + width=width, + font_size=self.title_font_size, + ) + y += self.title_font_size import hashlib for file in files: base_file = os.path.basename(file) - y += self.line_height + y += self.font_size + self.pad self.draw_label(base_file, x=x, y=y, bold=True, width=width) - y += self.line_height + y += self.font_size + self.pad - h = hashlib.sha1(open(file, "rb").read()).hexdigest() - self.draw_label("sha1: %s" % h, x=x + pad, y=y, width=width) - y += self.line_height + try: + h = hashlib.sha1(open(file, "rb").read()).hexdigest() + self.draw_label("sha1: %s" % h, x=x + pad, y=y, width=width) + y += self.font_size + except IsADirectoryError: + pass if file.endswith(".ttf"): ttFont = TTFont(file) @@ -184,8 +183,8 @@ class InterpolatablePlot: self.draw_label( "%s: %s" % (what, n), x=x + pad, y=y, width=width ) - y += self.line_height - elif file.endswith(".glyphs"): + y += self.font_size + self.pad + elif file.endswith((".glyphs", ".glyphspackage")): from glyphsLib import GSFont f = GSFont(file) @@ -200,7 +199,7 @@ class InterpolatablePlot: y=y, width=width, ) - y += self.line_height + y += self.font_size + self.pad self.draw_legend( show_tolerance=show_tolerance, tolerance=tolerance, kinkiness=kinkiness @@ -211,8 +210,8 @@ class InterpolatablePlot: cr = cairo.Context(self.surface) x = self.pad - y = self.total_height() - self.pad - self.line_height * 2 - width = self.total_width() - 2 * self.pad + y = self.height - self.pad - self.font_size * 2 + width = self.width - 2 * self.pad xx = x + self.pad * 2 xxx = x + self.pad * 4 @@ -221,10 +220,10 @@ class InterpolatablePlot: self.draw_label( "Tolerance: badness; closer to zero the worse", x=xxx, y=y, width=width ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Underweight contours", x=xxx, y=y, width=width) - cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.line_height) + cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.font_size) cr.set_source_rgb(*self.fill_color) cr.fill_preserve() if self.stroke_color: @@ -233,12 +232,12 @@ class InterpolatablePlot: cr.stroke_preserve() cr.set_source_rgba(*self.weight_issue_contour_color) cr.fill() - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label( "Colored contours: contours with the wrong order", x=xxx, y=y, width=width ) - cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.line_height) + cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.font_size) if self.fill_color: cr.set_source_rgb(*self.fill_color) cr.fill_preserve() @@ -248,38 +247,38 @@ class InterpolatablePlot: cr.stroke_preserve() cr.set_source_rgba(*self.contour_colors[0], self.contour_alpha) cr.fill() - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Kink artifact", x=xxx, y=y, width=width) self.draw_circle( cr, x=xx, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, diameter=self.kink_circle_size, stroke_width=self.kink_circle_stroke_width, color=self.kink_circle_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Point causing kink in the contour", x=xxx, y=y, width=width) self.draw_dot( cr, x=xx, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, diameter=self.kink_point_size, color=self.kink_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Suggested new contour start point", x=xxx, y=y, width=width) self.draw_dot( cr, x=xx, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, diameter=self.corrected_start_point_size, color=self.corrected_start_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label( "Contour start point in contours with wrong direction", @@ -290,10 +289,10 @@ class InterpolatablePlot: self.draw_arrow( cr, x=xx - self.start_arrow_length * 0.3, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, color=self.wrong_start_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label( "Contour start point when the first two points overlap", @@ -304,23 +303,23 @@ class InterpolatablePlot: self.draw_dot( cr, x=xx, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, diameter=self.corrected_start_point_size, color=self.start_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Contour start point and direction", x=xxx, y=y, width=width) self.draw_arrow( cr, x=xx - self.start_arrow_length * 0.3, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, color=self.start_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Legend:", x=x, y=y, width=width, bold=True) - y -= self.pad + self.line_height + y -= self.pad + self.font_size if kinkiness is not None: self.draw_label( @@ -329,7 +328,7 @@ class InterpolatablePlot: y=y, width=width, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size if tolerance is not None: self.draw_label( @@ -338,10 +337,87 @@ class InterpolatablePlot: y=y, width=width, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Parameters:", x=x, y=y, width=width, bold=True) - y -= self.pad + self.line_height + y -= self.pad + self.font_size + + def add_summary(self, problems): + pad = self.pad + width = self.width - 3 * self.pad + height = self.height - 2 * self.pad + x = y = pad + + self.draw_label( + "Summary of problems", + x=x, + y=y, + bold=True, + width=width, + font_size=self.title_font_size, + ) + y += self.title_font_size + + glyphs_per_problem = defaultdict(set) + for glyphname, problems in sorted(problems.items()): + for problem in problems: + glyphs_per_problem[problem["type"]].add(glyphname) + + if "nothing" in glyphs_per_problem: + del glyphs_per_problem["nothing"] + + for problem_type in sorted( + glyphs_per_problem, key=lambda x: InterpolatableProblem.severity[x] + ): + y += self.font_size + self.draw_label( + "%s: %d" % (problem_type, len(glyphs_per_problem[problem_type])), + x=x, + y=y, + width=width, + bold=True, + ) + y += self.font_size + + for glyphname in sorted(glyphs_per_problem[problem_type]): + if y + self.font_size > height: + self.show_page() + y = self.font_size + pad + self.draw_label(glyphname, x=x + 2 * pad, y=y, width=width - 2 * pad) + y += self.font_size + + self.show_page() + + def _add_listing(self, title, items): + pad = self.pad + width = self.width - 2 * self.pad + height = self.height - 2 * self.pad + x = y = pad + + self.draw_label( + title, x=x, y=y, bold=True, width=width, font_size=self.title_font_size + ) + y += self.title_font_size + self.pad + + last_glyphname = None + for page_no, (glyphname, problems) in items: + if glyphname == last_glyphname: + continue + last_glyphname = glyphname + if y + self.font_size > height: + self.show_page() + y = self.font_size + pad + self.draw_label(glyphname, x=x + 5 * pad, y=y, width=width - 2 * pad) + self.draw_label(str(page_no), x=x, y=y, width=4 * pad, align=1) + y += self.font_size + + self.show_page() + + def add_table_of_contents(self): + self._add_listing("Table of contents", sorted(self.toc.items())) + + def add_index(self): + self._add_listing("Index", sorted(self.toc.items(), key=lambda x: x[1][0])) def add_problems(self, problems, *, show_tolerance=True, show_page_number=True): for glyph, glyph_problems in problems.items(): @@ -383,6 +459,8 @@ class InterpolatablePlot: if type(problems) not in (list, tuple): problems = [problems] + self.toc[self.page_number] = (glyphname, problems) + problem_type = problems[0]["type"] problem_types = set(problem["type"] for problem in problems) if not all(pt == problem_type for pt in problem_types): @@ -397,14 +475,12 @@ class InterpolatablePlot: ) master_indices = [problems[0][k] for k in master_keys] - if problem_type == "missing": + if problem_type == InterpolatableProblem.MISSING: sample_glyph = next( i for i, m in enumerate(self.glyphsets) if m[glyphname] is not None ) master_indices.insert(0, sample_glyph) - self.set_size(self.total_width(), self.total_height()) - x = self.pad y = self.pad @@ -415,6 +491,7 @@ class InterpolatablePlot: color=self.head_color, align=0, bold=True, + font_size=self.title_font_size, ) tolerance = min(p.get("tolerance", 1) for p in problems) if tolerance < 1 and show_tolerance: @@ -422,29 +499,35 @@ class InterpolatablePlot: "tolerance: %.2f" % tolerance, x=x, y=y, - width=self.total_width() - 2 * self.pad, + width=self.width - 2 * self.pad, align=1, bold=True, ) - y += self.line_height + self.pad + y += self.title_font_size + self.pad self.draw_label( - problem_type, + "Problems: " + problem_type, x=x, y=y, - width=self.total_width() - 2 * self.pad, + width=self.width - 2 * self.pad, color=self.head_color, - align=0.5, bold=True, ) - y += self.line_height + self.pad + y += self.font_size + self.pad * 2 scales = [] for which, master_idx in enumerate(master_indices): glyphset = self.glyphsets[master_idx] name = self.names[master_idx] - self.draw_label(name, x=x, y=y, color=self.label_color, align=0.5) - y += self.line_height + self.pad + self.draw_label( + name, + x=x, + y=y, + color=self.label_color, + width=self.panel_width, + align=0.5, + ) + y += self.font_size + self.pad if glyphset[glyphname] is not None: scales.append( @@ -452,24 +535,24 @@ class InterpolatablePlot: ) else: self.draw_emoticon(self.shrug, x=x, y=y) - y += self.height + self.pad + y += self.panel_height + self.font_size + self.pad if any( pt in ( - "nothing", - "wrong_start_point", - "contour_order", - "kink", - "underweight", - "overweight", + InterpolatableProblem.NOTHING, + InterpolatableProblem.WRONG_START_POINT, + InterpolatableProblem.CONTOUR_ORDER, + InterpolatableProblem.KINK, + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, ) for pt in problem_types ): - x = self.pad + self.width + self.pad + x = self.pad + self.panel_width + self.pad y = self.pad - y += self.line_height + self.pad - y += self.line_height + self.pad + y += self.title_font_size + self.pad * 2 + y += self.font_size + self.pad glyphset1 = self.glyphsets[master_indices[0]] glyphset2 = self.glyphsets[master_indices[1]] @@ -477,9 +560,14 @@ class InterpolatablePlot: # Draw the mid-way of the two masters self.draw_label( - "midway interpolation", x=x, y=y, color=self.head_color, align=0.5 + "midway interpolation", + x=x, + y=y, + color=self.head_color, + width=self.panel_width, + align=0.5, ) - y += self.line_height + self.pad + y += self.font_size + self.pad midway_glyphset = LerpGlyphSet(glyphset1, glyphset2) self.draw_glyph( @@ -489,7 +577,12 @@ class InterpolatablePlot: + [ p for p in problems - if p["type"] in ("kink", "underweight", "overweight") + if p["type"] + in ( + InterpolatableProblem.KINK, + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, + ) ], None, x=x, @@ -497,21 +590,28 @@ class InterpolatablePlot: scale=min(scales), ) - y += self.height + self.pad + y += self.panel_height + self.font_size + self.pad if any( pt in ( - "wrong_start_point", - "contour_order", - "kink", + InterpolatableProblem.WRONG_START_POINT, + InterpolatableProblem.CONTOUR_ORDER, + InterpolatableProblem.KINK, ) for pt in problem_types ): # Draw the proposed fix - self.draw_label("proposed fix", x=x, y=y, color=self.head_color, align=0.5) - y += self.line_height + self.pad + self.draw_label( + "proposed fix", + x=x, + y=y, + color=self.head_color, + width=self.panel_width, + align=0.5, + ) + y += self.font_size + self.pad overriding1 = OverridingDict(glyphset1) overriding2 = OverridingDict(glyphset2) @@ -525,14 +625,14 @@ class InterpolatablePlot: glyphset2[glyphname].draw(perContourPen2) for problem in problems: - if problem["type"] == "contour_order": + if problem["type"] == InterpolatableProblem.CONTOUR_ORDER: fixed_contours = [ perContourPen2.value[i] for i in problems[0]["value_2"] ] perContourPen2.value = fixed_contours for problem in problems: - if problem["type"] == "wrong_start_point": + if problem["type"] == InterpolatableProblem.WRONG_START_POINT: # Save the wrong contours wrongContour1 = perContourPen1.value[problem["contour"]] wrongContour2 = perContourPen2.value[problem["contour"]] @@ -578,7 +678,7 @@ class InterpolatablePlot: for problem in problems: # If we have a kink, try to fix it. - if problem["type"] == "kink": + if problem["type"] == InterpolatableProblem.KINK: # Save the wrong contours wrongContour1 = perContourPen1.value[problem["contour"]] wrongContour2 = perContourPen2.value[problem["contour"]] @@ -669,15 +769,15 @@ class InterpolatablePlot: ) except ValueError: self.draw_emoticon(self.shrug, x=x, y=y) - y += self.height + self.pad + y += self.panel_height + self.pad else: emoticon = self.shrug - if "underweight" in problem_types: + if InterpolatableProblem.UNDERWEIGHT in problem_types: emoticon = self.underweight - elif "overweight" in problem_types: + elif InterpolatableProblem.OVERWEIGHT in problem_types: emoticon = self.overweight - elif "nothing" in problem_types: + elif InterpolatableProblem.NOTHING in problem_types: emoticon = self.yay self.draw_emoticon(emoticon, x=x, y=y) @@ -685,8 +785,8 @@ class InterpolatablePlot: self.draw_label( str(self.page_number), x=0, - y=self.total_height() - self.line_height, - width=self.total_width(), + y=self.height - self.font_size - self.pad, + width=self.width, color=self.head_color, align=0.5, ) @@ -702,20 +802,23 @@ class InterpolatablePlot: bold=False, width=None, height=None, + font_size=None, ): if width is None: width = self.width if height is None: height = self.height + if font_size is None: + font_size = self.font_size cr = cairo.Context(self.surface) cr.select_font_face( "@cairo:", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD if bold else cairo.FONT_WEIGHT_NORMAL, ) - cr.set_font_size(self.line_height) + cr.set_font_size(font_size) font_extents = cr.font_extents() - font_size = self.line_height * self.line_height / font_extents[2] + font_size = font_size * font_size / font_extents[2] cr.set_font_size(font_size) font_extents = cr.font_extents() @@ -762,14 +865,14 @@ class InterpolatablePlot: if glyph_width: if scale is None: - scale = self.width / glyph_width + scale = self.panel_width / glyph_width else: - scale = min(scale, self.height / glyph_height) + scale = min(scale, self.panel_height / glyph_height) if glyph_height: if scale is None: - scale = self.height / glyph_height + scale = self.panel_height / glyph_height else: - scale = min(scale, self.height / glyph_height) + scale = min(scale, self.panel_height / glyph_height) if scale is None: scale = 1 @@ -777,8 +880,8 @@ class InterpolatablePlot: cr.translate(x, y) # Center cr.translate( - (self.width - glyph_width * scale) / 2, - (self.height - glyph_height * scale) / 2, + (self.panel_width - glyph_width * scale) / 2, + (self.panel_height - glyph_height * scale) / 2, ) cr.scale(scale, -scale) cr.translate(-bounds[0], -bounds[3]) @@ -793,7 +896,7 @@ class InterpolatablePlot: pen = CairoPen(glyphset, cr) decomposedRecording.replay(pen) - if self.fill_color and problem_type != "open_path": + if self.fill_color and problem_type != InterpolatableProblem.OPEN_PATH: cr.set_source_rgb(*self.fill_color) cr.fill_preserve() @@ -804,11 +907,17 @@ class InterpolatablePlot: cr.new_path() - if "underweight" in problem_types or "overweight" in problem_types: + if ( + InterpolatableProblem.UNDERWEIGHT in problem_types + or InterpolatableProblem.OVERWEIGHT in problem_types + ): perContourPen = PerContourOrComponentPen(RecordingPen, glyphset=glyphset) recording.replay(perContourPen) for problem in problems: - if problem["type"] in ("underweight", "overweight"): + if problem["type"] in ( + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, + ): contour = perContourPen.value[problem["contour"]] contour.replay(CairoPen(glyphset, cr)) cr.set_source_rgba(*self.weight_issue_contour_color) @@ -817,9 +926,9 @@ class InterpolatablePlot: if any( t in problem_types for t in { - "nothing", - "node_count", - "node_incompatibility", + InterpolatableProblem.NOTHING, + InterpolatableProblem.NODE_COUNT, + InterpolatableProblem.NODE_INCOMPATIBILITY, } ): cr.set_line_cap(cairo.LINE_CAP_ROUND) @@ -873,7 +982,7 @@ class InterpolatablePlot: matching = None for problem in problems: - if problem["type"] == "contour_order": + if problem["type"] == InterpolatableProblem.CONTOUR_ORDER: matching = problem["value_2"] colors = cycle(self.contour_colors) perContourPen = PerContourOrComponentPen( @@ -889,7 +998,10 @@ class InterpolatablePlot: cr.fill() for problem in problems: - if problem["type"] in ("nothing", "wrong_start_point"): + if problem["type"] in ( + InterpolatableProblem.NOTHING, + InterpolatableProblem.WRONG_START_POINT, + ): idx = problem.get("contour") # Draw suggested point @@ -967,7 +1079,7 @@ class InterpolatablePlot: cr.restore() - if problem["type"] == "kink": + if problem["type"] == InterpolatableProblem.KINK: idx = problem.get("contour") perContourPen = PerContourOrComponentPen( RecordingPen, glyphset=glyphset @@ -1053,19 +1165,19 @@ class InterpolatablePlot: text = text.splitlines() cr = cairo.Context(self.surface) cr.set_source_rgb(*color) - cr.set_font_size(self.line_height) + cr.set_font_size(self.font_size) cr.select_font_face( "@cairo:monospace", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL ) text_width = 0 text_height = 0 font_extents = cr.font_extents() - font_line_height = font_extents[2] + font_font_size = font_extents[2] font_ascent = font_extents[0] for line in text: extents = cr.text_extents(line) text_width = max(text_width, extents.x_advance) - text_height += font_line_height + text_height += font_font_size if not text_width: return cr.translate(x, y) @@ -1080,45 +1192,44 @@ class InterpolatablePlot: for line in text: cr.move_to(0, 0) cr.show_text(line) - cr.translate(0, font_line_height) + cr.translate(0, font_font_size) def draw_cupcake(self): - self.set_size(self.total_width(), self.total_height()) - self.draw_label( self.no_issues_label, x=self.pad, y=self.pad, color=self.no_issues_label_color, - width=self.total_width() - 2 * self.pad, + width=self.width - 2 * self.pad, align=0.5, bold=True, + font_size=self.title_font_size, ) self.draw_text( self.cupcake, x=self.pad, - y=self.pad + self.line_height, - width=self.total_width() - 2 * self.pad, - height=self.total_height() - 2 * self.pad - self.line_height, + y=self.pad + self.font_size, + width=self.width - 2 * self.pad, + height=self.height - 2 * self.pad - self.font_size, color=self.cupcake_color, ) def draw_emoticon(self, emoticon, x=0, y=0): - self.draw_text(emoticon, x=x, y=y, color=self.emoticon_color) + self.draw_text( + emoticon, + x=x, + y=y, + color=self.emoticon_color, + width=self.panel_width, + height=self.panel_height, + ) class InterpolatablePostscriptLike(InterpolatablePlot): - @wraps(InterpolatablePlot.__init__) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def __exit__(self, type, value, traceback): self.surface.finish() - def set_size(self, width, height): - self.surface.set_size(width, height) - def show_page(self): super().show_page() self.surface.show_page() @@ -1141,24 +1252,18 @@ class InterpolatablePDF(InterpolatablePostscriptLike): class InterpolatableSVG(InterpolatablePlot): - @wraps(InterpolatablePlot.__init__) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def __enter__(self): - self.surface = None + self.sink = BytesIO() + self.surface = cairo.SVGSurface(self.sink, self.width, self.height) return self def __exit__(self, type, value, traceback): if self.surface is not None: self.show_page() - def set_size(self, width, height): - self.sink = BytesIO() - self.surface = cairo.SVGSurface(self.sink, width, height) - def show_page(self): super().show_page() self.surface.finish() self.out.append(self.sink.getvalue()) - self.surface = None + self.sink = BytesIO() + self.surface = cairo.SVGSurface(self.sink, self.width, self.height) diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatableTestContourOrder.py b/contrib/python/fonttools/fontTools/varLib/interpolatableTestContourOrder.py index d089e43576..9edb1afcb5 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatableTestContourOrder.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatableTestContourOrder.py @@ -1,4 +1,7 @@ from .interpolatableHelpers import * +import logging + +log = logging.getLogger("fontTools.varLib.interpolatable") def test_contour_order(glyph0, glyph1): @@ -71,4 +74,9 @@ def test_contour_order(glyph0, glyph1): matching_cost = matching_cost_green identity_cost = identity_cost_green - return matching, matching_cost, identity_cost + this_tolerance = matching_cost / identity_cost if identity_cost else 1 + log.debug( + "test-contour-order: tolerance %g", + this_tolerance, + ) + return this_tolerance, matching diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatableTestStartingPoint.py b/contrib/python/fonttools/fontTools/varLib/interpolatableTestStartingPoint.py index 9f742a14f5..e760006631 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatableTestStartingPoint.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatableTestStartingPoint.py @@ -9,18 +9,15 @@ def test_starting_point(glyph0, glyph1, ix, tolerance, matching): m0Vectors = glyph0.greenVectors m1Vectors = [glyph1.greenVectors[i] for i in matching] - proposed_point = 0 - reverse = False - min_cost = first_cost = 1 - c0 = contour0[0] # Next few lines duplicated below. costs = [vdiff_hypot2_complex(c0[0], c1[0]) for c1 in contour1] min_cost_idx, min_cost = min(enumerate(costs), key=lambda x: x[1]) first_cost = costs[0] + proposed_point = contour1[min_cost_idx][1] + reverse = contour1[min_cost_idx][2] if min_cost < first_cost * tolerance: - this_tolerance = min_cost / first_cost # c0 is the first isomorphism of the m0 master # contour1 is list of all isomorphisms of the m1 master # @@ -37,8 +34,6 @@ def test_starting_point(glyph0, glyph1, ix, tolerance, matching): # closest point again. If it matches this time, let it # pass. - proposed_point = contour1[min_cost_idx][1] - reverse = contour1[min_cost_idx][2] num_points = len(glyph1.points[ix]) leeway = 3 if not reverse and ( @@ -102,4 +97,9 @@ def test_starting_point(glyph0, glyph1, ix, tolerance, matching): # proposed_point = 0 # new_contour1[min_cost_idx][1] pass - return proposed_point, reverse, min_cost, first_cost + this_tolerance = min_cost / first_cost if first_cost else 1 + log.debug( + "test-starting-point: tolerance %g", + this_tolerance, + ) + return this_tolerance, proposed_point, reverse diff --git a/contrib/python/fonttools/fontTools/varLib/merger.py b/contrib/python/fonttools/fontTools/varLib/merger.py index b2c34016b3..96029166a7 100644 --- a/contrib/python/fonttools/fontTools/varLib/merger.py +++ b/contrib/python/fonttools/fontTools/varLib/merger.py @@ -1059,7 +1059,7 @@ class InstancerMerger(AligningMerger): Merger.__init__(self, font) self.model = model self.location = location - self.scalars = model.getScalars(location) + self.masterScalars = model.getMasterScalars(location) @InstancerMerger.merger(ot.CaretValue) @@ -1067,8 +1067,10 @@ def merge(merger, self, lst): assert self.Format == 1 Coords = [a.Coordinate for a in lst] model = merger.model - scalars = merger.scalars - self.Coordinate = otRound(model.interpolateFromMastersAndScalars(Coords, scalars)) + masterScalars = merger.masterScalars + self.Coordinate = otRound( + model.interpolateFromValuesAndScalars(Coords, masterScalars) + ) @InstancerMerger.merger(ot.Anchor) @@ -1077,15 +1079,19 @@ def merge(merger, self, lst): XCoords = [a.XCoordinate for a in lst] YCoords = [a.YCoordinate for a in lst] model = merger.model - scalars = merger.scalars - self.XCoordinate = otRound(model.interpolateFromMastersAndScalars(XCoords, scalars)) - self.YCoordinate = otRound(model.interpolateFromMastersAndScalars(YCoords, scalars)) + masterScalars = merger.masterScalars + self.XCoordinate = otRound( + model.interpolateFromValuesAndScalars(XCoords, masterScalars) + ) + self.YCoordinate = otRound( + model.interpolateFromValuesAndScalars(YCoords, masterScalars) + ) @InstancerMerger.merger(otBase.ValueRecord) def merge(merger, self, lst): model = merger.model - scalars = merger.scalars + masterScalars = merger.masterScalars # TODO Handle differing valueformats for name, tableName in [ ("XAdvance", "XAdvDevice"), @@ -1097,7 +1103,9 @@ def merge(merger, self, lst): if hasattr(self, name): values = [getattr(a, name, 0) for a in lst] - value = otRound(model.interpolateFromMastersAndScalars(values, scalars)) + value = otRound( + model.interpolateFromValuesAndScalars(values, masterScalars) + ) setattr(self, name, value) diff --git a/contrib/python/fonttools/fontTools/varLib/models.py b/contrib/python/fonttools/fontTools/varLib/models.py index 33deabe043..59815316f8 100644 --- a/contrib/python/fonttools/fontTools/varLib/models.py +++ b/contrib/python/fonttools/fontTools/varLib/models.py @@ -271,6 +271,12 @@ class VariationModel(object): self._subModels = {} def getSubModel(self, items): + """Return a sub-model and the items that are not None. + + The sub-model is necessary for working with the subset + of items when some are None. + + The sub-model is cached.""" if None not in items: return self, items key = tuple(v is not None for v in items) @@ -465,6 +471,10 @@ class VariationModel(object): return model.getDeltas(items, round=round), model.supports def getScalars(self, loc): + """Return scalars for each delta, for the given location. + If interpolating many master-values at the same location, + this function allows speed up by fetching the scalars once + and using them with interpolateFromMastersAndScalars().""" return [ supportScalar( loc, support, extrapolate=self.extrapolate, axisRanges=self.axisRanges @@ -472,29 +482,65 @@ class VariationModel(object): for support in self.supports ] + def getMasterScalars(self, targetLocation): + """Return multipliers for each master, for the given location. + If interpolating many master-values at the same location, + this function allows speed up by fetching the scalars once + and using them with interpolateFromValuesAndScalars(). + + Note that the scalars used in interpolateFromMastersAndScalars(), + are *not* the same as the ones returned here. They are the result + of getScalars().""" + out = self.getScalars(targetLocation) + for i, weights in reversed(list(enumerate(self.deltaWeights))): + for j, weight in weights.items(): + out[j] -= out[i] * weight + + out = [out[self.mapping[i]] for i in range(len(out))] + return out + @staticmethod - def interpolateFromDeltasAndScalars(deltas, scalars): + def interpolateFromValuesAndScalars(values, scalars): + """Interpolate from values and scalars coefficients. + + If the values are master-values, then the scalars should be + fetched from getMasterScalars(). + + If the values are deltas, then the scalars should be fetched + from getScalars(); in which case this is the same as + interpolateFromDeltasAndScalars(). + """ v = None - assert len(deltas) == len(scalars) - for delta, scalar in zip(deltas, scalars): + assert len(values) == len(scalars) + for value, scalar in zip(values, scalars): if not scalar: continue - contribution = delta * scalar + contribution = value * scalar if v is None: v = contribution else: v += contribution return v + @staticmethod + def interpolateFromDeltasAndScalars(deltas, scalars): + """Interpolate from deltas and scalars fetched from getScalars().""" + return VariationModel.interpolateFromValuesAndScalars(deltas, scalars) + def interpolateFromDeltas(self, loc, deltas): + """Interpolate from deltas, at location loc.""" scalars = self.getScalars(loc) return self.interpolateFromDeltasAndScalars(deltas, scalars) def interpolateFromMasters(self, loc, masterValues, *, round=noRound): - deltas = self.getDeltas(masterValues, round=round) - return self.interpolateFromDeltas(loc, deltas) + """Interpolate from master-values, at location loc.""" + scalars = self.getMasterScalars(loc) + return self.interpolateFromValuesAndScalars(masterValues, scalars) def interpolateFromMastersAndScalars(self, masterValues, scalars, *, round=noRound): + """Interpolate from master-values, and scalars fetched from + getScalars(), which is useful when you want to interpolate + multiple master-values with the same location.""" deltas = self.getDeltas(masterValues, round=round) return self.interpolateFromDeltasAndScalars(deltas, scalars) diff --git a/contrib/python/fonttools/ya.make b/contrib/python/fonttools/ya.make index c2de0cc284..6e76c94da1 100644 --- a/contrib/python/fonttools/ya.make +++ b/contrib/python/fonttools/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(4.46.0) +VERSION(4.47.0) LICENSE(MIT) diff --git a/contrib/python/freezegun/py3/.dist-info/METADATA b/contrib/python/freezegun/py3/.dist-info/METADATA index 0978a40fcb..d5971e9f78 100644 --- a/contrib/python/freezegun/py3/.dist-info/METADATA +++ b/contrib/python/freezegun/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: freezegun -Version: 1.3.1 +Version: 1.4.0 Summary: Let your Python tests travel through time Home-page: https://github.com/spulec/freezegun Author: Steve Pulec diff --git a/contrib/python/freezegun/py3/freezegun/__init__.py b/contrib/python/freezegun/py3/freezegun/__init__.py index 003b2bd878..8fcb18ddfe 100644 --- a/contrib/python/freezegun/py3/freezegun/__init__.py +++ b/contrib/python/freezegun/py3/freezegun/__init__.py @@ -9,7 +9,7 @@ from .api import freeze_time from .config import configure __title__ = 'freezegun' -__version__ = '1.3.1' +__version__ = '1.4.0' __author__ = 'Steve Pulec' __license__ = 'Apache License 2.0' __copyright__ = 'Copyright 2012 Steve Pulec' diff --git a/contrib/python/freezegun/py3/freezegun/api.py b/contrib/python/freezegun/py3/freezegun/api.py index f732ff8c24..2917fa1f75 100644 --- a/contrib/python/freezegun/py3/freezegun/api.py +++ b/contrib/python/freezegun/py3/freezegun/api.py @@ -544,7 +544,7 @@ class StepTickTimeFactory: class _freeze_time: - def __init__(self, time_to_freeze_str, tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds): + def __init__(self, time_to_freeze_str, tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds, real_asyncio): self.time_to_freeze = _parse_time_to_freeze(time_to_freeze_str) self.tz_offset = _parse_tz_offset(tz_offset) self.ignore = tuple(ignore) @@ -554,6 +554,7 @@ class _freeze_time: self.modules_at_start = set() self.as_arg = as_arg self.as_kwarg = as_kwarg + self.real_asyncio = real_asyncio def __call__(self, func): if inspect.isclass(func): @@ -727,20 +728,21 @@ class _freeze_time: setattr(module, attribute_name, fake) add_change((module, attribute_name, attribute_value)) - # To avoid breaking `asyncio.sleep()`, let asyncio event loops see real - # monotonic time even though we've just frozen `time.monotonic()` which - # is normally used there. If we didn't do this, `await asyncio.sleep()` - # would be hanging forever breaking many tests that use `freeze_time`. - # - # Note that we cannot statically tell the class of asyncio event loops - # because it is not officially documented and can actually be changed - # at run time using `asyncio.set_event_loop_policy`. That's why we check - # the type by creating a loop here and destroying it immediately. - event_loop = asyncio.new_event_loop() - event_loop.close() - EventLoopClass = type(event_loop) - add_change((EventLoopClass, "time", EventLoopClass.time)) - EventLoopClass.time = lambda self: real_monotonic() + if self.real_asyncio: + # To avoid breaking `asyncio.sleep()`, let asyncio event loops see real + # monotonic time even though we've just frozen `time.monotonic()` which + # is normally used there. If we didn't do this, `await asyncio.sleep()` + # would be hanging forever breaking many tests that use `freeze_time`. + # + # Note that we cannot statically tell the class of asyncio event loops + # because it is not officially documented and can actually be changed + # at run time using `asyncio.set_event_loop_policy`. That's why we check + # the type by creating a loop here and destroying it immediately. + event_loop = asyncio.new_event_loop() + event_loop.close() + EventLoopClass = type(event_loop) + add_change((EventLoopClass, "time", EventLoopClass.time)) + EventLoopClass.time = lambda self: real_monotonic() return freeze_factory @@ -830,7 +832,7 @@ class _freeze_time: def freeze_time(time_to_freeze=None, tz_offset=0, ignore=None, tick=False, as_arg=False, as_kwarg='', - auto_tick_seconds=0): + auto_tick_seconds=0, real_asyncio=False): acceptable_times = (type(None), str, datetime.date, datetime.timedelta, types.FunctionType, types.GeneratorType) @@ -845,14 +847,14 @@ def freeze_time(time_to_freeze=None, tz_offset=0, ignore=None, tick=False, as_ar raise SystemError('Calling freeze_time with tick=True is only compatible with CPython') if isinstance(time_to_freeze, types.FunctionType): - return freeze_time(time_to_freeze(), tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds) + return freeze_time(time_to_freeze(), tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds, real_asyncio=real_asyncio) if isinstance(time_to_freeze, types.GeneratorType): - return freeze_time(next(time_to_freeze), tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds) + return freeze_time(next(time_to_freeze), tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds, real_asyncio=real_asyncio) if MayaDT is not None and isinstance(time_to_freeze, MayaDT): return freeze_time(time_to_freeze.datetime(), tz_offset, ignore, - tick, as_arg, as_kwarg, auto_tick_seconds) + tick, as_arg, as_kwarg, auto_tick_seconds, real_asyncio=real_asyncio) if ignore is None: ignore = [] @@ -868,6 +870,7 @@ def freeze_time(time_to_freeze=None, tz_offset=0, ignore=None, tick=False, as_ar as_arg=as_arg, as_kwarg=as_kwarg, auto_tick_seconds=auto_tick_seconds, + real_asyncio=real_asyncio, ) diff --git a/contrib/python/freezegun/py3/ya.make b/contrib/python/freezegun/py3/ya.make index 4c2d15c637..d6c429a154 100644 --- a/contrib/python/freezegun/py3/ya.make +++ b/contrib/python/freezegun/py3/ya.make @@ -4,7 +4,7 @@ PY3_LIBRARY() PROVIDES(freezegun) -VERSION(1.3.1) +VERSION(1.4.0) LICENSE(Apache-2.0) diff --git a/contrib/python/httpx/.dist-info/METADATA b/contrib/python/httpx/.dist-info/METADATA index 247ac2e118..ab2b707fcd 100644 --- a/contrib/python/httpx/.dist-info/METADATA +++ b/contrib/python/httpx/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: httpx -Version: 0.25.2 +Version: 0.26.0 Summary: The next generation HTTP client. Project-URL: Changelog, https://github.com/encode/httpx/blob/master/CHANGELOG.md Project-URL: Documentation, https://www.python-httpx.org @@ -31,8 +31,8 @@ Requires-Dist: httpcore==1.* Requires-Dist: idna Requires-Dist: sniffio Provides-Extra: brotli -Requires-Dist: brotli; platform_python_implementation == 'CPython' and extra == 'brotli' -Requires-Dist: brotlicffi; platform_python_implementation != 'CPython' and extra == 'brotli' +Requires-Dist: brotli; (platform_python_implementation == 'CPython') and extra == 'brotli' +Requires-Dist: brotlicffi; (platform_python_implementation != 'CPython') and extra == 'brotli' Provides-Extra: cli Requires-Dist: click==8.*; extra == 'cli' Requires-Dist: pygments==2.*; extra == 'cli' @@ -196,7 +196,19 @@ inspiration around the lower-level networking details. ### Added -* Add missing type hints to few `__init__()` methods. (#2938) +* The `proxy` argument was added. You should use the `proxy` argument instead of the deprecated `proxies`, or use `mounts=` for more complex configurations. (#2879) + +### Deprecated + +* The `proxies` argument is now deprecated. It will still continue to work, but it will be removed in the future. (#2879) + +### Fixed + +* Fix cases of double escaping of URL path components. Allow / as a safe character in the query portion. (#2990) +* Handle `NO_PROXY` envvar cases when a fully qualified URL is supplied as the value. (#2741) +* Allow URLs where username or password contains unescaped '@'. (#2986) +* Ensure ASGI `raw_path` does not include URL query component. (#2999) +* Ensure `Response.iter_text()` cannot yield empty strings. (#2998) --- diff --git a/contrib/python/httpx/httpx/__version__.py b/contrib/python/httpx/httpx/__version__.py index c6bc0ac023..3edc842c69 100644 --- a/contrib/python/httpx/httpx/__version__.py +++ b/contrib/python/httpx/httpx/__version__.py @@ -1,3 +1,3 @@ __title__ = "httpx" __description__ = "A next generation HTTP client, for Python 3." -__version__ = "0.25.2" +__version__ = "0.26.0" diff --git a/contrib/python/httpx/httpx/_api.py b/contrib/python/httpx/httpx/_api.py index 571289cf2b..c7af947218 100644 --- a/contrib/python/httpx/httpx/_api.py +++ b/contrib/python/httpx/httpx/_api.py @@ -10,6 +10,7 @@ from ._types import ( CookieTypes, HeaderTypes, ProxiesTypes, + ProxyTypes, QueryParamTypes, RequestContent, RequestData, @@ -32,6 +33,7 @@ def request( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, @@ -63,6 +65,7 @@ def request( request. * **auth** - *(optional)* An authentication class to use when sending the request. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. * **proxies** - *(optional)* A dictionary mapping proxy keys to proxy URLs. * **timeout** - *(optional)* The timeout configuration to use when sending the request. @@ -91,6 +94,7 @@ def request( """ with Client( cookies=cookies, + proxy=proxy, proxies=proxies, cert=cert, verify=verify, @@ -124,6 +128,7 @@ def stream( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, @@ -143,6 +148,7 @@ def stream( """ with Client( cookies=cookies, + proxy=proxy, proxies=proxies, cert=cert, verify=verify, @@ -171,6 +177,7 @@ def get( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -193,6 +200,7 @@ def get( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -209,6 +217,7 @@ def options( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -231,6 +240,7 @@ def options( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -247,6 +257,7 @@ def head( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -269,6 +280,7 @@ def head( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -289,6 +301,7 @@ def post( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -312,6 +325,7 @@ def post( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -332,6 +346,7 @@ def put( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -355,6 +370,7 @@ def put( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -375,6 +391,7 @@ def patch( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -398,6 +415,7 @@ def patch( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -414,6 +432,7 @@ def delete( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -436,6 +455,7 @@ def delete( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, diff --git a/contrib/python/httpx/httpx/_client.py b/contrib/python/httpx/httpx/_client.py index 2c7ae030f5..2813a84f01 100644 --- a/contrib/python/httpx/httpx/_client.py +++ b/contrib/python/httpx/httpx/_client.py @@ -36,6 +36,7 @@ from ._types import ( CookieTypes, HeaderTypes, ProxiesTypes, + ProxyTypes, QueryParamTypes, RequestContent, RequestData, @@ -597,6 +598,7 @@ class Client(BaseClient): to authenticate the client. Either a path to an SSL certificate file, or two-tuple of (certificate file, key file), or a three-tuple of (certificate file, key file, password). + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. * **proxies** - *(optional)* A dictionary mapping proxy keys to proxy URLs. * **timeout** - *(optional)* The timeout configuration to use when sending @@ -628,8 +630,11 @@ class Client(BaseClient): cert: typing.Optional[CertTypes] = None, http1: bool = True, http2: bool = False, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, - mounts: typing.Optional[typing.Mapping[str, BaseTransport]] = None, + mounts: typing.Optional[ + typing.Mapping[str, typing.Optional[BaseTransport]] + ] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, limits: Limits = DEFAULT_LIMITS, @@ -666,8 +671,17 @@ class Client(BaseClient): "Make sure to install httpx using `pip install httpx[http2]`." ) from None + if proxies: + message = ( + "The 'proxies' argument is now deprecated." + " Use 'proxy' or 'mounts' instead." + ) + warnings.warn(message, DeprecationWarning) + if proxy: + raise RuntimeError("Use either `proxy` or 'proxies', not both.") + allow_env_proxies = trust_env and app is None and transport is None - proxy_map = self._get_proxy_map(proxies, allow_env_proxies) + proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) self._transport = self._init_transport( verify=verify, @@ -1264,7 +1278,9 @@ class Client(BaseClient): if self._state != ClientState.UNOPENED: msg = { ClientState.OPENED: "Cannot open a client instance more than once.", - ClientState.CLOSED: "Cannot reopen a client instance, once it has been closed.", + ClientState.CLOSED: ( + "Cannot reopen a client instance, once it has been closed." + ), }[self._state] raise RuntimeError(msg) @@ -1322,6 +1338,7 @@ class AsyncClient(BaseClient): file, key file, password). * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be enabled. Defaults to `False`. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. * **proxies** - *(optional)* A dictionary mapping HTTP protocols to proxy URLs. * **timeout** - *(optional)* The timeout configuration to use when sending @@ -1353,8 +1370,11 @@ class AsyncClient(BaseClient): cert: typing.Optional[CertTypes] = None, http1: bool = True, http2: bool = False, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, - mounts: typing.Optional[typing.Mapping[str, AsyncBaseTransport]] = None, + mounts: typing.Optional[ + typing.Mapping[str, typing.Optional[AsyncBaseTransport]] + ] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, limits: Limits = DEFAULT_LIMITS, @@ -1391,8 +1411,17 @@ class AsyncClient(BaseClient): "Make sure to install httpx using `pip install httpx[http2]`." ) from None + if proxies: + message = ( + "The 'proxies' argument is now deprecated." + " Use 'proxy' or 'mounts' instead." + ) + warnings.warn(message, DeprecationWarning) + if proxy: + raise RuntimeError("Use either `proxy` or 'proxies', not both.") + allow_env_proxies = trust_env and app is None and transport is None - proxy_map = self._get_proxy_map(proxies, allow_env_proxies) + proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) self._transport = self._init_transport( verify=verify, @@ -1980,7 +2009,9 @@ class AsyncClient(BaseClient): if self._state != ClientState.UNOPENED: msg = { ClientState.OPENED: "Cannot open a client instance more than once.", - ClientState.CLOSED: "Cannot reopen a client instance, once it has been closed.", + ClientState.CLOSED: ( + "Cannot reopen a client instance, once it has been closed." + ), }[self._state] raise RuntimeError(msg) diff --git a/contrib/python/httpx/httpx/_content.py b/contrib/python/httpx/httpx/_content.py index 0aaea33749..cd0d17f171 100644 --- a/contrib/python/httpx/httpx/_content.py +++ b/contrib/python/httpx/httpx/_content.py @@ -105,7 +105,7 @@ class UnattachedStream(AsyncByteStream, SyncByteStream): def encode_content( - content: Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] + content: Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]], ) -> Tuple[Dict[str, str], Union[SyncByteStream, AsyncByteStream]]: if isinstance(content, (bytes, str)): body = content.encode("utf-8") if isinstance(content, str) else content diff --git a/contrib/python/httpx/httpx/_decoders.py b/contrib/python/httpx/httpx/_decoders.py index b4ac9a44af..3f507c8e04 100644 --- a/contrib/python/httpx/httpx/_decoders.py +++ b/contrib/python/httpx/httpx/_decoders.py @@ -212,7 +212,7 @@ class TextChunker: def decode(self, content: str) -> typing.List[str]: if self._chunk_size is None: - return [content] + return [content] if content else [] self._buffer.write(content) if self._buffer.tell() >= self._chunk_size: @@ -259,7 +259,8 @@ class LineDecoder: """ Handles incrementally reading lines from text. - Has the same behaviour as the stdllib splitlines, but handling the input iteratively. + Has the same behaviour as the stdllib splitlines, + but handling the input iteratively. """ def __init__(self) -> None: @@ -279,7 +280,9 @@ class LineDecoder: text = text[:-1] if not text: - return [] + # NOTE: the edge case input of empty text doesn't occur in practice, + # because other httpx internals filter out this value + return [] # pragma: no cover trailing_newline = text[-1] in NEWLINE_CHARS lines = text.splitlines() diff --git a/contrib/python/httpx/httpx/_exceptions.py b/contrib/python/httpx/httpx/_exceptions.py index 24a4f8aba3..123692955b 100644 --- a/contrib/python/httpx/httpx/_exceptions.py +++ b/contrib/python/httpx/httpx/_exceptions.py @@ -313,7 +313,10 @@ class ResponseNotRead(StreamError): """ def __init__(self) -> None: - message = "Attempted to access streaming response content, without having called `read()`." + message = ( + "Attempted to access streaming response content," + " without having called `read()`." + ) super().__init__(message) @@ -323,7 +326,10 @@ class RequestNotRead(StreamError): """ def __init__(self) -> None: - message = "Attempted to access streaming request content, without having called `read()`." + message = ( + "Attempted to access streaming request content," + " without having called `read()`." + ) super().__init__(message) diff --git a/contrib/python/httpx/httpx/_main.py b/contrib/python/httpx/httpx/_main.py index 7c12ce841d..adb57d5fc0 100644 --- a/contrib/python/httpx/httpx/_main.py +++ b/contrib/python/httpx/httpx/_main.py @@ -63,20 +63,21 @@ def print_help() -> None: ) table.add_row( "--auth [cyan]<USER PASS>", - "Username and password to include in the request. Specify '-' for the password to use " - "a password prompt. Note that using --verbose/-v will expose the Authorization " - "header, including the password encoding in a trivially reversible format.", + "Username and password to include in the request. Specify '-' for the password" + " to use a password prompt. Note that using --verbose/-v will expose" + " the Authorization header, including the password encoding" + " in a trivially reversible format.", ) table.add_row( - "--proxies [cyan]URL", + "--proxy [cyan]URL", "Send the request via a proxy. Should be the URL giving the proxy address.", ) table.add_row( "--timeout [cyan]FLOAT", - "Timeout value to use for network operations, such as establishing the connection, " - "reading some data, etc... [Default: 5.0]", + "Timeout value to use for network operations, such as establishing the" + " connection, reading some data, etc... [Default: 5.0]", ) table.add_row("--follow-redirects", "Automatically follow redirects.") @@ -385,8 +386,8 @@ def handle_help( ), ) @click.option( - "--proxies", - "proxies", + "--proxy", + "proxy", type=str, default=None, help="Send the request via a proxy. Should be the URL giving the proxy address.", @@ -455,7 +456,7 @@ def main( headers: typing.List[typing.Tuple[str, str]], cookies: typing.List[typing.Tuple[str, str]], auth: typing.Optional[typing.Tuple[str, str]], - proxies: str, + proxy: str, timeout: float, follow_redirects: bool, verify: bool, @@ -472,7 +473,7 @@ def main( try: with Client( - proxies=proxies, + proxy=proxy, timeout=timeout, verify=verify, http2=http2, diff --git a/contrib/python/httpx/httpx/_models.py b/contrib/python/httpx/httpx/_models.py index 8a5e6280f3..b8617cdab5 100644 --- a/contrib/python/httpx/httpx/_models.py +++ b/contrib/python/httpx/httpx/_models.py @@ -358,7 +358,8 @@ class Request: # Using `content=...` implies automatically populated `Host` and content # headers, of either `Content-Length: ...` or `Transfer-Encoding: chunked`. # - # Using `stream=...` will not automatically include *any* auto-populated headers. + # Using `stream=...` will not automatically include *any* + # auto-populated headers. # # As an end-user you don't really need `stream=...`. It's only # useful when: @@ -852,7 +853,7 @@ class Response: yield chunk text_content = decoder.flush() for chunk in chunker.decode(text_content): - yield chunk + yield chunk # pragma: no cover for chunk in chunker.flush(): yield chunk @@ -956,7 +957,7 @@ class Response: yield chunk text_content = decoder.flush() for chunk in chunker.decode(text_content): - yield chunk + yield chunk # pragma: no cover for chunk in chunker.flush(): yield chunk diff --git a/contrib/python/httpx/httpx/_multipart.py b/contrib/python/httpx/httpx/_multipart.py index 6d5baa8639..5122d5114f 100644 --- a/contrib/python/httpx/httpx/_multipart.py +++ b/contrib/python/httpx/httpx/_multipart.py @@ -48,7 +48,8 @@ class DataField: ) if value is not None and not isinstance(value, (str, bytes, int, float)): raise TypeError( - f"Invalid type for value. Expected primitive type, got {type(value)}: {value!r}" + "Invalid type for value. Expected primitive type," + f" got {type(value)}: {value!r}" ) self.name = name self.value: typing.Union[str, bytes] = ( @@ -96,11 +97,13 @@ class FileField: content_type: typing.Optional[str] = None # This large tuple based API largely mirror's requests' API - # It would be good to think of better APIs for this that we could include in httpx 2.0 - # since variable length tuples (especially of 4 elements) are quite unwieldly + # It would be good to think of better APIs for this that we could + # include in httpx 2.0 since variable length tuples(especially of 4 elements) + # are quite unwieldly if isinstance(value, tuple): if len(value) == 2: - # neither the 3rd parameter (content_type) nor the 4th (headers) was included + # neither the 3rd parameter (content_type) nor the 4th (headers) + # was included filename, fileobj = value # type: ignore elif len(value) == 3: filename, fileobj, content_type = value # type: ignore @@ -116,9 +119,9 @@ class FileField: has_content_type_header = any("content-type" in key.lower() for key in headers) if content_type is not None and not has_content_type_header: - # note that unlike requests, we ignore the content_type - # provided in the 3rd tuple element if it is also included in the headers - # requests does the opposite (it overwrites the header with the 3rd tuple element) + # note that unlike requests, we ignore the content_type provided in the 3rd + # tuple element if it is also included in the headers requests does + # the opposite (it overwrites the headerwith the 3rd tuple element) headers["Content-Type"] = content_type if isinstance(fileobj, io.StringIO): diff --git a/contrib/python/httpx/httpx/_transports/asgi.py b/contrib/python/httpx/httpx/_transports/asgi.py index f67f0fbd5b..08cd392f75 100644 --- a/contrib/python/httpx/httpx/_transports/asgi.py +++ b/contrib/python/httpx/httpx/_transports/asgi.py @@ -103,7 +103,7 @@ class ASGITransport(AsyncBaseTransport): "headers": [(k.lower(), v) for (k, v) in request.headers.raw], "scheme": request.url.scheme, "path": request.url.path, - "raw_path": request.url.raw_path, + "raw_path": request.url.raw_path.split(b"?")[0], "query_string": request.url.query, "server": (request.url.host, request.url.port), "client": self.client, diff --git a/contrib/python/httpx/httpx/_transports/default.py b/contrib/python/httpx/httpx/_transports/default.py index 343c588f9f..14a087389a 100644 --- a/contrib/python/httpx/httpx/_transports/default.py +++ b/contrib/python/httpx/httpx/_transports/default.py @@ -47,7 +47,8 @@ from .._exceptions import ( WriteTimeout, ) from .._models import Request, Response -from .._types import AsyncByteStream, CertTypes, SyncByteStream, VerifyTypes +from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream, VerifyTypes +from .._urls import URL from .base import AsyncBaseTransport, BaseTransport T = typing.TypeVar("T", bound="HTTPTransport") @@ -124,13 +125,14 @@ class HTTPTransport(BaseTransport): http2: bool = False, limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, - proxy: typing.Optional[Proxy] = None, + proxy: typing.Optional[ProxyTypes] = None, uds: typing.Optional[str] = None, local_address: typing.Optional[str] = None, retries: int = 0, socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None, ) -> None: ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy if proxy is None: self._pool = httpcore.ConnectionPool( @@ -190,7 +192,8 @@ class HTTPTransport(BaseTransport): ) else: # pragma: no cover raise ValueError( - f"Proxy protocol must be either 'http', 'https', or 'socks5', but got {proxy.url.scheme!r}." + "Proxy protocol must be either 'http', 'https', or 'socks5'," + f" but got {proxy.url.scheme!r}." ) def __enter__(self: T) -> T: # Use generics for subclass support. @@ -263,13 +266,14 @@ class AsyncHTTPTransport(AsyncBaseTransport): http2: bool = False, limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, - proxy: typing.Optional[Proxy] = None, + proxy: typing.Optional[ProxyTypes] = None, uds: typing.Optional[str] = None, local_address: typing.Optional[str] = None, retries: int = 0, socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None, ) -> None: ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy if proxy is None: self._pool = httpcore.AsyncConnectionPool( @@ -328,7 +332,8 @@ class AsyncHTTPTransport(AsyncBaseTransport): ) else: # pragma: no cover raise ValueError( - f"Proxy protocol must be either 'http', 'https', or 'socks5', but got {proxy.url.scheme!r}." + "Proxy protocol must be either 'http', 'https', or 'socks5'," + " but got {proxy.url.scheme!r}." ) async def __aenter__(self: A) -> A: # Use generics for subclass support. diff --git a/contrib/python/httpx/httpx/_types.py b/contrib/python/httpx/httpx/_types.py index 83cf35a32a..649d101d54 100644 --- a/contrib/python/httpx/httpx/_types.py +++ b/contrib/python/httpx/httpx/_types.py @@ -78,7 +78,8 @@ TimeoutTypes = Union[ Tuple[Optional[float], Optional[float], Optional[float], Optional[float]], "Timeout", ] -ProxiesTypes = Union[URLTypes, "Proxy", Dict[URLTypes, Union[None, URLTypes, "Proxy"]]] +ProxyTypes = Union[URLTypes, "Proxy"] +ProxiesTypes = Union[ProxyTypes, Dict[URLTypes, Union[None, ProxyTypes]]] AuthTypes = Union[ Tuple[Union[str, bytes], Union[str, bytes]], diff --git a/contrib/python/httpx/httpx/_urlparse.py b/contrib/python/httpx/httpx/_urlparse.py index 8e060424e8..07bbea9070 100644 --- a/contrib/python/httpx/httpx/_urlparse.py +++ b/contrib/python/httpx/httpx/_urlparse.py @@ -62,8 +62,8 @@ AUTHORITY_REGEX = re.compile( ( r"(?:(?P<userinfo>{userinfo})@)?" r"(?P<host>{host})" r":?(?P<port>{port})?" ).format( - userinfo="[^@]*", # Any character sequence not including '@'. - host="(\\[.*\\]|[^:]*)", # Either any character sequence not including ':', + userinfo=".*", # Any character sequence. + host="(\\[.*\\]|[^:@]*)", # Either any character sequence excluding ':' or '@', # or an IPv6 address enclosed within square brackets. port=".*", # Any character sequence. ) @@ -260,10 +260,8 @@ def urlparse(url: str = "", **kwargs: typing.Optional[str]) -> ParseResult: # For 'path' we need to drop ? and # from the GEN_DELIMS set. parsed_path: str = quote(path, safe=SUB_DELIMS + ":/[]@") # For 'query' we need to drop '#' from the GEN_DELIMS set. - # We also exclude '/' because it is more robust to replace it with a percent - # encoding despite it not being a requirement of the spec. parsed_query: typing.Optional[str] = ( - None if query is None else quote(query, safe=SUB_DELIMS + ":?[]@") + None if query is None else quote(query, safe=SUB_DELIMS + ":/?[]@") ) # For 'fragment' we can include all of the GEN_DELIMS set. parsed_fragment: typing.Optional[str] = ( @@ -360,24 +358,25 @@ def normalize_port( def validate_path(path: str, has_scheme: bool, has_authority: bool) -> None: """ - Path validation rules that depend on if the URL contains a scheme or authority component. + Path validation rules that depend on if the URL contains + a scheme or authority component. See https://datatracker.ietf.org/doc/html/rfc3986.html#section-3.3 """ if has_authority: - # > If a URI contains an authority component, then the path component - # > must either be empty or begin with a slash ("/") character." + # If a URI contains an authority component, then the path component + # must either be empty or begin with a slash ("/") character." if path and not path.startswith("/"): raise InvalidURL("For absolute URLs, path must be empty or begin with '/'") else: - # > If a URI does not contain an authority component, then the path cannot begin - # > with two slash characters ("//"). + # If a URI does not contain an authority component, then the path cannot begin + # with two slash characters ("//"). if path.startswith("//"): raise InvalidURL( "URLs with no authority component cannot have a path starting with '//'" ) - # > In addition, a URI reference (Section 4.1) may be a relative-path reference, in which - # > case the first path segment cannot contain a colon (":") character. + # In addition, a URI reference (Section 4.1) may be a relative-path reference, + # in which case the first path segment cannot contain a colon (":") character. if path.startswith(":") and not has_scheme: raise InvalidURL( "URLs with no scheme component cannot have a path starting with ':'" @@ -431,13 +430,12 @@ def is_safe(string: str, safe: str = "/") -> bool: if char not in NON_ESCAPED_CHARS: return False - # Any '%' characters must be valid '%xx' escape sequences. - return string.count("%") == len(PERCENT_ENCODED_REGEX.findall(string)) + return True -def quote(string: str, safe: str = "/") -> str: +def percent_encoded(string: str, safe: str = "/") -> str: """ - Use percent-encoding to quote a string if required. + Use percent-encoding to quote a string. """ if is_safe(string, safe=safe): return string @@ -448,17 +446,57 @@ def quote(string: str, safe: str = "/") -> str: ) +def quote(string: str, safe: str = "/") -> str: + """ + Use percent-encoding to quote a string, omitting existing '%xx' escape sequences. + + See: https://www.rfc-editor.org/rfc/rfc3986#section-2.1 + + * `string`: The string to be percent-escaped. + * `safe`: A string containing characters that may be treated as safe, and do not + need to be escaped. Unreserved characters are always treated as safe. + See: https://www.rfc-editor.org/rfc/rfc3986#section-2.3 + """ + parts = [] + current_position = 0 + for match in re.finditer(PERCENT_ENCODED_REGEX, string): + start_position, end_position = match.start(), match.end() + matched_text = match.group(0) + # Add any text up to the '%xx' escape sequence. + if start_position != current_position: + leading_text = string[current_position:start_position] + parts.append(percent_encoded(leading_text, safe=safe)) + + # Add the '%xx' escape sequence. + parts.append(matched_text) + current_position = end_position + + # Add any text after the final '%xx' escape sequence. + if current_position != len(string): + trailing_text = string[current_position:] + parts.append(percent_encoded(trailing_text, safe=safe)) + + return "".join(parts) + + def urlencode(items: typing.List[typing.Tuple[str, str]]) -> str: - # We can use a much simpler version of the stdlib urlencode here because - # we don't need to handle a bunch of different typing cases, such as bytes vs str. - # - # https://github.com/python/cpython/blob/b2f7b2ef0b5421e01efb8c7bee2ef95d3bab77eb/Lib/urllib/parse.py#L926 - # - # Note that we use '%20' encoding for spaces. and '%2F for '/'. - # This is slightly different than `requests`, but is the behaviour that browsers use. - # - # See - # - https://github.com/encode/httpx/issues/2536 - # - https://github.com/encode/httpx/issues/2721 - # - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode - return "&".join([quote(k, safe="") + "=" + quote(v, safe="") for k, v in items]) + """ + We can use a much simpler version of the stdlib urlencode here because + we don't need to handle a bunch of different typing cases, such as bytes vs str. + + https://github.com/python/cpython/blob/b2f7b2ef0b5421e01efb8c7bee2ef95d3bab77eb/Lib/urllib/parse.py#L926 + + Note that we use '%20' encoding for spaces. and '%2F for '/'. + This is slightly different than `requests`, but is the behaviour that browsers use. + + See + - https://github.com/encode/httpx/issues/2536 + - https://github.com/encode/httpx/issues/2721 + - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode + """ + return "&".join( + [ + percent_encoded(k, safe="") + "=" + percent_encoded(v, safe="") + for k, v in items + ] + ) diff --git a/contrib/python/httpx/httpx/_urls.py b/contrib/python/httpx/httpx/_urls.py index b023941b62..26202e95db 100644 --- a/contrib/python/httpx/httpx/_urls.py +++ b/contrib/python/httpx/httpx/_urls.py @@ -51,21 +51,23 @@ class URL: assert url.raw_host == b"xn--fiqs8s.icom.museum" * `url.port` is either None or an integer. URLs that include the default port for - "http", "https", "ws", "wss", and "ftp" schemes have their port normalized to `None`. + "http", "https", "ws", "wss", and "ftp" schemes have their port + normalized to `None`. assert httpx.URL("http://example.com") == httpx.URL("http://example.com:80") assert httpx.URL("http://example.com").port is None assert httpx.URL("http://example.com:80").port is None - * `url.userinfo` is raw bytes, without URL escaping. Usually you'll want to work with - `url.username` and `url.password` instead, which handle the URL escaping. + * `url.userinfo` is raw bytes, without URL escaping. Usually you'll want to work + with `url.username` and `url.password` instead, which handle the URL escaping. * `url.raw_path` is raw bytes of both the path and query, without URL escaping. This portion is used as the target when constructing HTTP requests. Usually you'll want to work with `url.path` instead. - * `url.query` is raw bytes, without URL escaping. A URL query string portion can only - be properly URL escaped when decoding the parameter names and values themselves. + * `url.query` is raw bytes, without URL escaping. A URL query string portion can + only be properly URL escaped when decoding the parameter names and values + themselves. """ def __init__( @@ -115,7 +117,8 @@ class URL: self._uri_reference = url._uri_reference.copy_with(**kwargs) else: raise TypeError( - f"Invalid type for url. Expected str or httpx.URL, got {type(url)}: {url!r}" + "Invalid type for url. Expected str or httpx.URL," + f" got {type(url)}: {url!r}" ) @property @@ -305,7 +308,8 @@ class URL: Provides the (scheme, host, port, target) for the outgoing request. In older versions of `httpx` this was used in the low-level transport API. - We no longer use `RawURL`, and this property will be deprecated in a future release. + We no longer use `RawURL`, and this property will be deprecated + in a future release. """ return RawURL( self.raw_scheme, @@ -342,7 +346,9 @@ class URL: For example: - url = httpx.URL("https://www.example.com").copy_with(username="jo@gmail.com", password="a secret") + url = httpx.URL("https://www.example.com").copy_with( + username="jo@gmail.com", password="a secret" + ) assert url == "https://jo%40email.com:a%20secret@www.example.com" """ return URL(self, **kwargs) diff --git a/contrib/python/httpx/httpx/_utils.py b/contrib/python/httpx/httpx/_utils.py index ba5807c048..bc3cb001dd 100644 --- a/contrib/python/httpx/httpx/_utils.py +++ b/contrib/python/httpx/httpx/_utils.py @@ -152,7 +152,7 @@ SENSITIVE_HEADERS = {"authorization", "proxy-authorization"} def obfuscate_sensitive_headers( - items: typing.Iterable[typing.Tuple[typing.AnyStr, typing.AnyStr]] + items: typing.Iterable[typing.Tuple[typing.AnyStr, typing.AnyStr]], ) -> typing.Iterator[typing.Tuple[typing.AnyStr, typing.AnyStr]]: for k, v in items: if to_str(k.lower()) in SENSITIVE_HEADERS: @@ -227,7 +227,9 @@ def get_environment_proxies() -> typing.Dict[str, typing.Optional[str]]: # (But not "wwwgoogle.com") # NO_PROXY can include domains, IPv6, IPv4 addresses and "localhost" # NO_PROXY=example.com,::1,localhost,192.168.0.0/16 - if is_ipv4_hostname(hostname): + if "://" in hostname: + mounts[hostname] = None + elif is_ipv4_hostname(hostname): mounts[f"all://{hostname}"] = None elif is_ipv6_hostname(hostname): mounts[f"all://[{hostname}]"] = None @@ -293,14 +295,10 @@ class Timer: import trio return trio.current_time() - elif library == "curio": # pragma: no cover - import curio - - return typing.cast(float, await curio.clock()) - - import asyncio + else: + import asyncio - return asyncio.get_event_loop().time() + return asyncio.get_event_loop().time() def sync_start(self) -> None: self.started = time.perf_counter() diff --git a/contrib/python/httpx/ya.make b/contrib/python/httpx/ya.make index f07d3a08ce..2f8eaa9087 100644 --- a/contrib/python/httpx/ya.make +++ b/contrib/python/httpx/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(0.25.2) +VERSION(0.26.0) LICENSE(BSD-3-Clause) diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index c243f74513..b70d38e2c2 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hypothesis -Version: 6.92.0 +Version: 6.92.1 Summary: A library for property-based testing Home-page: https://hypothesis.works Author: David R. MacIver and Zac Hatfield-Dodds diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/compat.py b/contrib/python/hypothesis/py3/hypothesis/internal/compat.py index 3eaed1eba1..41e8ce61d1 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/compat.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/compat.py @@ -9,6 +9,8 @@ # obtain one at https://mozilla.org/MPL/2.0/. import codecs +import copy +import dataclasses import inspect import platform import sys @@ -188,3 +190,46 @@ def bad_django_TestCase(runner): from hypothesis.extra.django._impl import HypothesisTestCase return not isinstance(runner, HypothesisTestCase) + + +# see issue #3812 +if sys.version_info[:2] < (3, 12): + + def dataclass_asdict(obj, *, dict_factory=dict): + """ + A vendored variant of dataclasses.asdict. Includes the bugfix for + defaultdicts (cpython/32056) for all versions. See also issues/3812. + + This should be removed whenever we drop support for 3.11. We can use the + standard dataclasses.asdict after that point. + """ + if not dataclasses._is_dataclass_instance(obj): # pragma: no cover + raise TypeError("asdict() should be called on dataclass instances") + return _asdict_inner(obj, dict_factory) + +else: # pragma: no cover + dataclass_asdict = dataclasses.asdict + + +def _asdict_inner(obj, dict_factory): + if dataclasses._is_dataclass_instance(obj): + return dict_factory( + (f.name, _asdict_inner(getattr(obj, f.name), dict_factory)) + for f in dataclasses.fields(obj) + ) + elif isinstance(obj, tuple) and hasattr(obj, "_fields"): + return type(obj)(*[_asdict_inner(v, dict_factory) for v in obj]) + elif isinstance(obj, (list, tuple)): + return type(obj)(_asdict_inner(v, dict_factory) for v in obj) + elif isinstance(obj, dict): + if hasattr(type(obj), "default_factory"): + result = type(obj)(obj.default_factory) + for k, v in obj.items(): + result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) + return result + return type(obj)( + (_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) + for k, v in obj.items() + ) + else: + return copy.deepcopy(obj) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py index a5a862635a..03653c61e1 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py @@ -77,6 +77,7 @@ from hypothesis.internal.compat import ( from hypothesis.internal.conjecture.utils import calc_label_from_cls, check_sample from hypothesis.internal.entropy import get_seeder_and_restorer from hypothesis.internal.floats import float_of +from hypothesis.internal.observability import TESTCASE_CALLBACKS from hypothesis.internal.reflection import ( define_function_signature, get_pretty_function_description, @@ -2103,7 +2104,9 @@ class DataObject: self.count += 1 printer = RepresentationPrinter(context=current_build_context()) desc = f"Draw {self.count}{'' if label is None else f' ({label})'}: " - self.conjecture_data._observability_args[desc] = to_jsonable(result) + if TESTCASE_CALLBACKS: + self.conjecture_data._observability_args[desc] = to_jsonable(result) + printer.text(desc) printer.pretty(result) note(printer.getvalue()) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/utils.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/utils.py index 995b179b40..b2a7661cd6 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/utils.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/utils.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Callable, Dict import attr from hypothesis.internal.cache import LRUReusedCache +from hypothesis.internal.compat import dataclass_asdict from hypothesis.internal.floats import float_to_int from hypothesis.internal.reflection import proxies from hypothesis.vendor.pretty import pretty @@ -177,7 +178,7 @@ def to_jsonable(obj: object) -> object: and dcs.is_dataclass(obj) and not isinstance(obj, type) ): - return to_jsonable(dcs.asdict(obj)) + return to_jsonable(dataclass_asdict(obj)) if attr.has(type(obj)): return to_jsonable(attr.asdict(obj, recurse=False)) # type: ignore if (pyd := sys.modules.get("pydantic")) and isinstance(obj, pyd.BaseModel): diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index 8357097704..937332138d 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 92, 0) +__version_info__ = (6, 92, 1) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index 456dabd865..80e21c22c2 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.92.0) +VERSION(6.92.1) LICENSE(MPL-2.0) diff --git a/contrib/python/importlib-metadata/py3/.dist-info/METADATA b/contrib/python/importlib-metadata/py3/.dist-info/METADATA index a344859f17..c2bab66615 100644 --- a/contrib/python/importlib-metadata/py3/.dist-info/METADATA +++ b/contrib/python/importlib-metadata/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: importlib-metadata -Version: 7.0.0 +Version: 7.0.1 Summary: Read metadata from Python packages Home-page: https://github.com/python/importlib_metadata Author: Jason R. Coombs @@ -42,7 +42,7 @@ Requires-Dist: importlib-resources >=1.3 ; (python_version < "3.9") and extra == .. image:: https://img.shields.io/pypi/pyversions/importlib_metadata.svg -.. image:: https://github.com/python/importlib_metadata/workflows/tests/badge.svg +.. image:: https://github.com/python/importlib_metadata/actions/workflows/main.yml/badge.svg :target: https://github.com/python/importlib_metadata/actions?query=workflow%3A%22tests%22 :alt: tests @@ -82,6 +82,8 @@ were contributed to different versions in the standard library: * - importlib_metadata - stdlib + * - 7.0 + - 3.13 * - 6.5 - 3.12 * - 4.13 diff --git a/contrib/python/importlib-metadata/py3/README.rst b/contrib/python/importlib-metadata/py3/README.rst index 73ddc5fab4..6788a7e3d3 100644 --- a/contrib/python/importlib-metadata/py3/README.rst +++ b/contrib/python/importlib-metadata/py3/README.rst @@ -3,7 +3,7 @@ .. image:: https://img.shields.io/pypi/pyversions/importlib_metadata.svg -.. image:: https://github.com/python/importlib_metadata/workflows/tests/badge.svg +.. image:: https://github.com/python/importlib_metadata/actions/workflows/main.yml/badge.svg :target: https://github.com/python/importlib_metadata/actions?query=workflow%3A%22tests%22 :alt: tests @@ -43,6 +43,8 @@ were contributed to different versions in the standard library: * - importlib_metadata - stdlib + * - 7.0 + - 3.13 * - 6.5 - 3.12 * - 4.13 diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py b/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py index cd015707b1..4debbecb9d 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import re import abc @@ -20,7 +22,6 @@ from . import _adapters, _meta, _py39compat from ._collections import FreezableDefaultDict, Pair from ._compat import ( NullFinder, - StrPath, install, ) from ._functools import method_cache, pass_none @@ -320,14 +321,12 @@ class PackagePath(pathlib.PurePosixPath): dist: "Distribution" def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] - with self.locate().open(encoding=encoding) as stream: - return stream.read() + return self.locate().read_text(encoding=encoding) def read_binary(self) -> bytes: - with self.locate().open('rb') as stream: - return stream.read() + return self.locate().read_bytes() - def locate(self) -> pathlib.Path: + def locate(self) -> SimplePath: """Return a path-like object for this path""" return self.dist.locate_file(self) @@ -341,6 +340,7 @@ class FileHash: class DeprecatedNonAbstract: + # Required until Python 3.14 def __new__(cls, *args, **kwargs): all_names = { name for subclass in inspect.getmro(cls) for name in vars(subclass) @@ -360,20 +360,42 @@ class DeprecatedNonAbstract: class Distribution(DeprecatedNonAbstract): - """A Python distribution package.""" + """ + An abstract Python distribution package. + + Custom providers may derive from this class and define + the abstract methods to provide a concrete implementation + for their environment. + """ @abc.abstractmethod def read_text(self, filename) -> Optional[str]: """Attempt to load metadata file given by the name. + Python distribution metadata is organized by blobs of text + typically represented as "files" in the metadata directory + (e.g. package-1.0.dist-info). These files include things + like: + + - METADATA: The distribution metadata including fields + like Name and Version and Description. + - entry_points.txt: A series of entry points defined by + the Setuptools spec in an ini format with sections + representing the groups. + - RECORD: A record of files as installed by a typical + installer. + + A package may provide any set of files, including those + not listed here or none at all. + :param filename: The name of the file in the distribution info. :return: The text if found, otherwise None. """ @abc.abstractmethod - def locate_file(self, path: StrPath) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: """ - Given a path to a file in this distribution, return a path + Given a path to a file in this distribution, return a SimplePath to it. """ @@ -396,16 +418,18 @@ class Distribution(DeprecatedNonAbstract): raise PackageNotFoundError(name) @classmethod - def discover(cls, **kwargs) -> Iterable["Distribution"]: + def discover( + cls, *, context: Optional['DistributionFinder.Context'] = None, **kwargs + ) -> Iterable["Distribution"]: """Return an iterable of Distribution objects for all packages. Pass a ``context`` or pass keyword arguments for constructing a context. :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. + :return: Iterable of Distribution objects for packages matching + the context. """ - context = kwargs.pop('context', None) if context and kwargs: raise ValueError("cannot accept context and kwargs") context = context or DistributionFinder.Context(**kwargs) @@ -414,8 +438,8 @@ class Distribution(DeprecatedNonAbstract): ) @staticmethod - def at(path: StrPath) -> "Distribution": - """Return a Distribution for the indicated metadata path + def at(path: str | os.PathLike[str]) -> "Distribution": + """Return a Distribution for the indicated metadata path. :param path: a string or path-like object :return: a concrete Distribution instance for the path @@ -424,7 +448,7 @@ class Distribution(DeprecatedNonAbstract): @staticmethod def _discover_resolvers(): - """Search the meta_path for resolvers.""" + """Search the meta_path for resolvers (MetadataPathFinders).""" declared = ( getattr(finder, 'find_distributions', None) for finder in sys.meta_path ) @@ -822,7 +846,7 @@ class PathDistribution(Distribution): """ self._path = path - def read_text(self, filename: StrPath) -> Optional[str]: + def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: with suppress( FileNotFoundError, IsADirectoryError, @@ -836,7 +860,7 @@ class PathDistribution(Distribution): read_text.__doc__ = Distribution.read_text.__doc__ - def locate_file(self, path: StrPath) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: return self._path.parent / path @property diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/_compat.py b/contrib/python/importlib-metadata/py3/importlib_metadata/_compat.py index a48b609be6..1a3578e60f 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/_compat.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/_compat.py @@ -1,9 +1,6 @@ -import os import sys import platform -from typing import Union - __all__ = ['install', 'NullFinder'] @@ -44,7 +41,7 @@ def disable_stdlib_finder(): class NullFinder: """ - A "Finder" (aka "MetaClassFinder") that never finds any modules, + A "Finder" (aka "MetaPathFinder") that never finds any modules, but may find distributions. """ @@ -61,10 +58,3 @@ def pypy_partial(val): """ is_pypy = platform.python_implementation() == 'PyPy' return val + is_pypy - - -if sys.version_info >= (3, 9): - StrPath = Union[str, os.PathLike[str]] -else: - # PathLike is only subscriptable at runtime in 3.9+ - StrPath = Union[str, "os.PathLike[str]"] # pragma: no cover diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/_meta.py b/contrib/python/importlib-metadata/py3/importlib_metadata/_meta.py index f670016de7..1342d839ba 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/_meta.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/_meta.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import os from typing import Protocol from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload @@ -44,20 +47,26 @@ class PackageMetadata(Protocol): """ -class SimplePath(Protocol[_T]): +class SimplePath(Protocol): """ - A minimal subset of pathlib.Path required by PathDistribution. + A minimal subset of pathlib.Path required by Distribution. """ - def joinpath(self, other: Union[str, _T]) -> _T: + def joinpath(self, other: Union[str, os.PathLike[str]]) -> SimplePath: ... # pragma: no cover - def __truediv__(self, other: Union[str, _T]) -> _T: + def __truediv__(self, other: Union[str, os.PathLike[str]]) -> SimplePath: ... # pragma: no cover @property - def parent(self) -> _T: + def parent(self) -> SimplePath: + ... # pragma: no cover + + def read_text(self, encoding=None) -> str: + ... # pragma: no cover + + def read_bytes(self) -> bytes: ... # pragma: no cover - def read_text(self) -> str: + def exists(self) -> bool: ... # pragma: no cover diff --git a/contrib/python/importlib-metadata/py3/ya.make b/contrib/python/importlib-metadata/py3/ya.make index 4e4813824c..f967f7b41b 100644 --- a/contrib/python/importlib-metadata/py3/ya.make +++ b/contrib/python/importlib-metadata/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(7.0.0) +VERSION(7.0.1) LICENSE(Apache-2.0) diff --git a/contrib/python/ipython/py3/.dist-info/METADATA b/contrib/python/ipython/py3/.dist-info/METADATA index ded1f01e88..ff27b2e371 100644 --- a/contrib/python/ipython/py3/.dist-info/METADATA +++ b/contrib/python/ipython/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ipython -Version: 8.18.1 +Version: 8.19.0 Summary: IPython: Productive Interactive Computing Home-page: https://ipython.org Author: The IPython Development Team @@ -23,7 +23,7 @@ Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3 :: Only Classifier: Topic :: System :: Shells -Requires-Python: >=3.9 +Requires-Python: >=3.10 Description-Content-Type: text/x-rst License-File: LICENSE Requires-Dist: decorator @@ -46,10 +46,9 @@ Requires-Dist: sphinx-rtd-theme ; extra == 'all' Requires-Dist: docrepr ; extra == 'all' Requires-Dist: matplotlib ; extra == 'all' Requires-Dist: stack-data ; extra == 'all' -Requires-Dist: pytest <7 ; extra == 'all' +Requires-Dist: pytest ; extra == 'all' Requires-Dist: typing-extensions ; extra == 'all' Requires-Dist: exceptiongroup ; extra == 'all' -Requires-Dist: pytest <7.1 ; extra == 'all' Requires-Dist: pytest-asyncio <0.22 ; extra == 'all' Requires-Dist: testpath ; extra == 'all' Requires-Dist: pickleshare ; extra == 'all' @@ -61,7 +60,7 @@ Requires-Dist: ipyparallel ; extra == 'all' Requires-Dist: qtconsole ; extra == 'all' Requires-Dist: curio ; extra == 'all' Requires-Dist: matplotlib !=3.2.0 ; extra == 'all' -Requires-Dist: numpy >=1.22 ; extra == 'all' +Requires-Dist: numpy >=1.23 ; extra == 'all' Requires-Dist: pandas ; extra == 'all' Requires-Dist: trio ; extra == 'all' Provides-Extra: black @@ -74,10 +73,9 @@ Requires-Dist: sphinx-rtd-theme ; extra == 'doc' Requires-Dist: docrepr ; extra == 'doc' Requires-Dist: matplotlib ; extra == 'doc' Requires-Dist: stack-data ; extra == 'doc' -Requires-Dist: pytest <7 ; extra == 'doc' +Requires-Dist: pytest ; extra == 'doc' Requires-Dist: typing-extensions ; extra == 'doc' Requires-Dist: exceptiongroup ; extra == 'doc' -Requires-Dist: pytest <7.1 ; extra == 'doc' Requires-Dist: pytest-asyncio <0.22 ; extra == 'doc' Requires-Dist: testpath ; extra == 'doc' Requires-Dist: pickleshare ; extra == 'doc' @@ -96,19 +94,19 @@ Provides-Extra: qtconsole Requires-Dist: qtconsole ; extra == 'qtconsole' Provides-Extra: terminal Provides-Extra: test -Requires-Dist: pytest <7.1 ; extra == 'test' +Requires-Dist: pytest ; extra == 'test' Requires-Dist: pytest-asyncio <0.22 ; extra == 'test' Requires-Dist: testpath ; extra == 'test' Requires-Dist: pickleshare ; extra == 'test' Provides-Extra: test_extra -Requires-Dist: pytest <7.1 ; extra == 'test_extra' +Requires-Dist: pytest ; extra == 'test_extra' Requires-Dist: pytest-asyncio <0.22 ; extra == 'test_extra' Requires-Dist: testpath ; extra == 'test_extra' Requires-Dist: pickleshare ; extra == 'test_extra' Requires-Dist: curio ; extra == 'test_extra' Requires-Dist: matplotlib !=3.2.0 ; extra == 'test_extra' Requires-Dist: nbformat ; extra == 'test_extra' -Requires-Dist: numpy >=1.22 ; extra == 'test_extra' +Requires-Dist: numpy >=1.23 ; extra == 'test_extra' Requires-Dist: pandas ; extra == 'test_extra' Requires-Dist: trio ; extra == 'test_extra' diff --git a/contrib/python/ipython/py3/IPython/__init__.py b/contrib/python/ipython/py3/IPython/__init__.py index 3322562a16..b7235481f2 100644 --- a/contrib/python/ipython/py3/IPython/__init__.py +++ b/contrib/python/ipython/py3/IPython/__init__.py @@ -26,9 +26,10 @@ import sys #----------------------------------------------------------------------------- # Don't forget to also update setup.py when this changes! -if sys.version_info < (3, 9): +if sys.version_info < (3, 10): raise ImportError( """ +IPython 8.19+ supports Python 3.10 and above, following SPEC0. IPython 8.13+ supports Python 3.9 and above, following NEP 29. IPython 8.0-8.12 supports Python 3.8 and above, following NEP 29. When using Python 2.7, please install IPython 5.x LTS Long Term Support version. diff --git a/contrib/python/ipython/py3/IPython/core/completer.py b/contrib/python/ipython/py3/IPython/core/completer.py index cc5f6c4270..8e2bb7f810 100644 --- a/contrib/python/ipython/py3/IPython/core/completer.py +++ b/contrib/python/ipython/py3/IPython/core/completer.py @@ -2237,7 +2237,7 @@ class IPCompleter(Completer): self, cursor_column: int, cursor_line: int, text: str ) -> Iterator[_JediCompletionLike]: """ - Return a list of :any:`jedi.api.Completion`s object from a ``text`` and + Return a list of :any:`jedi.api.Completion`\\s object from a ``text`` and cursor position. Parameters diff --git a/contrib/python/ipython/py3/IPython/core/interactiveshell.py b/contrib/python/ipython/py3/IPython/core/interactiveshell.py index 67727404f9..1a73571f78 100644 --- a/contrib/python/ipython/py3/IPython/core/interactiveshell.py +++ b/contrib/python/ipython/py3/IPython/core/interactiveshell.py @@ -2639,7 +2639,10 @@ class InteractiveShell(SingletonConfigurable): """ cmd = self.var_expand(cmd, depth=1) # warn if there is an IPython magic alternative. - main_cmd = cmd.split()[0] + if cmd == "": + main_cmd = "" + else: + main_cmd = cmd.split()[0] has_magic_alternatives = ("pip", "conda", "cd") if main_cmd in has_magic_alternatives: diff --git a/contrib/python/ipython/py3/IPython/core/oinspect.py b/contrib/python/ipython/py3/IPython/core/oinspect.py index d77a732267..a8be281975 100644 --- a/contrib/python/ipython/py3/IPython/core/oinspect.py +++ b/contrib/python/ipython/py3/IPython/core/oinspect.py @@ -28,10 +28,7 @@ import warnings from typing import Any, Optional, Dict, Union, List, Tuple -if sys.version_info <= (3, 10): - from typing_extensions import TypeAlias -else: - from typing import TypeAlias +from typing import TypeAlias # IPython's own from IPython.core import page diff --git a/contrib/python/ipython/py3/IPython/core/release.py b/contrib/python/ipython/py3/IPython/core/release.py index d012ec426b..70d5a675a3 100644 --- a/contrib/python/ipython/py3/IPython/core/release.py +++ b/contrib/python/ipython/py3/IPython/core/release.py @@ -16,8 +16,8 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 8 -_version_minor = 18 -_version_patch = 1 +_version_minor = 19 +_version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" _version_extra = "" # Uncomment this for full releases diff --git a/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py b/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py index 997fd82250..532287f5e7 100644 --- a/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py +++ b/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py @@ -588,6 +588,17 @@ class TerminalInteractiveShell(InteractiveShell): help="Display the current vi mode (when using vi editing mode)." ).tag(config=True) + prompt_line_number_format = Unicode( + "", + help="The format for line numbering, will be passed `line` (int, 1 based)" + " the current line number and `rel_line` the relative line number." + " for example to display both you can use the following template string :" + " c.TerminalInteractiveShell.prompt_line_number_format='{line: 4d}/{rel_line:+03d} | '" + " This will display the current line number, with leading space and a width of at least 4" + " character, as well as the relative line number 0 padded and always with a + or - sign." + " Note that when using Emacs mode the prompt of the first line may not update.", + ).tag(config=True) + @observe('term_title') def init_term_title(self, change=None): # Enable or disable the terminal title. @@ -736,7 +747,7 @@ class TerminalInteractiveShell(InteractiveShell): def get_message(): return PygmentsTokens(self.prompts.in_prompt_tokens()) - if self.editing_mode == 'emacs': + if self.editing_mode == "emacs" and self.prompt_line_number_format == "": # with emacs mode the prompt is (usually) static, so we call only # the function once. With VI mode it can toggle between [ins] and # [nor] so we can't precompute. @@ -753,7 +764,7 @@ class TerminalInteractiveShell(InteractiveShell): "message": get_message, "prompt_continuation": ( lambda width, lineno, is_soft_wrap: PygmentsTokens( - self.prompts.continuation_prompt_tokens(width) + self.prompts.continuation_prompt_tokens(width, lineno=lineno) ) ), "multiline": True, diff --git a/contrib/python/ipython/py3/IPython/terminal/prompts.py b/contrib/python/ipython/py3/IPython/terminal/prompts.py index 3f5c07b980..ca56d91a40 100644 --- a/contrib/python/ipython/py3/IPython/terminal/prompts.py +++ b/contrib/python/ipython/py3/IPython/terminal/prompts.py @@ -25,11 +25,21 @@ class Prompts(object): return '['+mode+'] ' return '' + def current_line(self) -> int: + if self.shell.pt_app is not None: + return self.shell.pt_app.default_buffer.document.cursor_position_row or 0 + return 0 def in_prompt_tokens(self): return [ - (Token.Prompt, self.vi_mode() ), - (Token.Prompt, 'In ['), + (Token.Prompt, self.vi_mode()), + ( + Token.Prompt, + self.shell.prompt_line_number_format.format( + line=1, rel_line=-self.current_line() + ), + ), + (Token.Prompt, "In ["), (Token.PromptNum, str(self.shell.execution_count)), (Token.Prompt, ']: '), ] @@ -37,11 +47,20 @@ class Prompts(object): def _width(self): return fragment_list_width(self.in_prompt_tokens()) - def continuation_prompt_tokens(self, width=None): + def continuation_prompt_tokens(self, width=None, *, lineno=None): if width is None: width = self._width() + line = lineno + 1 if lineno is not None else 0 + prefix = " " * len( + self.vi_mode() + ) + self.shell.prompt_line_number_format.format( + line=line, rel_line=line - self.current_line() - 1 + ) return [ - (Token.Prompt, (' ' * (width - 5)) + '...: '), + ( + Token.Prompt, + prefix + (" " * (width - len(prefix) - 5)) + "...: ", + ), ] def rewrite_prompt_tokens(self): diff --git a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py index 8cb62ae3d0..02820dc9c1 100644 --- a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py +++ b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py @@ -1,2 +1,2 @@ # GENERATED BY setup.py -commit = "49914f938" +commit = "dc4369111" diff --git a/contrib/python/ipython/py3/IPython/utils/tz.py b/contrib/python/ipython/py3/IPython/utils/tz.py index dbe1e8e732..ba1199d274 100644 --- a/contrib/python/ipython/py3/IPython/utils/tz.py +++ b/contrib/python/ipython/py3/IPython/utils/tz.py @@ -3,29 +3,56 @@ Timezone utilities Just UTC-awareness right now + +Deprecated since IPython 8.19.0. """ -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Copyright (C) 2013 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Imports -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +import warnings from datetime import tzinfo, timedelta, datetime -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Code -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +__all__ = ["tzUTC", "utc_aware", "utcfromtimestamp", "utcnow"] + + # constant for zero offset ZERO = timedelta(0) + +def __getattr__(name): + if name not in __all__: + err = f"IPython.utils.tz is deprecated and has no attribute {name}" + raise AttributeError(err) + + _warn_deprecated() + + return getattr(name) + + +def _warn_deprecated(): + msg = "The module `IPython.utils.tz` is deprecated and will be completely removed." + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + + class tzUTC(tzinfo): - """tzinfo object for UTC (zero offset)""" + """tzinfo object for UTC (zero offset) + + Deprecated since IPython 8.19.0. + """ + + _warn_deprecated() def utcoffset(self, d): return ZERO @@ -38,11 +65,18 @@ UTC = tzUTC() # type: ignore[abstract] def utc_aware(unaware): - """decorator for adding UTC tzinfo to datetime's utcfoo methods""" + """decorator for adding UTC tzinfo to datetime's utcfoo methods + + Deprecated since IPython 8.19.0. + """ + def utc_method(*args, **kwargs): + _warn_deprecated() dt = unaware(*args, **kwargs) return dt.replace(tzinfo=UTC) + return utc_method + utcfromtimestamp = utc_aware(datetime.utcfromtimestamp) utcnow = utc_aware(datetime.utcnow) diff --git a/contrib/python/ipython/py3/ya.make b/contrib/python/ipython/py3/ya.make index 46c7cd63c3..e6f0887b76 100644 --- a/contrib/python/ipython/py3/ya.make +++ b/contrib/python/ipython/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(8.18.1) +VERSION(8.19.0) LICENSE(BSD-3-Clause) diff --git a/contrib/python/numpy/py3/.dist-info/METADATA b/contrib/python/numpy/py3/.dist-info/METADATA index 7341e57f9d..5e515025ec 100644 --- a/contrib/python/numpy/py3/.dist-info/METADATA +++ b/contrib/python/numpy/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: numpy -Version: 1.26.2 +Version: 1.26.3 Summary: Fundamental package for array computing in Python Home-page: https://numpy.org Author: Travis E. Oliphant et al. diff --git a/contrib/python/numpy/py3/numpy/__init__.py b/contrib/python/numpy/py3/numpy/__init__.py index 4120c2c116..a2fbd368be 100644 --- a/contrib/python/numpy/py3/numpy/__init__.py +++ b/contrib/python/numpy/py3/numpy/__init__.py @@ -388,20 +388,25 @@ else: pass if sys.platform == "darwin": + from . import exceptions with warnings.catch_warnings(record=True) as w: #_mac_os_check() # Throw runtime error, if the test failed Check for warning and error_message - error_message = "" if len(w) > 0: - error_message = "{}: {}".format(w[-1].category.__name__, str(w[-1].message)) - msg = ( - "Polyfit sanity test emitted a warning, most likely due " - "to using a buggy Accelerate backend." - "\nIf you compiled yourself, more information is available at:" - "\nhttps://numpy.org/doc/stable/user/building.html#accelerated-blas-lapack-libraries" - "\nOtherwise report this to the vendor " - "that provided NumPy.\n{}\n".format(error_message)) - raise RuntimeError(msg) + for _wn in w: + if _wn.category is exceptions.RankWarning: + # Ignore other warnings, they may not be relevant (see gh-25433). + error_message = f"{_wn.category.__name__}: {str(_wn.message)}" + msg = ( + "Polyfit sanity test emitted a warning, most likely due " + "to using a buggy Accelerate backend." + "\nIf you compiled yourself, more information is available at:" + "\nhttps://numpy.org/devdocs/building/index.html" + "\nOtherwise report this to the vendor " + "that provided NumPy.\n\n{}\n".format(error_message)) + raise RuntimeError(msg) + del _wn + del w del _mac_os_check # We usually use madvise hugepages support, but on some old kernels it diff --git a/contrib/python/numpy/py3/numpy/array_api/__init__.py b/contrib/python/numpy/py3/numpy/array_api/__init__.py index 964873faab..77f227882e 100644 --- a/contrib/python/numpy/py3/numpy/array_api/__init__.py +++ b/contrib/python/numpy/py3/numpy/array_api/__init__.py @@ -125,7 +125,7 @@ __array_api_version__ = "2022.12" __all__ = ["__array_api_version__"] -from ._constants import e, inf, nan, pi +from ._constants import e, inf, nan, pi, newaxis __all__ += ["e", "inf", "nan", "pi"] diff --git a/contrib/python/numpy/py3/numpy/array_api/_array_object.py b/contrib/python/numpy/py3/numpy/array_api/_array_object.py index ec465208e8..5aff9863d8 100644 --- a/contrib/python/numpy/py3/numpy/array_api/_array_object.py +++ b/contrib/python/numpy/py3/numpy/array_api/_array_object.py @@ -543,7 +543,11 @@ class Array: def __getitem__( self: Array, key: Union[ - int, slice, ellipsis, Tuple[Union[int, slice, ellipsis], ...], Array + int, + slice, + ellipsis, + Tuple[Union[int, slice, ellipsis, None], ...], + Array, ], /, ) -> Array: diff --git a/contrib/python/numpy/py3/numpy/array_api/_constants.py b/contrib/python/numpy/py3/numpy/array_api/_constants.py index 9541941e7c..15ab81d16d 100644 --- a/contrib/python/numpy/py3/numpy/array_api/_constants.py +++ b/contrib/python/numpy/py3/numpy/array_api/_constants.py @@ -4,3 +4,4 @@ e = np.e inf = np.inf nan = np.nan pi = np.pi +newaxis = np.newaxis diff --git a/contrib/python/numpy/py3/numpy/array_api/linalg.py b/contrib/python/numpy/py3/numpy/array_api/linalg.py index 58320db55c..09af9dfc3a 100644 --- a/contrib/python/numpy/py3/numpy/array_api/linalg.py +++ b/contrib/python/numpy/py3/numpy/array_api/linalg.py @@ -318,8 +318,9 @@ def _solve(a, b): # This does nothing currently but is left in because it will be relevant # when complex dtype support is added to the spec in 2022. signature = 'DD->D' if isComplexType(t) else 'dd->d' - extobj = get_linalg_error_extobj(_raise_linalgerror_singular) - r = gufunc(a, b, signature=signature, extobj=extobj) + with np.errstate(call=_raise_linalgerror_singular, invalid='call', + over='ignore', divide='ignore', under='ignore'): + r = gufunc(a, b, signature=signature) return wrap(r.astype(result_t, copy=False)) diff --git a/contrib/python/numpy/py3/numpy/core/src/common/half.hpp b/contrib/python/numpy/py3/numpy/core/src/common/half.hpp index ff9a547766..484750ad84 100644 --- a/contrib/python/numpy/py3/numpy/core/src/common/half.hpp +++ b/contrib/python/numpy/py3/numpy/core/src/common/half.hpp @@ -59,7 +59,11 @@ class Half final { __vector float vf32 = vec_splats(f); __vector unsigned short vf16; __asm__ __volatile__ ("xvcvsphp %x0,%x1" : "=wa" (vf16) : "wa" (vf32)); + #ifdef __BIG_ENDIAN__ + bits_ = vec_extract(vf16, 1); + #else bits_ = vec_extract(vf16, 0); + #endif #else bits_ = half_private::FromFloatBits(BitCast<uint32_t>(f)); #endif diff --git a/contrib/python/numpy/py3/numpy/core/src/common/simd/neon/memory.h b/contrib/python/numpy/py3/numpy/core/src/common/simd/neon/memory.h index 2dc21e5a43..e7503b822e 100644 --- a/contrib/python/numpy/py3/numpy/core/src/common/simd/neon/memory.h +++ b/contrib/python/numpy/py3/numpy/core/src/common/simd/neon/memory.h @@ -52,21 +52,12 @@ NPYV_IMPL_NEON_MEM(f64, double) ***************************/ NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) { - switch (stride) { - case 2: - return vld2q_s32((const int32_t*)ptr).val[0]; - case 3: - return vld3q_s32((const int32_t*)ptr).val[0]; - case 4: - return vld4q_s32((const int32_t*)ptr).val[0]; - default:; - int32x2_t ax = vcreate_s32(*ptr); - int32x4_t a = vcombine_s32(ax, ax); - a = vld1q_lane_s32((const int32_t*)ptr + stride, a, 1); - a = vld1q_lane_s32((const int32_t*)ptr + stride*2, a, 2); - a = vld1q_lane_s32((const int32_t*)ptr + stride*3, a, 3); - return a; - } + int32x4_t a = vdupq_n_s32(0); + a = vld1q_lane_s32((const int32_t*)ptr, a, 0); + a = vld1q_lane_s32((const int32_t*)ptr + stride, a, 1); + a = vld1q_lane_s32((const int32_t*)ptr + stride*2, a, 2); + a = vld1q_lane_s32((const int32_t*)ptr + stride*3, a, 3); + return a; } NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) diff --git a/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c b/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c index 58e8b3778b..efa4c271cf 100644 --- a/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c +++ b/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c @@ -21408,7 +21408,7 @@ OBJECT_nonzero (PyObject **ip, PyArrayObject *ap) } else { PyObject *obj; - memcpy(&obj, ip, sizeof(obj)); + memcpy(&obj, (void *)ip, sizeof(obj)); if (obj == NULL) { return NPY_FALSE; } diff --git a/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c.src b/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c.src index bc3f743727..f3feee63da 100644 --- a/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c.src +++ b/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c.src @@ -2889,7 +2889,7 @@ OBJECT_nonzero (PyObject **ip, PyArrayObject *ap) } else { PyObject *obj; - memcpy(&obj, ip, sizeof(obj)); + memcpy(&obj, (void *)ip, sizeof(obj)); if (obj == NULL) { return NPY_FALSE; } diff --git a/contrib/python/numpy/py3/numpy/core/src/multiarray/nditer_constr.c b/contrib/python/numpy/py3/numpy/core/src/multiarray/nditer_constr.c index dfa84c51cd..427dd3d876 100644 --- a/contrib/python/numpy/py3/numpy/core/src/multiarray/nditer_constr.c +++ b/contrib/python/numpy/py3/numpy/core/src/multiarray/nditer_constr.c @@ -198,6 +198,9 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, /* Allocate memory for the iterator */ iter = (NpyIter*) PyObject_Malloc(NIT_SIZEOF_ITERATOR(itflags, ndim, nop)); + if (iter == NULL) { + return NULL; + } NPY_IT_TIME_POINT(c_malloc); diff --git a/contrib/python/numpy/py3/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp b/contrib/python/numpy/py3/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp index 3f5099758c..a75f882ff4 100644 --- a/contrib/python/numpy/py3/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp +++ b/contrib/python/numpy/py3/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp @@ -9,6 +9,7 @@ #if defined(NPY_HAVE_AVX512_SPR) && !defined(_MSC_VER) #include "x86-simd-sort/src/avx512fp16-16bit-qsort.hpp" + #include "x86-simd-sort/src/avx512-16bit-qsort.hpp" #elif defined(NPY_HAVE_AVX512_ICL) && !defined(_MSC_VER) #include "x86-simd-sort/src/avx512-16bit-qsort.hpp" #endif diff --git a/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512-16bit-qsort.hpp b/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512-16bit-qsort.hpp index 606f870645..2eb445422b 100644 --- a/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512-16bit-qsort.hpp +++ b/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512-16bit-qsort.hpp @@ -350,7 +350,7 @@ struct zmm_vector<uint16_t> { }; template <> -bool comparison_func<zmm_vector<float16>>(const uint16_t &a, const uint16_t &b) +inline bool comparison_func<zmm_vector<float16>>(const uint16_t &a, const uint16_t &b) { uint16_t signa = a & 0x8000, signb = b & 0x8000; uint16_t expa = a & 0x7c00, expb = b & 0x7c00; @@ -406,7 +406,7 @@ replace_inf_with_nan(uint16_t *arr, int64_t arrsize, int64_t nan_count) } template <> -void avx512_qselect(int16_t *arr, int64_t k, int64_t arrsize) +inline void avx512_qselect(int16_t *arr, int64_t k, int64_t arrsize) { if (arrsize > 1) { qselect_16bit_<zmm_vector<int16_t>, int16_t>( @@ -415,7 +415,7 @@ void avx512_qselect(int16_t *arr, int64_t k, int64_t arrsize) } template <> -void avx512_qselect(uint16_t *arr, int64_t k, int64_t arrsize) +inline void avx512_qselect(uint16_t *arr, int64_t k, int64_t arrsize) { if (arrsize > 1) { qselect_16bit_<zmm_vector<uint16_t>, uint16_t>( @@ -423,7 +423,7 @@ void avx512_qselect(uint16_t *arr, int64_t k, int64_t arrsize) } } -void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize) +inline void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize) { if (arrsize > 1) { int64_t nan_count = replace_nan_with_inf(arr, arrsize); @@ -434,7 +434,7 @@ void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize) } template <> -void avx512_qsort(int16_t *arr, int64_t arrsize) +inline void avx512_qsort(int16_t *arr, int64_t arrsize) { if (arrsize > 1) { qsort_16bit_<zmm_vector<int16_t>, int16_t>( @@ -443,7 +443,7 @@ void avx512_qsort(int16_t *arr, int64_t arrsize) } template <> -void avx512_qsort(uint16_t *arr, int64_t arrsize) +inline void avx512_qsort(uint16_t *arr, int64_t arrsize) { if (arrsize > 1) { qsort_16bit_<zmm_vector<uint16_t>, uint16_t>( @@ -451,7 +451,7 @@ void avx512_qsort(uint16_t *arr, int64_t arrsize) } } -void avx512_qsort_fp16(uint16_t *arr, int64_t arrsize) +inline void avx512_qsort_fp16(uint16_t *arr, int64_t arrsize) { if (arrsize > 1) { int64_t nan_count = replace_nan_with_inf(arr, arrsize); diff --git a/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512fp16-16bit-qsort.hpp b/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512fp16-16bit-qsort.hpp index 8a9a49ede9..1206f8223d 100644 --- a/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512fp16-16bit-qsort.hpp +++ b/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512fp16-16bit-qsort.hpp @@ -145,7 +145,7 @@ replace_inf_with_nan(_Float16 *arr, int64_t arrsize, int64_t nan_count) } template <> -void avx512_qselect(_Float16 *arr, int64_t k, int64_t arrsize) +inline void avx512_qselect(_Float16 *arr, int64_t k, int64_t arrsize) { if (arrsize > 1) { int64_t nan_count = replace_nan_with_inf(arr, arrsize); @@ -156,7 +156,7 @@ void avx512_qselect(_Float16 *arr, int64_t k, int64_t arrsize) } template <> -void avx512_qsort(_Float16 *arr, int64_t arrsize) +inline void avx512_qsort(_Float16 *arr, int64_t arrsize) { if (arrsize > 1) { int64_t nan_count = replace_nan_with_inf(arr, arrsize); diff --git a/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c b/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c index 695d488250..19706ae07f 100644 --- a/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c +++ b/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c @@ -68,7 +68,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_add) #endif return; } -#if NPY_SIMD_F32 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F32 if (len > npyv_nlanes_f32*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -132,8 +154,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_add) npyv_store_f32((npy_float*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 0 + #if 0 npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, 1.0f); + #elif 0 + npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, NPY_NANF); #else npyv_f32 a = npyv_load_tillz_f32((const npy_float*)src0, len); #endif @@ -204,7 +228,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_subtract) #endif return; } -#if NPY_SIMD_F32 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F32 if (len > npyv_nlanes_f32*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -268,8 +314,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_subtract) npyv_store_f32((npy_float*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 0 + #if 0 npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, 1.0f); + #elif 0 + npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, NPY_NANF); #else npyv_f32 a = npyv_load_tillz_f32((const npy_float*)src0, len); #endif @@ -340,7 +388,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_multiply) #endif return; } -#if NPY_SIMD_F32 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F32 if (len > npyv_nlanes_f32*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -404,8 +474,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_multiply) npyv_store_f32((npy_float*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 1 + #if 1 npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, 1.0f); + #elif 0 + npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, NPY_NANF); #else npyv_f32 a = npyv_load_tillz_f32((const npy_float*)src0, len); #endif @@ -476,7 +548,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_divide) #endif return; } -#if NPY_SIMD_F32 +#if 1 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F32 if (len > npyv_nlanes_f32*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -540,8 +634,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_divide) npyv_store_f32((npy_float*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 1 || 0 + #if 0 npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, 1.0f); + #elif 1 + npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, NPY_NANF); #else npyv_f32 a = npyv_load_tillz_f32((const npy_float*)src0, len); #endif @@ -614,7 +710,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_add) #endif return; } -#if NPY_SIMD_F64 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F64 if (len > npyv_nlanes_f64*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -678,8 +796,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_add) npyv_store_f64((npy_double*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 0 + #if 0 npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, 1.0); + #elif 0 + npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, NPY_NAN); #else npyv_f64 a = npyv_load_tillz_f64((const npy_double*)src0, len); #endif @@ -750,7 +870,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_subtract) #endif return; } -#if NPY_SIMD_F64 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F64 if (len > npyv_nlanes_f64*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -814,8 +956,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_subtract) npyv_store_f64((npy_double*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 0 + #if 0 npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, 1.0); + #elif 0 + npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, NPY_NAN); #else npyv_f64 a = npyv_load_tillz_f64((const npy_double*)src0, len); #endif @@ -886,7 +1030,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_multiply) #endif return; } -#if NPY_SIMD_F64 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F64 if (len > npyv_nlanes_f64*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -950,8 +1116,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_multiply) npyv_store_f64((npy_double*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 1 + #if 1 npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, 1.0); + #elif 0 + npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, NPY_NAN); #else npyv_f64 a = npyv_load_tillz_f64((const npy_double*)src0, len); #endif @@ -1022,7 +1190,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_divide) #endif return; } -#if NPY_SIMD_F64 +#if 1 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F64 if (len > npyv_nlanes_f64*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -1086,8 +1276,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_divide) npyv_store_f64((npy_double*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 1 || 0 + #if 0 npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, 1.0); + #elif 1 + npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, NPY_NAN); #else npyv_f64 a = npyv_load_tillz_f64((const npy_double*)src0, len); #endif @@ -1220,7 +1412,7 @@ simd_csquare_f64(npyv_f64 x) { return simd_cmul_f64(x, x); } #endif -#line 286 +#line 310 #if NPY_SIMD_F32 NPY_FINLINE npyv_f32 simd_cabsolute_f32(npyv_f32 re, npyv_f32 im) @@ -1264,7 +1456,7 @@ simd_cabsolute_f32(npyv_f32 re, npyv_f32 im) } #endif // VECTOR -#line 286 +#line 310 #if NPY_SIMD_F64 NPY_FINLINE npyv_f64 simd_cabsolute_f64(npyv_f64 re, npyv_f64 im) @@ -1312,8 +1504,8 @@ simd_cabsolute_f64(npyv_f64 re, npyv_f64 im) /******************************************************************************** ** Defining ufunc inner functions ********************************************************************************/ -#line 342 -#line 350 +#line 366 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_add) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -1565,7 +1757,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CFLOAT_add_indexed) return 0; } -#line 350 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_subtract) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -1817,7 +2009,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CFLOAT_subtract_indexed) return 0; } -#line 350 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_multiply) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2070,7 +2262,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CFLOAT_multiply_indexed) } -#line 606 +#line 630 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_conjugate) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2158,7 +2350,7 @@ loop_scalar: } } -#line 606 +#line 630 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_square) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2247,8 +2439,8 @@ loop_scalar: } -#line 342 -#line 350 +#line 366 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_add) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2500,7 +2692,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CDOUBLE_add_indexed) return 0; } -#line 350 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_subtract) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2752,7 +2944,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CDOUBLE_subtract_indexed) return 0; } -#line 350 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_multiply) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -3005,7 +3197,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CDOUBLE_multiply_indexed) } -#line 606 +#line 630 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_conjugate) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -3093,7 +3285,7 @@ loop_scalar: } } -#line 606 +#line 630 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_square) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { diff --git a/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src b/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src index 30111258d6..0d0de90125 100644 --- a/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src +++ b/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src @@ -74,7 +74,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_@kind@) #endif return; } -#if @VECTOR@ +#if @is_div@ && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif @VECTOR@ if (len > npyv_nlanes_@sfx@*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -138,8 +160,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_@kind@) npyv_store_@sfx@((@type@*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if @is_div@ || @is_mul@ + #if @is_mul@ npyv_@sfx@ a = npyv_load_till_@sfx@((const @type@*)src0, len, 1.0@c@); + #elif @is_div@ + npyv_@sfx@ a = npyv_load_till_@sfx@((const @type@*)src0, len, NPY_NAN@C@); #else npyv_@sfx@ a = npyv_load_tillz_@sfx@((const @type@*)src0, len); #endif diff --git a/contrib/python/numpy/py3/numpy/core/tests/examples/cython/meson.build b/contrib/python/numpy/py3/numpy/core/tests/examples/cython/meson.build index 12fc640b88..836b74ac38 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/examples/cython/meson.build +++ b/contrib/python/numpy/py3/numpy/core/tests/examples/cython/meson.build @@ -14,6 +14,15 @@ npy_include_path = run_command(py, [ 'import os; os.chdir(".."); import numpy; print(os.path.abspath(numpy.get_include()))' ], check: true).stdout().strip() +npy_path = run_command(py, [ + '-c', + 'import os; os.chdir(".."); import numpy; print(os.path.dirname(numpy.__file__).removesuffix("numpy"))' + ], check: true).stdout().strip() + +# TODO: This is a hack due to gh-25135, where cython may not find the right +# __init__.pyd file. +add_project_arguments('-I', npy_path, language : 'cython') + py.extension_module( 'checks', 'checks.pyx', diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_cpu_features.py b/contrib/python/numpy/py3/numpy/core/tests/test_cpu_features.py index 3a7c7a80fd..da010df63e 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_cpu_features.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_cpu_features.py @@ -352,6 +352,7 @@ class Test_X86_Features(AbstractTest): SSE3="PNI", SSE41="SSE4_1", SSE42="SSE4_2", FMA3="FMA", AVX512VNNI="AVX512_VNNI", AVX512BITALG="AVX512_BITALG", AVX512VBMI2="AVX512_VBMI2", AVX5124FMAPS="AVX512_4FMAPS", AVX5124VNNIW="AVX512_4VNNIW", AVX512VPOPCNTDQ="AVX512_VPOPCNTDQ", + AVX512FP16="AVX512_FP16", ) def load_flags(self): self.load_flags_cpuinfo("flags") diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_cython.py b/contrib/python/numpy/py3/numpy/core/tests/test_cython.py index 99dd57e4c6..0e0d00c250 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_cython.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_cython.py @@ -28,14 +28,14 @@ else: pytestmark = pytest.mark.skipif(cython is None, reason="requires cython") -@pytest.fixture -def install_temp(tmp_path): +@pytest.fixture(scope='module') +def install_temp(tmpdir_factory): # Based in part on test_cython from random.tests.test_extending if IS_WASM: pytest.skip("No subprocess") srcdir = os.path.join(os.path.dirname(__file__), 'examples', 'cython') - build_dir = tmp_path / "build" + build_dir = tmpdir_factory.mktemp("cython_test") / "build" os.makedirs(build_dir, exist_ok=True) try: subprocess.check_call(["meson", "--version"]) diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_mem_policy.py b/contrib/python/numpy/py3/numpy/core/tests/test_mem_policy.py index bc3f330dc1..a381fa1d89 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_mem_policy.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_mem_policy.py @@ -440,3 +440,4 @@ def test_owner_is_base(get_module): with pytest.warns(UserWarning, match='warn_on_free'): del a gc.collect() + gc.collect() diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_nditer.py b/contrib/python/numpy/py3/numpy/core/tests/test_nditer.py index 35bd6e97e0..8c1a770cd2 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_nditer.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_nditer.py @@ -3203,7 +3203,6 @@ def test_warn_noclose(): assert len(sup.log) == 1 -@pytest.mark.skip @pytest.mark.skipif(sys.version_info[:2] == (3, 9) and sys.platform == "win32", reason="Errors with Python 3.9 on Windows") @pytest.mark.parametrize(["in_dtype", "buf_dtype"], diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_umath.py b/contrib/python/numpy/py3/numpy/core/tests/test_umath.py index 59c670ffed..963e740d8d 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_umath.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_umath.py @@ -17,7 +17,8 @@ from numpy.testing import ( assert_, assert_equal, assert_raises, assert_raises_regex, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_array_max_ulp, assert_allclose, assert_no_warnings, suppress_warnings, - _gen_alignment_data, assert_array_almost_equal_nulp, IS_WASM, IS_MUSL + _gen_alignment_data, assert_array_almost_equal_nulp, IS_WASM, IS_MUSL, + IS_PYPY ) from numpy.testing._private.utils import _glibc_older_than @@ -1825,6 +1826,18 @@ class TestSpecialFloats: with assert_no_warnings(): ufunc(array) + @pytest.mark.parametrize("dtype", ('e', 'f', 'd')) + def test_divide_spurious_fpexception(self, dtype): + dt = np.dtype(dtype) + dt_info = np.finfo(dt) + subnorm = dt_info.smallest_subnormal + # Verify a bug fix caused due to filling the remaining lanes of the + # partially loaded dividend SIMD vector with ones, which leads to + # raising an overflow warning when the divisor is denormal. + # see https://github.com/numpy/numpy/issues/25097 + with assert_no_warnings(): + np.zeros(128 + 1, dtype=dt) / subnorm + class TestFPClass: @pytest.mark.parametrize("stride", [-5, -4, -3, -2, -1, 1, 2, 4, 5, 6, 7, 8, 9, 10]) @@ -4180,7 +4193,10 @@ class TestComplexFunctions: for p in points: a = complex(func(np.complex_(p))) b = cfunc(p) - assert_(abs(a - b) < atol, "%s %s: %s; cmath: %s" % (fname, p, a, b)) + assert_( + abs(a - b) < atol, + "%s %s: %s; cmath: %s" % (fname, p, a, b) + ) @pytest.mark.xfail( # manylinux2014 uses glibc2.17 diff --git a/contrib/python/numpy/py3/numpy/distutils/checks/cpu_avx512_knl.c b/contrib/python/numpy/py3/numpy/distutils/checks/cpu_avx512_knl.c index b3f4f69765..cb55e57aa2 100644 --- a/contrib/python/numpy/py3/numpy/distutils/checks/cpu_avx512_knl.c +++ b/contrib/python/numpy/py3/numpy/distutils/checks/cpu_avx512_knl.c @@ -15,7 +15,7 @@ int main(int argc, char **argv) { - int base[128]; + int base[128]={}; __m512d ad = _mm512_loadu_pd((const __m512d*)argv[argc-1]); /* ER */ __m512i a = _mm512_castpd_si512(_mm512_exp2a23_pd(ad)); diff --git a/contrib/python/numpy/py3/numpy/f2py/__init__.py b/contrib/python/numpy/py3/numpy/f2py/__init__.py index dbe3df27f6..e583250f70 100644 --- a/contrib/python/numpy/py3/numpy/f2py/__init__.py +++ b/contrib/python/numpy/py3/numpy/f2py/__init__.py @@ -1,13 +1,21 @@ #!/usr/bin/env python3 """Fortran to Python Interface Generator. +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +Permission to use, modify, and distribute this software is given under the terms +of the NumPy License. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. """ __all__ = ['run_main', 'compile', 'get_include'] import sys import subprocess import os +import warnings +from numpy.exceptions import VisibleDeprecationWarning from . import f2py2e from . import diagnose diff --git a/contrib/python/numpy/py3/numpy/f2py/_backends/_distutils.py b/contrib/python/numpy/py3/numpy/f2py/_backends/_distutils.py index e548fc5430..e9b22a3921 100644 --- a/contrib/python/numpy/py3/numpy/f2py/_backends/_distutils.py +++ b/contrib/python/numpy/py3/numpy/f2py/_backends/_distutils.py @@ -13,7 +13,7 @@ import warnings class DistutilsBackend(Backend): def __init__(sef, *args, **kwargs): warnings.warn( - "distutils has been deprecated since NumPy 1.26." + "distutils has been deprecated since NumPy 1.26.x" "Use the Meson backend instead, or generate wrappers" "without -c and use a custom build script", VisibleDeprecationWarning, diff --git a/contrib/python/numpy/py3/numpy/f2py/_backends/_meson.py b/contrib/python/numpy/py3/numpy/f2py/_backends/_meson.py index 3176a5e08f..f324e0f595 100644 --- a/contrib/python/numpy/py3/numpy/f2py/_backends/_meson.py +++ b/contrib/python/numpy/py3/numpy/f2py/_backends/_meson.py @@ -1,12 +1,15 @@ from __future__ import annotations +import os import errno import shutil import subprocess +import sys from pathlib import Path from ._backend import Backend from string import Template +from itertools import chain import warnings @@ -19,10 +22,14 @@ class MesonTemplate: modulename: str, sources: list[Path], deps: list[str], + libraries: list[str], + library_dirs: list[Path], + include_dirs: list[Path], object_files: list[Path], linker_args: list[str], c_args: list[str], build_type: str, + python_exe: str, ): self.modulename = modulename self.build_template_path = ( @@ -30,14 +37,23 @@ class MesonTemplate: ) self.sources = sources self.deps = deps + self.libraries = libraries + self.library_dirs = library_dirs + if include_dirs is not None: + self.include_dirs = include_dirs + else: + self.include_dirs = [] self.substitutions = {} self.objects = object_files self.pipeline = [ self.initialize_template, self.sources_substitution, self.deps_substitution, + self.include_substitution, + self.libraries_substitution, ] self.build_type = build_type + self.python_exe = python_exe def meson_build_template(self) -> str: if not self.build_template_path.is_file(): @@ -52,17 +68,47 @@ class MesonTemplate: def initialize_template(self) -> None: self.substitutions["modulename"] = self.modulename self.substitutions["buildtype"] = self.build_type + self.substitutions["python"] = self.python_exe def sources_substitution(self) -> None: indent = " " * 21 self.substitutions["source_list"] = f",\n{indent}".join( - [f"'{source}'" for source in self.sources] + [f"{indent}'{source}'" for source in self.sources] ) def deps_substitution(self) -> None: indent = " " * 21 self.substitutions["dep_list"] = f",\n{indent}".join( - [f"dependency('{dep}')" for dep in self.deps] + [f"{indent}dependency('{dep}')" for dep in self.deps] + ) + + def libraries_substitution(self) -> None: + self.substitutions["lib_dir_declarations"] = "\n".join( + [ + f"lib_dir_{i} = declare_dependency(link_args : ['-L{lib_dir}'])" + for i, lib_dir in enumerate(self.library_dirs) + ] + ) + + self.substitutions["lib_declarations"] = "\n".join( + [ + f"{lib} = declare_dependency(link_args : ['-l{lib}'])" + for lib in self.libraries + ] + ) + + indent = " " * 21 + self.substitutions["lib_list"] = f"\n{indent}".join( + [f"{indent}{lib}," for lib in self.libraries] + ) + self.substitutions["lib_dir_list"] = f"\n{indent}".join( + [f"{indent}lib_dir_{i}," for i in range(len(self.library_dirs))] + ) + + def include_substitution(self) -> None: + indent = " " * 21 + self.substitutions["inc_list"] = f",\n{indent}".join( + [f"{indent}'{inc}'" for inc in self.include_dirs] ) def generate_meson_build(self): @@ -83,16 +129,18 @@ class MesonBackend(Backend): def _move_exec_to_root(self, build_dir: Path): walk_dir = Path(build_dir) / self.meson_build_dir - path_objects = walk_dir.glob(f"{self.modulename}*.so") + path_objects = chain( + walk_dir.glob(f"{self.modulename}*.so"), + walk_dir.glob(f"{self.modulename}*.pyd"), + ) + # Same behavior as distutils + # https://github.com/numpy/numpy/issues/24874#issuecomment-1835632293 for path_object in path_objects: - shutil.move(path_object, Path.cwd()) - - def _get_build_command(self): - return [ - "meson", - "setup", - self.meson_build_dir, - ] + dest_path = Path.cwd() / path_object.name + if dest_path.exists(): + dest_path.unlink() + shutil.copy2(path_object, dest_path) + os.remove(path_object) def write_meson_build(self, build_dir: Path) -> None: """Writes the meson build file at specified location""" @@ -100,10 +148,14 @@ class MesonBackend(Backend): self.modulename, self.sources, self.dependencies, + self.libraries, + self.library_dirs, + self.include_dirs, self.extra_objects, self.flib_flags, self.fc_flags, self.build_type, + sys.executable, ) src = meson_template.generate_meson_build() Path(build_dir).mkdir(parents=True, exist_ok=True) @@ -111,19 +163,14 @@ class MesonBackend(Backend): meson_build_file.write_text(src) return meson_build_file + def _run_subprocess_command(self, command, cwd): + subprocess.run(command, cwd=cwd, check=True) + def run_meson(self, build_dir: Path): - completed_process = subprocess.run(self._get_build_command(), cwd=build_dir) - if completed_process.returncode != 0: - raise subprocess.CalledProcessError( - completed_process.returncode, completed_process.args - ) - completed_process = subprocess.run( - ["meson", "compile", "-C", self.meson_build_dir], cwd=build_dir - ) - if completed_process.returncode != 0: - raise subprocess.CalledProcessError( - completed_process.returncode, completed_process.args - ) + setup_command = ["meson", "setup", self.meson_build_dir] + self._run_subprocess_command(setup_command, build_dir) + compile_command = ["meson", "compile", "-C", self.meson_build_dir] + self._run_subprocess_command(compile_command, build_dir) def compile(self) -> None: self.sources = _prepare_sources(self.modulename, self.sources, self.build_dir) @@ -137,7 +184,8 @@ def _prepare_sources(mname, sources, bdir): Path(bdir).mkdir(parents=True, exist_ok=True) # Copy sources for source in sources: - shutil.copy(source, bdir) + if Path(source).exists() and Path(source).is_file(): + shutil.copy(source, bdir) generated_sources = [ Path(f"{mname}module.c"), Path(f"{mname}-f2pywrappers2.f90"), diff --git a/contrib/python/numpy/py3/numpy/f2py/_backends/meson.build.template b/contrib/python/numpy/py3/numpy/f2py/_backends/meson.build.template index 545e399521..8e34fdc8d4 100644 --- a/contrib/python/numpy/py3/numpy/f2py/_backends/meson.build.template +++ b/contrib/python/numpy/py3/numpy/f2py/_backends/meson.build.template @@ -6,8 +6,9 @@ project('${modulename}', 'warning_level=1', 'buildtype=${buildtype}' ]) +fc = meson.get_compiler('fortran') -py = import('python').find_installation(pure: false) +py = import('python').find_installation('${python}', pure: false) py_dep = py.dependency() incdir_numpy = run_command(py, @@ -28,15 +29,26 @@ inc_f2py = include_directories(incdir_f2py) fortranobject_c = incdir_f2py / 'fortranobject.c' inc_np = include_directories(incdir_numpy, incdir_f2py) +# gh-25000 +quadmath_dep = fc.find_library('quadmath', required: false) + +${lib_declarations} +${lib_dir_declarations} py.extension_module('${modulename}', [ ${source_list}, fortranobject_c ], - include_directories: [inc_np], + include_directories: [ + inc_np, +${inc_list} + ], dependencies : [ py_dep, + quadmath_dep, ${dep_list} +${lib_list} +${lib_dir_list} ], install : true) diff --git a/contrib/python/numpy/py3/numpy/f2py/_isocbind.py b/contrib/python/numpy/py3/numpy/f2py/_isocbind.py index 81f52fb4de..3043c5d916 100644 --- a/contrib/python/numpy/py3/numpy/f2py/_isocbind.py +++ b/contrib/python/numpy/py3/numpy/f2py/_isocbind.py @@ -1,45 +1,61 @@ +""" +ISO_C_BINDING maps for f2py2e. +Only required declarations/macros/functions will be used. + +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +Permission to use, modify, and distribute this software is given under the +terms of the NumPy License. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. +""" +# These map to keys in c2py_map, via forced casting for now, see gh-25229 iso_c_binding_map = { 'integer': { 'c_int': 'int', - 'c_short': 'short int', - 'c_long': 'long int', - 'c_long_long': 'long long int', - 'c_signed_char': 'signed char', - 'c_size_t': 'size_t', - 'c_int8_t': 'int8_t', - 'c_int16_t': 'int16_t', - 'c_int32_t': 'int32_t', - 'c_int64_t': 'int64_t', - 'c_int_least8_t': 'int_least8_t', - 'c_int_least16_t': 'int_least16_t', - 'c_int_least32_t': 'int_least32_t', - 'c_int_least64_t': 'int_least64_t', - 'c_int_fast8_t': 'int_fast8_t', - 'c_int_fast16_t': 'int_fast16_t', - 'c_int_fast32_t': 'int_fast32_t', - 'c_int_fast64_t': 'int_fast64_t', - 'c_intmax_t': 'intmax_t', - 'c_intptr_t': 'intptr_t', - 'c_ptrdiff_t': 'intptr_t', + 'c_short': 'short', # 'short' <=> 'int' for now + 'c_long': 'long', # 'long' <=> 'int' for now + 'c_long_long': 'long_long', + 'c_signed_char': 'signed_char', + 'c_size_t': 'unsigned', # size_t <=> 'unsigned' for now + 'c_int8_t': 'signed_char', # int8_t <=> 'signed_char' for now + 'c_int16_t': 'short', # int16_t <=> 'short' for now + 'c_int32_t': 'int', # int32_t <=> 'int' for now + 'c_int64_t': 'long_long', + 'c_int_least8_t': 'signed_char', # int_least8_t <=> 'signed_char' for now + 'c_int_least16_t': 'short', # int_least16_t <=> 'short' for now + 'c_int_least32_t': 'int', # int_least32_t <=> 'int' for now + 'c_int_least64_t': 'long_long', + 'c_int_fast8_t': 'signed_char', # int_fast8_t <=> 'signed_char' for now + 'c_int_fast16_t': 'short', # int_fast16_t <=> 'short' for now + 'c_int_fast32_t': 'int', # int_fast32_t <=> 'int' for now + 'c_int_fast64_t': 'long_long', + 'c_intmax_t': 'long_long', # intmax_t <=> 'long_long' for now + 'c_intptr_t': 'long', # intptr_t <=> 'long' for now + 'c_ptrdiff_t': 'long', # ptrdiff_t <=> 'long' for now }, 'real': { 'c_float': 'float', 'c_double': 'double', - 'c_long_double': 'long double' + 'c_long_double': 'long_double' }, 'complex': { - 'c_float_complex': 'float _Complex', - 'c_double_complex': 'double _Complex', - 'c_long_double_complex': 'long double _Complex' + 'c_float_complex': 'complex_float', + 'c_double_complex': 'complex_double', + 'c_long_double_complex': 'complex_long_double' }, 'logical': { - 'c_bool': '_Bool' + 'c_bool': 'unsigned_char' # _Bool <=> 'unsigned_char' for now }, 'character': { 'c_char': 'char' } } +# TODO: See gh-25229 +isoc_c2pycode_map = {} +iso_c2py_map = {} + isoc_kindmap = {} for fortran_type, c_type_dict in iso_c_binding_map.items(): for c_type in c_type_dict.keys(): diff --git a/contrib/python/numpy/py3/numpy/f2py/_src_pyf.py b/contrib/python/numpy/py3/numpy/f2py/_src_pyf.py new file mode 100644 index 0000000000..6247b95bfe --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/_src_pyf.py @@ -0,0 +1,239 @@ +import re + +# START OF CODE VENDORED FROM `numpy.distutils.from_template` +############################################################# +""" +process_file(filename) + + takes templated file .xxx.src and produces .xxx file where .xxx + is .pyf .f90 or .f using the following template rules: + + '<..>' denotes a template. + + All function and subroutine blocks in a source file with names that + contain '<..>' will be replicated according to the rules in '<..>'. + + The number of comma-separated words in '<..>' will determine the number of + replicates. + + '<..>' may have two different forms, named and short. For example, + + named: + <p=d,s,z,c> where anywhere inside a block '<p>' will be replaced with + 'd', 's', 'z', and 'c' for each replicate of the block. + + <_c> is already defined: <_c=s,d,c,z> + <_t> is already defined: <_t=real,double precision,complex,double complex> + + short: + <s,d,c,z>, a short form of the named, useful when no <p> appears inside + a block. + + In general, '<..>' contains a comma separated list of arbitrary + expressions. If these expression must contain a comma|leftarrow|rightarrow, + then prepend the comma|leftarrow|rightarrow with a backslash. + + If an expression matches '\\<index>' then it will be replaced + by <index>-th expression. + + Note that all '<..>' forms in a block must have the same number of + comma-separated entries. + + Predefined named template rules: + <prefix=s,d,c,z> + <ftype=real,double precision,complex,double complex> + <ftypereal=real,double precision,\\0,\\1> + <ctype=float,double,complex_float,complex_double> + <ctypereal=float,double,\\0,\\1> +""" + +routine_start_re = re.compile(r'(\n|\A)(( (\$|\*))|)\s*(subroutine|function)\b', re.I) +routine_end_re = re.compile(r'\n\s*end\s*(subroutine|function)\b.*(\n|\Z)', re.I) +function_start_re = re.compile(r'\n (\$|\*)\s*function\b', re.I) + +def parse_structure(astr): + """ Return a list of tuples for each function or subroutine each + tuple is the start and end of a subroutine or function to be + expanded. + """ + + spanlist = [] + ind = 0 + while True: + m = routine_start_re.search(astr, ind) + if m is None: + break + start = m.start() + if function_start_re.match(astr, start, m.end()): + while True: + i = astr.rfind('\n', ind, start) + if i==-1: + break + start = i + if astr[i:i+7]!='\n $': + break + start += 1 + m = routine_end_re.search(astr, m.end()) + ind = end = m and m.end()-1 or len(astr) + spanlist.append((start, end)) + return spanlist + +template_re = re.compile(r"<\s*(\w[\w\d]*)\s*>") +named_re = re.compile(r"<\s*(\w[\w\d]*)\s*=\s*(.*?)\s*>") +list_re = re.compile(r"<\s*((.*?))\s*>") + +def find_repl_patterns(astr): + reps = named_re.findall(astr) + names = {} + for rep in reps: + name = rep[0].strip() or unique_key(names) + repl = rep[1].replace(r'\,', '@comma@') + thelist = conv(repl) + names[name] = thelist + return names + +def find_and_remove_repl_patterns(astr): + names = find_repl_patterns(astr) + astr = re.subn(named_re, '', astr)[0] + return astr, names + +item_re = re.compile(r"\A\\(?P<index>\d+)\Z") +def conv(astr): + b = astr.split(',') + l = [x.strip() for x in b] + for i in range(len(l)): + m = item_re.match(l[i]) + if m: + j = int(m.group('index')) + l[i] = l[j] + return ','.join(l) + +def unique_key(adict): + """ Obtain a unique key given a dictionary.""" + allkeys = list(adict.keys()) + done = False + n = 1 + while not done: + newkey = '__l%s' % (n) + if newkey in allkeys: + n += 1 + else: + done = True + return newkey + + +template_name_re = re.compile(r'\A\s*(\w[\w\d]*)\s*\Z') +def expand_sub(substr, names): + substr = substr.replace(r'\>', '@rightarrow@') + substr = substr.replace(r'\<', '@leftarrow@') + lnames = find_repl_patterns(substr) + substr = named_re.sub(r"<\1>", substr) # get rid of definition templates + + def listrepl(mobj): + thelist = conv(mobj.group(1).replace(r'\,', '@comma@')) + if template_name_re.match(thelist): + return "<%s>" % (thelist) + name = None + for key in lnames.keys(): # see if list is already in dictionary + if lnames[key] == thelist: + name = key + if name is None: # this list is not in the dictionary yet + name = unique_key(lnames) + lnames[name] = thelist + return "<%s>" % name + + substr = list_re.sub(listrepl, substr) # convert all lists to named templates + # newnames are constructed as needed + + numsubs = None + base_rule = None + rules = {} + for r in template_re.findall(substr): + if r not in rules: + thelist = lnames.get(r, names.get(r, None)) + if thelist is None: + raise ValueError('No replicates found for <%s>' % (r)) + if r not in names and not thelist.startswith('_'): + names[r] = thelist + rule = [i.replace('@comma@', ',') for i in thelist.split(',')] + num = len(rule) + + if numsubs is None: + numsubs = num + rules[r] = rule + base_rule = r + elif num == numsubs: + rules[r] = rule + else: + print("Mismatch in number of replacements (base <{}={}>) " + "for <{}={}>. Ignoring.".format(base_rule, ','.join(rules[base_rule]), r, thelist)) + if not rules: + return substr + + def namerepl(mobj): + name = mobj.group(1) + return rules.get(name, (k+1)*[name])[k] + + newstr = '' + for k in range(numsubs): + newstr += template_re.sub(namerepl, substr) + '\n\n' + + newstr = newstr.replace('@rightarrow@', '>') + newstr = newstr.replace('@leftarrow@', '<') + return newstr + +def process_str(allstr): + newstr = allstr + writestr = '' + + struct = parse_structure(newstr) + + oldend = 0 + names = {} + names.update(_special_names) + for sub in struct: + cleanedstr, defs = find_and_remove_repl_patterns(newstr[oldend:sub[0]]) + writestr += cleanedstr + names.update(defs) + writestr += expand_sub(newstr[sub[0]:sub[1]], names) + oldend = sub[1] + writestr += newstr[oldend:] + + return writestr + +include_src_re = re.compile(r"(\n|\A)\s*include\s*['\"](?P<name>[\w\d./\\]+\.src)['\"]", re.I) + +def resolve_includes(source): + d = os.path.dirname(source) + with open(source) as fid: + lines = [] + for line in fid: + m = include_src_re.match(line) + if m: + fn = m.group('name') + if not os.path.isabs(fn): + fn = os.path.join(d, fn) + if os.path.isfile(fn): + lines.extend(resolve_includes(fn)) + else: + lines.append(line) + else: + lines.append(line) + return lines + +def process_file(source): + lines = resolve_includes(source) + return process_str(''.join(lines)) + +_special_names = find_repl_patterns(''' +<_c=s,d,c,z> +<_t=real,double precision,complex,double complex> +<prefix=s,d,c,z> +<ftype=real,double precision,complex,double complex> +<ctype=float,double,complex_float,complex_double> +<ftypereal=real,double precision,\\0,\\1> +<ctypereal=float,double,\\0,\\1> +''') + +# END OF CODE VENDORED FROM `numpy.distutils.from_template` +########################################################### diff --git a/contrib/python/numpy/py3/numpy/f2py/auxfuncs.py b/contrib/python/numpy/py3/numpy/f2py/auxfuncs.py index 0c08e0a5e2..13a1074b44 100644 --- a/contrib/python/numpy/py3/numpy/f2py/auxfuncs.py +++ b/contrib/python/numpy/py3/numpy/f2py/auxfuncs.py @@ -1,18 +1,12 @@ -#!/usr/bin/env python3 """ - Auxiliary functions for f2py2e. -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy (BSD style) LICENSE. - NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/07/24 19:01:55 $ -Pearu Peterson - """ import pprint import sys @@ -50,7 +44,7 @@ __all__ = [ 'isunsigned_long_longarray', 'isunsigned_short', 'isunsigned_shortarray', 'l_and', 'l_not', 'l_or', 'outmess', 'replace', 'show', 'stripcomma', 'throw_error', 'isattr_value', - 'deep_merge' + 'getuseblocks', 'process_f2cmap_dict' ] @@ -899,28 +893,6 @@ def applyrules(rules, d, var={}): del ret[k] return ret -def deep_merge(dict1, dict2): - """Recursively merge two dictionaries into a new dictionary. - - Parameters: - - dict1: The base dictionary. - - dict2: The dictionary to merge into a copy of dict1. - If a key exists in both, the dict2 value will take precedence. - - Returns: - - A new merged dictionary. - """ - merged_dict = deepcopy(dict1) - for key, value in dict2.items(): - if key in merged_dict: - if isinstance(merged_dict[key], dict) and isinstance(value, dict): - merged_dict[key] = deep_merge(merged_dict[key], value) - else: - merged_dict[key] = value - else: - merged_dict[key] = value - return merged_dict - _f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P<name>[\w_]+)', re.I).match _f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P<name>[\w_]*?' @@ -937,3 +909,80 @@ def get_f2py_modulename(source): name = m.group('name') break return name + +def getuseblocks(pymod): + all_uses = [] + for inner in pymod['body']: + for modblock in inner['body']: + if modblock.get('use'): + all_uses.extend([x for x in modblock.get("use").keys() if "__" not in x]) + return all_uses + +def process_f2cmap_dict(f2cmap_all, new_map, c2py_map, verbose = False): + """ + Update the Fortran-to-C type mapping dictionary with new mappings and + return a list of successfully mapped C types. + + This function integrates a new mapping dictionary into an existing + Fortran-to-C type mapping dictionary. It ensures that all keys are in + lowercase and validates new entries against a given C-to-Python mapping + dictionary. Redefinitions and invalid entries are reported with a warning. + + Parameters + ---------- + f2cmap_all : dict + The existing Fortran-to-C type mapping dictionary that will be updated. + It should be a dictionary of dictionaries where the main keys represent + Fortran types and the nested dictionaries map Fortran type specifiers + to corresponding C types. + + new_map : dict + A dictionary containing new type mappings to be added to `f2cmap_all`. + The structure should be similar to `f2cmap_all`, with keys representing + Fortran types and values being dictionaries of type specifiers and their + C type equivalents. + + c2py_map : dict + A dictionary used for validating the C types in `new_map`. It maps C + types to corresponding Python types and is used to ensure that the C + types specified in `new_map` are valid. + + verbose : boolean + A flag used to provide information about the types mapped + + Returns + ------- + tuple of (dict, list) + The updated Fortran-to-C type mapping dictionary and a list of + successfully mapped C types. + """ + f2cmap_mapped = [] + + new_map_lower = {} + for k, d1 in new_map.items(): + d1_lower = {k1.lower(): v1 for k1, v1 in d1.items()} + new_map_lower[k.lower()] = d1_lower + + for k, d1 in new_map_lower.items(): + if k not in f2cmap_all: + f2cmap_all[k] = {} + + for k1, v1 in d1.items(): + if v1 in c2py_map: + if k1 in f2cmap_all[k]: + outmess( + "\tWarning: redefinition of {'%s':{'%s':'%s'->'%s'}}\n" + % (k, k1, f2cmap_all[k][k1], v1) + ) + f2cmap_all[k][k1] = v1 + if verbose: + outmess('\tMapping "%s(kind=%s)" to "%s"\n' % (k, k1, v1)) + f2cmap_mapped.append(v1) + else: + if verbose: + errmess( + "\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" + % (k, k1, v1, v1, list(c2py_map.keys())) + ) + + return f2cmap_all, f2cmap_mapped diff --git a/contrib/python/numpy/py3/numpy/f2py/capi_maps.py b/contrib/python/numpy/py3/numpy/f2py/capi_maps.py index 32b6db5c59..fa477a5b9a 100644 --- a/contrib/python/numpy/py3/numpy/f2py/capi_maps.py +++ b/contrib/python/numpy/py3/numpy/f2py/capi_maps.py @@ -1,15 +1,10 @@ -#!/usr/bin/env python3 """ - -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 10:57:33 $ -Pearu Peterson - """ from . import __version__ f2py_version = __version__.version @@ -19,7 +14,7 @@ import re import os from .crackfortran import markoutercomma from . import cb_rules -from ._isocbind import iso_c_binding_map +from ._isocbind import iso_c_binding_map, isoc_c2pycode_map, iso_c2py_map # The environment provided by auxfuncs.py is needed for some calls to eval. # As the needed functions cannot be determined by static inspection of the @@ -29,7 +24,7 @@ from .auxfuncs import * __all__ = [ 'getctype', 'getstrlength', 'getarrdims', 'getpydocsign', 'getarrdocsign', 'getinit', 'sign2map', 'routsign2map', 'modsign2map', - 'cb_sign2map', 'cb_routsign2map', 'common_sign2map' + 'cb_sign2map', 'cb_routsign2map', 'common_sign2map', 'process_f2cmap_dict' ] @@ -131,13 +126,17 @@ f2cmap_all = {'real': {'': 'float', '4': 'float', '8': 'double', 'byte': {'': 'char'}, } -f2cmap_all = deep_merge(f2cmap_all, iso_c_binding_map) +# Add ISO_C handling +c2pycode_map.update(isoc_c2pycode_map) +c2py_map.update(iso_c2py_map) +f2cmap_all, _ = process_f2cmap_dict(f2cmap_all, iso_c_binding_map, c2py_map) +# End ISO_C handling f2cmap_default = copy.deepcopy(f2cmap_all) f2cmap_mapped = [] def load_f2cmap_file(f2cmap_file): - global f2cmap_all + global f2cmap_all, f2cmap_mapped f2cmap_all = copy.deepcopy(f2cmap_default) @@ -156,29 +155,11 @@ def load_f2cmap_file(f2cmap_file): outmess('Reading f2cmap from {!r} ...\n'.format(f2cmap_file)) with open(f2cmap_file) as f: d = eval(f.read().lower(), {}, {}) - for k, d1 in d.items(): - for k1 in d1.keys(): - d1[k1.lower()] = d1[k1] - d[k.lower()] = d[k] - for k in d.keys(): - if k not in f2cmap_all: - f2cmap_all[k] = {} - for k1 in d[k].keys(): - if d[k][k1] in c2py_map: - if k1 in f2cmap_all[k]: - outmess( - "\tWarning: redefinition of {'%s':{'%s':'%s'->'%s'}}\n" % (k, k1, f2cmap_all[k][k1], d[k][k1])) - f2cmap_all[k][k1] = d[k][k1] - outmess('\tMapping "%s(kind=%s)" to "%s"\n' % - (k, k1, d[k][k1])) - f2cmap_mapped.append(d[k][k1]) - else: - errmess("\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" % ( - k, k1, d[k][k1], d[k][k1], list(c2py_map.keys()))) + f2cmap_all, f2cmap_mapped = process_f2cmap_dict(f2cmap_all, d, c2py_map, True) outmess('Successfully applied user defined f2cmap changes\n') except Exception as msg: - errmess( - 'Failed to apply user defined f2cmap changes: %s. Skipping.\n' % (msg)) + errmess('Failed to apply user defined f2cmap changes: %s. Skipping.\n' % (msg)) + cformat_map = {'double': '%g', 'float': '%g', diff --git a/contrib/python/numpy/py3/numpy/f2py/cb_rules.py b/contrib/python/numpy/py3/numpy/f2py/cb_rules.py index 761831e004..721e075b6c 100644 --- a/contrib/python/numpy/py3/numpy/f2py/cb_rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/cb_rules.py @@ -1,17 +1,12 @@ -#!/usr/bin/env python3 """ - Build call-back mechanism for f2py2e. -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/07/20 11:27:58 $ -Pearu Peterson - """ from . import __version__ from .auxfuncs import ( diff --git a/contrib/python/numpy/py3/numpy/f2py/cfuncs.py b/contrib/python/numpy/py3/numpy/f2py/cfuncs.py index f89793061b..4328a6e500 100644 --- a/contrib/python/numpy/py3/numpy/f2py/cfuncs.py +++ b/contrib/python/numpy/py3/numpy/f2py/cfuncs.py @@ -1,18 +1,14 @@ #!/usr/bin/env python3 """ - C declarations, CPP macros, and C functions for f2py2e. Only required declarations/macros/functions will be used. -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 11:42:34 $ -Pearu Peterson - """ import sys import copy @@ -64,7 +60,7 @@ typedefs['unsigned_char'] = 'typedef unsigned char unsigned_char;' typedefs['unsigned_short'] = 'typedef unsigned short unsigned_short;' typedefs['unsigned_long'] = 'typedef unsigned long unsigned_long;' typedefs['signed_char'] = 'typedef signed char signed_char;' -typedefs['long_long'] = """\ +typedefs['long_long'] = """ #if defined(NPY_OS_WIN32) typedef __int64 long_long; #else @@ -72,14 +68,14 @@ typedef long long long_long; typedef unsigned long long unsigned_long_long; #endif """ -typedefs['unsigned_long_long'] = """\ +typedefs['unsigned_long_long'] = """ #if defined(NPY_OS_WIN32) typedef __uint64 long_long; #else typedef unsigned long long unsigned_long_long; #endif """ -typedefs['long_double'] = """\ +typedefs['long_double'] = """ #ifndef _LONG_DOUBLE typedef long double long_double; #endif @@ -93,7 +89,7 @@ typedefs['character'] = """typedef char character;""" ############### CPP macros #################### -cppmacros['CFUNCSMESS'] = """\ +cppmacros['CFUNCSMESS'] = """ #ifdef DEBUGCFUNCS #define CFUNCSMESS(mess) fprintf(stderr,\"debug-capi:\"mess); #define CFUNCSMESSPY(mess,obj) CFUNCSMESS(mess) \\ @@ -104,7 +100,7 @@ cppmacros['CFUNCSMESS'] = """\ #define CFUNCSMESSPY(mess,obj) #endif """ -cppmacros['F_FUNC'] = """\ +cppmacros['F_FUNC'] = """ #if defined(PREPEND_FORTRAN) #if defined(NO_APPEND_FORTRAN) #if defined(UPPERCASE_FORTRAN) @@ -140,7 +136,7 @@ cppmacros['F_FUNC'] = """\ #define F_FUNC_US(f,F) F_FUNC(f,F) #endif """ -cppmacros['F_WRAPPEDFUNC'] = """\ +cppmacros['F_WRAPPEDFUNC'] = """ #if defined(PREPEND_FORTRAN) #if defined(NO_APPEND_FORTRAN) #if defined(UPPERCASE_FORTRAN) @@ -176,7 +172,7 @@ cppmacros['F_WRAPPEDFUNC'] = """\ #define F_WRAPPEDFUNC_US(f,F) F_WRAPPEDFUNC(f,F) #endif """ -cppmacros['F_MODFUNC'] = """\ +cppmacros['F_MODFUNC'] = """ #if defined(F90MOD2CCONV1) /*E.g. Compaq Fortran */ #if defined(NO_APPEND_FORTRAN) #define F_MODFUNCNAME(m,f) $ ## m ## $ ## f @@ -210,12 +206,12 @@ cppmacros['F_MODFUNC'] = """\ #define F_MODFUNC(m,f) (*(f2pymodstruct##m##.##f)) """ -cppmacros['SWAPUNSAFE'] = """\ +cppmacros['SWAPUNSAFE'] = """ #define SWAP(a,b) (size_t)(a) = ((size_t)(a) ^ (size_t)(b));\\ (size_t)(b) = ((size_t)(a) ^ (size_t)(b));\\ (size_t)(a) = ((size_t)(a) ^ (size_t)(b)) """ -cppmacros['SWAP'] = """\ +cppmacros['SWAP'] = """ #define SWAP(a,b,t) {\\ t *c;\\ c = a;\\ @@ -224,13 +220,13 @@ cppmacros['SWAP'] = """\ """ # cppmacros['ISCONTIGUOUS']='#define ISCONTIGUOUS(m) (PyArray_FLAGS(m) & # NPY_ARRAY_C_CONTIGUOUS)' -cppmacros['PRINTPYOBJERR'] = """\ +cppmacros['PRINTPYOBJERR'] = """ #define PRINTPYOBJERR(obj)\\ fprintf(stderr,\"#modulename#.error is related to \");\\ PyObject_Print((PyObject *)obj,stderr,Py_PRINT_RAW);\\ fprintf(stderr,\"\\n\"); """ -cppmacros['MINMAX'] = """\ +cppmacros['MINMAX'] = """ #ifndef max #define max(a,b) ((a > b) ? (a) : (b)) #endif @@ -244,7 +240,7 @@ cppmacros['MINMAX'] = """\ #define MIN(a,b) ((a < b) ? (a) : (b)) #endif """ -cppmacros['len..'] = """\ +cppmacros['len..'] = """ /* See fortranobject.h for definitions. The macros here are provided for BC. */ #define rank f2py_rank #define shape f2py_shape @@ -254,16 +250,21 @@ cppmacros['len..'] = """\ #define slen f2py_slen #define size f2py_size """ -cppmacros[ - 'pyobj_from_char1'] = '#define pyobj_from_char1(v) (PyLong_FromLong(v))' -cppmacros[ - 'pyobj_from_short1'] = '#define pyobj_from_short1(v) (PyLong_FromLong(v))' +cppmacros['pyobj_from_char1'] = r""" +#define pyobj_from_char1(v) (PyLong_FromLong(v)) +""" +cppmacros['pyobj_from_short1'] = r""" +#define pyobj_from_short1(v) (PyLong_FromLong(v)) +""" needs['pyobj_from_int1'] = ['signed_char'] -cppmacros['pyobj_from_int1'] = '#define pyobj_from_int1(v) (PyLong_FromLong(v))' -cppmacros[ - 'pyobj_from_long1'] = '#define pyobj_from_long1(v) (PyLong_FromLong(v))' +cppmacros['pyobj_from_int1'] = r""" +#define pyobj_from_int1(v) (PyLong_FromLong(v)) +""" +cppmacros['pyobj_from_long1'] = r""" +#define pyobj_from_long1(v) (PyLong_FromLong(v)) +""" needs['pyobj_from_long_long1'] = ['long_long'] -cppmacros['pyobj_from_long_long1'] = """\ +cppmacros['pyobj_from_long_long1'] = """ #ifdef HAVE_LONG_LONG #define pyobj_from_long_long1(v) (PyLong_FromLongLong(v)) #else @@ -272,29 +273,29 @@ cppmacros['pyobj_from_long_long1'] = """\ #endif """ needs['pyobj_from_long_double1'] = ['long_double'] -cppmacros[ - 'pyobj_from_long_double1'] = '#define pyobj_from_long_double1(v) (PyFloat_FromDouble(v))' -cppmacros[ - 'pyobj_from_double1'] = '#define pyobj_from_double1(v) (PyFloat_FromDouble(v))' -cppmacros[ - 'pyobj_from_float1'] = '#define pyobj_from_float1(v) (PyFloat_FromDouble(v))' +cppmacros['pyobj_from_long_double1'] = """ +#define pyobj_from_long_double1(v) (PyFloat_FromDouble(v))""" +cppmacros['pyobj_from_double1'] = """ +#define pyobj_from_double1(v) (PyFloat_FromDouble(v))""" +cppmacros['pyobj_from_float1'] = """ +#define pyobj_from_float1(v) (PyFloat_FromDouble(v))""" needs['pyobj_from_complex_long_double1'] = ['complex_long_double'] -cppmacros[ - 'pyobj_from_complex_long_double1'] = '#define pyobj_from_complex_long_double1(v) (PyComplex_FromDoubles(v.r,v.i))' +cppmacros['pyobj_from_complex_long_double1'] = """ +#define pyobj_from_complex_long_double1(v) (PyComplex_FromDoubles(v.r,v.i))""" needs['pyobj_from_complex_double1'] = ['complex_double'] -cppmacros[ - 'pyobj_from_complex_double1'] = '#define pyobj_from_complex_double1(v) (PyComplex_FromDoubles(v.r,v.i))' +cppmacros['pyobj_from_complex_double1'] = """ +#define pyobj_from_complex_double1(v) (PyComplex_FromDoubles(v.r,v.i))""" needs['pyobj_from_complex_float1'] = ['complex_float'] -cppmacros[ - 'pyobj_from_complex_float1'] = '#define pyobj_from_complex_float1(v) (PyComplex_FromDoubles(v.r,v.i))' +cppmacros['pyobj_from_complex_float1'] = """ +#define pyobj_from_complex_float1(v) (PyComplex_FromDoubles(v.r,v.i))""" needs['pyobj_from_string1'] = ['string'] -cppmacros[ - 'pyobj_from_string1'] = '#define pyobj_from_string1(v) (PyUnicode_FromString((char *)v))' +cppmacros['pyobj_from_string1'] = """ +#define pyobj_from_string1(v) (PyUnicode_FromString((char *)v))""" needs['pyobj_from_string1size'] = ['string'] -cppmacros[ - 'pyobj_from_string1size'] = '#define pyobj_from_string1size(v,len) (PyUnicode_FromStringAndSize((char *)v, len))' +cppmacros['pyobj_from_string1size'] = """ +#define pyobj_from_string1size(v,len) (PyUnicode_FromStringAndSize((char *)v, len))""" needs['TRYPYARRAYTEMPLATE'] = ['PRINTPYOBJERR'] -cppmacros['TRYPYARRAYTEMPLATE'] = """\ +cppmacros['TRYPYARRAYTEMPLATE'] = """ /* New SciPy */ #define TRYPYARRAYTEMPLATECHAR case NPY_STRING: *(char *)(PyArray_DATA(arr))=*v; break; #define TRYPYARRAYTEMPLATELONG case NPY_LONG: *(long *)(PyArray_DATA(arr))=*v; break; @@ -331,7 +332,7 @@ cppmacros['TRYPYARRAYTEMPLATE'] = """\ """ needs['TRYCOMPLEXPYARRAYTEMPLATE'] = ['PRINTPYOBJERR'] -cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """\ +cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """ #define TRYCOMPLEXPYARRAYTEMPLATEOBJECT case NPY_OBJECT: PyArray_SETITEM(arr, PyArray_DATA(arr), pyobj_from_complex_ ## ctype ## 1((*v))); break; #define TRYCOMPLEXPYARRAYTEMPLATE(ctype,typecode)\\ PyArrayObject *arr = NULL;\\ @@ -372,7 +373,7 @@ cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """\ };\\ return -1; """ -# cppmacros['NUMFROMARROBJ']="""\ +# cppmacros['NUMFROMARROBJ']=""" # define NUMFROMARROBJ(typenum,ctype) \\ # if (PyArray_Check(obj)) arr = (PyArrayObject *)obj;\\ # else arr = (PyArrayObject *)PyArray_ContiguousFromObject(obj,typenum,0,0);\\ @@ -388,7 +389,7 @@ cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """\ # } # """ # XXX: Note that CNUMFROMARROBJ is identical with NUMFROMARROBJ -# cppmacros['CNUMFROMARROBJ']="""\ +# cppmacros['CNUMFROMARROBJ']=""" # define CNUMFROMARROBJ(typenum,ctype) \\ # if (PyArray_Check(obj)) arr = (PyArrayObject *)obj;\\ # else arr = (PyArrayObject *)PyArray_ContiguousFromObject(obj,typenum,0,0);\\ @@ -406,7 +407,7 @@ cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """\ needs['GETSTRFROMPYTUPLE'] = ['STRINGCOPYN', 'PRINTPYOBJERR'] -cppmacros['GETSTRFROMPYTUPLE'] = """\ +cppmacros['GETSTRFROMPYTUPLE'] = """ #define GETSTRFROMPYTUPLE(tuple,index,str,len) {\\ PyObject *rv_cb_str = PyTuple_GetItem((tuple),(index));\\ if (rv_cb_str == NULL)\\ @@ -421,7 +422,7 @@ cppmacros['GETSTRFROMPYTUPLE'] = """\ }\\ } """ -cppmacros['GETSCALARFROMPYTUPLE'] = """\ +cppmacros['GETSCALARFROMPYTUPLE'] = """ #define GETSCALARFROMPYTUPLE(tuple,index,var,ctype,mess) {\\ if ((capi_tmp = PyTuple_GetItem((tuple),(index)))==NULL) goto capi_fail;\\ if (!(ctype ## _from_pyobj((var),capi_tmp,mess)))\\ @@ -429,7 +430,7 @@ cppmacros['GETSCALARFROMPYTUPLE'] = """\ } """ -cppmacros['FAILNULL'] = """\\ +cppmacros['FAILNULL'] = """\ #define FAILNULL(p) do { \\ if ((p) == NULL) { \\ PyErr_SetString(PyExc_MemoryError, "NULL pointer found"); \\ @@ -438,11 +439,11 @@ cppmacros['FAILNULL'] = """\\ } while (0) """ needs['MEMCOPY'] = ['string.h', 'FAILNULL'] -cppmacros['MEMCOPY'] = """\ +cppmacros['MEMCOPY'] = """ #define MEMCOPY(to,from,n)\\ do { FAILNULL(to); FAILNULL(from); (void)memcpy(to,from,n); } while (0) """ -cppmacros['STRINGMALLOC'] = """\ +cppmacros['STRINGMALLOC'] = """ #define STRINGMALLOC(str,len)\\ if ((str = (string)malloc(len+1)) == NULL) {\\ PyErr_SetString(PyExc_MemoryError, \"out of memory\");\\ @@ -451,11 +452,11 @@ cppmacros['STRINGMALLOC'] = """\ (str)[len] = '\\0';\\ } """ -cppmacros['STRINGFREE'] = """\ +cppmacros['STRINGFREE'] = """ #define STRINGFREE(str) do {if (!(str == NULL)) free(str);} while (0) """ needs['STRINGPADN'] = ['string.h'] -cppmacros['STRINGPADN'] = """\ +cppmacros['STRINGPADN'] = """ /* STRINGPADN replaces null values with padding values from the right. @@ -476,7 +477,7 @@ STRINGPADN(to, N, PADDING, NULLVALUE) is an inverse operation. } while (0) """ needs['STRINGCOPYN'] = ['string.h', 'FAILNULL'] -cppmacros['STRINGCOPYN'] = """\ +cppmacros['STRINGCOPYN'] = """ /* STRINGCOPYN copies N bytes. @@ -492,23 +493,23 @@ STRINGCOPYN copies N bytes. } while (0) """ needs['STRINGCOPY'] = ['string.h', 'FAILNULL'] -cppmacros['STRINGCOPY'] = """\ +cppmacros['STRINGCOPY'] = """ #define STRINGCOPY(to,from)\\ do { FAILNULL(to); FAILNULL(from); (void)strcpy(to,from); } while (0) """ -cppmacros['CHECKGENERIC'] = """\ +cppmacros['CHECKGENERIC'] = """ #define CHECKGENERIC(check,tcheck,name) \\ if (!(check)) {\\ PyErr_SetString(#modulename#_error,\"(\"tcheck\") failed for \"name);\\ /*goto capi_fail;*/\\ } else """ -cppmacros['CHECKARRAY'] = """\ +cppmacros['CHECKARRAY'] = """ #define CHECKARRAY(check,tcheck,name) \\ if (!(check)) {\\ PyErr_SetString(#modulename#_error,\"(\"tcheck\") failed for \"name);\\ /*goto capi_fail;*/\\ } else """ -cppmacros['CHECKSTRING'] = """\ +cppmacros['CHECKSTRING'] = """ #define CHECKSTRING(check,tcheck,name,show,var)\\ if (!(check)) {\\ char errstring[256];\\ @@ -516,7 +517,7 @@ cppmacros['CHECKSTRING'] = """\ PyErr_SetString(#modulename#_error, errstring);\\ /*goto capi_fail;*/\\ } else """ -cppmacros['CHECKSCALAR'] = """\ +cppmacros['CHECKSCALAR'] = """ #define CHECKSCALAR(check,tcheck,name,show,var)\\ if (!(check)) {\\ char errstring[256];\\ @@ -524,7 +525,7 @@ cppmacros['CHECKSCALAR'] = """\ PyErr_SetString(#modulename#_error,errstring);\\ /*goto capi_fail;*/\\ } else """ -# cppmacros['CHECKDIMS']="""\ +# cppmacros['CHECKDIMS']=""" # define CHECKDIMS(dims,rank) \\ # for (int i=0;i<(rank);i++)\\ # if (dims[i]<0) {\\ @@ -534,12 +535,12 @@ cppmacros['CHECKSCALAR'] = """\ # """ cppmacros[ 'ARRSIZE'] = '#define ARRSIZE(dims,rank) (_PyArray_multiply_list(dims,rank))' -cppmacros['OLDPYNUM'] = """\ +cppmacros['OLDPYNUM'] = """ #ifdef OLDPYNUM #error You need to install NumPy version 0.13 or higher. See https://scipy.org/install.html #endif """ -cppmacros["F2PY_THREAD_LOCAL_DECL"] = """\ +cppmacros["F2PY_THREAD_LOCAL_DECL"] = """ #ifndef F2PY_THREAD_LOCAL_DECL #if defined(_MSC_VER) #define F2PY_THREAD_LOCAL_DECL __declspec(thread) @@ -565,21 +566,21 @@ cppmacros["F2PY_THREAD_LOCAL_DECL"] = """\ """ ################# C functions ############### -cfuncs['calcarrindex'] = """\ +cfuncs['calcarrindex'] = """ static int calcarrindex(int *i,PyArrayObject *arr) { int k,ii = i[0]; for (k=1; k < PyArray_NDIM(arr); k++) ii += (ii*(PyArray_DIM(arr,k) - 1)+i[k]); /* assuming contiguous arr */ return ii; }""" -cfuncs['calcarrindextr'] = """\ +cfuncs['calcarrindextr'] = """ static int calcarrindextr(int *i,PyArrayObject *arr) { int k,ii = i[PyArray_NDIM(arr)-1]; for (k=1; k < PyArray_NDIM(arr); k++) ii += (ii*(PyArray_DIM(arr,PyArray_NDIM(arr)-k-1) - 1)+i[PyArray_NDIM(arr)-k-1]); /* assuming contiguous arr */ return ii; }""" -cfuncs['forcomb'] = """\ +cfuncs['forcomb'] = """ static struct { int nd;npy_intp *d;int *i,*i_tr,tr; } forcombcache; static int initforcomb(npy_intp *dims,int nd,int tr) { int k; @@ -620,7 +621,7 @@ static int *nextforcomb(void) { return i; }""" needs['try_pyarr_from_string'] = ['STRINGCOPYN', 'PRINTPYOBJERR', 'string'] -cfuncs['try_pyarr_from_string'] = """\ +cfuncs['try_pyarr_from_string'] = """ /* try_pyarr_from_string copies str[:len(obj)] to the data of an `ndarray`. @@ -634,6 +635,9 @@ static int try_pyarr_from_string(PyObject *obj, fprintf(stderr, "try_pyarr_from_string(str='%s', len=%d, obj=%p)\\n", (char*)str,len, obj); #endif + if (!obj) return -2; /* Object missing */ + if (obj == Py_None) return -1; /* None */ + if (!PyArray_Check(obj)) goto capi_fail; /* not an ndarray */ if (PyArray_Check(obj)) { PyArrayObject *arr = (PyArrayObject *)obj; assert(ISCONTIGUOUS(arr)); @@ -656,7 +660,7 @@ capi_fail: } """ needs['string_from_pyobj'] = ['string', 'STRINGMALLOC', 'STRINGCOPYN'] -cfuncs['string_from_pyobj'] = """\ +cfuncs['string_from_pyobj'] = """ /* Create a new string buffer `str` of at most length `len` from a Python string-like object `obj`. @@ -756,7 +760,7 @@ capi_fail: } """ -cfuncs['character_from_pyobj'] = """\ +cfuncs['character_from_pyobj'] = """ static int character_from_pyobj(character* v, PyObject *obj, const char *errmess) { if (PyBytes_Check(obj)) { @@ -823,8 +827,10 @@ character_from_pyobj(character* v, PyObject *obj, const char *errmess) { } """ +# TODO: These should be dynamically generated, too many mapped to int things, +# see note in _isocbind.py needs['char_from_pyobj'] = ['int_from_pyobj'] -cfuncs['char_from_pyobj'] = """\ +cfuncs['char_from_pyobj'] = """ static int char_from_pyobj(char* v, PyObject *obj, const char *errmess) { int i = 0; @@ -838,7 +844,7 @@ char_from_pyobj(char* v, PyObject *obj, const char *errmess) { needs['signed_char_from_pyobj'] = ['int_from_pyobj', 'signed_char'] -cfuncs['signed_char_from_pyobj'] = """\ +cfuncs['signed_char_from_pyobj'] = """ static int signed_char_from_pyobj(signed_char* v, PyObject *obj, const char *errmess) { int i = 0; @@ -852,7 +858,7 @@ signed_char_from_pyobj(signed_char* v, PyObject *obj, const char *errmess) { needs['short_from_pyobj'] = ['int_from_pyobj'] -cfuncs['short_from_pyobj'] = """\ +cfuncs['short_from_pyobj'] = """ static int short_from_pyobj(short* v, PyObject *obj, const char *errmess) { int i = 0; @@ -865,7 +871,7 @@ short_from_pyobj(short* v, PyObject *obj, const char *errmess) { """ -cfuncs['int_from_pyobj'] = """\ +cfuncs['int_from_pyobj'] = """ static int int_from_pyobj(int* v, PyObject *obj, const char *errmess) { @@ -915,7 +921,7 @@ int_from_pyobj(int* v, PyObject *obj, const char *errmess) """ -cfuncs['long_from_pyobj'] = """\ +cfuncs['long_from_pyobj'] = """ static int long_from_pyobj(long* v, PyObject *obj, const char *errmess) { PyObject* tmp = NULL; @@ -964,7 +970,7 @@ long_from_pyobj(long* v, PyObject *obj, const char *errmess) { needs['long_long_from_pyobj'] = ['long_long'] -cfuncs['long_long_from_pyobj'] = """\ +cfuncs['long_long_from_pyobj'] = """ static int long_long_from_pyobj(long_long* v, PyObject *obj, const char *errmess) { @@ -1014,7 +1020,7 @@ long_long_from_pyobj(long_long* v, PyObject *obj, const char *errmess) needs['long_double_from_pyobj'] = ['double_from_pyobj', 'long_double'] -cfuncs['long_double_from_pyobj'] = """\ +cfuncs['long_double_from_pyobj'] = """ static int long_double_from_pyobj(long_double* v, PyObject *obj, const char *errmess) { @@ -1038,7 +1044,7 @@ long_double_from_pyobj(long_double* v, PyObject *obj, const char *errmess) """ -cfuncs['double_from_pyobj'] = """\ +cfuncs['double_from_pyobj'] = """ static int double_from_pyobj(double* v, PyObject *obj, const char *errmess) { @@ -1082,7 +1088,7 @@ double_from_pyobj(double* v, PyObject *obj, const char *errmess) needs['float_from_pyobj'] = ['double_from_pyobj'] -cfuncs['float_from_pyobj'] = """\ +cfuncs['float_from_pyobj'] = """ static int float_from_pyobj(float* v, PyObject *obj, const char *errmess) { @@ -1098,7 +1104,7 @@ float_from_pyobj(float* v, PyObject *obj, const char *errmess) needs['complex_long_double_from_pyobj'] = ['complex_long_double', 'long_double', 'complex_double_from_pyobj', 'npy_math.h'] -cfuncs['complex_long_double_from_pyobj'] = """\ +cfuncs['complex_long_double_from_pyobj'] = """ static int complex_long_double_from_pyobj(complex_long_double* v, PyObject *obj, const char *errmess) { @@ -1125,7 +1131,7 @@ complex_long_double_from_pyobj(complex_long_double* v, PyObject *obj, const char needs['complex_double_from_pyobj'] = ['complex_double', 'npy_math.h'] -cfuncs['complex_double_from_pyobj'] = """\ +cfuncs['complex_double_from_pyobj'] = """ static int complex_double_from_pyobj(complex_double* v, PyObject *obj, const char *errmess) { Py_complex c; @@ -1202,7 +1208,7 @@ complex_double_from_pyobj(complex_double* v, PyObject *obj, const char *errmess) needs['complex_float_from_pyobj'] = [ 'complex_float', 'complex_double_from_pyobj'] -cfuncs['complex_float_from_pyobj'] = """\ +cfuncs['complex_float_from_pyobj'] = """ static int complex_float_from_pyobj(complex_float* v,PyObject *obj,const char *errmess) { @@ -1217,7 +1223,7 @@ complex_float_from_pyobj(complex_float* v,PyObject *obj,const char *errmess) """ -cfuncs['try_pyarr_from_character'] = """\ +cfuncs['try_pyarr_from_character'] = """ static int try_pyarr_from_character(PyObject* obj, character* v) { PyArrayObject *arr = (PyArrayObject*)obj; if (!obj) return -2; @@ -1282,7 +1288,7 @@ cfuncs[ needs['create_cb_arglist'] = ['CFUNCSMESS', 'PRINTPYOBJERR', 'MINMAX'] # create the list of arguments to be used when calling back to python -cfuncs['create_cb_arglist'] = """\ +cfuncs['create_cb_arglist'] = """ static int create_cb_arglist(PyObject* fun, PyTupleObject* xa , const int maxnofargs, const int nofoptargs, int *nofargs, PyTupleObject **args, diff --git a/contrib/python/numpy/py3/numpy/f2py/common_rules.py b/contrib/python/numpy/py3/numpy/f2py/common_rules.py index 5a488bf5a5..64347b7374 100644 --- a/contrib/python/numpy/py3/numpy/f2py/common_rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/common_rules.py @@ -1,23 +1,18 @@ -#!/usr/bin/env python3 """ - Build common block mechanism for f2py2e. -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 10:57:33 $ -Pearu Peterson - """ from . import __version__ f2py_version = __version__.version from .auxfuncs import ( - hasbody, hascommon, hasnote, isintent_hide, outmess + hasbody, hascommon, hasnote, isintent_hide, outmess, getuseblocks ) from . import capi_maps from . import func2subr @@ -78,6 +73,8 @@ def buildhooks(m): outmess('\t\tConstructing COMMON block support for "%s"...\n\t\t %s\n' % ( name, ','.join(inames))) fadd('subroutine f2pyinit%s(setupfunc)' % name) + for usename in getuseblocks(m): + fadd(f'use {usename}') fadd('external setupfunc') for n in vnames: fadd(func2subr.var2fixfortran(vars, n)) diff --git a/contrib/python/numpy/py3/numpy/f2py/crackfortran.py b/contrib/python/numpy/py3/numpy/f2py/crackfortran.py index f352bbaa27..8d3fc27608 100755 --- a/contrib/python/numpy/py3/numpy/f2py/crackfortran.py +++ b/contrib/python/numpy/py3/numpy/f2py/crackfortran.py @@ -2,14 +2,12 @@ """ crackfortran --- read fortran (77,90) code and extract declaration information. -Copyright 1999-2004 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/09/27 07:13:49 $ -Pearu Peterson Usage of crackfortran: @@ -995,6 +993,16 @@ def _resolvenameargspattern(line): def analyzeline(m, case, line): + """ + Reads each line in the input file in sequence and updates global vars. + + Effectively reads and collects information from the input file to the + global variable groupcache, a dictionary containing info about each part + of the fortran module. + + At the end of analyzeline, information is filtered into the correct dict + keys, but parameter values and dimensions are not yet interpreted. + """ global groupcounter, groupname, groupcache, grouplist, filepositiontext global currentfilename, f77modulename, neededinterface, neededmodule global expectbegin, gotnextfile, previous_context @@ -1681,10 +1689,18 @@ def markinnerspaces(line): def updatevars(typespec, selector, attrspec, entitydecl): + """ + Returns last_name, the variable name without special chars, parenthesis + or dimension specifiers. + + Alters groupcache to add the name, typespec, attrspec (and possibly value) + of current variable. + """ global groupcache, groupcounter last_name = None kindselect, charselect, typename = cracktypespec(typespec, selector) + # Clean up outer commas, whitespace and undesired chars from attrspec if attrspec: attrspec = [x.strip() for x in markoutercomma(attrspec).split('@,@')] l = [] @@ -2398,8 +2414,6 @@ def _calc_depend_dict(vars): def get_sorted_names(vars): - """ - """ depend_dict = _calc_depend_dict(vars) names = [] for name in list(depend_dict.keys()): @@ -2452,7 +2466,7 @@ def _selected_real_kind_func(p, r=0, radix=0): if p < 16: return 8 machine = platform.machine().lower() - if machine.startswith(('aarch64', 'arm64', 'loongarch', 'power', 'ppc', 'riscv', 's390x', 'sparc')): + if machine.startswith(('aarch64', 'alpha', 'arm64', 'loongarch', 'mips', 'power', 'ppc', 'riscv', 's390x', 'sparc')): if p <= 33: return 16 else: @@ -2491,6 +2505,7 @@ def get_parameters(vars, global_params={}): # TODO: test .eq., .neq., etc replacements. ]: v = v.replace(*repl) + v = kind_re.sub(r'kind("\1")', v) v = selected_int_kind_re.sub(r'selected_int_kind(\1)', v) @@ -2499,14 +2514,17 @@ def get_parameters(vars, global_params={}): # then we may easily remove those specifiers. # However, it may be that the user uses other specifiers...(!) is_replaced = False + if 'kindselector' in vars[n]: + # Remove kind specifier (including those defined + # by parameters) if 'kind' in vars[n]['kindselector']: orig_v_len = len(v) v = v.replace('_' + vars[n]['kindselector']['kind'], '') # Again, this will be true if even a single specifier # has been replaced, see comment above. is_replaced = len(v) < orig_v_len - + if not is_replaced: if not selected_kind_re.match(v): v_ = v.split('_') @@ -2533,6 +2551,10 @@ def get_parameters(vars, global_params={}): outmess(f'get_parameters[TODO]: ' f'implement evaluation of complex expression {v}\n') + dimspec = ([s.lstrip('dimension').strip() + for s in vars[n]['attrspec'] + if s.startswith('dimension')] or [None])[0] + # Handle _dp for gh-6624 # Also fixes gh-20460 if real16pattern.search(v): @@ -2540,11 +2562,11 @@ def get_parameters(vars, global_params={}): elif real8pattern.search(v): v = 4 try: - params[n] = eval(v, g_params, params) - + params[n] = param_eval(v, g_params, params, dimspec=dimspec) except Exception as msg: params[n] = v - outmess('get_parameters: got "%s" on %s\n' % (msg, repr(v))) + outmess(f'get_parameters: got "{msg}" on {n!r}\n') + if isstring(vars[n]) and isinstance(params[n], int): params[n] = chr(params[n]) nl = n.lower() @@ -2552,8 +2574,7 @@ def get_parameters(vars, global_params={}): params[nl] = params[n] else: print(vars[n]) - outmess( - 'get_parameters:parameter %s does not have value?!\n' % (repr(n))) + outmess(f'get_parameters:parameter {n!r} does not have value?!\n') return params @@ -2562,6 +2583,7 @@ def _eval_length(length, params): return '(*)' return _eval_scalar(length, params) + _is_kind_number = re.compile(r'\d+_').match @@ -2582,6 +2604,10 @@ def _eval_scalar(value, params): def analyzevars(block): + """ + Sets correct dimension information for each variable/parameter + """ + global f90modulevars setmesstext(block) @@ -2610,7 +2636,8 @@ def analyzevars(block): svars.append(n) params = get_parameters(vars, get_useparameters(block)) - + # At this point, params are read and interpreted, but + # the params used to define vars are not yet parsed dep_matches = {} name_match = re.compile(r'[A-Za-z][\w$]*').match for v in list(vars.keys()): @@ -2709,27 +2736,30 @@ def analyzevars(block): check = None if dim and 'dimension' not in vars[n]: vars[n]['dimension'] = [] - for d in rmbadname([x.strip() for x in markoutercomma(dim).split('@,@')]): - star = ':' if d == ':' else '*' + for d in rmbadname( + [x.strip() for x in markoutercomma(dim).split('@,@')] + ): + # d is the expression inside the dimension declaration # Evaluate `d` with respect to params - if d in params: - d = str(params[d]) - for p in params: - re_1 = re.compile(r'(?P<before>.*?)\b' + p + r'\b(?P<after>.*)', re.I) - m = re_1.match(d) - while m: - d = m.group('before') + \ - str(params[p]) + m.group('after') - m = re_1.match(d) - - if d == star: - dl = [star] + try: + # the dimension for this variable depends on a + # previously defined parameter + d = param_parse(d, params) + except (ValueError, IndexError, KeyError): + outmess( + ('analyzevars: could not parse dimension for ' + f'variable {d!r}\n') + ) + + dim_char = ':' if d == ':' else '*' + if d == dim_char: + dl = [dim_char] else: dl = markoutercomma(d, ':').split('@:@') if len(dl) == 2 and '*' in dl: # e.g. dimension(5:*) dl = ['*'] d = '*' - if len(dl) == 1 and dl[0] != star: + if len(dl) == 1 and dl[0] != dim_char: dl = ['1', dl[0]] if len(dl) == 2: d1, d2 = map(symbolic.Expr.parse, dl) @@ -2963,9 +2993,152 @@ def analyzevars(block): del vars[n] return vars + analyzeargs_re_1 = re.compile(r'\A[a-z]+[\w$]*\Z', re.I) +def param_eval(v, g_params, params, dimspec=None): + """ + Creates a dictionary of indices and values for each parameter in a + parameter array to be evaluated later. + + WARNING: It is not possible to initialize multidimensional array + parameters e.g. dimension(-3:1, 4, 3:5) at this point. This is because in + Fortran initialization through array constructor requires the RESHAPE + intrinsic function. Since the right-hand side of the parameter declaration + is not executed in f2py, but rather at the compiled c/fortran extension, + later, it is not possible to execute a reshape of a parameter array. + One issue remains: if the user wants to access the array parameter from + python, we should either + 1) allow them to access the parameter array using python standard indexing + (which is often incompatible with the original fortran indexing) + 2) allow the parameter array to be accessed in python as a dictionary with + fortran indices as keys + We are choosing 2 for now. + """ + if dimspec is None: + try: + p = eval(v, g_params, params) + except Exception as msg: + p = v + outmess(f'param_eval: got "{msg}" on {v!r}\n') + return p + + # This is an array parameter. + # First, we parse the dimension information + if len(dimspec) < 2 or dimspec[::len(dimspec)-1] != "()": + raise ValueError(f'param_eval: dimension {dimspec} can\'t be parsed') + dimrange = dimspec[1:-1].split(',') + if len(dimrange) == 1: + # e.g. dimension(2) or dimension(-1:1) + dimrange = dimrange[0].split(':') + # now, dimrange is a list of 1 or 2 elements + if len(dimrange) == 1: + bound = param_parse(dimrange[0], params) + dimrange = range(1, int(bound)+1) + else: + lbound = param_parse(dimrange[0], params) + ubound = param_parse(dimrange[1], params) + dimrange = range(int(lbound), int(ubound)+1) + else: + raise ValueError(f'param_eval: multidimensional array parameters ' + '{dimspec} not supported') + + # Parse parameter value + v = (v[2:-2] if v.startswith('(/') else v).split(',') + v_eval = [] + for item in v: + try: + item = eval(item, g_params, params) + except Exception as msg: + outmess(f'param_eval: got "{msg}" on {item!r}\n') + v_eval.append(item) + + p = dict(zip(dimrange, v_eval)) + + return p + + +def param_parse(d, params): + """Recursively parse array dimensions. + + Parses the declaration of an array variable or parameter + `dimension` keyword, and is called recursively if the + dimension for this array is a previously defined parameter + (found in `params`). + + Parameters + ---------- + d : str + Fortran expression describing the dimension of an array. + params : dict + Previously parsed parameters declared in the Fortran source file. + + Returns + ------- + out : str + Parsed dimension expression. + + Examples + -------- + + * If the line being analyzed is + + `integer, parameter, dimension(2) :: pa = (/ 3, 5 /)` + + then `d = 2` and we return immediately, with + + >>> d = '2' + >>> param_parse(d, params) + 2 + + * If the line being analyzed is + + `integer, parameter, dimension(pa) :: pb = (/1, 2, 3/)` + + then `d = 'pa'`; since `pa` is a previously parsed parameter, + and `pa = 3`, we call `param_parse` recursively, to obtain + + >>> d = 'pa' + >>> params = {'pa': 3} + >>> param_parse(d, params) + 3 + + * If the line being analyzed is + + `integer, parameter, dimension(pa(1)) :: pb = (/1, 2, 3/)` + + then `d = 'pa(1)'`; since `pa` is a previously parsed parameter, + and `pa(1) = 3`, we call `param_parse` recursively, to obtain + + >>> d = 'pa(1)' + >>> params = dict(pa={1: 3, 2: 5}) + >>> param_parse(d, params) + 3 + """ + if "(" in d: + # this dimension expression is an array + dname = d[:d.find("(")] + ddims = d[d.find("(")+1:d.rfind(")")] + # this dimension expression is also a parameter; + # parse it recursively + index = int(param_parse(ddims, params)) + return str(params[dname][index]) + elif d in params: + return str(params[d]) + else: + for p in params: + re_1 = re.compile( + r'(?P<before>.*?)\b' + p + r'\b(?P<after>.*)', re.I + ) + m = re_1.match(d) + while m: + d = m.group('before') + \ + str(params[p]) + m.group('after') + m = re_1.match(d) + return d + + def expr2name(a, block, args=[]): orig_a = a a_is_expr = not analyzeargs_re_1.match(a) @@ -3218,11 +3391,6 @@ def true_intent_list(var): def vars2fortran(block, vars, args, tab='', as_interface=False): - """ - TODO: - public sub - ... - """ setmesstext(block) ret = '' nout = [] diff --git a/contrib/python/numpy/py3/numpy/f2py/f2py2e.py b/contrib/python/numpy/py3/numpy/f2py/f2py2e.py index 1cfe8cddd6..ce22b2d8a9 100755 --- a/contrib/python/numpy/py3/numpy/f2py/f2py2e.py +++ b/contrib/python/numpy/py3/numpy/f2py/f2py2e.py @@ -4,15 +4,12 @@ f2py2e - Fortran to Python C/API generator. 2nd Edition. See __usage__ below. -Copyright 1999--2011 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@cens.ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 08:31:19 $ -Pearu Peterson - """ import sys import os @@ -21,6 +18,7 @@ import re from pathlib import Path from itertools import dropwhile import argparse +import copy from . import crackfortran from . import rules @@ -38,6 +36,7 @@ errmess = sys.stderr.write # outmess=sys.stdout.write show = pprint.pprint outmess = auxfuncs.outmess +MESON_ONLY_VER = (sys.version_info >= (3, 12)) __usage__ =\ f"""Usage: @@ -63,12 +62,6 @@ Description: This program generates a Python C/API file (<modulename>module.c) Options: - --2d-numpy Use numpy.f2py tool with NumPy support. [DEFAULT] - --2d-numeric Use f2py2e tool with Numeric support. - --2d-numarray Use f2py2e tool with Numarray support. - --g3-numpy Use 3rd generation f2py from the separate f2py package. - [NOT AVAILABLE YET] - -h <filename> Write signatures of the fortran routines to file <filename> and exit. You can then edit <filename> and use it instead of <fortran files>. If <filename>==stdout then the @@ -129,20 +122,22 @@ Options: -v Print f2py version ID and exit. -build backend options (only effective with -c): +build backend options (only effective with -c) +[NO_MESON] is used to indicate an option not meant to be used +with the meson backend or above Python 3.12: - --fcompiler= Specify Fortran compiler type by vendor - --compiler= Specify C compiler type (as defined by distutils) + --fcompiler= Specify Fortran compiler type by vendor [NO_MESON] + --compiler= Specify distutils C compiler type [NO_MESON] - --help-fcompiler List available Fortran compilers and exit - --f77exec= Specify the path to F77 compiler - --f90exec= Specify the path to F90 compiler + --help-fcompiler List available Fortran compilers and exit [NO_MESON] + --f77exec= Specify the path to F77 compiler [NO_MESON] + --f90exec= Specify the path to F90 compiler [NO_MESON] --f77flags= Specify F77 compiler flags --f90flags= Specify F90 compiler flags - --opt= Specify optimization flags - --arch= Specify architecture specific optimization flags - --noopt Compile without optimization - --noarch Compile without arch-dependent optimization + --opt= Specify optimization flags [NO_MESON] + --arch= Specify architecture specific optimization flags [NO_MESON] + --noopt Compile without optimization [NO_MESON] + --noarch Compile without arch-dependent optimization [NO_MESON] --debug Compile with debugging information --dep <dependency> @@ -167,7 +162,7 @@ Extra options (only effective with -c): by numpy.distutils/system_info.py. E.g. to link with optimized LAPACK libraries (vecLib on MacOSX, ATLAS elsewhere), use --link-lapack_opt. - See also --help-link switch. + See also --help-link switch. [NO_MESON] -L/path/to/lib/ -l<libname> -D<define> -U<name> @@ -189,15 +184,15 @@ Extra options (only effective with -c): Version: {f2py_version} numpy Version: {numpy_version} -Requires: Python 3.5 or higher. License: NumPy license (see LICENSE.txt in the NumPy source code) -Copyright 1999 - 2011 Pearu Peterson all rights reserved. -https://web.archive.org/web/20140822061353/http://cens.ioc.ee/projects/f2py2e""" +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +https://numpy.org/doc/stable/f2py/index.html\n""" def scaninputline(inputline): files, skipfuncs, onlyfuncs, debug = [], [], [], [] - f, f2, f3, f5, f6, f7, f8, f9, f10 = 1, 0, 0, 0, 0, 0, 0, 0, 0 + f, f2, f3, f5, f6, f8, f9, f10 = 1, 0, 0, 0, 0, 0, 0, 0 verbose = 1 emptygen = True dolc = -1 @@ -205,7 +200,7 @@ def scaninputline(inputline): dorestdoc = 0 wrapfuncs = 1 buildpath = '.' - include_paths = [] + include_paths, inputline = get_includes(inputline) signsfile, modulename = None, None options = {'buildpath': buildpath, 'coutput': None, @@ -265,14 +260,6 @@ def scaninputline(inputline): elif l[:8] == '-include': cfuncs.outneeds['userincludes'].append(l[9:-1]) cfuncs.userincludes[l[9:-1]] = '#include ' + l[8:] - elif l[:15] in '--include_paths': - outmess( - 'f2py option --include_paths is deprecated, use --include-paths instead.\n') - f7 = 1 - elif l[:15] in '--include-paths': - # Similar to using -I with -c, however this is - # also used during generation of wrappers - f7 = 1 elif l == '--skip-empty-wrappers': emptygen = False elif l[0] == '-': @@ -287,9 +274,6 @@ def scaninputline(inputline): elif f6: f6 = 0 buildpath = l - elif f7: - f7 = 0 - include_paths.extend(l.split(os.pathsep)) elif f8: f8 = 0 options["coutput"] = l @@ -456,6 +440,23 @@ def run_main(comline_list): f2pydir = os.path.dirname(os.path.abspath(cfuncs.__file__)) fobjhsrc = os.path.join(f2pydir, 'src', 'fortranobject.h') fobjcsrc = os.path.join(f2pydir, 'src', 'fortranobject.c') + # gh-22819 -- begin + parser = make_f2py_compile_parser() + args, comline_list = parser.parse_known_args(comline_list) + pyf_files, _ = filter_files("", "[.]pyf([.]src|)", comline_list) + # Checks that no existing modulename is defined in a pyf file + # TODO: Remove all this when scaninputline is replaced + if args.module_name: + if "-h" in comline_list: + modname = ( + args.module_name + ) # Directly use from args when -h is present + else: + modname = validate_modulename( + pyf_files, args.module_name + ) # Validate modname when -h is not present + comline_list += ['-m', modname] # needed for the rest of scaninputline + # gh-22819 -- end files, options = scaninputline(comline_list) auxfuncs.options = options capi_maps.load_f2cmap_file(options['f2cmap_file']) @@ -522,24 +523,59 @@ def get_prefix(module): p = os.path.dirname(os.path.dirname(module.__file__)) return p -def preparse_sysargv(): - # To keep backwards bug compatibility, newer flags are handled by argparse, - # and `sys.argv` is passed to the rest of `f2py` as is. + +class CombineIncludePaths(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + include_paths_set = set(getattr(namespace, 'include_paths', []) or []) + if option_string == "--include_paths": + outmess("Use --include-paths or -I instead of --include_paths which will be removed") + if option_string == "--include-paths" or option_string == "--include_paths": + include_paths_set.update(values.split(':')) + else: + include_paths_set.add(values) + setattr(namespace, 'include_paths', list(include_paths_set)) + +def include_parser(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("-I", dest="include_paths", action=CombineIncludePaths) + parser.add_argument("--include-paths", dest="include_paths", action=CombineIncludePaths) + parser.add_argument("--include_paths", dest="include_paths", action=CombineIncludePaths) + return parser + +def get_includes(iline): + iline = (' '.join(iline)).split() + parser = include_parser() + args, remain = parser.parse_known_args(iline) + ipaths = args.include_paths + if args.include_paths is None: + ipaths = [] + return ipaths, remain + +def make_f2py_compile_parser(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--dep", action="append", dest="dependencies") parser.add_argument("--backend", choices=['meson', 'distutils'], default='distutils') + parser.add_argument("-m", dest="module_name") + return parser + +def preparse_sysargv(): + # To keep backwards bug compatibility, newer flags are handled by argparse, + # and `sys.argv` is passed to the rest of `f2py` as is. + parser = make_f2py_compile_parser() args, remaining_argv = parser.parse_known_args() sys.argv = [sys.argv[0]] + remaining_argv backend_key = args.backend - if sys.version_info >= (3, 12) and backend_key == 'distutils': - outmess('Cannot use distutils backend with Python 3.12, using meson backend instead.') - backend_key = 'meson' + if MESON_ONLY_VER and backend_key == 'distutils': + outmess("Cannot use distutils backend with Python>=3.12," + " using meson backend instead.\n") + backend_key = "meson" return { "dependencies": args.dependencies or [], - "backend": backend_key + "backend": backend_key, + "modulename": args.module_name, } def run_compile(): @@ -550,11 +586,13 @@ def run_compile(): # Collect dependency flags, preprocess sys.argv argy = preparse_sysargv() + modulename = argy["modulename"] + if modulename is None: + modulename = 'untitled' dependencies = argy["dependencies"] backend_key = argy["backend"] build_backend = f2py_build_generator(backend_key) - i = sys.argv.index('-c') del sys.argv[i] @@ -607,21 +645,27 @@ def run_compile(): for s in flib_flags: v = '--fcompiler=' if s[:len(v)] == v: - from numpy.distutils import fcompiler - fcompiler.load_all_fcompiler_classes() - allowed_keys = list(fcompiler.fcompiler_class.keys()) - nv = ov = s[len(v):].lower() - if ov not in allowed_keys: - vmap = {} # XXX - try: - nv = vmap[ov] - except KeyError: - if ov not in vmap.values(): - print('Unknown vendor: "%s"' % (s[len(v):])) - nv = ov - i = flib_flags.index(s) - flib_flags[i] = '--fcompiler=' + nv - continue + if MESON_ONLY_VER or backend_key == 'meson': + outmess( + "--fcompiler cannot be used with meson," + "set compiler with the FC environment variable\n" + ) + else: + from numpy.distutils import fcompiler + fcompiler.load_all_fcompiler_classes() + allowed_keys = list(fcompiler.fcompiler_class.keys()) + nv = ov = s[len(v):].lower() + if ov not in allowed_keys: + vmap = {} # XXX + try: + nv = vmap[ov] + except KeyError: + if ov not in vmap.values(): + print('Unknown vendor: "%s"' % (s[len(v):])) + nv = ov + i = flib_flags.index(s) + flib_flags[i] = '--fcompiler=' + nv + continue for s in del_list: i = flib_flags.index(s) del flib_flags[i] @@ -634,32 +678,19 @@ def run_compile(): if '--quiet' in f2py_flags: setup_flags.append('--quiet') - modulename = 'untitled' + # Ugly filter to remove everything but sources sources = sys.argv[1:] - - for optname in ['--include_paths', '--include-paths', '--f2cmap']: - if optname in sys.argv: - i = sys.argv.index(optname) - f2py_flags.extend(sys.argv[i:i + 2]) - del sys.argv[i + 1], sys.argv[i] - sources = sys.argv[1:] - - pyf_files = [] - if '-m' in sys.argv: - i = sys.argv.index('-m') - modulename = sys.argv[i + 1] + f2cmapopt = '--f2cmap' + if f2cmapopt in sys.argv: + i = sys.argv.index(f2cmapopt) + f2py_flags.extend(sys.argv[i:i + 2]) del sys.argv[i + 1], sys.argv[i] sources = sys.argv[1:] - else: - pyf_files, _sources = filter_files('', '[.]pyf([.]src|)', sources) - sources = pyf_files + _sources - for f in pyf_files: - modulename = auxfuncs.get_f2py_modulename(f) - if modulename: - break + pyf_files, _sources = filter_files("", "[.]pyf([.]src|)", sources) + sources = pyf_files + _sources + modulename = validate_modulename(pyf_files, modulename) extra_objects, sources = filter_files('', '[.](o|a|so|dylib)', sources) - include_dirs, sources = filter_files('-I', '', sources, remove_prefix=1) library_dirs, sources = filter_files('-L', '', sources, remove_prefix=1) libraries, sources = filter_files('-l', '', sources, remove_prefix=1) undef_macros, sources = filter_files('-U', '', sources, remove_prefix=1) @@ -675,13 +706,15 @@ def run_compile(): # Construct wrappers / signatures / things if backend_key == 'meson': - outmess('Using meson backend\nWill pass --lower to f2py\nSee https://numpy.org/doc/stable/f2py/buildtools/meson.html') - f2py_flags.append('--lower') - if pyf_files: - run_main(f" {' '.join(f2py_flags)} {' '.join(pyf_files)}".split()) - else: + if not pyf_files: + outmess('Using meson backend\nWill pass --lower to f2py\nSee https://numpy.org/doc/stable/f2py/buildtools/meson.html\n') + f2py_flags.append('--lower') run_main(f" {' '.join(f2py_flags)} -m {modulename} {' '.join(sources)}".split()) + else: + run_main(f" {' '.join(f2py_flags)} {' '.join(pyf_files)}".split()) + # Order matters here, includes are needed for run_main above + include_dirs, sources = get_includes(sources) # Now use the builder builder = build_backend( modulename, @@ -704,30 +737,31 @@ def run_compile(): builder.compile() + +def validate_modulename(pyf_files, modulename='untitled'): + if len(pyf_files) > 1: + raise ValueError("Only one .pyf file per call") + if pyf_files: + pyff = pyf_files[0] + pyf_modname = auxfuncs.get_f2py_modulename(pyff) + if modulename != pyf_modname: + outmess( + f"Ignoring -m {modulename}.\n" + f"{pyff} defines {pyf_modname} to be the modulename.\n" + ) + modulename = pyf_modname + return modulename + def main(): if '--help-link' in sys.argv[1:]: sys.argv.remove('--help-link') - from numpy.distutils.system_info import show_all - show_all() + if MESON_ONLY_VER: + outmess("Use --dep for meson builds\n") + else: + from numpy.distutils.system_info import show_all + show_all() return - # Probably outdated options that were not working before 1.16 - if '--g3-numpy' in sys.argv[1:]: - sys.stderr.write("G3 f2py support is not implemented, yet.\\n") - sys.exit(1) - elif '--2e-numeric' in sys.argv[1:]: - sys.argv.remove('--2e-numeric') - elif '--2e-numarray' in sys.argv[1:]: - # Note that this errors becaust the -DNUMARRAY argument is - # not recognized. Just here for back compatibility and the - # error message. - sys.argv.append("-DNUMARRAY") - sys.argv.remove('--2e-numarray') - elif '--2e-numpy' in sys.argv[1:]: - sys.argv.remove('--2e-numpy') - else: - pass - if '-c' in sys.argv[1:]: run_compile() else: diff --git a/contrib/python/numpy/py3/numpy/f2py/f90mod_rules.py b/contrib/python/numpy/py3/numpy/f2py/f90mod_rules.py index 1c47bee02b..2f8a8dc187 100644 --- a/contrib/python/numpy/py3/numpy/f2py/f90mod_rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/f90mod_rules.py @@ -1,17 +1,12 @@ -#!/usr/bin/env python3 """ - Build F90 module support for f2py2e. -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/02/03 19:30:23 $ -Pearu Peterson - """ __version__ = "$Revision: 1.27 $"[10:-1] @@ -99,6 +94,8 @@ def buildhooks(pymod): def dadd(line, s=doc): s[0] = '%s\n%s' % (s[0], line) + + usenames = getuseblocks(pymod) for m in findf90modules(pymod): sargs, fargs, efargs, modobjs, notvars, onlyvars = [], [], [], [], [ m['name']], [] @@ -115,6 +112,9 @@ def buildhooks(pymod): mfargs.append(n) outmess('\t\tConstructing F90 module support for "%s"...\n' % (m['name'])) + if m['name'] in usenames and not onlyvars: + outmess(f"\t\t\tSkipping {m['name']} since it is in 'use'...\n") + continue if onlyvars: outmess('\t\t Variables: %s\n' % (' '.join(onlyvars))) chooks = [''] diff --git a/contrib/python/numpy/py3/numpy/f2py/func2subr.py b/contrib/python/numpy/py3/numpy/f2py/func2subr.py index 2eedc0ade8..b9aa9fc007 100644 --- a/contrib/python/numpy/py3/numpy/f2py/func2subr.py +++ b/contrib/python/numpy/py3/numpy/f2py/func2subr.py @@ -1,17 +1,13 @@ -#!/usr/bin/env python3 """ Rules for building C/API module with f2py2e. -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2004/11/26 11:13:06 $ -Pearu Peterson - """ import copy diff --git a/contrib/python/numpy/py3/numpy/f2py/rules.py b/contrib/python/numpy/py3/numpy/f2py/rules.py index 1bac871024..009365e047 100755 --- a/contrib/python/numpy/py3/numpy/f2py/rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/rules.py @@ -40,15 +40,12 @@ wrapper_function(args) return buildvalue -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/08/30 08:58:42 $ -Pearu Peterson - """ import os, sys import time diff --git a/contrib/python/numpy/py3/numpy/f2py/setup.py b/contrib/python/numpy/py3/numpy/f2py/setup.py index 98f1e9aaae..05bef30001 100644 --- a/contrib/python/numpy/py3/numpy/f2py/setup.py +++ b/contrib/python/numpy/py3/numpy/f2py/setup.py @@ -31,7 +31,7 @@ def configuration(parent_package='', top_path=None): config.add_data_files( 'src/fortranobject.c', 'src/fortranobject.h', - 'backends/meson.build.template', + '_backends/meson.build.template', ) config.add_data_files('*.pyi') return config diff --git a/contrib/python/numpy/py3/numpy/f2py/symbolic.py b/contrib/python/numpy/py3/numpy/f2py/symbolic.py index b1b9f5b6a1..67120d79a5 100644 --- a/contrib/python/numpy/py3/numpy/f2py/symbolic.py +++ b/contrib/python/numpy/py3/numpy/f2py/symbolic.py @@ -2,6 +2,13 @@ References: - J3/21-007: Draft Fortran 202x. https://j3-fortran.org/doc/year/21/21-007.pdf + +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +Permission to use, modify, and distribute this software is given under the +terms of the NumPy License. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. """ # To analyze Fortran expressions to solve dimensions specifications, diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.f b/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.f new file mode 100644 index 0000000000..ba727a10a0 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.f @@ -0,0 +1,10 @@ + SUBROUTINE FOO(FUN,R) + EXTERNAL FUN + INTEGER I + REAL*8 R, FUN +Cf2py intent(out) r + R = 0D0 + DO I=-5,5 + R = R + FUN(I) + ENDDO + END diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.pyf b/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.pyf new file mode 100644 index 0000000000..f120111533 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.pyf @@ -0,0 +1,18 @@ +python module __user__routines + interface + function fun(i) result (r) + integer :: i + real*8 :: r + end function fun + end interface +end python module __user__routines + +python module callback2 + interface + subroutine foo(f,r) + use __user__routines, f=>fun + external f + real*8 intent(out) :: r + end subroutine foo + end interface +end python module callback2 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/cli/gh_22819.pyf b/contrib/python/numpy/py3/numpy/f2py/tests/src/cli/gh_22819.pyf new file mode 100644 index 0000000000..8eb5bb106a --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/cli/gh_22819.pyf @@ -0,0 +1,6 @@ +python module test_22819 + interface + subroutine hello() + end subroutine hello + end interface +end python module test_22819 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/common/gh19161.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/common/gh19161.f90 new file mode 100644 index 0000000000..a2f40735ad --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/common/gh19161.f90 @@ -0,0 +1,10 @@ +module typedefmod + use iso_fortran_env, only: real32 +end module typedefmod + +module data + use typedefmod, only: real32 + implicit none + real(kind=real32) :: x + common/test/x +end module data diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/isocintrin/isoCtests.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/isocintrin/isoCtests.f90 index 42db6cccc1..765f7c1ce6 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/src/isocintrin/isoCtests.f90 +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/isocintrin/isoCtests.f90 @@ -1,5 +1,5 @@ module coddity - use iso_c_binding, only: c_double, c_int + use iso_c_binding, only: c_double, c_int, c_int64_t implicit none contains subroutine c_add(a, b, c) bind(c, name="c_add") @@ -14,4 +14,21 @@ z = x + 7 end function wat + ! gh-25207 + subroutine c_add_int64(a, b, c) bind(c) + integer(c_int64_t), intent(in) :: a, b + integer(c_int64_t), intent(out) :: c + c = a + b + end subroutine c_add_int64 + ! gh-25207 + subroutine add_arr(A, B, C) + integer(c_int64_t), intent(in) :: A(3) + integer(c_int64_t), intent(in) :: B(3) + integer(c_int64_t), intent(out) :: C(3) + integer :: j + + do j = 1, 3 + C(j) = A(j)+B(j) + end do + end subroutine end module coddity diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/data.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/data.f90 new file mode 100644 index 0000000000..483d13ceb9 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/data.f90 @@ -0,0 +1,8 @@ +module data + real(8) :: shift +contains + subroutine set_shift(in_shift) + real(8), intent(in) :: in_shift + shift = in_shift + end subroutine set_shift +end module data diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/use_data.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/use_data.f90 new file mode 100644 index 0000000000..b3fae8b875 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/use_data.f90 @@ -0,0 +1,6 @@ +subroutine shift_a(dim_a, a) + use data, only: shift + integer, intent(in) :: dim_a + real(8), intent(inout), dimension(dim_a) :: a + a = a + shift +end subroutine shift_a diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh24662.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh24662.f90 new file mode 100644 index 0000000000..ca53413cc9 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh24662.f90 @@ -0,0 +1,7 @@ +subroutine string_inout_optional(output) + implicit none + character*(32), optional, intent(inout) :: output + if (present(output)) then + output="output string" + endif +end subroutine diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.f90 new file mode 100644 index 0000000000..db1c7100d2 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.f90 @@ -0,0 +1,14 @@ +subroutine charint(trans, info) + character, intent(in) :: trans + integer, intent(out) :: info + if (trans == 'N') then + info = 1 + else if (trans == 'T') then + info = 2 + else if (trans == 'C') then + info = 3 + else + info = -1 + end if + +end subroutine charint diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.pyf b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.pyf new file mode 100644 index 0000000000..7b9609071b --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.pyf @@ -0,0 +1,12 @@ +python module _char_handling_test + interface + subroutine charint(trans, info) + callstatement (*f2py_func)(&trans, &info) + callprotoargument char*, int* + + character, intent(in), check(trans=='N'||trans=='T'||trans=='C') :: trans = 'N' + integer intent(out) :: info + + end subroutine charint + end interface +end python module _char_handling_test diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286_bc.pyf b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286_bc.pyf new file mode 100644 index 0000000000..e7b10fa921 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286_bc.pyf @@ -0,0 +1,12 @@ +python module _char_handling_test + interface + subroutine charint(trans, info) + callstatement (*f2py_func)(&trans, &info) + callprotoargument char*, int* + + character, intent(in), check(*trans=='N'||*trans=='T'||*trans=='C') :: trans = 'N' + integer intent(out) :: info + + end subroutine charint + end interface +end python module _char_handling_test diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_callback.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_callback.py index 018cea4fd5..5b6c294d33 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_callback.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_callback.py @@ -228,3 +228,16 @@ class TestGH18335(util.F2PyTest): r = self.module.gh18335(foo) assert r == 123 + 1 + + +class TestGH25211(util.F2PyTest): + sources = [util.getpath("tests", "src", "callback", "gh25211.f"), + util.getpath("tests", "src", "callback", "gh25211.pyf")] + module_name = "callback2" + + def test_gh18335(self): + def bar(x): + return x*x + + res = self.module.foo(bar) + assert res == 110 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_character.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_character.py index 373262bf96..e55b1b6b23 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_character.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_character.py @@ -595,3 +595,42 @@ class TestStringAssumedLength(util.F2PyTest): def test_gh24008(self): self.module.greet("joe", "bob") + +class TestStringOptionalInOut(util.F2PyTest): + sources = [util.getpath("tests", "src", "string", "gh24662.f90")] + + def test_gh24662(self): + self.module.string_inout_optional() + a = np.array('hi', dtype='S32') + self.module.string_inout_optional(a) + assert "output string" in a.tobytes().decode() + with pytest.raises(Exception): + aa = "Hi" + self.module.string_inout_optional(aa) + + +@pytest.mark.slow +class TestNewCharHandling(util.F2PyTest): + # from v1.24 onwards, gh-19388 + sources = [ + util.getpath("tests", "src", "string", "gh25286.pyf"), + util.getpath("tests", "src", "string", "gh25286.f90") + ] + module_name = "_char_handling_test" + + def test_gh25286(self): + info = self.module.charint('T') + assert info == 2 + +@pytest.mark.slow +class TestBCCharHandling(util.F2PyTest): + # SciPy style, "incorrect" bindings with a hook + sources = [ + util.getpath("tests", "src", "string", "gh25286_bc.pyf"), + util.getpath("tests", "src", "string", "gh25286.f90") + ] + module_name = "_char_handling_test" + + def test_gh25286(self): + info = self.module.charint('T') + assert info == 2 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_common.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_common.py index 8a4b221ef8..68c1b3b31c 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_common.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_common.py @@ -16,3 +16,12 @@ class TestCommonBlock(util.F2PyTest): assert self.module.block.long_bn == np.array(1.0, dtype=np.float64) assert self.module.block.string_bn == np.array("2", dtype="|S1") assert self.module.block.ok == np.array(3, dtype=np.int32) + + +class TestCommonWithUse(util.F2PyTest): + sources = [util.getpath("tests", "src", "common", "gh19161.f90")] + + @pytest.mark.skipif(sys.platform == "win32", + reason="Fails with MinGW64 Gfortran (Issue #9673)") + def test_common_gh19161(self): + assert self.module.data.x == 0 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_f2py2e.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_f2py2e.py index 5f7b56a68a..659e0e96bd 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_f2py2e.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_f2py2e.py @@ -1,6 +1,7 @@ import textwrap, re, sys, subprocess, shlex from pathlib import Path from collections import namedtuple +import platform import pytest @@ -72,6 +73,15 @@ def gh23598_warn(tmpdir_factory): @pytest.fixture(scope="session") +def gh22819_cli(tmpdir_factory): + """F90 file for testing disallowed CLI arguments in ghff819""" + fdat = util.getpath("tests", "src", "cli", "gh_22819.pyf").read_text() + fn = tmpdir_factory.getbasetemp() / "gh_22819.pyf" + fn.write_text(fdat, encoding="ascii") + return fn + + +@pytest.fixture(scope="session") def hello_world_f77(tmpdir_factory): """Generates a single f77 file for testing""" fdat = util.getpath("tests", "src", "cli", "hi77.f").read_text() @@ -100,6 +110,38 @@ def f2cmap_f90(tmpdir_factory): return fn +def test_gh22819_cli(capfd, gh22819_cli, monkeypatch): + """Check that module names are handled correctly + gh-22819 + Essentially, the -m name cannot be used to import the module, so the module + named in the .pyf needs to be used instead + + CLI :: -m and a .pyf file + """ + ipath = Path(gh22819_cli) + monkeypatch.setattr(sys, "argv", f"f2py -m blah {ipath}".split()) + with util.switchdir(ipath.parent): + f2pycli() + gen_paths = [item.name for item in ipath.parent.rglob("*") if item.is_file()] + assert "blahmodule.c" not in gen_paths # shouldn't be generated + assert "blah-f2pywrappers.f" not in gen_paths + assert "test_22819-f2pywrappers.f" in gen_paths + assert "test_22819module.c" in gen_paths + assert "Ignoring blah" + + +def test_gh22819_many_pyf(capfd, gh22819_cli, monkeypatch): + """Only one .pyf file allowed + gh-22819 + CLI :: .pyf files + """ + ipath = Path(gh22819_cli) + monkeypatch.setattr(sys, "argv", f"f2py -m blah {ipath} hello.pyf".split()) + with util.switchdir(ipath.parent): + with pytest.raises(ValueError, match="Only one .pyf file per call"): + f2pycli() + + def test_gh23598_warn(capfd, gh23598_warn, monkeypatch): foutl = get_io_paths(gh23598_warn, mname="test") ipath = foutl.f90inp @@ -156,6 +198,53 @@ def test_gen_pyf_no_overwrite(capfd, hello_world_f90, monkeypatch): assert "Use --overwrite-signature to overwrite" in err +@pytest.mark.skipif((platform.system() != 'Linux') or (sys.version_info <= (3, 12)), + reason='Compiler and 3.12 required') +def test_untitled_cli(capfd, hello_world_f90, monkeypatch): + """Check that modules are named correctly + + CLI :: defaults + """ + ipath = Path(hello_world_f90) + monkeypatch.setattr(sys, "argv", f"f2py --backend meson -c {ipath}".split()) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "untitledmodule.c" in out + + +@pytest.mark.skipif((platform.system() != 'Linux') or (sys.version_info <= (3, 12)), reason='Compiler and 3.12 required') +def test_no_py312_distutils_fcompiler(capfd, hello_world_f90, monkeypatch): + """Check that no distutils imports are performed on 3.12 + CLI :: --fcompiler --help-link --backend distutils + """ + MNAME = "hi" + foutl = get_io_paths(hello_world_f90, mname=MNAME) + ipath = foutl.f90inp + monkeypatch.setattr( + sys, "argv", f"f2py {ipath} -c --fcompiler=gfortran -m {MNAME}".split() + ) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "--fcompiler cannot be used with meson" in out + monkeypatch.setattr( + sys, "argv", f"f2py --help-link".split() + ) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "Use --dep for meson builds" in out + MNAME = "hi2" # Needs to be different for a new -c + monkeypatch.setattr( + sys, "argv", f"f2py {ipath} -c -m {MNAME} --backend distutils".split() + ) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "Cannot use distutils backend with Python>=3.12" in out + + @pytest.mark.xfail def test_f2py_skip(capfd, retreal_f77, monkeypatch): """Tests that functions can be skipped @@ -250,6 +339,22 @@ def test_mod_gen_f77(capfd, hello_world_f90, monkeypatch): assert Path.exists(foutl.wrap77) +def test_mod_gen_gh25263(capfd, hello_world_f77, monkeypatch): + """Check that pyf files are correctly generated with module structure + CLI :: -m <name> -h pyf_file + BUG: numpy-gh #20520 + """ + MNAME = "hi" + foutl = get_io_paths(hello_world_f77, mname=MNAME) + ipath = foutl.finp + monkeypatch.setattr(sys, "argv", f'f2py {ipath} -m {MNAME} -h hi.pyf'.split()) + with util.switchdir(ipath.parent): + f2pycli() + with Path('hi.pyf').open() as hipyf: + pyfdat = hipyf.read() + assert "python module hi" in pyfdat + + def test_lower_cmod(capfd, hello_world_f77, monkeypatch): """Lowers cases by flag or when -h is present diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_isoc.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_isoc.py index 7e189bd7b8..594bd7caea 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_isoc.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_isoc.py @@ -1,5 +1,7 @@ from . import util import numpy as np +import pytest +from numpy.testing import assert_allclose class TestISOC(util.F2PyTest): sources = [ @@ -17,3 +19,34 @@ class TestISOC(util.F2PyTest): out = self.module.coddity.wat(1, 20) exp_out = 8 assert out == exp_out + + # gh-25207 + def test_bindc_kinds(self): + out = self.module.coddity.c_add_int64(1, 20) + exp_out = 21 + assert out == exp_out + + # gh-25207 + def test_bindc_add_arr(self): + a = np.array([1,2,3]) + b = np.array([1,2,3]) + out = self.module.coddity.add_arr(a, b) + exp_out = a*2 + assert_allclose(out, exp_out) + + +def test_process_f2cmap_dict(): + from numpy.f2py.auxfuncs import process_f2cmap_dict + + f2cmap_all = {"integer": {"8": "rubbish_type"}} + new_map = {"INTEGER": {"4": "int"}} + c2py_map = {"int": "int", "rubbish_type": "long"} + + exp_map, exp_maptyp = ({"integer": {"8": "rubbish_type", "4": "int"}}, ["int"]) + + # Call the function + res_map, res_maptyp = process_f2cmap_dict(f2cmap_all, new_map, c2py_map) + + # Assert the result is as expected + assert res_map == exp_map + assert res_maptyp == exp_maptyp diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_pyf_src.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_pyf_src.py new file mode 100644 index 0000000000..f77ded2f31 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_pyf_src.py @@ -0,0 +1,44 @@ +# This test is ported from numpy.distutils +from numpy.f2py._src_pyf import process_str +from numpy.testing import assert_equal + + +pyf_src = """ +python module foo + <_rd=real,double precision> + interface + subroutine <s,d>foosub(tol) + <_rd>, intent(in,out) :: tol + end subroutine <s,d>foosub + end interface +end python module foo +""" + +expected_pyf = """ +python module foo + interface + subroutine sfoosub(tol) + real, intent(in,out) :: tol + end subroutine sfoosub + subroutine dfoosub(tol) + double precision, intent(in,out) :: tol + end subroutine dfoosub + end interface +end python module foo +""" + + +def normalize_whitespace(s): + """ + Remove leading and trailing whitespace, and convert internal + stretches of whitespace to a single space. + """ + return ' '.join(s.split()) + + +def test_from_template(): + """Regression test for gh-10712.""" + pyf = process_str(pyf_src) + normalized_pyf = normalize_whitespace(pyf) + normalized_expected_pyf = normalize_whitespace(expected_pyf) + assert_equal(normalized_pyf, normalized_expected_pyf) diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_regression.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_regression.py index 044f952f22..1c10978309 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_regression.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_regression.py @@ -64,3 +64,14 @@ def test_include_path(): fnames_in_dir = os.listdir(incdir) for fname in ("fortranobject.c", "fortranobject.h"): assert fname in fnames_in_dir + + +class TestModuleAndSubroutine(util.F2PyTest): + module_name = "example" + sources = [util.getpath("tests", "src", "regression", "gh25337", "data.f90"), + util.getpath("tests", "src", "regression", "gh25337", "use_data.f90")] + + @pytest.mark.slow + def test_gh25337(self): + self.module.data.set_shift(3) + assert "data" in dir(self.module) diff --git a/contrib/python/numpy/py3/numpy/f2py/use_rules.py b/contrib/python/numpy/py3/numpy/f2py/use_rules.py index f1b71e83c2..808b3dd97e 100644 --- a/contrib/python/numpy/py3/numpy/f2py/use_rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/use_rules.py @@ -1,19 +1,12 @@ -#!/usr/bin/env python3 """ - Build 'use others module data' mechanism for f2py2e. -Unfinished. - -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2000/09/10 12:35:43 $ -Pearu Peterson - """ __version__ = "$Revision: 1.3 $"[10:-1] diff --git a/contrib/python/numpy/py3/numpy/random/_generator.pyx b/contrib/python/numpy/py3/numpy/random/_generator.pyx index 1bdba33565..ba3118d9c7 100644 --- a/contrib/python/numpy/py3/numpy/random/_generator.pyx +++ b/contrib/python/numpy/py3/numpy/random/_generator.pyx @@ -3757,7 +3757,7 @@ cdef class Generator: if size is None: shape = [] - elif isinstance(size, (int, long, np.integer)): + elif isinstance(size, (int, np.integer)): shape = [size] else: shape = size diff --git a/contrib/python/numpy/py3/numpy/random/mtrand.pyx b/contrib/python/numpy/py3/numpy/random/mtrand.pyx index 9ffaa572d8..690dea1485 100644 --- a/contrib/python/numpy/py3/numpy/random/mtrand.pyx +++ b/contrib/python/numpy/py3/numpy/random/mtrand.pyx @@ -367,15 +367,16 @@ cdef class RandomState: else: if not isinstance(state, (tuple, list)): raise TypeError('state must be a dict or a tuple.') - if state[0] != 'MT19937': - raise ValueError('set_state can only be used with legacy MT19937' - 'state instances.') - st = {'bit_generator': state[0], - 'state': {'key': state[1], 'pos': state[2]}} - if len(state) > 3: - st['has_gauss'] = state[3] - st['gauss'] = state[4] - value = st + with cython.boundscheck(True): + if state[0] != 'MT19937': + raise ValueError('set_state can only be used with legacy ' + 'MT19937 state instances.') + st = {'bit_generator': state[0], + 'state': {'key': state[1], 'pos': state[2]}} + if len(state) > 3: + st['has_gauss'] = state[3] + st['gauss'] = state[4] + value = st self._aug_state.gauss = st.get('gauss', 0.0) self._aug_state.has_gauss = st.get('has_gauss', 0) diff --git a/contrib/python/numpy/py3/numpy/random/tests/test_random.py b/contrib/python/numpy/py3/numpy/random/tests/test_random.py index e64ace7119..3d081fe1db 100644 --- a/contrib/python/numpy/py3/numpy/random/tests/test_random.py +++ b/contrib/python/numpy/py3/numpy/random/tests/test_random.py @@ -147,6 +147,11 @@ class TestSetState: # arguments without truncation. self.prng.negative_binomial(0.5, 0.5) + def test_set_invalid_state(self): + # gh-25402 + with pytest.raises(IndexError): + self.prng.set_state(()) + class TestRandint: diff --git a/contrib/python/numpy/py3/numpy/tests/test_warnings.py b/contrib/python/numpy/py3/numpy/tests/test_warnings.py index d7a6d880cb..ee5124c5d5 100644 --- a/contrib/python/numpy/py3/numpy/tests/test_warnings.py +++ b/contrib/python/numpy/py3/numpy/tests/test_warnings.py @@ -5,6 +5,7 @@ all of these occurrences but should catch almost all. import pytest from pathlib import Path +import sys import ast import tokenize import numpy @@ -56,6 +57,8 @@ class FindFuncs(ast.NodeVisitor): @pytest.mark.slow +@pytest.mark.skipif(sys.version_info >= (3, 12), + reason="Deprecation warning in ast") def test_warning_calls(): # combined "ignore" and stacklevel error base = Path(numpy.__file__).parent diff --git a/contrib/python/numpy/py3/numpy/version.py b/contrib/python/numpy/py3/numpy/version.py index 54fc637f26..692240a486 100644 --- a/contrib/python/numpy/py3/numpy/version.py +++ b/contrib/python/numpy/py3/numpy/version.py @@ -1,5 +1,5 @@ -version = "1.26.2" +version = "1.26.3" __version__ = version full_version = version diff --git a/contrib/python/numpy/py3/ya.make b/contrib/python/numpy/py3/ya.make index cc5ca1691b..92042220c3 100644 --- a/contrib/python/numpy/py3/ya.make +++ b/contrib/python/numpy/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() PROVIDES(numpy) -VERSION(1.26.2) +VERSION(1.26.3) LICENSE(BSD-3-Clause) @@ -441,6 +441,7 @@ PY_SRCS( numpy/f2py/_backends/_distutils.py numpy/f2py/_backends/_meson.py numpy/f2py/_isocbind.py + numpy/f2py/_src_pyf.py numpy/f2py/auxfuncs.py numpy/f2py/capi_maps.py numpy/f2py/cb_rules.py diff --git a/contrib/python/pytest/py3/.dist-info/METADATA b/contrib/python/pytest/py3/.dist-info/METADATA index 8e45f6d805..20b02c2453 100644 --- a/contrib/python/pytest/py3/.dist-info/METADATA +++ b/contrib/python/pytest/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pytest -Version: 7.4.2 +Version: 7.4.4 Summary: pytest: simple powerful testing with Python Home-page: https://docs.pytest.org/en/latest/ Author: Holger Krekel, Bruno Oliveira, Ronny Pfannschmidt, Floris Bruynooghe, Brianna Laugher, Florian Bruhin and others diff --git a/contrib/python/pytest/py3/AUTHORS b/contrib/python/pytest/py3/AUTHORS index e8456d92b3..950672930f 100644 --- a/contrib/python/pytest/py3/AUTHORS +++ b/contrib/python/pytest/py3/AUTHORS @@ -47,6 +47,7 @@ Ariel Pillemer Armin Rigo Aron Coyle Aron Curzon +Arthur Richard Ashish Kurmi Aviral Verma Aviv Palivoda @@ -231,6 +232,7 @@ Maho Maik Figura Mandeep Bhutani Manuel Krebber +Marc Mueller Marc Schlaich Marcelo Duarte Trevisani Marcin Bachry @@ -322,6 +324,7 @@ Ronny Pfannschmidt Ross Lawley Ruaridh Williamson Russel Winder +Ryan Puddephatt Ryan Wooden Saiprasad Kale Samuel Colvin @@ -336,6 +339,7 @@ Serhii Mozghovyi Seth Junot Shantanu Jain Shubham Adep +Simon Blanchard Simon Gomizelj Simon Holesch Simon Kerr @@ -373,6 +377,7 @@ Tomer Keren Tony Narlock Tor Colvin Trevor Bekolay +Tushar Sadhwani Tyler Goodlet Tyler Smart Tzu-ping Chung diff --git a/contrib/python/pytest/py3/_pytest/_version.py b/contrib/python/pytest/py3/_pytest/_version.py index a7f556f8a6..458d065928 100644 --- a/contrib/python/pytest/py3/_pytest/_version.py +++ b/contrib/python/pytest/py3/_pytest/_version.py @@ -1,4 +1,16 @@ # file generated by setuptools_scm # don't change, don't track in version control -__version__ = version = '7.4.2' -__version_tuple__ = version_tuple = (7, 4, 2) +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple, Union + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '7.4.4' +__version_tuple__ = version_tuple = (7, 4, 4) diff --git a/contrib/python/pytest/py3/_pytest/assertion/rewrite.py b/contrib/python/pytest/py3/_pytest/assertion/rewrite.py index ab83fee32b..d1974bb3b4 100644 --- a/contrib/python/pytest/py3/_pytest/assertion/rewrite.py +++ b/contrib/python/pytest/py3/_pytest/assertion/rewrite.py @@ -13,6 +13,7 @@ import struct import sys import tokenize import types +from collections import defaultdict from pathlib import Path from pathlib import PurePath from typing import Callable @@ -56,6 +57,10 @@ else: astNum = ast.Num +class Sentinel: + pass + + assertstate_key = StashKey["AssertionState"]() # pytest caches rewritten pycs in pycache dirs @@ -63,6 +68,9 @@ PYTEST_TAG = f"{sys.implementation.cache_tag}-pytest-{version}" PYC_EXT = ".py" + (__debug__ and "c" or "o") PYC_TAIL = "." + PYTEST_TAG + PYC_EXT +# Special marker that denotes we have just left a scope definition +_SCOPE_END_MARKER = Sentinel() + class AssertionRewritingHook(importlib.abc.MetaPathFinder, importlib.abc.Loader): """PEP302/PEP451 import hook which rewrites asserts.""" @@ -596,6 +604,13 @@ def _get_assertion_exprs(src: bytes) -> Dict[int, str]: return ret +def _get_ast_constant_value(value: astStr) -> object: + if sys.version_info >= (3, 8): + return value.value + else: + return value.s + + class AssertionRewriter(ast.NodeVisitor): """Assertion rewriting implementation. @@ -645,6 +660,8 @@ class AssertionRewriter(ast.NodeVisitor): .push_format_context() and .pop_format_context() which allows to build another %-formatted string while already building one. + :scope: A tuple containing the current scope used for variables_overwrite. + :variables_overwrite: A dict filled with references to variables that change value within an assert. This happens when a variable is reassigned with the walrus operator @@ -666,7 +683,10 @@ class AssertionRewriter(ast.NodeVisitor): else: self.enable_assertion_pass_hook = False self.source = source - self.variables_overwrite: Dict[str, str] = {} + self.scope: tuple[ast.AST, ...] = () + self.variables_overwrite: defaultdict[ + tuple[ast.AST, ...], Dict[str, str] + ] = defaultdict(dict) def run(self, mod: ast.Module) -> None: """Find all assert statements in *mod* and rewrite them.""" @@ -687,11 +707,10 @@ class AssertionRewriter(ast.NodeVisitor): expect_docstring and isinstance(item, ast.Expr) and isinstance(item.value, astStr) + and isinstance(_get_ast_constant_value(item.value), str) ): - if sys.version_info >= (3, 8): - doc = item.value.value - else: - doc = item.value.s + doc = _get_ast_constant_value(item.value) + assert isinstance(doc, str) if self.is_rewrite_disabled(doc): return expect_docstring = False @@ -732,9 +751,17 @@ class AssertionRewriter(ast.NodeVisitor): mod.body[pos:pos] = imports # Collect asserts. - nodes: List[ast.AST] = [mod] + self.scope = (mod,) + nodes: List[Union[ast.AST, Sentinel]] = [mod] while nodes: node = nodes.pop() + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + self.scope = tuple((*self.scope, node)) + nodes.append(_SCOPE_END_MARKER) + if node == _SCOPE_END_MARKER: + self.scope = self.scope[:-1] + continue + assert isinstance(node, ast.AST) for name, field in ast.iter_fields(node): if isinstance(field, list): new: List[ast.AST] = [] @@ -1005,7 +1032,7 @@ class AssertionRewriter(ast.NodeVisitor): ] ): pytest_temp = self.variable() - self.variables_overwrite[ + self.variables_overwrite[self.scope][ v.left.target.id ] = v.left # type:ignore[assignment] v.left.target.id = pytest_temp @@ -1048,17 +1075,20 @@ class AssertionRewriter(ast.NodeVisitor): new_args = [] new_kwargs = [] for arg in call.args: - if isinstance(arg, ast.Name) and arg.id in self.variables_overwrite: - arg = self.variables_overwrite[arg.id] # type:ignore[assignment] + if isinstance(arg, ast.Name) and arg.id in self.variables_overwrite.get( + self.scope, {} + ): + arg = self.variables_overwrite[self.scope][ + arg.id + ] # type:ignore[assignment] res, expl = self.visit(arg) arg_expls.append(expl) new_args.append(res) for keyword in call.keywords: - if ( - isinstance(keyword.value, ast.Name) - and keyword.value.id in self.variables_overwrite - ): - keyword.value = self.variables_overwrite[ + if isinstance( + keyword.value, ast.Name + ) and keyword.value.id in self.variables_overwrite.get(self.scope, {}): + keyword.value = self.variables_overwrite[self.scope][ keyword.value.id ] # type:ignore[assignment] res, expl = self.visit(keyword.value) @@ -1094,12 +1124,14 @@ class AssertionRewriter(ast.NodeVisitor): def visit_Compare(self, comp: ast.Compare) -> Tuple[ast.expr, str]: self.push_format_context() # We first check if we have overwritten a variable in the previous assert - if isinstance(comp.left, ast.Name) and comp.left.id in self.variables_overwrite: - comp.left = self.variables_overwrite[ + if isinstance( + comp.left, ast.Name + ) and comp.left.id in self.variables_overwrite.get(self.scope, {}): + comp.left = self.variables_overwrite[self.scope][ comp.left.id ] # type:ignore[assignment] if isinstance(comp.left, namedExpr): - self.variables_overwrite[ + self.variables_overwrite[self.scope][ comp.left.target.id ] = comp.left # type:ignore[assignment] left_res, left_expl = self.visit(comp.left) @@ -1119,7 +1151,7 @@ class AssertionRewriter(ast.NodeVisitor): and next_operand.target.id == left_res.id ): next_operand.target.id = self.variable() - self.variables_overwrite[ + self.variables_overwrite[self.scope][ left_res.id ] = next_operand # type:ignore[assignment] next_res, next_expl = self.visit(next_operand) diff --git a/contrib/python/pytest/py3/_pytest/assertion/util.py b/contrib/python/pytest/py3/_pytest/assertion/util.py index fc5dfdbd5b..39ca5403e0 100644 --- a/contrib/python/pytest/py3/_pytest/assertion/util.py +++ b/contrib/python/pytest/py3/_pytest/assertion/util.py @@ -132,7 +132,7 @@ def isiterable(obj: Any) -> bool: try: iter(obj) return not istext(obj) - except TypeError: + except Exception: return False diff --git a/contrib/python/pytest/py3/_pytest/compat.py b/contrib/python/pytest/py3/_pytest/compat.py index a1f9d37722..6fdb5a1d8e 100644 --- a/contrib/python/pytest/py3/_pytest/compat.py +++ b/contrib/python/pytest/py3/_pytest/compat.py @@ -380,15 +380,24 @@ else: def get_user_id() -> int | None: - """Return the current user id, or None if we cannot get it reliably on the current platform.""" - # win32 does not have a getuid() function. - # On Emscripten, getuid() is a stub that always returns 0. - if sys.platform in ("win32", "emscripten"): + """Return the current process's real user id or None if it could not be + determined. + + :return: The user id or None if it could not be determined. + """ + # mypy follows the version and platform checking expectation of PEP 484: + # https://mypy.readthedocs.io/en/stable/common_issues.html?highlight=platform#python-version-and-system-platform-checks + # Containment checks are too complex for mypy v1.5.0 and cause failure. + if sys.platform == "win32" or sys.platform == "emscripten": + # win32 does not have a getuid() function. + # Emscripten has a return 0 stub. return None - # getuid shouldn't fail, but cpython defines such a case. - # Let's hope for the best. - uid = os.getuid() - return uid if uid != -1 else None + else: + # On other platforms, a return value of -1 is assumed to indicate that + # the current process's real user id could not be determined. + ERROR = -1 + uid = os.getuid() + return uid if uid != ERROR else None # Perform exhaustiveness checking. diff --git a/contrib/python/pytest/py3/_pytest/faulthandler.py b/contrib/python/pytest/py3/_pytest/faulthandler.py index af879aa44c..d8c7e9fd3b 100644 --- a/contrib/python/pytest/py3/_pytest/faulthandler.py +++ b/contrib/python/pytest/py3/_pytest/faulthandler.py @@ -1,4 +1,3 @@ -import io import os import sys from typing import Generator @@ -10,8 +9,8 @@ from _pytest.nodes import Item from _pytest.stash import StashKey +fault_handler_original_stderr_fd_key = StashKey[int]() fault_handler_stderr_fd_key = StashKey[int]() -fault_handler_originally_enabled_key = StashKey[bool]() def pytest_addoption(parser: Parser) -> None: @@ -25,8 +24,15 @@ def pytest_addoption(parser: Parser) -> None: def pytest_configure(config: Config) -> None: import faulthandler - config.stash[fault_handler_stderr_fd_key] = os.dup(get_stderr_fileno()) - config.stash[fault_handler_originally_enabled_key] = faulthandler.is_enabled() + # at teardown we want to restore the original faulthandler fileno + # but faulthandler has no api to return the original fileno + # so here we stash the stderr fileno to be used at teardown + # sys.stderr and sys.__stderr__ may be closed or patched during the session + # so we can't rely on their values being good at that point (#11572). + stderr_fileno = get_stderr_fileno() + if faulthandler.is_enabled(): + config.stash[fault_handler_original_stderr_fd_key] = stderr_fileno + config.stash[fault_handler_stderr_fd_key] = os.dup(stderr_fileno) faulthandler.enable(file=config.stash[fault_handler_stderr_fd_key]) @@ -38,9 +44,10 @@ def pytest_unconfigure(config: Config) -> None: if fault_handler_stderr_fd_key in config.stash: os.close(config.stash[fault_handler_stderr_fd_key]) del config.stash[fault_handler_stderr_fd_key] - if config.stash.get(fault_handler_originally_enabled_key, False): - # Re-enable the faulthandler if it was originally enabled. - faulthandler.enable(file=get_stderr_fileno()) + # Re-enable the faulthandler if it was originally enabled. + if fault_handler_original_stderr_fd_key in config.stash: + faulthandler.enable(config.stash[fault_handler_original_stderr_fd_key]) + del config.stash[fault_handler_original_stderr_fd_key] def get_stderr_fileno() -> int: @@ -51,7 +58,7 @@ def get_stderr_fileno() -> int: if fileno == -1: raise AttributeError() return fileno - except (AttributeError, io.UnsupportedOperation): + except (AttributeError, ValueError): # pytest-xdist monkeypatches sys.stderr with an object that is not an actual file. # https://docs.python.org/3/library/faulthandler.html#issue-with-file-descriptors # This is potentially dangerous, but the best we can do. diff --git a/contrib/python/pytest/py3/_pytest/junitxml.py b/contrib/python/pytest/py3/_pytest/junitxml.py index ed259e4c41..9ee35b84e8 100644 --- a/contrib/python/pytest/py3/_pytest/junitxml.py +++ b/contrib/python/pytest/py3/_pytest/junitxml.py @@ -369,7 +369,7 @@ def record_testsuite_property(request: FixtureRequest) -> Callable[[str, object] __tracebackhide__ = True def record_func(name: str, value: object) -> None: - """No-op function in case --junitxml was not passed in the command-line.""" + """No-op function in case --junit-xml was not passed in the command-line.""" __tracebackhide__ = True _check_record_param_type("name", name) diff --git a/contrib/python/pytest/py3/_pytest/mark/structures.py b/contrib/python/pytest/py3/_pytest/mark/structures.py index 42fb294c6d..32bdc7e38b 100644 --- a/contrib/python/pytest/py3/_pytest/mark/structures.py +++ b/contrib/python/pytest/py3/_pytest/mark/structures.py @@ -373,7 +373,9 @@ def get_unpacked_marks( if not consider_mro: mark_lists = [obj.__dict__.get("pytestmark", [])] else: - mark_lists = [x.__dict__.get("pytestmark", []) for x in obj.__mro__] + mark_lists = [ + x.__dict__.get("pytestmark", []) for x in reversed(obj.__mro__) + ] mark_list = [] for item in mark_lists: if isinstance(item, list): diff --git a/contrib/python/pytest/py3/_pytest/nodes.py b/contrib/python/pytest/py3/_pytest/nodes.py index 667a02b77a..a5313cb765 100644 --- a/contrib/python/pytest/py3/_pytest/nodes.py +++ b/contrib/python/pytest/py3/_pytest/nodes.py @@ -567,7 +567,7 @@ class Collector(Node): ntraceback = traceback.cut(path=self.path) if ntraceback == traceback: ntraceback = ntraceback.cut(excludepath=tracebackcutdir) - return excinfo.traceback.filter(excinfo) + return ntraceback.filter(excinfo) return excinfo.traceback diff --git a/contrib/python/pytest/py3/_pytest/outcomes.py b/contrib/python/pytest/py3/_pytest/outcomes.py index 1be97dda4e..53c3e1511c 100644 --- a/contrib/python/pytest/py3/_pytest/outcomes.py +++ b/contrib/python/pytest/py3/_pytest/outcomes.py @@ -123,7 +123,7 @@ def exit( only because `msg` is deprecated. :param returncode: - Return code to be used when exiting pytest. + Return code to be used when exiting pytest. None means the same as ``0`` (no error), same as :func:`sys.exit`. :param msg: Same as ``reason``, but deprecated. Will be removed in a future version, use ``reason`` instead. diff --git a/contrib/python/pytest/py3/_pytest/pathlib.py b/contrib/python/pytest/py3/_pytest/pathlib.py index 5c765c6834..c2f8535f5f 100644 --- a/contrib/python/pytest/py3/_pytest/pathlib.py +++ b/contrib/python/pytest/py3/_pytest/pathlib.py @@ -623,8 +623,9 @@ def module_name_from_path(path: Path, root: Path) -> str: # Use the parts for the relative path to the root path. path_parts = relative_path.parts - # Module name for packages do not contain the __init__ file. - if path_parts[-1] == "__init__": + # Module name for packages do not contain the __init__ file, unless + # the `__init__.py` file is at the root. + if len(path_parts) >= 2 and path_parts[-1] == "__init__": path_parts = path_parts[:-1] return ".".join(path_parts) diff --git a/contrib/python/pytest/py3/_pytest/pytester.py b/contrib/python/pytest/py3/_pytest/pytester.py index cdfc2c04ae..0771065e06 100644 --- a/contrib/python/pytest/py3/_pytest/pytester.py +++ b/contrib/python/pytest/py3/_pytest/pytester.py @@ -1074,7 +1074,7 @@ class Pytester: return self.inline_run(*values) def inline_genitems(self, *args) -> Tuple[List[Item], HookRecorder]: - """Run ``pytest.main(['--collectonly'])`` in-process. + """Run ``pytest.main(['--collect-only'])`` in-process. Runs the :py:func:`pytest.main` function to run all of pytest inside the test process itself like :py:meth:`inline_run`, but returns a diff --git a/contrib/python/pytest/py3/ya.make b/contrib/python/pytest/py3/ya.make index 8bef9919d6..a1ab790ee2 100644 --- a/contrib/python/pytest/py3/ya.make +++ b/contrib/python/pytest/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(7.4.2) +VERSION(7.4.4) LICENSE(MIT) diff --git a/contrib/python/setuptools/py3/.dist-info/METADATA b/contrib/python/setuptools/py3/.dist-info/METADATA index ebe368510a..a12855b091 100644 --- a/contrib/python/setuptools/py3/.dist-info/METADATA +++ b/contrib/python/setuptools/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: setuptools -Version: 69.0.2 +Version: 69.0.3 Summary: Easily download, build, install, upgrade, and uninstall Python packages Home-page: https://github.com/pypa/setuptools Author: Python Packaging Authority @@ -75,7 +75,7 @@ Requires-Dist: pytest-perf ; (sys_platform != "cygwin") and extra == 'testing' .. image:: https://img.shields.io/pypi/pyversions/setuptools.svg -.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg +.. image:: https://github.com/pypa/setuptools/actions/workflows/main.yml/badge.svg :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22 :alt: tests diff --git a/contrib/python/setuptools/py3/README.rst b/contrib/python/setuptools/py3/README.rst index 699576551d..92c7dc6e87 100644 --- a/contrib/python/setuptools/py3/README.rst +++ b/contrib/python/setuptools/py3/README.rst @@ -3,7 +3,7 @@ .. image:: https://img.shields.io/pypi/pyversions/setuptools.svg -.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg +.. image:: https://github.com/pypa/setuptools/actions/workflows/main.yml/badge.svg :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22 :alt: tests diff --git a/contrib/python/setuptools/py3/setuptools/_normalization.py b/contrib/python/setuptools/py3/setuptools/_normalization.py index eee4fb7746..aa9274f093 100644 --- a/contrib/python/setuptools/py3/setuptools/_normalization.py +++ b/contrib/python/setuptools/py3/setuptools/_normalization.py @@ -12,7 +12,7 @@ _Path = Union[str, Path] # https://packaging.python.org/en/latest/specifications/core-metadata/#name _VALID_NAME = re.compile(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.I) -_UNSAFE_NAME_CHARS = re.compile(r"[^A-Z0-9.]+", re.I) +_UNSAFE_NAME_CHARS = re.compile(r"[^A-Z0-9._-]+", re.I) _NON_ALPHANUMERIC = re.compile(r"[^A-Z0-9]+", re.I) _PEP440_FALLBACK = re.compile(r"^v?(?P<safe>(?:[0-9]+!)?[0-9]+(?:\.[0-9]+)*)", re.I) @@ -35,6 +35,8 @@ def safe_name(component: str) -> str: 'hello-world' >>> safe_name("hello?world") 'hello-world' + >>> safe_name("hello_world") + 'hello_world' """ # See pkg_resources.safe_name return _UNSAFE_NAME_CHARS.sub("-", component) diff --git a/contrib/python/setuptools/py3/ya.make b/contrib/python/setuptools/py3/ya.make index 87de186610..2cb2a8b26c 100644 --- a/contrib/python/setuptools/py3/ya.make +++ b/contrib/python/setuptools/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(69.0.2) +VERSION(69.0.3) LICENSE(MIT) diff --git a/contrib/python/types-protobuf/.dist-info/METADATA b/contrib/python/types-protobuf/.dist-info/METADATA index 1eaca69326..a54cbbdacd 100644 --- a/contrib/python/types-protobuf/.dist-info/METADATA +++ b/contrib/python/types-protobuf/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: types-protobuf -Version: 3.20.4.1 +Version: 4.24.0.4 Summary: Typing stubs for protobuf Home-page: https://github.com/python/typeshed License: Apache-2.0 license @@ -11,17 +11,33 @@ Project-URL: Chat, https://gitter.im/python/typing Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python :: 3 Classifier: Typing :: Stubs Only +Requires-Python: >=3.7 Description-Content-Type: text/markdown ## Typing stubs for protobuf -This is a PEP 561 type stub package for the `protobuf` package. -It can be used by type-checking tools like mypy, PyCharm, pytype etc. to check code -that uses `protobuf`. The source for this package can be found at -https://github.com/python/typeshed/tree/master/stubs/protobuf. All fixes for +This is a [PEP 561](https://peps.python.org/pep-0561/) +type stub package for the [`protobuf`](https://github.com/protocolbuffers/protobuf) package. +It can be used by type-checking tools like +[mypy](https://github.com/python/mypy/), +[pyright](https://github.com/microsoft/pyright), +[pytype](https://github.com/google/pytype/), +PyCharm, etc. to check code that uses +`protobuf`. + +This version of `types-protobuf` aims to provide accurate annotations +for `protobuf==4.24.*`. +The source for this package can be found at +https://github.com/python/typeshed/tree/main/stubs/protobuf. All fixes for types and metadata should be contributed there. -Generated with aid from mypy-protobuf v3.4.0 +Generated using [mypy-protobuf==3.5.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.5.0) on protobuf==4.21.8 + +This stub package is marked as [partial](https://peps.python.org/pep-0561/#partial-stub-packages). +If you find that annotations are missing, feel free to contribute and help complete them. + -See https://github.com/python/typeshed/blob/master/README.md for more details. -This package was generated from typeshed commit `f43ee406f155b563856551fb7215cb03fdef8e2d`. +See https://github.com/python/typeshed/blob/main/README.md for more details. +This package was generated from typeshed commit `9d345b4df42939b697a84ee461a8760eb674050e` and was tested +with mypy 1.6.1, pyright 1.1.332, and +pytype 2023.10.17. diff --git a/contrib/python/types-protobuf/google-stubs/METADATA.toml b/contrib/python/types-protobuf/google-stubs/METADATA.toml index 06628e4dcc..7ee1107f03 100644 --- a/contrib/python/types-protobuf/google-stubs/METADATA.toml +++ b/contrib/python/types-protobuf/google-stubs/METADATA.toml @@ -1,2 +1,7 @@ -version = "3.20.*" -extra_description = "Generated with aid from mypy-protobuf v3.4.0" +version = "4.24.*" +upstream_repository = "https://github.com/protocolbuffers/protobuf" +extra_description = "Generated using [mypy-protobuf==3.5.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.5.0) on protobuf==4.21.8" +partial_stub = true + +[tool.stubtest] +ignore_missing_stub = true diff --git a/contrib/python/types-protobuf/ya.make b/contrib/python/types-protobuf/ya.make index 5e5ba3f5c1..4fb5c85b88 100644 --- a/contrib/python/types-protobuf/ya.make +++ b/contrib/python/types-protobuf/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.20.4.1) +VERSION(4.24.0.4) LICENSE(Apache-2.0) @@ -10,7 +10,6 @@ NO_LINT() PY_SRCS( TOP_LEVEL - google-stubs/__init__.pyi google-stubs/protobuf/__init__.pyi google-stubs/protobuf/any_pb2.pyi google-stubs/protobuf/api_pb2.pyi @@ -24,6 +23,7 @@ PY_SRCS( google-stubs/protobuf/field_mask_pb2.pyi google-stubs/protobuf/internal/__init__.pyi google-stubs/protobuf/internal/api_implementation.pyi + google-stubs/protobuf/internal/builder.pyi google-stubs/protobuf/internal/containers.pyi google-stubs/protobuf/internal/decoder.pyi google-stubs/protobuf/internal/encoder.pyi diff --git a/contrib/restricted/llhttp/README.md b/contrib/restricted/llhttp/README.md index e982ed0ae7..4960dbb5a8 100644 --- a/contrib/restricted/llhttp/README.md +++ b/contrib/restricted/llhttp/README.md @@ -363,6 +363,16 @@ With this flag only a CR is required to terminate such sections. **Enabling this flag can pose a security issue since you will be exposed to request smuggling attacks. USE WITH CAUTION!** +### `void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled)` + +Enables/disables lenient handling of line separators. + +Normally `llhttp` would error when a LF is not preceded by CR when terminating the +request line, the status line, the headers, a chunk header or a chunk data. +With this flag only a LF is required to terminate such sections. + +**Enabling this flag can pose a security issue since you will be exposed to request smuggling attacks. USE WITH CAUTION!** + ### `void llhttp_set_lenient_optional_crlf_after_chunk(llhttp_t* parser, int enabled)` Enables/disables lenient handling of chunks not separated via CRLF. @@ -373,6 +383,15 @@ With this flag the new chunk can start immediately after the previous one. **Enabling this flag can pose a security issue since you will be exposed to request smuggling attacks. USE WITH CAUTION!** +### `void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled)` + +Enables/disables lenient handling of spaces after chunk size. + +Normally `llhttp` would error when after a chunk size is followed by one or more spaces are present instead of a CRLF or `;`. +With this flag this check is disabled. + +**Enabling this flag can pose a security issue since you will be exposed to request smuggling attacks. USE WITH CAUTION!** + ## Build Instructions Make sure you have [Node.js](https://nodejs.org/), npm and npx installed. Then under project directory run: diff --git a/contrib/restricted/llhttp/include/llhttp.h b/contrib/restricted/llhttp/include/llhttp.h index 6588ae5e36..46a7213c12 100644 --- a/contrib/restricted/llhttp/include/llhttp.h +++ b/contrib/restricted/llhttp/include/llhttp.h @@ -3,8 +3,8 @@ #define INCLUDE_LLHTTP_H_ #define LLHTTP_VERSION_MAJOR 9 -#define LLHTTP_VERSION_MINOR 0 -#define LLHTTP_VERSION_PATCH 1 +#define LLHTTP_VERSION_MINOR 1 +#define LLHTTP_VERSION_PATCH 3 #ifndef INCLUDE_LLHTTP_ITSELF_H_ #define INCLUDE_LLHTTP_ITSELF_H_ @@ -30,7 +30,7 @@ struct llhttp__internal_s { uint8_t http_major; uint8_t http_minor; uint8_t header_state; - uint8_t lenient_flags; + uint16_t lenient_flags; uint8_t upgrade; uint8_t finish; uint16_t flags; @@ -115,7 +115,9 @@ enum llhttp_lenient_flags { LENIENT_VERSION = 0x10, LENIENT_DATA_AFTER_CLOSE = 0x20, LENIENT_OPTIONAL_LF_AFTER_CR = 0x40, - LENIENT_OPTIONAL_CRLF_AFTER_CHUNK = 0x80 + LENIENT_OPTIONAL_CRLF_AFTER_CHUNK = 0x80, + LENIENT_OPTIONAL_CR_BEFORE_LF = 0x100, + LENIENT_SPACES_AFTER_CHUNK_SIZE = 0x200 }; typedef enum llhttp_lenient_flags llhttp_lenient_flags_t; @@ -807,7 +809,7 @@ void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled); * avoid request smuggling. * With this flag the extra value will be parsed normally. * - * **Enabling this flag can pose a security issue since you will be exposed to + * **Enabling this flag can pose a security issue since you will be exposed to * request smuggling attacks. USE WITH CAUTION!** */ LLHTTP_EXPORT @@ -850,6 +852,19 @@ void llhttp_set_lenient_data_after_close(llhttp_t* parser, int enabled); LLHTTP_EXPORT void llhttp_set_lenient_optional_lf_after_cr(llhttp_t* parser, int enabled); +/* + * Enables/disables lenient handling of line separators. + * + * Normally `llhttp` would error when a LF is not preceded by CR when terminating the + * request line, the status line, the headers, a chunk header or a chunk data. + * With this flag only a LF is required to terminate such sections. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled); + /* Enables/disables lenient handling of chunks not separated via CRLF. * * Normally `llhttp` would error when after a chunk data a CRLF is missing before @@ -862,6 +877,18 @@ void llhttp_set_lenient_optional_lf_after_cr(llhttp_t* parser, int enabled); LLHTTP_EXPORT void llhttp_set_lenient_optional_crlf_after_chunk(llhttp_t* parser, int enabled); +/* Enables/disables lenient handling of spaces after chunk size. + * + * Normally `llhttp` would error when after a chunk size is followed by one or more + * spaces are present instead of a CRLF or `;`. + * With this flag this check is disabled. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/contrib/restricted/llhttp/src/api.c b/contrib/restricted/llhttp/src/api.c index e0d0385051..8c2ce3dc5c 100644 --- a/contrib/restricted/llhttp/src/api.c +++ b/contrib/restricted/llhttp/src/api.c @@ -126,7 +126,7 @@ void llhttp_reset(llhttp_t* parser) { llhttp_type_t type = parser->type; const llhttp_settings_t* settings = parser->settings; void* data = parser->data; - uint8_t lenient_flags = parser->lenient_flags; + uint16_t lenient_flags = parser->lenient_flags; llhttp__internal_init(parser); @@ -315,6 +315,22 @@ void llhttp_set_lenient_optional_crlf_after_chunk(llhttp_t* parser, int enabled) } } +void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_OPTIONAL_CR_BEFORE_LF; + } else { + parser->lenient_flags &= ~LENIENT_OPTIONAL_CR_BEFORE_LF; + } +} + +void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_SPACES_AFTER_CHUNK_SIZE; + } else { + parser->lenient_flags &= ~LENIENT_SPACES_AFTER_CHUNK_SIZE; + } +} + /* Callbacks */ diff --git a/contrib/restricted/llhttp/src/http.c b/contrib/restricted/llhttp/src/http.c index 3a66044f5f..1ab91a5579 100644 --- a/contrib/restricted/llhttp/src/http.c +++ b/contrib/restricted/llhttp/src/http.c @@ -39,13 +39,33 @@ int llhttp__after_headers_complete(llhttp_t* parser, const char* p, int hasBody; hasBody = parser->flags & F_CHUNKED || parser->content_length > 0; - if (parser->upgrade && (parser->method == HTTP_CONNECT || - (parser->flags & F_SKIPBODY) || !hasBody)) { + if ( + (parser->upgrade && (parser->method == HTTP_CONNECT || + (parser->flags & F_SKIPBODY) || !hasBody)) || + /* See RFC 2616 section 4.4 - 1xx e.g. Continue */ + (parser->type == HTTP_RESPONSE && parser->status_code == 101) + ) { /* Exit, the rest of the message is in a different protocol. */ return 1; } - if (parser->flags & F_SKIPBODY) { + if (parser->type == HTTP_RESPONSE && parser->status_code == 100) { + /* No body, restart as the message is complete */ + return 0; + } + + /* See RFC 2616 section 4.4 */ + if ( + parser->flags & F_SKIPBODY || /* response to a HEAD request */ + ( + parser->type == HTTP_RESPONSE && ( + parser->status_code == 102 || /* Processing */ + parser->status_code == 103 || /* Early Hints */ + parser->status_code == 204 || /* No Content */ + parser->status_code == 304 /* Not Modified */ + ) + ) + ) { return 0; } else if (parser->flags & F_CHUNKED) { /* chunked encoding - ignore Content-Length header, prepare for a chunk */ diff --git a/contrib/restricted/llhttp/src/llhttp.c b/contrib/restricted/llhttp/src/llhttp.c index 03f6d57c8c..af84c42425 100644 --- a/contrib/restricted/llhttp/src/llhttp.c +++ b/contrib/restricted/llhttp/src/llhttp.c @@ -22,197 +22,194 @@ typedef int (*llhttp__internal__span_cb)( llhttp__internal_t*, const char*, const char*); static const unsigned char llparse_blob0[] = { - 0xd, 0xa -}; -static const unsigned char llparse_blob1[] = { 'o', 'n' }; -static const unsigned char llparse_blob2[] = { +static const unsigned char llparse_blob1[] = { 'e', 'c', 't', 'i', 'o', 'n' }; -static const unsigned char llparse_blob3[] = { +static const unsigned char llparse_blob2[] = { 'l', 'o', 's', 'e' }; -static const unsigned char llparse_blob4[] = { +static const unsigned char llparse_blob3[] = { 'e', 'e', 'p', '-', 'a', 'l', 'i', 'v', 'e' }; -static const unsigned char llparse_blob5[] = { +static const unsigned char llparse_blob4[] = { 'p', 'g', 'r', 'a', 'd', 'e' }; -static const unsigned char llparse_blob6[] = { +static const unsigned char llparse_blob5[] = { 'c', 'h', 'u', 'n', 'k', 'e', 'd' }; #ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob7[] = { +static const unsigned char ALIGN(16) llparse_blob6[] = { 0x9, 0x9, ' ', '~', 0x80, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; #endif /* __SSE4_2__ */ #ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob8[] = { +static const unsigned char ALIGN(16) llparse_blob7[] = { '!', '!', '#', '\'', '*', '+', '-', '.', '0', '9', 'A', 'Z', '^', 'z', '|', '|' }; #endif /* __SSE4_2__ */ #ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob9[] = { +static const unsigned char ALIGN(16) llparse_blob8[] = { '~', '~', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; #endif /* __SSE4_2__ */ -static const unsigned char llparse_blob10[] = { +static const unsigned char llparse_blob9[] = { 'e', 'n', 't', '-', 'l', 'e', 'n', 'g', 't', 'h' }; -static const unsigned char llparse_blob11[] = { +static const unsigned char llparse_blob10[] = { 'r', 'o', 'x', 'y', '-', 'c', 'o', 'n', 'n', 'e', 'c', 't', 'i', 'o', 'n' }; -static const unsigned char llparse_blob12[] = { +static const unsigned char llparse_blob11[] = { 'r', 'a', 'n', 's', 'f', 'e', 'r', '-', 'e', 'n', 'c', 'o', 'd', 'i', 'n', 'g' }; -static const unsigned char llparse_blob13[] = { +static const unsigned char llparse_blob12[] = { 'p', 'g', 'r', 'a', 'd', 'e' }; -static const unsigned char llparse_blob14[] = { +static const unsigned char llparse_blob13[] = { 'T', 'T', 'P', '/' }; -static const unsigned char llparse_blob15[] = { +static const unsigned char llparse_blob14[] = { 0xd, 0xa, 0xd, 0xa, 'S', 'M', 0xd, 0xa, 0xd, 0xa }; -static const unsigned char llparse_blob16[] = { +static const unsigned char llparse_blob15[] = { 'C', 'E', '/' }; -static const unsigned char llparse_blob17[] = { +static const unsigned char llparse_blob16[] = { 'T', 'S', 'P', '/' }; -static const unsigned char llparse_blob18[] = { +static const unsigned char llparse_blob17[] = { 'N', 'O', 'U', 'N', 'C', 'E' }; -static const unsigned char llparse_blob19[] = { +static const unsigned char llparse_blob18[] = { 'I', 'N', 'D' }; -static const unsigned char llparse_blob20[] = { +static const unsigned char llparse_blob19[] = { 'E', 'C', 'K', 'O', 'U', 'T' }; -static const unsigned char llparse_blob21[] = { +static const unsigned char llparse_blob20[] = { 'N', 'E', 'C', 'T' }; -static const unsigned char llparse_blob22[] = { +static const unsigned char llparse_blob21[] = { 'E', 'T', 'E' }; -static const unsigned char llparse_blob23[] = { +static const unsigned char llparse_blob22[] = { 'C', 'R', 'I', 'B', 'E' }; -static const unsigned char llparse_blob24[] = { +static const unsigned char llparse_blob23[] = { 'L', 'U', 'S', 'H' }; -static const unsigned char llparse_blob25[] = { +static const unsigned char llparse_blob24[] = { 'E', 'T' }; -static const unsigned char llparse_blob26[] = { +static const unsigned char llparse_blob25[] = { 'P', 'A', 'R', 'A', 'M', 'E', 'T', 'E', 'R' }; -static const unsigned char llparse_blob27[] = { +static const unsigned char llparse_blob26[] = { 'E', 'A', 'D' }; -static const unsigned char llparse_blob28[] = { +static const unsigned char llparse_blob27[] = { 'N', 'K' }; -static const unsigned char llparse_blob29[] = { +static const unsigned char llparse_blob28[] = { 'C', 'K' }; -static const unsigned char llparse_blob30[] = { +static const unsigned char llparse_blob29[] = { 'S', 'E', 'A', 'R', 'C', 'H' }; -static const unsigned char llparse_blob31[] = { +static const unsigned char llparse_blob30[] = { 'R', 'G', 'E' }; -static const unsigned char llparse_blob32[] = { +static const unsigned char llparse_blob31[] = { 'C', 'T', 'I', 'V', 'I', 'T', 'Y' }; -static const unsigned char llparse_blob33[] = { +static const unsigned char llparse_blob32[] = { 'L', 'E', 'N', 'D', 'A', 'R' }; -static const unsigned char llparse_blob34[] = { +static const unsigned char llparse_blob33[] = { 'V', 'E' }; -static const unsigned char llparse_blob35[] = { +static const unsigned char llparse_blob34[] = { 'O', 'T', 'I', 'F', 'Y' }; -static const unsigned char llparse_blob36[] = { +static const unsigned char llparse_blob35[] = { 'P', 'T', 'I', 'O', 'N', 'S' }; -static const unsigned char llparse_blob37[] = { +static const unsigned char llparse_blob36[] = { 'C', 'H' }; -static const unsigned char llparse_blob38[] = { +static const unsigned char llparse_blob37[] = { 'S', 'E' }; -static const unsigned char llparse_blob39[] = { +static const unsigned char llparse_blob38[] = { 'A', 'Y' }; -static const unsigned char llparse_blob40[] = { +static const unsigned char llparse_blob39[] = { 'S', 'T' }; -static const unsigned char llparse_blob41[] = { +static const unsigned char llparse_blob40[] = { 'I', 'N', 'D' }; -static const unsigned char llparse_blob42[] = { +static const unsigned char llparse_blob41[] = { 'A', 'T', 'C', 'H' }; -static const unsigned char llparse_blob43[] = { +static const unsigned char llparse_blob42[] = { 'G', 'E' }; -static const unsigned char llparse_blob44[] = { +static const unsigned char llparse_blob43[] = { 'I', 'N', 'D' }; -static const unsigned char llparse_blob45[] = { +static const unsigned char llparse_blob44[] = { 'O', 'R', 'D' }; -static const unsigned char llparse_blob46[] = { +static const unsigned char llparse_blob45[] = { 'I', 'R', 'E', 'C', 'T' }; -static const unsigned char llparse_blob47[] = { +static const unsigned char llparse_blob46[] = { 'O', 'R', 'T' }; -static const unsigned char llparse_blob48[] = { +static const unsigned char llparse_blob47[] = { 'R', 'C', 'H' }; -static const unsigned char llparse_blob49[] = { +static const unsigned char llparse_blob48[] = { 'P', 'A', 'R', 'A', 'M', 'E', 'T', 'E', 'R' }; -static const unsigned char llparse_blob50[] = { +static const unsigned char llparse_blob49[] = { 'U', 'R', 'C', 'E' }; -static const unsigned char llparse_blob51[] = { +static const unsigned char llparse_blob50[] = { 'B', 'S', 'C', 'R', 'I', 'B', 'E' }; -static const unsigned char llparse_blob52[] = { +static const unsigned char llparse_blob51[] = { 'A', 'R', 'D', 'O', 'W', 'N' }; -static const unsigned char llparse_blob53[] = { +static const unsigned char llparse_blob52[] = { 'A', 'C', 'E' }; -static const unsigned char llparse_blob54[] = { +static const unsigned char llparse_blob53[] = { 'I', 'N', 'D' }; -static const unsigned char llparse_blob55[] = { +static const unsigned char llparse_blob54[] = { 'N', 'K' }; -static const unsigned char llparse_blob56[] = { +static const unsigned char llparse_blob55[] = { 'C', 'K' }; -static const unsigned char llparse_blob57[] = { +static const unsigned char llparse_blob56[] = { 'U', 'B', 'S', 'C', 'R', 'I', 'B', 'E' }; -static const unsigned char llparse_blob58[] = { +static const unsigned char llparse_blob57[] = { 'H', 'T', 'T', 'P', '/' }; -static const unsigned char llparse_blob59[] = { +static const unsigned char llparse_blob58[] = { 'A', 'D' }; -static const unsigned char llparse_blob60[] = { +static const unsigned char llparse_blob59[] = { 'T', 'P', '/' }; @@ -229,7 +226,7 @@ struct llparse_match_s { }; typedef struct llparse_match_s llparse_match_t; -static llparse_match_t llparse__match_sequence_id( +static llparse_match_t llparse__match_sequence_to_lower( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp, const unsigned char* seq, uint32_t seq_len) { @@ -240,7 +237,7 @@ static llparse_match_t llparse__match_sequence_id( for (; p != endp; p++) { unsigned char current; - current = *p; + current = ((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p)); if (current == seq[index]) { if (++index == seq_len) { res.status = kMatchComplete; @@ -261,7 +258,7 @@ reset: return res; } -static llparse_match_t llparse__match_sequence_to_lower( +static llparse_match_t llparse__match_sequence_to_lower_unsafe( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp, const unsigned char* seq, uint32_t seq_len) { @@ -272,7 +269,7 @@ static llparse_match_t llparse__match_sequence_to_lower( for (; p != endp; p++) { unsigned char current; - current = ((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p)); + current = ((*p) | 0x20); if (current == seq[index]) { if (++index == seq_len) { res.status = kMatchComplete; @@ -293,7 +290,7 @@ reset: return res; } -static llparse_match_t llparse__match_sequence_to_lower_unsafe( +static llparse_match_t llparse__match_sequence_id( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp, const unsigned char* seq, uint32_t seq_len) { @@ -304,7 +301,7 @@ static llparse_match_t llparse__match_sequence_to_lower_unsafe( for (; p != endp; p++) { unsigned char current; - current = ((*p) | 0x20); + current = *p; if (current == seq[index]) { if (++index == seq_len) { res.status = kMatchComplete; @@ -330,23 +327,32 @@ enum llparse_state_e { s_n_llhttp__internal__n_closed, s_n_llhttp__internal__n_invoke_llhttp__after_message_complete, s_n_llhttp__internal__n_pause_1, + s_n_llhttp__internal__n_invoke_is_equal_upgrade, + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2, + s_n_llhttp__internal__n_chunk_data_almost_done_1, s_n_llhttp__internal__n_chunk_data_almost_done, s_n_llhttp__internal__n_consume_content_length, s_n_llhttp__internal__n_span_start_llhttp__on_body, s_n_llhttp__internal__n_invoke_is_equal_content_length, s_n_llhttp__internal__n_chunk_size_almost_done, + s_n_llhttp__internal__n_invoke_test_lenient_flags_9, s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete, s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_1, + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2, + s_n_llhttp__internal__n_invoke_test_lenient_flags_10, s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete, - s_n_llhttp__internal__n_chunk_extension_quoted_value_done, s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1, - s_n_llhttp__internal__n_error_21, - s_n_llhttp__internal__n_chunk_extension_quoted_value, + s_n_llhttp__internal__n_chunk_extension_quoted_value_done, s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2, - s_n_llhttp__internal__n_error_23, + s_n_llhttp__internal__n_error_30, + s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair, + s_n_llhttp__internal__n_error_31, + s_n_llhttp__internal__n_chunk_extension_quoted_value, + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3, + s_n_llhttp__internal__n_error_33, s_n_llhttp__internal__n_chunk_extension_value, s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_value, - s_n_llhttp__internal__n_error_24, + s_n_llhttp__internal__n_error_34, s_n_llhttp__internal__n_chunk_extension_name, s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_name, s_n_llhttp__internal__n_chunk_extensions, @@ -354,8 +360,6 @@ enum llparse_state_e { s_n_llhttp__internal__n_chunk_size, s_n_llhttp__internal__n_chunk_size_digit, s_n_llhttp__internal__n_invoke_update_content_length_1, - s_n_llhttp__internal__n_invoke_is_equal_upgrade, - s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2, s_n_llhttp__internal__n_consume_content_length_1, s_n_llhttp__internal__n_span_start_llhttp__on_body_1, s_n_llhttp__internal__n_eof, @@ -370,8 +374,9 @@ enum llparse_state_e { s_n_llhttp__internal__n_header_value_discard_ws_almost_done, s_n_llhttp__internal__n_header_value_lws, s_n_llhttp__internal__n_header_value_almost_done, + s_n_llhttp__internal__n_invoke_test_lenient_flags_17, s_n_llhttp__internal__n_header_value_lenient, - s_n_llhttp__internal__n_error_41, + s_n_llhttp__internal__n_error_52, s_n_llhttp__internal__n_header_value_otherwise, s_n_llhttp__internal__n_header_value_connection_token, s_n_llhttp__internal__n_header_value_connection_ws, @@ -379,12 +384,12 @@ enum llparse_state_e { s_n_llhttp__internal__n_header_value_connection_2, s_n_llhttp__internal__n_header_value_connection_3, s_n_llhttp__internal__n_header_value_connection, - s_n_llhttp__internal__n_error_43, - s_n_llhttp__internal__n_error_44, + s_n_llhttp__internal__n_error_54, + s_n_llhttp__internal__n_error_55, s_n_llhttp__internal__n_header_value_content_length_ws, s_n_llhttp__internal__n_header_value_content_length, - s_n_llhttp__internal__n_error_46, - s_n_llhttp__internal__n_error_45, + s_n_llhttp__internal__n_error_57, + s_n_llhttp__internal__n_error_56, s_n_llhttp__internal__n_header_value_te_token_ows, s_n_llhttp__internal__n_header_value, s_n_llhttp__internal__n_header_value_te_token, @@ -392,6 +397,7 @@ enum llparse_state_e { s_n_llhttp__internal__n_header_value_te_chunked, s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1, s_n_llhttp__internal__n_header_value_discard_ws, + s_n_llhttp__internal__n_invoke_load_header_state, s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete, s_n_llhttp__internal__n_header_field_general_otherwise, s_n_llhttp__internal__n_header_field_general, @@ -416,12 +422,12 @@ enum llparse_state_e { s_n_llhttp__internal__n_req_http_complete, s_n_llhttp__internal__n_invoke_load_method_1, s_n_llhttp__internal__n_invoke_llhttp__on_version_complete, - s_n_llhttp__internal__n_error_51, - s_n_llhttp__internal__n_error_57, + s_n_llhttp__internal__n_error_64, + s_n_llhttp__internal__n_error_71, s_n_llhttp__internal__n_req_http_minor, - s_n_llhttp__internal__n_error_58, + s_n_llhttp__internal__n_error_72, s_n_llhttp__internal__n_req_http_dot, - s_n_llhttp__internal__n_error_59, + s_n_llhttp__internal__n_error_73, s_n_llhttp__internal__n_req_http_major, s_n_llhttp__internal__n_span_start_llhttp__on_version, s_n_llhttp__internal__n_req_http_start_1, @@ -524,21 +530,21 @@ enum llparse_state_e { s_n_llhttp__internal__n_after_start_req, s_n_llhttp__internal__n_span_start_llhttp__on_method_1, s_n_llhttp__internal__n_res_line_almost_done, + s_n_llhttp__internal__n_invoke_test_lenient_flags_29, s_n_llhttp__internal__n_res_status, s_n_llhttp__internal__n_span_start_llhttp__on_status, - s_n_llhttp__internal__n_res_status_start, s_n_llhttp__internal__n_res_status_code_otherwise, s_n_llhttp__internal__n_res_status_code_digit_3, s_n_llhttp__internal__n_res_status_code_digit_2, s_n_llhttp__internal__n_res_status_code_digit_1, s_n_llhttp__internal__n_res_after_version, s_n_llhttp__internal__n_invoke_llhttp__on_version_complete_1, - s_n_llhttp__internal__n_error_73, - s_n_llhttp__internal__n_error_85, + s_n_llhttp__internal__n_error_87, + s_n_llhttp__internal__n_error_101, s_n_llhttp__internal__n_res_http_minor, - s_n_llhttp__internal__n_error_86, + s_n_llhttp__internal__n_error_102, s_n_llhttp__internal__n_res_http_dot, - s_n_llhttp__internal__n_error_87, + s_n_llhttp__internal__n_error_103, s_n_llhttp__internal__n_res_http_major, s_n_llhttp__internal__n_span_start_llhttp__on_version_1, s_n_llhttp__internal__n_start_res, @@ -669,7 +675,21 @@ int llhttp__internal__c_test_lenient_flags( return (state->lenient_flags & 1) == 1; } -int llhttp__after_headers_complete( +int llhttp__internal__c_test_lenient_flags_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 256) == 256; +} + +int llhttp__internal__c_test_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 128) == 128; +} + +int llhttp__on_chunk_complete( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); @@ -677,6 +697,13 @@ int llhttp__on_message_complete( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); +int llhttp__internal__c_is_equal_upgrade( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->upgrade == 1; +} + int llhttp__after_message_complete( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); @@ -719,6 +746,18 @@ int llhttp__internal__c_test_lenient_flags_3( return (state->lenient_flags & 32) == 32; } +int llhttp__before_headers_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_headers_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__after_headers_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + int llhttp__internal__c_mul_add_content_length( llhttp__internal_t* state, const unsigned char* p, @@ -745,6 +784,13 @@ int llhttp__internal__c_mul_add_content_length( return 0; } +int llhttp__internal__c_test_lenient_flags_4( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 512) == 512; +} + int llhttp__on_chunk_header( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); @@ -756,11 +802,7 @@ int llhttp__internal__c_is_equal_content_length( return state->content_length == 0; } -int llhttp__on_chunk_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_test_lenient_flags_4( +int llhttp__internal__c_test_lenient_flags_7( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -775,7 +817,7 @@ int llhttp__internal__c_or_flags( return 0; } -int llhttp__internal__c_test_lenient_flags_5( +int llhttp__internal__c_test_lenient_flags_8( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -790,13 +832,6 @@ int llhttp__on_chunk_extension_value_complete( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); -int llhttp__internal__c_is_equal_upgrade( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return state->upgrade == 1; -} - int llhttp__internal__c_update_finish_3( llhttp__internal_t* state, const unsigned char* p, @@ -805,35 +840,6 @@ int llhttp__internal__c_update_finish_3( return 0; } -int llhttp__internal__c_test_flags( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->flags & 128) == 128; -} - -int llhttp__internal__c_test_flags_1( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->flags & 544) == 544; -} - -int llhttp__internal__c_test_lenient_flags_6( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->lenient_flags & 2) == 2; -} - -int llhttp__before_headers_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__on_headers_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - int llhttp__internal__c_or_flags_1( llhttp__internal_t* state, const unsigned char* p, @@ -870,7 +876,21 @@ int llhttp__internal__c_load_header_state( return state->header_state; } -int llhttp__internal__c_or_flags_3( +int llhttp__internal__c_test_flags_4( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 512) == 512; +} + +int llhttp__internal__c_test_lenient_flags_21( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 2) == 2; +} + +int llhttp__internal__c_or_flags_5( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -890,7 +910,7 @@ int llhttp__on_header_value_complete( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); -int llhttp__internal__c_or_flags_4( +int llhttp__internal__c_or_flags_6( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -898,7 +918,7 @@ int llhttp__internal__c_or_flags_4( return 0; } -int llhttp__internal__c_or_flags_5( +int llhttp__internal__c_or_flags_7( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -906,7 +926,7 @@ int llhttp__internal__c_or_flags_5( return 0; } -int llhttp__internal__c_or_flags_6( +int llhttp__internal__c_or_flags_8( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -979,7 +999,7 @@ int llhttp__internal__c_mul_add_content_length_1( return 0; } -int llhttp__internal__c_or_flags_15( +int llhttp__internal__c_or_flags_17( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -994,14 +1014,14 @@ int llhttp__internal__c_test_flags_3( return (state->flags & 8) == 8; } -int llhttp__internal__c_test_lenient_flags_13( +int llhttp__internal__c_test_lenient_flags_19( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { return (state->lenient_flags & 8) == 8; } -int llhttp__internal__c_or_flags_16( +int llhttp__internal__c_or_flags_18( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -1025,7 +1045,7 @@ int llhttp__internal__c_update_header_state_8( return 0; } -int llhttp__internal__c_or_flags_18( +int llhttp__internal__c_or_flags_20( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -1058,7 +1078,7 @@ int llhttp__internal__c_store_http_minor( return 0; } -int llhttp__internal__c_test_lenient_flags_15( +int llhttp__internal__c_test_lenient_flags_23( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -1192,25 +1212,63 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } + case s_n_llhttp__internal__n_invoke_is_equal_upgrade: + s_n_llhttp__internal__n_invoke_is_equal_upgrade: { + switch (llhttp__internal__c_is_equal_upgrade(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; + default: + goto s_n_llhttp__internal__n_pause_1; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: { + switch (llhttp__on_message_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_is_equal_upgrade; + case 21: + goto s_n_llhttp__internal__n_pause_13; + default: + goto s_n_llhttp__internal__n_error_38; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_data_almost_done_1: + s_n_llhttp__internal__n_chunk_data_almost_done_1: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_data_almost_done_1; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; + } + default: { + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_7; + } + } + /* UNREACHABLE */; + abort(); + } case s_n_llhttp__internal__n_chunk_data_almost_done: s_n_llhttp__internal__n_chunk_data_almost_done: { - llparse_match_t match_seq; - if (p == endp) { return s_n_llhttp__internal__n_chunk_data_almost_done; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob0, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { + switch (*p) { + case 10: { p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_6; } - case kMatchPause: { - return s_n_llhttp__internal__n_chunk_data_almost_done; + case 13: { + p++; + goto s_n_llhttp__internal__n_chunk_data_almost_done_1; } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_4; + default: { + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_7; } } /* UNREACHABLE */; @@ -1267,21 +1325,32 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header; } default: { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_5; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_8; } } /* UNREACHABLE */; abort(); } + case s_n_llhttp__internal__n_invoke_test_lenient_flags_9: + s_n_llhttp__internal__n_invoke_test_lenient_flags_9: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + default: + goto s_n_llhttp__internal__n_error_20; + } + /* UNREACHABLE */; + abort(); + } case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete: s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete: { switch (llhttp__on_chunk_extension_name_complete(state, p, endp)) { case 0: - goto s_n_llhttp__internal__n_chunk_size_almost_done; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_9; case 21: goto s_n_llhttp__internal__n_pause_5; default: - goto s_n_llhttp__internal__n_error_15; + goto s_n_llhttp__internal__n_error_19; } /* UNREACHABLE */; abort(); @@ -1290,11 +1359,35 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_1: { switch (llhttp__on_chunk_extension_name_complete(state, p, endp)) { case 0: - goto s_n_llhttp__internal__n_chunk_extensions; + goto s_n_llhttp__internal__n_chunk_size_almost_done; case 21: goto s_n_llhttp__internal__n_pause_6; default: - goto s_n_llhttp__internal__n_error_16; + goto s_n_llhttp__internal__n_error_21; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2: { + switch (llhttp__on_chunk_extension_name_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_chunk_extensions; + case 21: + goto s_n_llhttp__internal__n_pause_7; + default: + goto s_n_llhttp__internal__n_error_22; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_test_lenient_flags_10: + s_n_llhttp__internal__n_invoke_test_lenient_flags_10: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + default: + goto s_n_llhttp__internal__n_error_25; } /* UNREACHABLE */; abort(); @@ -1303,11 +1396,24 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete: { switch (llhttp__on_chunk_extension_value_complete(state, p, endp)) { case 0: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_10; + case 21: + goto s_n_llhttp__internal__n_pause_8; + default: + goto s_n_llhttp__internal__n_error_24; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1: { + switch (llhttp__on_chunk_extension_value_complete(state, p, endp)) { + case 0: goto s_n_llhttp__internal__n_chunk_size_almost_done; case 21: - goto s_n_llhttp__internal__n_pause_7; + goto s_n_llhttp__internal__n_pause_9; default: - goto s_n_llhttp__internal__n_error_18; + goto s_n_llhttp__internal__n_error_26; } /* UNREACHABLE */; abort(); @@ -1318,6 +1424,9 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_chunk_extension_quoted_value_done; } switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_11; + } case 13: { p++; goto s_n_llhttp__internal__n_chunk_size_almost_done; @@ -1327,27 +1436,72 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_chunk_extensions; } default: { - goto s_n_llhttp__internal__n_error_20; + goto s_n_llhttp__internal__n_error_29; } } /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1: - s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1: { + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2: { switch (llhttp__on_chunk_extension_value_complete(state, p, endp)) { case 0: goto s_n_llhttp__internal__n_chunk_extension_quoted_value_done; case 21: - goto s_n_llhttp__internal__n_pause_8; + goto s_n_llhttp__internal__n_pause_10; default: - goto s_n_llhttp__internal__n_error_19; + goto s_n_llhttp__internal__n_error_27; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_error_30: + s_n_llhttp__internal__n_error_30: { + state->error = 0x2; + state->reason = "Invalid quoted-pair in chunk extensions quoted value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair: + s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_chunk_extension_quoted_value; + } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_3; + } } /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_21: - s_n_llhttp__internal__n_error_21: { + case s_n_llhttp__internal__n_error_31: + s_n_llhttp__internal__n_error_31: { state->error = 0x2; state->reason = "Invalid character in chunk extensions quoted value"; state->error_pos = (const char*) p; @@ -1364,7 +1518,7 @@ static llparse_state_t llhttp__internal__run( 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -1386,30 +1540,34 @@ static llparse_state_t llhttp__internal__run( } case 2: { p++; - goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_1; + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_2; + } + case 3: { + p++; + goto s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair; } default: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_2; + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_4; } } /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2: - s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2: { + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3: { switch (llhttp__on_chunk_extension_value_complete(state, p, endp)) { case 0: - goto s_n_llhttp__internal__n_chunk_size_otherwise; + goto s_n_llhttp__internal__n_chunk_extensions; case 21: - goto s_n_llhttp__internal__n_pause_9; + goto s_n_llhttp__internal__n_pause_11; default: - goto s_n_llhttp__internal__n_error_22; + goto s_n_llhttp__internal__n_error_32; } /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_23: - s_n_llhttp__internal__n_error_23: { + case s_n_llhttp__internal__n_error_33: + s_n_llhttp__internal__n_error_33: { state->error = 0x2; state->reason = "Invalid character in chunk extensions value"; state->error_pos = (const char*) p; @@ -1421,14 +1579,14 @@ static llparse_state_t llhttp__internal__run( case s_n_llhttp__internal__n_chunk_extension_value: s_n_llhttp__internal__n_chunk_extension_value: { static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 3, 2, 2, 2, 2, 2, 0, 0, 2, 2, 0, 2, 2, 0, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 4, 0, 0, 0, 0, - 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 2, 0, 2, 0, + 0, 3, 4, 3, 3, 3, 3, 3, 0, 0, 3, 3, 0, 3, 3, 0, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 5, 0, 0, 0, 0, + 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1446,18 +1604,21 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value; } case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_1; + } + case 3: { p++; goto s_n_llhttp__internal__n_chunk_extension_value; } - case 3: { + case 4: { p++; goto s_n_llhttp__internal__n_chunk_extension_quoted_value; } - case 4: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_3; + case 5: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_5; } default: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_4; + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_6; } } /* UNREACHABLE */; @@ -1470,12 +1631,12 @@ static llparse_state_t llhttp__internal__run( } state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_chunk_extension_value; - goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_3; /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_24: - s_n_llhttp__internal__n_error_24: { + case s_n_llhttp__internal__n_error_34: + s_n_llhttp__internal__n_error_34: { state->error = 0x2; state->reason = "Invalid character in chunk extensions name"; state->error_pos = (const char*) p; @@ -1487,14 +1648,14 @@ static llparse_state_t llhttp__internal__run( case s_n_llhttp__internal__n_chunk_extension_name: s_n_llhttp__internal__n_chunk_extension_name: { static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 0, 2, 2, 2, 2, 2, 0, 0, 2, 2, 0, 2, 2, 0, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 3, 0, 4, 0, 0, - 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 2, 0, 2, 0, + 0, 3, 0, 3, 3, 3, 3, 3, 0, 0, 3, 3, 0, 3, 3, 0, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 4, 0, 5, 0, 0, + 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1512,18 +1673,21 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name; } case 2: { - p++; - goto s_n_llhttp__internal__n_chunk_extension_name; + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_1; } case 3: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_1; + p++; + goto s_n_llhttp__internal__n_chunk_extension_name; } case 4: { goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_2; } - default: { + case 5: { goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_3; } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_4; + } } /* UNREACHABLE */; abort(); @@ -1547,11 +1711,11 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 13: { p++; - goto s_n_llhttp__internal__n_error_13; + goto s_n_llhttp__internal__n_error_17; } case ' ': { p++; - goto s_n_llhttp__internal__n_error_14; + goto s_n_llhttp__internal__n_error_18; } default: { goto s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_name; @@ -1566,16 +1730,28 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_chunk_size_otherwise; } switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_4; + } + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_5; + } case 13: { p++; goto s_n_llhttp__internal__n_chunk_size_almost_done; } + case ' ': { + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_4; + } case ';': { p++; goto s_n_llhttp__internal__n_chunk_extensions; } default: { - goto s_n_llhttp__internal__n_error_25; + goto s_n_llhttp__internal__n_error_35; } } /* UNREACHABLE */; @@ -1821,7 +1997,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_mul_add_content_length; } default: { - goto s_n_llhttp__internal__n_error_27; + goto s_n_llhttp__internal__n_error_37; } } /* UNREACHABLE */; @@ -1836,30 +2012,6 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_invoke_is_equal_upgrade: - s_n_llhttp__internal__n_invoke_is_equal_upgrade: { - switch (llhttp__internal__c_is_equal_upgrade(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; - default: - goto s_n_llhttp__internal__n_pause_1; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: - s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: { - switch (llhttp__on_message_complete(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_is_equal_upgrade; - case 21: - goto s_n_llhttp__internal__n_pause_11; - default: - goto s_n_llhttp__internal__n_error_28; - } - /* UNREACHABLE */; - abort(); - } case s_n_llhttp__internal__n_consume_content_length_1: s_n_llhttp__internal__n_consume_content_length_1: { size_t avail; @@ -1922,7 +2074,7 @@ static llparse_state_t llhttp__internal__run( case 4: goto s_n_llhttp__internal__n_invoke_update_finish_3; case 5: - goto s_n_llhttp__internal__n_error_29; + goto s_n_llhttp__internal__n_error_39; default: goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete; } @@ -1947,10 +2099,10 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 10: { p++; - goto s_n_llhttp__internal__n_invoke_test_flags; + goto s_n_llhttp__internal__n_invoke_test_flags_1; } default: { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_7; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_12; } } /* UNREACHABLE */; @@ -1979,9 +2131,9 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_header_field_start; case 21: - goto s_n_llhttp__internal__n_pause_14; + goto s_n_llhttp__internal__n_pause_18; default: - goto s_n_llhttp__internal__n_error_37; + goto s_n_llhttp__internal__n_error_47; } /* UNREACHABLE */; abort(); @@ -2005,14 +2157,14 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 9: { p++; - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_10; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_15; } case ' ': { p++; - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_10; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_15; } default: { - goto s_n_llhttp__internal__n_invoke_load_header_state; + goto s_n_llhttp__internal__n_invoke_load_header_state_1; } } /* UNREACHABLE */; @@ -2029,7 +2181,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_discard_lws; } default: { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_11; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_16; } } /* UNREACHABLE */; @@ -2042,13 +2194,13 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 9: { - goto s_n_llhttp__internal__n_invoke_load_header_state_3; + goto s_n_llhttp__internal__n_invoke_load_header_state_4; } case ' ': { - goto s_n_llhttp__internal__n_invoke_load_header_state_3; + goto s_n_llhttp__internal__n_invoke_load_header_state_4; } default: { - goto s_n_llhttp__internal__n_invoke_load_header_state_4; + goto s_n_llhttp__internal__n_invoke_load_header_state_5; } } /* UNREACHABLE */; @@ -2065,12 +2217,23 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_lws; } default: { - goto s_n_llhttp__internal__n_error_40; + goto s_n_llhttp__internal__n_error_51; } } /* UNREACHABLE */; abort(); } + case s_n_llhttp__internal__n_invoke_test_lenient_flags_17: + s_n_llhttp__internal__n_invoke_test_lenient_flags_17: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_value_almost_done; + default: + goto s_n_llhttp__internal__n_error_50; + } + /* UNREACHABLE */; + abort(); + } case s_n_llhttp__internal__n_header_value_lenient: s_n_llhttp__internal__n_header_value_lenient: { if (p == endp) { @@ -2078,10 +2241,10 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 10: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4; } case 13: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5; } default: { p++; @@ -2091,8 +2254,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_41: - s_n_llhttp__internal__n_error_41: { + case s_n_llhttp__internal__n_error_52: + s_n_llhttp__internal__n_error_52: { state->error = 0xa; state->reason = "Invalid header value char"; state->error_pos = (const char*) p; @@ -2107,11 +2270,14 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_header_value_otherwise; } switch (*p) { - case 13: { + case 10: { goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1; } + case 13: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2; + } default: { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_12; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_18; } } /* UNREACHABLE */; @@ -2174,7 +2340,7 @@ static llparse_state_t llhttp__internal__run( } case ',': { p++; - goto s_n_llhttp__internal__n_invoke_load_header_state_5; + goto s_n_llhttp__internal__n_invoke_load_header_state_6; } default: { goto s_n_llhttp__internal__n_invoke_update_header_state_5; @@ -2190,7 +2356,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_value_connection_1; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob3, 4); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob2, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2214,7 +2380,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_value_connection_2; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob4, 9); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob3, 9); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2238,7 +2404,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_value_connection_3; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob5, 6); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob4, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2288,8 +2454,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_43: - s_n_llhttp__internal__n_error_43: { + case s_n_llhttp__internal__n_error_54: + s_n_llhttp__internal__n_error_54: { state->error = 0xb; state->reason = "Content-Length overflow"; state->error_pos = (const char*) p; @@ -2298,8 +2464,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_44: - s_n_llhttp__internal__n_error_44: { + case s_n_llhttp__internal__n_error_55: + s_n_llhttp__internal__n_error_55: { state->error = 0xb; state->reason = "Invalid character in Content-Length"; state->error_pos = (const char*) p; @@ -2315,17 +2481,17 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 10: { - goto s_n_llhttp__internal__n_invoke_or_flags_15; + goto s_n_llhttp__internal__n_invoke_or_flags_17; } case 13: { - goto s_n_llhttp__internal__n_invoke_or_flags_15; + goto s_n_llhttp__internal__n_invoke_or_flags_17; } case ' ': { p++; goto s_n_llhttp__internal__n_header_value_content_length_ws; } default: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_6; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_7; } } /* UNREACHABLE */; @@ -2394,8 +2560,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_46: - s_n_llhttp__internal__n_error_46: { + case s_n_llhttp__internal__n_error_57: + s_n_llhttp__internal__n_error_57: { state->error = 0xf; state->reason = "Invalid `Transfer-Encoding` header value"; state->error_pos = (const char*) p; @@ -2404,8 +2570,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_45: - s_n_llhttp__internal__n_error_45: { + case s_n_llhttp__internal__n_error_56: + s_n_llhttp__internal__n_error_56: { state->error = 0xf; state->reason = "Invalid `Transfer-Encoding` header value"; state->error_pos = (const char*) p; @@ -2467,7 +2633,7 @@ static llparse_state_t llhttp__internal__run( /* Load input */ input = _mm_loadu_si128((__m128i const*) p); - ranges = _mm_loadu_si128((__m128i const*) llparse_blob7); + ranges = _mm_loadu_si128((__m128i const*) llparse_blob6); /* Find first character that does not match `ranges` */ match_len = _mm_cmpestri(ranges, 6, @@ -2566,7 +2732,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_value_te_chunked; } - match_seq = llparse__match_sequence_to_lower_unsafe(state, p, endp, llparse_blob6, 7); + match_seq = llparse__match_sequence_to_lower_unsafe(state, p, endp, llparse_blob5, 7); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2590,7 +2756,7 @@ static llparse_state_t llhttp__internal__run( } state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_header_value; - goto s_n_llhttp__internal__n_invoke_load_header_state_2; + goto s_n_llhttp__internal__n_invoke_load_header_state_3; /* UNREACHABLE */; abort(); } @@ -2606,7 +2772,7 @@ static llparse_state_t llhttp__internal__run( } case 10: { p++; - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_9; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_14; } case 13: { p++; @@ -2623,15 +2789,28 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } + case s_n_llhttp__internal__n_invoke_load_header_state: + s_n_llhttp__internal__n_invoke_load_header_state: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 2: + goto s_n_llhttp__internal__n_invoke_test_flags_4; + case 3: + goto s_n_llhttp__internal__n_invoke_test_flags_5; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + /* UNREACHABLE */; + abort(); + } case s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: { switch (llhttp__on_header_field_complete(state, p, endp)) { case 0: - goto s_n_llhttp__internal__n_header_value_discard_ws; + goto s_n_llhttp__internal__n_invoke_load_header_state; case 21: - goto s_n_llhttp__internal__n_pause_15; + goto s_n_llhttp__internal__n_pause_19; default: - goto s_n_llhttp__internal__n_error_34; + goto s_n_llhttp__internal__n_error_44; } /* UNREACHABLE */; abort(); @@ -2646,7 +2825,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field_2; } default: { - goto s_n_llhttp__internal__n_error_47; + goto s_n_llhttp__internal__n_error_60; } } /* UNREACHABLE */; @@ -2684,7 +2863,7 @@ static llparse_state_t llhttp__internal__run( /* Load input */ input = _mm_loadu_si128((__m128i const*) p); - ranges = _mm_loadu_si128((__m128i const*) llparse_blob8); + ranges = _mm_loadu_si128((__m128i const*) llparse_blob7); /* Find first character that does not match `ranges` */ match_len = _mm_cmpestri(ranges, 16, @@ -2696,7 +2875,7 @@ static llparse_state_t llhttp__internal__run( p += match_len; goto s_n_llhttp__internal__n_header_field_general; } - ranges = _mm_loadu_si128((__m128i const*) llparse_blob9); + ranges = _mm_loadu_si128((__m128i const*) llparse_blob8); /* Find first character that does not match `ranges` */ match_len = _mm_cmpestri(ranges, 2, @@ -2730,7 +2909,7 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case ' ': { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_8; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_13; } case ':': { goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1; @@ -2749,7 +2928,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_field_3; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob2, 6); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob1, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2774,7 +2953,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_field_4; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob10, 10); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob9, 10); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2820,7 +2999,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_field_1; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob1, 2); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob0, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2844,7 +3023,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_field_5; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob11, 15); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob10, 15); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2869,7 +3048,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_field_6; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob12, 16); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob11, 16); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2894,7 +3073,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_header_field_7; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob13, 6); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob12, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -3044,7 +3223,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_to_http_09; } default: { - goto s_n_llhttp__internal__n_error_48; + goto s_n_llhttp__internal__n_error_61; } } /* UNREACHABLE */; @@ -3069,7 +3248,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_skip_lf_to_http09_1; } default: { - goto s_n_llhttp__internal__n_error_48; + goto s_n_llhttp__internal__n_error_61; } } /* UNREACHABLE */; @@ -3082,18 +3261,18 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_req_pri_upgrade; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob15, 10); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob14, 10); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_error_55; + goto s_n_llhttp__internal__n_error_69; } case kMatchPause: { return s_n_llhttp__internal__n_req_pri_upgrade; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; + goto s_n_llhttp__internal__n_error_70; } } /* UNREACHABLE */; @@ -3110,7 +3289,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_headers_start; } default: { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_16; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_25; } } /* UNREACHABLE */; @@ -3122,12 +3301,16 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_req_http_complete; } switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_24; + } case 13: { p++; goto s_n_llhttp__internal__n_req_http_complete_crlf; } default: { - goto s_n_llhttp__internal__n_error_54; + goto s_n_llhttp__internal__n_error_68; } } /* UNREACHABLE */; @@ -3150,15 +3333,15 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_invoke_load_method_1; case 21: - goto s_n_llhttp__internal__n_pause_17; + goto s_n_llhttp__internal__n_pause_21; default: - goto s_n_llhttp__internal__n_error_52; + goto s_n_llhttp__internal__n_error_65; } /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_51: - s_n_llhttp__internal__n_error_51: { + case s_n_llhttp__internal__n_error_64: + s_n_llhttp__internal__n_error_64: { state->error = 0x9; state->reason = "Invalid HTTP version"; state->error_pos = (const char*) p; @@ -3167,8 +3350,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_57: - s_n_llhttp__internal__n_error_57: { + case s_n_llhttp__internal__n_error_71: + s_n_llhttp__internal__n_error_71: { state->error = 0x9; state->reason = "Invalid minor version"; state->error_pos = (const char*) p; @@ -3240,8 +3423,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_58: - s_n_llhttp__internal__n_error_58: { + case s_n_llhttp__internal__n_error_72: + s_n_llhttp__internal__n_error_72: { state->error = 0x9; state->reason = "Expected dot"; state->error_pos = (const char*) p; @@ -3267,8 +3450,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_59: - s_n_llhttp__internal__n_error_59: { + case s_n_llhttp__internal__n_error_73: + s_n_llhttp__internal__n_error_73: { state->error = 0x9; state->reason = "Invalid major version"; state->error_pos = (const char*) p; @@ -3358,7 +3541,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_req_http_start_1; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob14, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob13, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -3369,7 +3552,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_req_http_start_1; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_62; + goto s_n_llhttp__internal__n_error_76; } } /* UNREACHABLE */; @@ -3382,7 +3565,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_req_http_start_2; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob16, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob15, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -3393,7 +3576,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_req_http_start_2; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_62; + goto s_n_llhttp__internal__n_error_76; } } /* UNREACHABLE */; @@ -3406,7 +3589,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_req_http_start_3; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob17, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob16, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -3417,7 +3600,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_req_http_start_3; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_62; + goto s_n_llhttp__internal__n_error_76; } } /* UNREACHABLE */; @@ -3446,7 +3629,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_req_http_start_3; } default: { - goto s_n_llhttp__internal__n_error_62; + goto s_n_llhttp__internal__n_error_76; } } /* UNREACHABLE */; @@ -3537,7 +3720,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_fragment; } default: { - goto s_n_llhttp__internal__n_error_63; + goto s_n_llhttp__internal__n_error_77; } } /* UNREACHABLE */; @@ -3598,7 +3781,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_span_end_stub_query_3; } default: { - goto s_n_llhttp__internal__n_error_64; + goto s_n_llhttp__internal__n_error_78; } } /* UNREACHABLE */; @@ -3636,7 +3819,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_query; } default: { - goto s_n_llhttp__internal__n_error_65; + goto s_n_llhttp__internal__n_error_79; } } /* UNREACHABLE */; @@ -3761,10 +3944,10 @@ static llparse_state_t llhttp__internal__run( } case 8: { p++; - goto s_n_llhttp__internal__n_error_66; + goto s_n_llhttp__internal__n_error_80; } default: { - goto s_n_llhttp__internal__n_error_67; + goto s_n_llhttp__internal__n_error_81; } } /* UNREACHABLE */; @@ -3823,7 +4006,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_server_with_at; } default: { - goto s_n_llhttp__internal__n_error_68; + goto s_n_llhttp__internal__n_error_82; } } /* UNREACHABLE */; @@ -3840,7 +4023,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_server; } default: { - goto s_n_llhttp__internal__n_error_69; + goto s_n_llhttp__internal__n_error_83; } } /* UNREACHABLE */; @@ -3877,7 +4060,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_schema_delim_1; } default: { - goto s_n_llhttp__internal__n_error_69; + goto s_n_llhttp__internal__n_error_83; } } /* UNREACHABLE */; @@ -3929,7 +4112,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_schema; } default: { - goto s_n_llhttp__internal__n_error_70; + goto s_n_llhttp__internal__n_error_84; } } /* UNREACHABLE */; @@ -3970,7 +4153,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_schema; } default: { - goto s_n_llhttp__internal__n_error_71; + goto s_n_llhttp__internal__n_error_85; } } /* UNREACHABLE */; @@ -4068,7 +4251,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_req_spaces_before_url; } default: { - goto s_n_llhttp__internal__n_error_72; + goto s_n_llhttp__internal__n_error_86; } } /* UNREACHABLE */; @@ -4080,9 +4263,9 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_req_first_space_before_url; case 21: - goto s_n_llhttp__internal__n_pause_22; + goto s_n_llhttp__internal__n_pause_26; default: - goto s_n_llhttp__internal__n_error_89; + goto s_n_llhttp__internal__n_error_105; } /* UNREACHABLE */; abort(); @@ -4099,7 +4282,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4112,7 +4295,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_3; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob18, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob17, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4124,7 +4307,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_3; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4145,7 +4328,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_3; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4158,7 +4341,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_4; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob19, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob18, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4170,7 +4353,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_4; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4183,7 +4366,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_6; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob20, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob19, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4195,7 +4378,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_6; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4208,7 +4391,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_8; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob21, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob20, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4220,7 +4403,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_8; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4238,7 +4421,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4259,7 +4442,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_9; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4280,7 +4463,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_7; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4293,7 +4476,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_12; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob22, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob21, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4305,7 +4488,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_12; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4318,7 +4501,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_13; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob23, 5); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob22, 5); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4330,7 +4513,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_13; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4351,7 +4534,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_13; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4368,7 +4551,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_11; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4381,7 +4564,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_14; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob24, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob23, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4393,7 +4576,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_14; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4406,7 +4589,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_17; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob26, 9); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob25, 9); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4418,7 +4601,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_17; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4449,7 +4632,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_15; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob25, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob24, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4460,7 +4643,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_15; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4473,7 +4656,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_18; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob27, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob26, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4485,7 +4668,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_18; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4498,7 +4681,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_20; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob28, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob27, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4510,7 +4693,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_20; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4523,7 +4706,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_21; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob29, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob28, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4535,7 +4718,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_21; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4556,7 +4739,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_21; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4569,7 +4752,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_23; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob30, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob29, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4581,7 +4764,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_23; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4594,7 +4777,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_24; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob31, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob30, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4606,7 +4789,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_24; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4619,7 +4802,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_26; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob32, 7); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob31, 7); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4631,7 +4814,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_26; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4644,7 +4827,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_28; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob33, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob32, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4656,7 +4839,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_28; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4674,7 +4857,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4695,7 +4878,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_29; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4716,7 +4899,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_27; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4729,7 +4912,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_30; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob34, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob33, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4741,7 +4924,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_30; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4770,7 +4953,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_30; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4783,7 +4966,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_31; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob35, 5); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob34, 5); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4795,7 +4978,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_31; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4808,7 +4991,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_32; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob36, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob35, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4820,7 +5003,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_32; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4833,7 +5016,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_35; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob37, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob36, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4845,7 +5028,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_35; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4858,7 +5041,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_36; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob38, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob37, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4870,7 +5053,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_36; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4891,7 +5074,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_36; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4904,7 +5087,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_37; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob39, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob38, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4916,7 +5099,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_37; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4929,7 +5112,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_38; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob40, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob39, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4941,7 +5124,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_38; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4954,7 +5137,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_42; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob41, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob40, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4966,7 +5149,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_42; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -4979,7 +5162,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_43; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob42, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob41, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -4991,7 +5174,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_43; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5012,7 +5195,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_43; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5029,7 +5212,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_41; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5051,7 +5234,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_40; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5064,7 +5247,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_45; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob43, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob42, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5076,7 +5259,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_45; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5098,7 +5281,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5131,7 +5314,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_44; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5144,7 +5327,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_48; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob44, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob43, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5156,7 +5339,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_48; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5169,7 +5352,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_49; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob45, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob44, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5181,7 +5364,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_49; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5194,7 +5377,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_50; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob46, 5); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob45, 5); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5206,7 +5389,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_50; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5219,7 +5402,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_51; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob47, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob46, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5231,7 +5414,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_51; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5260,7 +5443,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_51; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5277,7 +5460,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_47; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5290,7 +5473,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_54; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob48, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob47, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5302,7 +5485,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_54; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5320,7 +5503,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5333,7 +5516,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_57; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob49, 9); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob48, 9); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5345,7 +5528,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_57; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5366,7 +5549,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_57; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5387,7 +5570,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_55; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5400,7 +5583,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_58; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob50, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob49, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5412,7 +5595,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_58; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5425,7 +5608,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_59; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob51, 7); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob50, 7); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5437,7 +5620,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_59; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5462,7 +5645,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_59; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5475,7 +5658,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_61; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob52, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob51, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5487,7 +5670,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_61; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5500,7 +5683,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_62; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob53, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob52, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5512,7 +5695,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_62; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5533,7 +5716,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_62; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5546,7 +5729,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_65; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob54, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob53, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5558,7 +5741,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_65; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5571,7 +5754,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_67; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob55, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob54, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5583,7 +5766,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_67; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5596,7 +5779,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_68; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob56, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob55, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5608,7 +5791,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_68; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5629,7 +5812,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_68; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5642,7 +5825,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_after_start_req_69; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob57, 8); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob56, 8); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -5654,7 +5837,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_after_start_req_69; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5679,7 +5862,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_69; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5696,7 +5879,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_64; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5773,7 +5956,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_after_start_req_63; } default: { - goto s_n_llhttp__internal__n_error_90; + goto s_n_llhttp__internal__n_error_106; } } /* UNREACHABLE */; @@ -5805,12 +5988,23 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; } default: { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_18; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_28; } } /* UNREACHABLE */; abort(); } + case s_n_llhttp__internal__n_invoke_test_lenient_flags_29: + s_n_llhttp__internal__n_invoke_test_lenient_flags_29: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + default: + goto s_n_llhttp__internal__n_error_92; + } + /* UNREACHABLE */; + abort(); + } case s_n_llhttp__internal__n_res_status: s_n_llhttp__internal__n_res_status: { if (p == endp) { @@ -5842,27 +6036,6 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_res_status_start: - s_n_llhttp__internal__n_res_status_start: { - if (p == endp) { - return s_n_llhttp__internal__n_res_status_start; - } - switch (*p) { - case 10: { - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; - } - case 13: { - p++; - goto s_n_llhttp__internal__n_res_line_almost_done; - } - default: { - goto s_n_llhttp__internal__n_span_start_llhttp__on_status; - } - } - /* UNREACHABLE */; - abort(); - } case s_n_llhttp__internal__n_res_status_code_otherwise: s_n_llhttp__internal__n_res_status_code_otherwise: { if (p == endp) { @@ -5870,17 +6043,19 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 10: { - goto s_n_llhttp__internal__n_res_status_start; + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_27; } case 13: { - goto s_n_llhttp__internal__n_res_status_start; + p++; + goto s_n_llhttp__internal__n_res_line_almost_done; } case ' ': { p++; - goto s_n_llhttp__internal__n_res_status_start; + goto s_n_llhttp__internal__n_span_start_llhttp__on_status; } default: { - goto s_n_llhttp__internal__n_error_77; + goto s_n_llhttp__internal__n_error_93; } } /* UNREACHABLE */; @@ -5943,7 +6118,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; } default: { - goto s_n_llhttp__internal__n_error_79; + goto s_n_llhttp__internal__n_error_95; } } /* UNREACHABLE */; @@ -6006,7 +6181,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; } default: { - goto s_n_llhttp__internal__n_error_81; + goto s_n_llhttp__internal__n_error_97; } } /* UNREACHABLE */; @@ -6069,7 +6244,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_mul_add_status_code; } default: { - goto s_n_llhttp__internal__n_error_83; + goto s_n_llhttp__internal__n_error_99; } } /* UNREACHABLE */; @@ -6086,7 +6261,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_update_status_code; } default: { - goto s_n_llhttp__internal__n_error_84; + goto s_n_llhttp__internal__n_error_100; } } /* UNREACHABLE */; @@ -6098,15 +6273,15 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_res_after_version; case 21: - goto s_n_llhttp__internal__n_pause_21; + goto s_n_llhttp__internal__n_pause_25; default: - goto s_n_llhttp__internal__n_error_74; + goto s_n_llhttp__internal__n_error_88; } /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_73: - s_n_llhttp__internal__n_error_73: { + case s_n_llhttp__internal__n_error_87: + s_n_llhttp__internal__n_error_87: { state->error = 0x9; state->reason = "Invalid HTTP version"; state->error_pos = (const char*) p; @@ -6115,8 +6290,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_85: - s_n_llhttp__internal__n_error_85: { + case s_n_llhttp__internal__n_error_101: + s_n_llhttp__internal__n_error_101: { state->error = 0x9; state->reason = "Invalid minor version"; state->error_pos = (const char*) p; @@ -6188,8 +6363,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_86: - s_n_llhttp__internal__n_error_86: { + case s_n_llhttp__internal__n_error_102: + s_n_llhttp__internal__n_error_102: { state->error = 0x9; state->reason = "Expected dot"; state->error_pos = (const char*) p; @@ -6215,8 +6390,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - case s_n_llhttp__internal__n_error_87: - s_n_llhttp__internal__n_error_87: { + case s_n_llhttp__internal__n_error_103: + s_n_llhttp__internal__n_error_103: { state->error = 0x9; state->reason = "Invalid major version"; state->error_pos = (const char*) p; @@ -6306,7 +6481,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_start_res; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob58, 5); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob57, 5); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -6317,7 +6492,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_start_res; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_91; + goto s_n_llhttp__internal__n_error_107; } } /* UNREACHABLE */; @@ -6329,7 +6504,7 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_req_first_space_before_url; case 21: - goto s_n_llhttp__internal__n_pause_19; + goto s_n_llhttp__internal__n_pause_23; default: goto s_n_llhttp__internal__n_error_1; } @@ -6343,7 +6518,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_req_or_res_method_2; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob59, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob58, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -6355,7 +6530,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_req_or_res_method_2; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_88; + goto s_n_llhttp__internal__n_error_104; } } /* UNREACHABLE */; @@ -6377,7 +6552,7 @@ static llparse_state_t llhttp__internal__run( if (p == endp) { return s_n_llhttp__internal__n_req_or_res_method_3; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob60, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob59, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -6388,7 +6563,7 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_req_or_res_method_3; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_88; + goto s_n_llhttp__internal__n_error_104; } } /* UNREACHABLE */; @@ -6409,7 +6584,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_req_or_res_method_3; } default: { - goto s_n_llhttp__internal__n_error_88; + goto s_n_llhttp__internal__n_error_104; } } /* UNREACHABLE */; @@ -6426,7 +6601,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_req_or_res_method_1; } default: { - goto s_n_llhttp__internal__n_error_88; + goto s_n_llhttp__internal__n_error_104; } } /* UNREACHABLE */; @@ -6539,7 +6714,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_7: { + s_n_llhttp__internal__n_error_8: { state->error = 0x5; state->reason = "Data after `Connection: close`"; state->error_pos = (const char*) p; @@ -6553,7 +6728,7 @@ static llparse_state_t llhttp__internal__run( case 1: goto s_n_llhttp__internal__n_closed; default: - goto s_n_llhttp__internal__n_error_7; + goto s_n_llhttp__internal__n_error_8; } /* UNREACHABLE */; abort(); @@ -6576,6 +6751,54 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } + s_n_llhttp__internal__n_pause_13: { + state->error = 0x15; + state->reason = "on_message_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_upgrade; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_38: { + state->error = 0x12; + state->reason = "`on_message_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_15: { + state->error = 0x15; + state->reason = "on_chunk_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_40: { + state->error = 0x14; + state->reason = "`on_chunk_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1: { + switch (llhttp__on_chunk_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + case 21: + goto s_n_llhttp__internal__n_pause_15; + default: + goto s_n_llhttp__internal__n_error_40; + } + /* UNREACHABLE */; + abort(); + } s_n_llhttp__internal__n_pause_2: { state->error = 0x15; state->reason = "on_message_complete pause"; @@ -6585,7 +6808,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_8: { + s_n_llhttp__internal__n_error_9: { state->error = 0x12; state->reason = "`on_message_complete` callback error"; state->error_pos = (const char*) p; @@ -6601,12 +6824,12 @@ static llparse_state_t llhttp__internal__run( case 21: goto s_n_llhttp__internal__n_pause_2; default: - goto s_n_llhttp__internal__n_error_8; + goto s_n_llhttp__internal__n_error_9; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_26: { + s_n_llhttp__internal__n_error_36: { state->error = 0xc; state->reason = "Chunk size overflow"; state->error_pos = (const char*) p; @@ -6615,6 +6838,25 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } + s_n_llhttp__internal__n_error_10: { + state->error = 0xc; + state->reason = "Invalid character in chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_4: { + switch (llhttp__internal__c_test_lenient_flags_4(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_otherwise; + default: + goto s_n_llhttp__internal__n_error_10; + } + /* UNREACHABLE */; + abort(); + } s_n_llhttp__internal__n_pause_3: { state->error = 0x15; state->reason = "on_chunk_complete pause"; @@ -6624,7 +6866,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_10: { + s_n_llhttp__internal__n_error_14: { state->error = 0x14; state->reason = "`on_chunk_complete` callback error"; state->error_pos = (const char*) p; @@ -6640,12 +6882,31 @@ static llparse_state_t llhttp__internal__run( case 21: goto s_n_llhttp__internal__n_pause_3; default: - goto s_n_llhttp__internal__n_error_10; + goto s_n_llhttp__internal__n_error_14; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_11: { + s_n_llhttp__internal__n_error_13: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk data"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_6: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; + default: + goto s_n_llhttp__internal__n_error_13; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_15: { state->error = 0x2; state->reason = "Expected LF after chunk data"; state->error_pos = (const char*) p; @@ -6654,12 +6915,12 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_4: { - switch (llhttp__internal__c_test_lenient_flags_4(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_7: { + switch (llhttp__internal__c_test_lenient_flags_7(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; default: - goto s_n_llhttp__internal__n_error_11; + goto s_n_llhttp__internal__n_error_15; } /* UNREACHABLE */; abort(); @@ -6698,7 +6959,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_9: { + s_n_llhttp__internal__n_error_12: { state->error = 0x13; state->reason = "`on_chunk_header` callback error"; state->error_pos = (const char*) p; @@ -6714,12 +6975,12 @@ static llparse_state_t llhttp__internal__run( case 21: goto s_n_llhttp__internal__n_pause_4; default: - goto s_n_llhttp__internal__n_error_9; + goto s_n_llhttp__internal__n_error_12; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_12: { + s_n_llhttp__internal__n_error_16: { state->error = 0x2; state->reason = "Expected LF after chunk size"; state->error_pos = (const char*) p; @@ -6728,17 +6989,36 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_5: { - switch (llhttp__internal__c_test_lenient_flags_5(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_8: { + switch (llhttp__internal__c_test_lenient_flags_8(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header; default: - goto s_n_llhttp__internal__n_error_12; + goto s_n_llhttp__internal__n_error_16; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_13: { + s_n_llhttp__internal__n_error_11: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_5: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + default: + goto s_n_llhttp__internal__n_error_11; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_17: { state->error = 0x2; state->reason = "Invalid character in chunk extensions"; state->error_pos = (const char*) p; @@ -6747,7 +7027,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_14: { + s_n_llhttp__internal__n_error_18: { state->error = 0x2; state->reason = "Invalid character in chunk extensions"; state->error_pos = (const char*) p; @@ -6756,16 +7036,25 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } + s_n_llhttp__internal__n_error_20: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk extension name"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } s_n_llhttp__internal__n_pause_5: { state->error = 0x15; state->reason = "on_chunk_extension_name pause"; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_size_almost_done; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_test_lenient_flags_9; return s_error; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_15: { + s_n_llhttp__internal__n_error_19: { state->error = 0x22; state->reason = "`on_chunk_extension_name` callback error"; state->error_pos = (const char*) p; @@ -6783,11 +7072,10 @@ static llparse_state_t llhttp__internal__run( err = llhttp__on_chunk_extension_name(state, start, p); if (err != 0) { state->error = err; - state->error_pos = (const char*) (p + 1); + state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete; return s_error; } - p++; goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete; /* UNREACHABLE */; abort(); @@ -6796,12 +7084,12 @@ static llparse_state_t llhttp__internal__run( state->error = 0x15; state->reason = "on_chunk_extension_name pause"; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_extensions; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_size_almost_done; return s_error; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_16: { + s_n_llhttp__internal__n_error_21: { state->error = 0x22; state->reason = "`on_chunk_extension_name` callback error"; state->error_pos = (const char*) p; @@ -6830,14 +7118,59 @@ static llparse_state_t llhttp__internal__run( } s_n_llhttp__internal__n_pause_7: { state->error = 0x15; + state->reason = "on_chunk_extension_name pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_extensions; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_22: { + state->error = 0x22; + state->reason = "`on_chunk_extension_name` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_name(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_25: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk extension value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_8: { + state->error = 0x15; state->reason = "on_chunk_extension_value pause"; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_size_almost_done; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_test_lenient_flags_10; return s_error; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_18: { + s_n_llhttp__internal__n_error_24: { state->error = 0x23; state->reason = "`on_chunk_extension_value` callback error"; state->error_pos = (const char*) p; @@ -6855,16 +7188,70 @@ static llparse_state_t llhttp__internal__run( err = llhttp__on_chunk_extension_value(state, start, p); if (err != 0) { state->error = err; - state->error_pos = (const char*) (p + 1); + state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete; return s_error; } - p++; goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_20: { + s_n_llhttp__internal__n_pause_9: { + state->error = 0x15; + state->reason = "on_chunk_extension_value pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_size_almost_done; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_26: { + state->error = 0x23; + state->reason = "`on_chunk_extension_value` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_28: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk extension value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_11: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + default: + goto s_n_llhttp__internal__n_error_28; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_29: { state->error = 0x2; state->reason = "Invalid character in chunk extensions quote value"; state->error_pos = (const char*) p; @@ -6873,7 +7260,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_8: { + s_n_llhttp__internal__n_pause_10: { state->error = 0x15; state->reason = "on_chunk_extension_value pause"; state->error_pos = (const char*) p; @@ -6882,7 +7269,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_19: { + s_n_llhttp__internal__n_error_27: { state->error = 0x23; state->reason = "`on_chunk_extension_value` callback error"; state->error_pos = (const char*) p; @@ -6891,7 +7278,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_1: { + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_2: { const unsigned char* start; int err; @@ -6901,14 +7288,14 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2; return s_error; } - goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_2: { + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_3: { const unsigned char* start; int err; @@ -6918,24 +7305,42 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_21; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_30; return s_error; } p++; - goto s_n_llhttp__internal__n_error_21; + goto s_n_llhttp__internal__n_error_30; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_9: { + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_4: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_31; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_error_31; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_11: { state->error = 0x15; state->reason = "on_chunk_extension_value pause"; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_size_otherwise; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_extensions; return s_error; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_22: { + s_n_llhttp__internal__n_error_32: { state->error = 0x23; state->reason = "`on_chunk_extension_value` callback error"; state->error_pos = (const char*) p; @@ -6944,7 +7349,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_3: { + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_5: { const unsigned char* start; int err; @@ -6954,15 +7359,15 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3; return s_error; } p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_4: { + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_6: { const unsigned char* start; int err; @@ -6972,15 +7377,15 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_23; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_33; return s_error; } p++; - goto s_n_llhttp__internal__n_error_23; + goto s_n_llhttp__internal__n_error_33; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_10: { + s_n_llhttp__internal__n_pause_12: { state->error = 0x15; state->reason = "on_chunk_extension_name pause"; state->error_pos = (const char*) p; @@ -6989,7 +7394,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_17: { + s_n_llhttp__internal__n_error_23: { state->error = 0x22; state->reason = "`on_chunk_extension_name` callback error"; state->error_pos = (const char*) p; @@ -6998,19 +7403,19 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2: { + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_3: { switch (llhttp__on_chunk_extension_name_complete(state, p, endp)) { case 0: goto s_n_llhttp__internal__n_chunk_extension_value; case 21: - goto s_n_llhttp__internal__n_pause_10; + goto s_n_llhttp__internal__n_pause_12; default: - goto s_n_llhttp__internal__n_error_17; + goto s_n_llhttp__internal__n_error_23; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_2: { + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_3: { const unsigned char* start; int err; @@ -7028,7 +7433,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_3: { + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_4: { const unsigned char* start; int err; @@ -7038,15 +7443,15 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_24; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_34; return s_error; } p++; - goto s_n_llhttp__internal__n_error_24; + goto s_n_llhttp__internal__n_error_34; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_25: { + s_n_llhttp__internal__n_error_35: { state->error = 0xc; state->reason = "Invalid character in chunk size"; state->error_pos = (const char*) p; @@ -7058,14 +7463,14 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_mul_add_content_length: { switch (llhttp__internal__c_mul_add_content_length(state, p, endp, match)) { case 1: - goto s_n_llhttp__internal__n_error_26; + goto s_n_llhttp__internal__n_error_36; default: goto s_n_llhttp__internal__n_chunk_size; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_27: { + s_n_llhttp__internal__n_error_37: { state->error = 0xc; state->reason = "Invalid character in chunk size"; state->error_pos = (const char*) p; @@ -7074,24 +7479,6 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_11: { - state->error = 0x15; - state->reason = "on_message_complete pause"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_upgrade; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_28: { - state->error = 0x12; - state->reason = "`on_message_complete` callback error"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } s_n_llhttp__internal__n_span_end_llhttp__on_body_1: { const unsigned char* start; int err; @@ -7117,7 +7504,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_29: { + s_n_llhttp__internal__n_error_39: { state->error = 0xf; state->reason = "Request has invalid `Transfer-Encoding`"; state->error_pos = (const char*) p; @@ -7135,7 +7522,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_6: { + s_n_llhttp__internal__n_error_7: { state->error = 0x12; state->reason = "`on_message_complete` callback error"; state->error_pos = (const char*) p; @@ -7151,22 +7538,98 @@ static llparse_state_t llhttp__internal__run( case 21: goto s_n_llhttp__internal__n_pause; default: + goto s_n_llhttp__internal__n_error_7; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_1: { + switch (llhttp__internal__c_or_flags_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_2: { + switch (llhttp__internal__c_or_flags_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_upgrade: { + switch (llhttp__internal__c_update_upgrade(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_or_flags_2; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_14: { + state->error = 0x15; + state->reason = "Paused by on_headers_complete"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_6: { + state->error = 0x11; + state->reason = "User callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete: { + switch (llhttp__on_headers_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + case 1: + goto s_n_llhttp__internal__n_invoke_or_flags_1; + case 2: + goto s_n_llhttp__internal__n_invoke_update_upgrade; + case 21: + goto s_n_llhttp__internal__n_pause_14; + default: goto s_n_llhttp__internal__n_error_6; } /* UNREACHABLE */; abort(); } + s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete: { + switch (llhttp__before_headers_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_flags: { + switch (llhttp__internal__c_test_flags(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; + } + /* UNREACHABLE */; + abort(); + } s_n_llhttp__internal__n_invoke_test_lenient_flags_1: { - switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { case 1: - goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + goto s_n_llhttp__internal__n_invoke_test_flags; default: goto s_n_llhttp__internal__n_error_5; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_13: { + s_n_llhttp__internal__n_pause_17: { state->error = 0x15; state->reason = "on_chunk_complete pause"; state->error_pos = (const char*) p; @@ -7175,7 +7638,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_32: { + s_n_llhttp__internal__n_error_42: { state->error = 0x14; state->reason = "`on_chunk_complete` callback error"; state->error_pos = (const char*) p; @@ -7184,28 +7647,19 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1: { + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_2: { switch (llhttp__on_chunk_complete(state, p, endp)) { case 0: goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; case 21: - goto s_n_llhttp__internal__n_pause_13; + goto s_n_llhttp__internal__n_pause_17; default: - goto s_n_llhttp__internal__n_error_32; + goto s_n_llhttp__internal__n_error_42; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_31: { - state->error = 0x4; - state->reason = "Content-Length can't be present with Transfer-Encoding"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_1: { + s_n_llhttp__internal__n_invoke_or_flags_3: { switch (llhttp__internal__c_or_flags_1(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; @@ -7213,7 +7667,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_2: { + s_n_llhttp__internal__n_invoke_or_flags_4: { switch (llhttp__internal__c_or_flags_1(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; @@ -7221,15 +7675,15 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_update_upgrade: { + s_n_llhttp__internal__n_invoke_update_upgrade_1: { switch (llhttp__internal__c_update_upgrade(state, p, endp)) { default: - goto s_n_llhttp__internal__n_invoke_or_flags_2; + goto s_n_llhttp__internal__n_invoke_or_flags_4; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_12: { + s_n_llhttp__internal__n_pause_16: { state->error = 0x15; state->reason = "Paused by on_headers_complete"; state->error_pos = (const char*) p; @@ -7238,7 +7692,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_30: { + s_n_llhttp__internal__n_error_41: { state->error = 0x11; state->reason = "User callback error"; state->error_pos = (const char*) p; @@ -7247,61 +7701,41 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete: { + s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete_1: { switch (llhttp__on_headers_complete(state, p, endp)) { case 0: goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; case 1: - goto s_n_llhttp__internal__n_invoke_or_flags_1; + goto s_n_llhttp__internal__n_invoke_or_flags_3; case 2: - goto s_n_llhttp__internal__n_invoke_update_upgrade; + goto s_n_llhttp__internal__n_invoke_update_upgrade_1; case 21: - goto s_n_llhttp__internal__n_pause_12; + goto s_n_llhttp__internal__n_pause_16; default: - goto s_n_llhttp__internal__n_error_30; + goto s_n_llhttp__internal__n_error_41; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete: { + s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete_1: { switch (llhttp__before_headers_complete(state, p, endp)) { default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_lenient_flags_6: { - switch (llhttp__internal__c_test_lenient_flags_6(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_error_31; - default: - goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; + goto s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete_1; } /* UNREACHABLE */; abort(); } s_n_llhttp__internal__n_invoke_test_flags_1: { - switch (llhttp__internal__c_test_flags_1(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_6; - default: - goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_flags: { switch (llhttp__internal__c_test_flags(state, p, endp)) { case 1: - goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_2; default: - goto s_n_llhttp__internal__n_invoke_test_flags_1; + goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete_1; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_33: { + s_n_llhttp__internal__n_error_43: { state->error = 0x2; state->reason = "Expected LF after headers"; state->error_pos = (const char*) p; @@ -7310,12 +7744,12 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_7: { - switch (llhttp__internal__c_test_lenient_flags_5(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_12: { + switch (llhttp__internal__c_test_lenient_flags_8(state, p, endp)) { case 1: - goto s_n_llhttp__internal__n_invoke_test_flags; + goto s_n_llhttp__internal__n_invoke_test_flags_1; default: - goto s_n_llhttp__internal__n_error_33; + goto s_n_llhttp__internal__n_error_43; } /* UNREACHABLE */; abort(); @@ -7338,7 +7772,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_8: { + s_n_llhttp__internal__n_invoke_test_lenient_flags_13: { switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_header_field_colon_discard_ws; @@ -7348,7 +7782,16 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_36: { + s_n_llhttp__internal__n_error_58: { + state->error = 0xb; + state->reason = "Content-Length can't be present with Transfer-Encoding"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_46: { state->error = 0xa; state->reason = "Invalid header value char"; state->error_pos = (const char*) p; @@ -7357,17 +7800,17 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_10: { + s_n_llhttp__internal__n_invoke_test_lenient_flags_15: { switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_header_value_discard_ws; default: - goto s_n_llhttp__internal__n_error_36; + goto s_n_llhttp__internal__n_error_46; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_38: { + s_n_llhttp__internal__n_error_48: { state->error = 0xb; state->reason = "Empty Content-Length"; state->error_pos = (const char*) p; @@ -7376,7 +7819,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_14: { + s_n_llhttp__internal__n_pause_18: { state->error = 0x15; state->reason = "on_header_value_complete pause"; state->error_pos = (const char*) p; @@ -7385,7 +7828,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_37: { + s_n_llhttp__internal__n_error_47: { state->error = 0x1d; state->reason = "`on_header_value_complete` callback error"; state->error_pos = (const char*) p; @@ -7419,65 +7862,65 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_3: { - switch (llhttp__internal__c_or_flags_3(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_5: { + switch (llhttp__internal__c_or_flags_5(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_4: { - switch (llhttp__internal__c_or_flags_4(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_6: { + switch (llhttp__internal__c_or_flags_6(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_5: { - switch (llhttp__internal__c_or_flags_5(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_7: { + switch (llhttp__internal__c_or_flags_7(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_6: { - switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_8: { + switch (llhttp__internal__c_or_flags_8(state, p, endp)) { default: goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_load_header_state_1: { + s_n_llhttp__internal__n_invoke_load_header_state_2: { switch (llhttp__internal__c_load_header_state(state, p, endp)) { case 5: - goto s_n_llhttp__internal__n_invoke_or_flags_3; + goto s_n_llhttp__internal__n_invoke_or_flags_5; case 6: - goto s_n_llhttp__internal__n_invoke_or_flags_4; + goto s_n_llhttp__internal__n_invoke_or_flags_6; case 7: - goto s_n_llhttp__internal__n_invoke_or_flags_5; + goto s_n_llhttp__internal__n_invoke_or_flags_7; case 8: - goto s_n_llhttp__internal__n_invoke_or_flags_6; + goto s_n_llhttp__internal__n_invoke_or_flags_8; default: goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_load_header_state: { + s_n_llhttp__internal__n_invoke_load_header_state_1: { switch (llhttp__internal__c_load_header_state(state, p, endp)) { case 2: - goto s_n_llhttp__internal__n_error_38; + goto s_n_llhttp__internal__n_error_48; default: - goto s_n_llhttp__internal__n_invoke_load_header_state_1; + goto s_n_llhttp__internal__n_invoke_load_header_state_2; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_35: { + s_n_llhttp__internal__n_error_45: { state->error = 0xa; state->reason = "Invalid header value char"; state->error_pos = (const char*) p; @@ -7486,17 +7929,17 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_9: { - switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_14: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_header_value_discard_lws; default: - goto s_n_llhttp__internal__n_error_35; + goto s_n_llhttp__internal__n_error_45; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_39: { + s_n_llhttp__internal__n_error_49: { state->error = 0x2; state->reason = "Expected LF after CR"; state->error_pos = (const char*) p; @@ -7505,12 +7948,12 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_11: { + s_n_llhttp__internal__n_invoke_test_lenient_flags_16: { switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_header_value_discard_lws; default: - goto s_n_llhttp__internal__n_error_39; + goto s_n_llhttp__internal__n_error_49; } /* UNREACHABLE */; abort(); @@ -7523,7 +7966,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_load_header_state_3: { + s_n_llhttp__internal__n_invoke_load_header_state_4: { switch (llhttp__internal__c_load_header_state(state, p, endp)) { case 8: goto s_n_llhttp__internal__n_invoke_update_header_state_1; @@ -7541,55 +7984,55 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_7: { - switch (llhttp__internal__c_or_flags_3(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_9: { + switch (llhttp__internal__c_or_flags_5(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state_2; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_8: { - switch (llhttp__internal__c_or_flags_4(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_10: { + switch (llhttp__internal__c_or_flags_6(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state_2; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_9: { - switch (llhttp__internal__c_or_flags_5(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_11: { + switch (llhttp__internal__c_or_flags_7(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state_2; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_10: { - switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_12: { + switch (llhttp__internal__c_or_flags_8(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_load_header_state_4: { + s_n_llhttp__internal__n_invoke_load_header_state_5: { switch (llhttp__internal__c_load_header_state(state, p, endp)) { case 5: - goto s_n_llhttp__internal__n_invoke_or_flags_7; + goto s_n_llhttp__internal__n_invoke_or_flags_9; case 6: - goto s_n_llhttp__internal__n_invoke_or_flags_8; + goto s_n_llhttp__internal__n_invoke_or_flags_10; case 7: - goto s_n_llhttp__internal__n_invoke_or_flags_9; + goto s_n_llhttp__internal__n_invoke_or_flags_11; case 8: - goto s_n_llhttp__internal__n_invoke_or_flags_10; + goto s_n_llhttp__internal__n_invoke_or_flags_12; default: goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_40: { + s_n_llhttp__internal__n_error_51: { state->error = 0x3; state->reason = "Missing expected LF after header value"; state->error_pos = (const char*) p; @@ -7598,6 +8041,15 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } + s_n_llhttp__internal__n_error_50: { + state->error = 0x19; + state->reason = "Missing expected CR after header value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1: { const unsigned char* start; int err; @@ -7607,6 +8059,23 @@ static llparse_state_t llhttp__internal__run( err = llhttp__on_header_value(state, start, p); if (err != 0) { state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_test_lenient_flags_17; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_17; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; state->error_pos = (const char*) (p + 1); state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; return s_error; @@ -7616,7 +8085,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4: { const unsigned char* start; int err; @@ -7633,7 +8102,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5: { const unsigned char* start; int err; @@ -7651,7 +8120,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3: { const unsigned char* start; int err; @@ -7661,19 +8130,19 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_41; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_52; return s_error; } - goto s_n_llhttp__internal__n_error_41; + goto s_n_llhttp__internal__n_error_52; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_12: { + s_n_llhttp__internal__n_invoke_test_lenient_flags_18: { switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_header_value_lenient; default: - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3; } /* UNREACHABLE */; abort(); @@ -7686,48 +8155,48 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_11: { - switch (llhttp__internal__c_or_flags_3(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_13: { + switch (llhttp__internal__c_or_flags_5(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state_4; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_12: { - switch (llhttp__internal__c_or_flags_4(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_14: { + switch (llhttp__internal__c_or_flags_6(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state_4; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_13: { - switch (llhttp__internal__c_or_flags_5(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_15: { + switch (llhttp__internal__c_or_flags_7(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state_4; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_14: { - switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_16: { + switch (llhttp__internal__c_or_flags_8(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_connection; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_load_header_state_5: { + s_n_llhttp__internal__n_invoke_load_header_state_6: { switch (llhttp__internal__c_load_header_state(state, p, endp)) { case 5: - goto s_n_llhttp__internal__n_invoke_or_flags_11; + goto s_n_llhttp__internal__n_invoke_or_flags_13; case 6: - goto s_n_llhttp__internal__n_invoke_or_flags_12; + goto s_n_llhttp__internal__n_invoke_or_flags_14; case 7: - goto s_n_llhttp__internal__n_invoke_or_flags_13; + goto s_n_llhttp__internal__n_invoke_or_flags_15; case 8: - goto s_n_llhttp__internal__n_invoke_or_flags_14; + goto s_n_llhttp__internal__n_invoke_or_flags_16; default: goto s_n_llhttp__internal__n_header_value_connection; } @@ -7766,7 +8235,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_6: { const unsigned char* start; int err; @@ -7776,32 +8245,32 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_43; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_54; return s_error; } - goto s_n_llhttp__internal__n_error_43; + goto s_n_llhttp__internal__n_error_54; /* UNREACHABLE */; abort(); } s_n_llhttp__internal__n_invoke_mul_add_content_length_1: { switch (llhttp__internal__c_mul_add_content_length_1(state, p, endp, match)) { case 1: - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_6; default: goto s_n_llhttp__internal__n_header_value_content_length; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_15: { - switch (llhttp__internal__c_or_flags_15(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_17: { + switch (llhttp__internal__c_or_flags_17(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_otherwise; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_6: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_7: { const unsigned char* start; int err; @@ -7811,14 +8280,14 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_44; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_55; return s_error; } - goto s_n_llhttp__internal__n_error_44; + goto s_n_llhttp__internal__n_error_55; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_42: { + s_n_llhttp__internal__n_error_53: { state->error = 0x4; state->reason = "Duplicate Content-Length"; state->error_pos = (const char*) p; @@ -7832,12 +8301,12 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_header_value_content_length; default: - goto s_n_llhttp__internal__n_error_42; + goto s_n_llhttp__internal__n_error_53; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_8: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_9: { const unsigned char* start; int err; @@ -7847,11 +8316,11 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_46; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_57; return s_error; } p++; - goto s_n_llhttp__internal__n_error_46; + goto s_n_llhttp__internal__n_error_57; /* UNREACHABLE */; abort(); } @@ -7863,7 +8332,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_7: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_8: { const unsigned char* start; int err; @@ -7873,18 +8342,18 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_45; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_56; return s_error; } p++; - goto s_n_llhttp__internal__n_error_45; + goto s_n_llhttp__internal__n_error_56; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_13: { - switch (llhttp__internal__c_test_lenient_flags_13(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_19: { + switch (llhttp__internal__c_test_lenient_flags_19(state, p, endp)) { case 0: - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_7; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_8; default: goto s_n_llhttp__internal__n_header_value_te_chunked; } @@ -7894,7 +8363,7 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_load_type_1: { switch (llhttp__internal__c_load_type(state, p, endp)) { case 1: - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_13; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_19; default: goto s_n_llhttp__internal__n_header_value_te_chunked; } @@ -7917,20 +8386,20 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_17: { - switch (llhttp__internal__c_or_flags_16(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_19: { + switch (llhttp__internal__c_or_flags_18(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_and_flags; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_14: { - switch (llhttp__internal__c_test_lenient_flags_13(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_20: { + switch (llhttp__internal__c_test_lenient_flags_19(state, p, endp)) { case 0: - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_8; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_9; default: - goto s_n_llhttp__internal__n_invoke_or_flags_17; + goto s_n_llhttp__internal__n_invoke_or_flags_19; } /* UNREACHABLE */; abort(); @@ -7938,15 +8407,15 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_load_type_2: { switch (llhttp__internal__c_load_type(state, p, endp)) { case 1: - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_14; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_20; default: - goto s_n_llhttp__internal__n_invoke_or_flags_17; + goto s_n_llhttp__internal__n_invoke_or_flags_19; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_16: { - switch (llhttp__internal__c_or_flags_16(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_18: { + switch (llhttp__internal__c_or_flags_18(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_and_flags; } @@ -7958,20 +8427,20 @@ static llparse_state_t llhttp__internal__run( case 1: goto s_n_llhttp__internal__n_invoke_load_type_2; default: - goto s_n_llhttp__internal__n_invoke_or_flags_16; + goto s_n_llhttp__internal__n_invoke_or_flags_18; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_or_flags_18: { - switch (llhttp__internal__c_or_flags_18(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_20: { + switch (llhttp__internal__c_or_flags_20(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state_9; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_load_header_state_2: { + s_n_llhttp__internal__n_invoke_load_header_state_3: { switch (llhttp__internal__c_load_header_state(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_header_value_connection; @@ -7980,23 +8449,72 @@ static llparse_state_t llhttp__internal__run( case 3: goto s_n_llhttp__internal__n_invoke_test_flags_3; case 4: - goto s_n_llhttp__internal__n_invoke_or_flags_18; + goto s_n_llhttp__internal__n_invoke_or_flags_20; default: goto s_n_llhttp__internal__n_header_value; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_15: { + s_n_llhttp__internal__n_invoke_test_lenient_flags_21: { + switch (llhttp__internal__c_test_lenient_flags_21(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_error_58; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_flags_4: { + switch (llhttp__internal__c_test_flags_4(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_21; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_59: { + state->error = 0xf; + state->reason = "Transfer-Encoding can't be present with Content-Length"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_22: { + switch (llhttp__internal__c_test_lenient_flags_21(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_error_59; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_flags_5: { + switch (llhttp__internal__c_test_flags_2(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_22; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_19: { state->error = 0x15; state->reason = "on_header_field_complete pause"; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_discard_ws; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_header_state; return s_error; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_34: { + s_n_llhttp__internal__n_error_44: { state->error = 0x1c; state->reason = "`on_header_field_complete` callback error"; state->error_pos = (const char*) p; @@ -8041,7 +8559,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_47: { + s_n_llhttp__internal__n_error_60: { state->error = 0xa; state->reason = "Invalid header token"; state->error_pos = (const char*) p; @@ -8093,7 +8611,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_16: { + s_n_llhttp__internal__n_pause_20: { state->error = 0x15; state->reason = "on_url_complete pause"; state->error_pos = (const char*) p; @@ -8116,7 +8634,7 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_headers_start; case 21: - goto s_n_llhttp__internal__n_pause_16; + goto s_n_llhttp__internal__n_pause_20; default: goto s_n_llhttp__internal__n_error_3; } @@ -8156,7 +8674,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_48: { + s_n_llhttp__internal__n_error_61: { state->error = 0x7; state->reason = "Expected CRLF"; state->error_pos = (const char*) p; @@ -8182,7 +8700,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_55: { + s_n_llhttp__internal__n_error_69: { state->error = 0x17; state->reason = "Pause on PRI/Upgrade"; state->error_pos = (const char*) p; @@ -8191,7 +8709,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_56: { + s_n_llhttp__internal__n_error_70: { state->error = 0x9; state->reason = "Expected HTTP/2 Connection Preface"; state->error_pos = (const char*) p; @@ -8200,7 +8718,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_53: { + s_n_llhttp__internal__n_error_67: { state->error = 0x2; state->reason = "Expected CRLF after version"; state->error_pos = (const char*) p; @@ -8209,17 +8727,17 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_16: { - switch (llhttp__internal__c_test_lenient_flags_5(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_25: { + switch (llhttp__internal__c_test_lenient_flags_8(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_headers_start; default: - goto s_n_llhttp__internal__n_error_53; + goto s_n_llhttp__internal__n_error_67; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_54: { + s_n_llhttp__internal__n_error_66: { state->error = 0x9; state->reason = "Expected CRLF after version"; state->error_pos = (const char*) p; @@ -8228,7 +8746,26 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_17: { + s_n_llhttp__internal__n_invoke_test_lenient_flags_24: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_req_http_complete_crlf; + default: + goto s_n_llhttp__internal__n_error_66; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_68: { + state->error = 0x9; + state->reason = "Expected CRLF after version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_21: { state->error = 0x15; state->reason = "on_version_complete pause"; state->error_pos = (const char*) p; @@ -8237,7 +8774,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_52: { + s_n_llhttp__internal__n_error_65: { state->error = 0x21; state->reason = "`on_version_complete` callback error"; state->error_pos = (const char*) p; @@ -8273,10 +8810,10 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_51; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_64; return s_error; } - goto s_n_llhttp__internal__n_error_51; + goto s_n_llhttp__internal__n_error_64; /* UNREACHABLE */; abort(); } @@ -8326,8 +8863,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_15: { - switch (llhttp__internal__c_test_lenient_flags_15(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_23: { + switch (llhttp__internal__c_test_lenient_flags_23(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_span_end_llhttp__on_version_1; default: @@ -8339,7 +8876,7 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_store_http_minor: { switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { default: - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_15; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_23; } /* UNREACHABLE */; abort(); @@ -8354,10 +8891,10 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_57; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_71; return s_error; } - goto s_n_llhttp__internal__n_error_57; + goto s_n_llhttp__internal__n_error_71; /* UNREACHABLE */; abort(); } @@ -8371,10 +8908,10 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_58; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_72; return s_error; } - goto s_n_llhttp__internal__n_error_58; + goto s_n_llhttp__internal__n_error_72; /* UNREACHABLE */; abort(); } @@ -8396,14 +8933,14 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_59; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_73; return s_error; } - goto s_n_llhttp__internal__n_error_59; + goto s_n_llhttp__internal__n_error_73; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_50: { + s_n_llhttp__internal__n_error_63: { state->error = 0x8; state->reason = "Invalid method for HTTP/x.x request"; state->error_pos = (const char*) p; @@ -8485,12 +9022,12 @@ static llparse_state_t llhttp__internal__run( case 34: goto s_n_llhttp__internal__n_span_start_llhttp__on_version; default: - goto s_n_llhttp__internal__n_error_50; + goto s_n_llhttp__internal__n_error_63; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_62: { + s_n_llhttp__internal__n_error_76: { state->error = 0x8; state->reason = "Expected HTTP/"; state->error_pos = (const char*) p; @@ -8499,7 +9036,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_60: { + s_n_llhttp__internal__n_error_74: { state->error = 0x8; state->reason = "Expected SOURCE method for ICE/x.x request"; state->error_pos = (const char*) p; @@ -8513,12 +9050,12 @@ static llparse_state_t llhttp__internal__run( case 33: goto s_n_llhttp__internal__n_span_start_llhttp__on_version; default: - goto s_n_llhttp__internal__n_error_60; + goto s_n_llhttp__internal__n_error_74; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_61: { + s_n_llhttp__internal__n_error_75: { state->error = 0x8; state->reason = "Invalid method for RTSP/x.x request"; state->error_pos = (const char*) p; @@ -8558,12 +9095,12 @@ static llparse_state_t llhttp__internal__run( case 45: goto s_n_llhttp__internal__n_span_start_llhttp__on_version; default: - goto s_n_llhttp__internal__n_error_61; + goto s_n_llhttp__internal__n_error_75; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_18: { + s_n_llhttp__internal__n_pause_22: { state->error = 0x15; state->reason = "on_url_complete pause"; state->error_pos = (const char*) p; @@ -8572,7 +9109,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_49: { + s_n_llhttp__internal__n_error_62: { state->error = 0x1a; state->reason = "`on_url_complete` callback error"; state->error_pos = (const char*) p; @@ -8586,9 +9123,9 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_req_http_start; case 21: - goto s_n_llhttp__internal__n_pause_18; + goto s_n_llhttp__internal__n_pause_22; default: - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_62; } /* UNREACHABLE */; abort(); @@ -8661,7 +9198,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_63: { + s_n_llhttp__internal__n_error_77: { state->error = 0x7; state->reason = "Invalid char in url fragment start"; state->error_pos = (const char*) p; @@ -8721,7 +9258,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_64: { + s_n_llhttp__internal__n_error_78: { state->error = 0x7; state->reason = "Invalid char in url query"; state->error_pos = (const char*) p; @@ -8730,7 +9267,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_65: { + s_n_llhttp__internal__n_error_79: { state->error = 0x7; state->reason = "Invalid char in url path"; state->error_pos = (const char*) p; @@ -8841,7 +9378,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_66: { + s_n_llhttp__internal__n_error_80: { state->error = 0x7; state->reason = "Double @ in url"; state->error_pos = (const char*) p; @@ -8850,7 +9387,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_67: { + s_n_llhttp__internal__n_error_81: { state->error = 0x7; state->reason = "Unexpected char in url server"; state->error_pos = (const char*) p; @@ -8859,7 +9396,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_68: { + s_n_llhttp__internal__n_error_82: { state->error = 0x7; state->reason = "Unexpected char in url server"; state->error_pos = (const char*) p; @@ -8868,7 +9405,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_69: { + s_n_llhttp__internal__n_error_83: { state->error = 0x7; state->reason = "Unexpected char in url schema"; state->error_pos = (const char*) p; @@ -8877,7 +9414,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_70: { + s_n_llhttp__internal__n_error_84: { state->error = 0x7; state->reason = "Unexpected char in url schema"; state->error_pos = (const char*) p; @@ -8886,7 +9423,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_71: { + s_n_llhttp__internal__n_error_85: { state->error = 0x7; state->reason = "Unexpected start char in url"; state->error_pos = (const char*) p; @@ -8905,7 +9442,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_72: { + s_n_llhttp__internal__n_error_86: { state->error = 0x6; state->reason = "Expected space after method"; state->error_pos = (const char*) p; @@ -8914,7 +9451,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_22: { + s_n_llhttp__internal__n_pause_26: { state->error = 0x15; state->reason = "on_method_complete pause"; state->error_pos = (const char*) p; @@ -8923,7 +9460,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_89: { + s_n_llhttp__internal__n_error_105: { state->error = 0x20; state->reason = "`on_method_complete` callback error"; state->error_pos = (const char*) p; @@ -8957,7 +9494,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_90: { + s_n_llhttp__internal__n_error_106: { state->error = 0x6; state->reason = "Invalid method encountered"; state->error_pos = (const char*) p; @@ -8966,7 +9503,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_82: { + s_n_llhttp__internal__n_error_98: { state->error = 0xd; state->reason = "Invalid status code"; state->error_pos = (const char*) p; @@ -8975,7 +9512,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_80: { + s_n_llhttp__internal__n_error_96: { state->error = 0xd; state->reason = "Invalid status code"; state->error_pos = (const char*) p; @@ -8984,7 +9521,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_78: { + s_n_llhttp__internal__n_error_94: { state->error = 0xd; state->reason = "Invalid status code"; state->error_pos = (const char*) p; @@ -8993,7 +9530,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_20: { + s_n_llhttp__internal__n_pause_24: { state->error = 0x15; state->reason = "on_status_complete pause"; state->error_pos = (const char*) p; @@ -9002,7 +9539,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_75: { + s_n_llhttp__internal__n_error_90: { state->error = 0x1b; state->reason = "`on_status_complete` callback error"; state->error_pos = (const char*) p; @@ -9016,14 +9553,33 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_headers_start; case 21: - goto s_n_llhttp__internal__n_pause_20; + goto s_n_llhttp__internal__n_pause_24; default: - goto s_n_llhttp__internal__n_error_75; + goto s_n_llhttp__internal__n_error_90; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_76: { + s_n_llhttp__internal__n_error_89: { + state->error = 0xd; + state->reason = "Invalid response status"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_27: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + default: + goto s_n_llhttp__internal__n_error_89; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_91: { state->error = 0x2; state->reason = "Expected LF after CR"; state->error_pos = (const char*) p; @@ -9032,16 +9588,25 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_18: { - switch (llhttp__internal__c_test_lenient_flags_5(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_28: { + switch (llhttp__internal__c_test_lenient_flags_8(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; default: - goto s_n_llhttp__internal__n_error_76; + goto s_n_llhttp__internal__n_error_91; } /* UNREACHABLE */; abort(); } + s_n_llhttp__internal__n_error_92: { + state->error = 0x19; + state->reason = "Missing expected CR after response line"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } s_n_llhttp__internal__n_span_end_llhttp__on_status: { const unsigned char* start; int err; @@ -9052,11 +9617,11 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_res_line_almost_done; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_test_lenient_flags_29; return s_error; } p++; - goto s_n_llhttp__internal__n_res_line_almost_done; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_29; /* UNREACHABLE */; abort(); } @@ -9078,7 +9643,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_77: { + s_n_llhttp__internal__n_error_93: { state->error = 0xd; state->reason = "Invalid response status"; state->error_pos = (const char*) p; @@ -9090,14 +9655,14 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_mul_add_status_code_2: { switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { case 1: - goto s_n_llhttp__internal__n_error_78; + goto s_n_llhttp__internal__n_error_94; default: goto s_n_llhttp__internal__n_res_status_code_otherwise; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_79: { + s_n_llhttp__internal__n_error_95: { state->error = 0xd; state->reason = "Invalid status code"; state->error_pos = (const char*) p; @@ -9109,14 +9674,14 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_mul_add_status_code_1: { switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { case 1: - goto s_n_llhttp__internal__n_error_80; + goto s_n_llhttp__internal__n_error_96; default: goto s_n_llhttp__internal__n_res_status_code_digit_3; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_81: { + s_n_llhttp__internal__n_error_97: { state->error = 0xd; state->reason = "Invalid status code"; state->error_pos = (const char*) p; @@ -9128,14 +9693,14 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_mul_add_status_code: { switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { case 1: - goto s_n_llhttp__internal__n_error_82; + goto s_n_llhttp__internal__n_error_98; default: goto s_n_llhttp__internal__n_res_status_code_digit_2; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_83: { + s_n_llhttp__internal__n_error_99: { state->error = 0xd; state->reason = "Invalid status code"; state->error_pos = (const char*) p; @@ -9152,7 +9717,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_84: { + s_n_llhttp__internal__n_error_100: { state->error = 0x9; state->reason = "Expected space after version"; state->error_pos = (const char*) p; @@ -9161,7 +9726,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_21: { + s_n_llhttp__internal__n_pause_25: { state->error = 0x15; state->reason = "on_version_complete pause"; state->error_pos = (const char*) p; @@ -9170,7 +9735,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_74: { + s_n_llhttp__internal__n_error_88: { state->error = 0x21; state->reason = "`on_version_complete` callback error"; state->error_pos = (const char*) p; @@ -9206,10 +9771,10 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_73; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_87; return s_error; } - goto s_n_llhttp__internal__n_error_73; + goto s_n_llhttp__internal__n_error_87; /* UNREACHABLE */; abort(); } @@ -9259,8 +9824,8 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_invoke_test_lenient_flags_17: { - switch (llhttp__internal__c_test_lenient_flags_15(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_26: { + switch (llhttp__internal__c_test_lenient_flags_23(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_span_end_llhttp__on_version_6; default: @@ -9272,7 +9837,7 @@ static llparse_state_t llhttp__internal__run( s_n_llhttp__internal__n_invoke_store_http_minor_1: { switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { default: - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_17; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_26; } /* UNREACHABLE */; abort(); @@ -9287,10 +9852,10 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_85; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_101; return s_error; } - goto s_n_llhttp__internal__n_error_85; + goto s_n_llhttp__internal__n_error_101; /* UNREACHABLE */; abort(); } @@ -9304,10 +9869,10 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_86; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_102; return s_error; } - goto s_n_llhttp__internal__n_error_86; + goto s_n_llhttp__internal__n_error_102; /* UNREACHABLE */; abort(); } @@ -9329,14 +9894,14 @@ static llparse_state_t llhttp__internal__run( if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_87; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_103; return s_error; } - goto s_n_llhttp__internal__n_error_87; + goto s_n_llhttp__internal__n_error_103; /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_91: { + s_n_llhttp__internal__n_error_107: { state->error = 0x8; state->reason = "Expected HTTP/"; state->error_pos = (const char*) p; @@ -9345,7 +9910,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_19: { + s_n_llhttp__internal__n_pause_23: { state->error = 0x15; state->reason = "on_method_complete pause"; state->error_pos = (const char*) p; @@ -9396,7 +9961,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_88: { + s_n_llhttp__internal__n_error_104: { state->error = 0x8; state->reason = "Invalid word encountered"; state->error_pos = (const char*) p; @@ -9430,7 +9995,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_23: { + s_n_llhttp__internal__n_pause_27: { state->error = 0x15; state->reason = "on_message_begin pause"; state->error_pos = (const char*) p; @@ -9453,14 +10018,14 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_invoke_load_type; case 21: - goto s_n_llhttp__internal__n_pause_23; + goto s_n_llhttp__internal__n_pause_27; default: goto s_n_llhttp__internal__n_error; } /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_pause_24: { + s_n_llhttp__internal__n_pause_28: { state->error = 0x15; state->reason = "on_reset pause"; state->error_pos = (const char*) p; @@ -9469,7 +10034,7 @@ static llparse_state_t llhttp__internal__run( /* UNREACHABLE */; abort(); } - s_n_llhttp__internal__n_error_92: { + s_n_llhttp__internal__n_error_108: { state->error = 0x1f; state->reason = "`on_reset` callback error"; state->error_pos = (const char*) p; @@ -9483,9 +10048,9 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_invoke_update_finish; case 21: - goto s_n_llhttp__internal__n_pause_24; + goto s_n_llhttp__internal__n_pause_28; default: - goto s_n_llhttp__internal__n_error_92; + goto s_n_llhttp__internal__n_error_108; } /* UNREACHABLE */; abort(); diff --git a/contrib/restricted/llhttp/ya.make b/contrib/restricted/llhttp/ya.make index 749a6b7ce2..fe272c3b7a 100644 --- a/contrib/restricted/llhttp/ya.make +++ b/contrib/restricted/llhttp/ya.make @@ -6,9 +6,9 @@ LICENSE(MIT) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(9.0.1) +VERSION(9.1.3) -ORIGINAL_SOURCE(https://github.com/nodejs/llhttp/archive/release/v9.0.1.tar.gz) +ORIGINAL_SOURCE(https://github.com/nodejs/llhttp/archive/release/v9.1.3.tar.gz) ADDINCL( GLOBAL contrib/restricted/llhttp/include diff --git a/library/cpp/http/io/benchmark/main.cpp b/library/cpp/http/io/benchmark/main.cpp new file mode 100644 index 0000000000..2a3a07b3bb --- /dev/null +++ b/library/cpp/http/io/benchmark/main.cpp @@ -0,0 +1,81 @@ +#include <library/cpp/http/io/headers.h> + +#include <benchmark/benchmark.h> + +#include <util/stream/str.h> + +void FindHeaderFirstMatch(benchmark::State& state) { + THttpHeaders headers; + headers.AddHeader("Host", "example.com"); + Y_ENSURE(headers.FindHeader("Host")); + for (auto _ : state) { + auto header = headers.FindHeader("Host"); + benchmark::DoNotOptimize(header); + } +} + +void FindHeaderNoMatchSameSize(benchmark::State& state) { + THttpHeaders headers; + for (char c = 'a'; c <= 'z'; ++c) { + headers.AddHeader(TString::Join(c, "aaa"), "some value"); + } + Y_ENSURE(!headers.FindHeader("Host")); + for (auto _ : state) { + auto header = headers.FindHeader("Host"); + benchmark::DoNotOptimize(header); + } +} + +void FindHeaderNoMatchDifferentSizesNoCommonPrefix(benchmark::State& state) { + THttpHeaders headers; + for (char c = 'a'; c <= 'z'; ++c) { // same number of headers as above + headers.AddHeader("aaaaa", "some value"); + } + Y_ENSURE(!headers.FindHeader("Host")); + for (auto _ : state) { + auto header = headers.FindHeader("Host"); + benchmark::DoNotOptimize(header); + } +} + +void FindHeaderNoMatchDifferentSizesCommonPrefix(benchmark::State& state) { + THttpHeaders headers; + for (char c = 'a'; c <= 'z'; ++c) { + headers.AddHeader("Host2", "some value"); + } + Y_ENSURE(!headers.FindHeader("Host")); + for (auto _ : state) { + auto header = headers.FindHeader("Host"); + benchmark::DoNotOptimize(header); + } +} + +void FindHeaderMoreRealisticUseCase(benchmark::State& state) { + TString requestHeaders(R"(Host: yandex.ru +User-Agent: Mozilla/5.0 ... +Accept: */* +Accept-Language: en-US,en;q=0.5 +Accept-Encoding: gzip, deflate, br +Content-Type: text/plain;charset=UTF-8 +Content-Length: 1234 +Origin: https://a.yandex-team.ru +Connection: keep-alive +Referer: https://a.yandex-team.ru/ +Sec-Fetch-Dest: empty +Sec-Fetch-Mode: no-cors +Sec-Fetch-Site: cross-site +TE: trailers)"); + TStringInput stream(requestHeaders); + THttpHeaders headers(&stream); + Y_ENSURE(headers.FindHeader("Content-Type")); + for (auto _ : state) { + auto header = headers.FindHeader("Content-Type"); + benchmark::DoNotOptimize(header); + } +} + +BENCHMARK(FindHeaderFirstMatch); +BENCHMARK(FindHeaderNoMatchSameSize); +BENCHMARK(FindHeaderNoMatchDifferentSizesNoCommonPrefix); +BENCHMARK(FindHeaderNoMatchDifferentSizesCommonPrefix); +BENCHMARK(FindHeaderMoreRealisticUseCase); diff --git a/library/cpp/http/io/benchmark/ya.make b/library/cpp/http/io/benchmark/ya.make new file mode 100644 index 0000000000..6e1d252df0 --- /dev/null +++ b/library/cpp/http/io/benchmark/ya.make @@ -0,0 +1,11 @@ +G_BENCHMARK() + +PEERDIR( + library/cpp/http/io +) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/http/io/headers.cpp b/library/cpp/http/io/headers.cpp index 4ec27a29e8..f2baf64021 100644 --- a/library/cpp/http/io/headers.cpp +++ b/library/cpp/http/io/headers.cpp @@ -12,6 +12,13 @@ static inline TStringBuf Trim(const char* b, const char* e) noexcept { return StripString(TStringBuf(b, e)); } +static inline bool HeaderNameEqual(TStringBuf headerName, TStringBuf expectedName) noexcept { + // Most headers names have distinct sizes. + // Size comparison adds small overhead if all headers have the same size (~4% or lower with size = 4), + // but significantly speeds up the case where sizes are different (~4.5x for expectedName.size() = 4 and headerName.size() = 5) + return headerName.size() == expectedName.size() && AsciiCompareIgnoreCase(headerName, expectedName) == 0; +} + THttpInputHeader::THttpInputHeader(const TStringBuf header) { size_t pos = header.find(':'); @@ -65,7 +72,7 @@ bool THttpHeaders::HasHeader(const TStringBuf header) const { const THttpInputHeader* THttpHeaders::FindHeader(const TStringBuf header) const { for (const auto& hdr : Headers_) { - if (AsciiCompareIgnoreCase(hdr.Name(), header) == 0) { + if (HeaderNameEqual(hdr.Name(), header)) { return &hdr; } } @@ -74,7 +81,7 @@ const THttpInputHeader* THttpHeaders::FindHeader(const TStringBuf header) const void THttpHeaders::RemoveHeader(const TStringBuf header) { for (auto h = Headers_.begin(); h != Headers_.end(); ++h) { - if (AsciiCompareIgnoreCase(h->Name(), header) == 0) { + if (HeaderNameEqual(h->Name(), header)) { Headers_.erase(h); return; } @@ -82,8 +89,9 @@ void THttpHeaders::RemoveHeader(const TStringBuf header) { } void THttpHeaders::AddOrReplaceHeader(const THttpInputHeader& header) { + TStringBuf name = header.Name(); for (auto& hdr : Headers_) { - if (AsciiCompareIgnoreCase(hdr.Name(), header.Name()) == 0) { + if (HeaderNameEqual(hdr.Name(), name)) { hdr = header; return; } diff --git a/library/cpp/http/io/ya.make b/library/cpp/http/io/ya.make index 0390aecf55..20e014d011 100644 --- a/library/cpp/http/io/ya.make +++ b/library/cpp/http/io/ya.make @@ -17,7 +17,11 @@ SRCS( END() RECURSE( - fuzz list_codings +) + +RECURSE_FOR_TESTS( + benchmark + fuzz ut ) diff --git a/library/cpp/yt/string/unittests/enum_ut.cpp b/library/cpp/yt/string/unittests/enum_ut.cpp index f30aec1cab..4059b3d2e6 100644 --- a/library/cpp/yt/string/unittests/enum_ut.cpp +++ b/library/cpp/yt/string/unittests/enum_ut.cpp @@ -71,6 +71,15 @@ TEST(TParseEnumTest, ParseEnumWithCustomDomainName) EXPECT_EQ(std::nullopt, TryParseEnum<ECustomDomainName>("b")); } +TEST(TParseEnumTest, ParseBitEnum) +{ + EXPECT_EQ(ELangs::None, TryParseEnum<ELangs>("")); + EXPECT_EQ(ELangs::Cpp, TryParseEnum<ELangs>("cpp")); + EXPECT_EQ(ELangs::Cpp | ELangs::Rust, TryParseEnum<ELangs>("cpp|rust")); + EXPECT_EQ(ELangs::Cpp | ELangs::Rust, TryParseEnum<ELangs>("cpp | rust")); + EXPECT_EQ(std::nullopt, TryParseEnum<ELangs>("unk | rust")); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/util/string/strspn_ut.cpp b/util/string/strspn_ut.cpp new file mode 100644 index 0000000000..975374cba4 --- /dev/null +++ b/util/string/strspn_ut.cpp @@ -0,0 +1,73 @@ +#include "strspn.h" + +#include <library/cpp/testing/unittest/registar.h> + +Y_UNIT_TEST_SUITE(TStrSpnTest) { + Y_UNIT_TEST(FindFirstOf) { + const TString s("some text!"); + + UNIT_ASSERT_EQUAL(TCompactStrSpn("mos").FindFirstOf(s.begin(), s.end()), s.begin()); + UNIT_ASSERT_EQUAL(TCompactStrSpn("mos").FindFirstOf(s.c_str()), s.begin()); + + UNIT_ASSERT_EQUAL(TCompactStrSpn("xt").FindFirstOf(s.begin(), s.end()), s.begin() + 5); + UNIT_ASSERT_EQUAL(TCompactStrSpn("xt").FindFirstOf(s.c_str()), s.begin() + 5); + + UNIT_ASSERT_EQUAL(TCompactStrSpn(".?!").FindFirstOf(s.begin(), s.end()), s.end() - 1); + UNIT_ASSERT_EQUAL(TCompactStrSpn(".?!").FindFirstOf(s.c_str()), s.end() - 1); + + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstOf(s.begin(), s.end()), s.end()); + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstOf(s.c_str()), s.end()); + + // Must be const. If not, non-const begin() will clone the shared empty string + // and the next assertion will possibly use invalidated end iterator. + const TString empty; + + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstOf(empty.begin(), empty.end()), empty.end()); + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstOf(empty.c_str()), empty.end()); + } + + Y_UNIT_TEST(FindFirstNotOf) { + const TString s("abacabaxyz"); + + UNIT_ASSERT_EQUAL(TCompactStrSpn("123").FindFirstNotOf(s.begin(), s.end()), s.begin()); + UNIT_ASSERT_EQUAL(TCompactStrSpn("123").FindFirstNotOf(s.c_str()), s.begin()); + + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstNotOf(s.begin(), s.end()), s.begin() + 7); + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstNotOf(s.c_str()), s.begin() + 7); + + UNIT_ASSERT_EQUAL(TCompactStrSpn("abcxy").FindFirstNotOf(s.begin(), s.end()), s.end() - 1); + UNIT_ASSERT_EQUAL(TCompactStrSpn("abcxy").FindFirstNotOf(s.c_str()), s.end() - 1); + + UNIT_ASSERT_EQUAL(TCompactStrSpn("abcxyz").FindFirstNotOf(s.begin(), s.end()), s.end()); + UNIT_ASSERT_EQUAL(TCompactStrSpn("abcxyz").FindFirstNotOf(s.c_str()), s.end()); + + const TString empty; + + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstNotOf(empty.begin(), empty.end()), empty.end()); + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstNotOf(empty.c_str()), empty.end()); + } + + Y_UNIT_TEST(FindFirstOfReverse) { + TStringBuf s("some text"); + + UNIT_ASSERT_EQUAL(TCompactStrSpn("xt").FindFirstOf(s.rbegin(), s.rend()), s.rbegin()); + UNIT_ASSERT_EQUAL(TCompactStrSpn("mos").FindFirstOf(s.rbegin(), s.rend()), s.rend() - 3); + UNIT_ASSERT_EQUAL(TCompactStrSpn("s").FindFirstOf(s.rbegin(), s.rend()), s.rend() - 1); + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstOf(s.rbegin(), s.rend()), s.rend()); + + TStringBuf empty; + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstOf(empty.rbegin(), empty.rend()), empty.rend()); + } + + Y_UNIT_TEST(FindFirstNotOfReverse) { + TStringBuf s("_abacabaxyz"); + + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstNotOf(s.rbegin(), s.rend()), s.rbegin()); + UNIT_ASSERT_EQUAL(TCompactStrSpn("xyz").FindFirstNotOf(s.rbegin(), s.rend()), s.rbegin() + 3); + UNIT_ASSERT_EQUAL(TCompactStrSpn("abcxyz").FindFirstNotOf(s.rbegin(), s.rend()), s.rend() - 1); + UNIT_ASSERT_EQUAL(TCompactStrSpn("abcxyz_").FindFirstNotOf(s.rbegin(), s.rend()), s.rend()); + + TStringBuf empty; + UNIT_ASSERT_EQUAL(TCompactStrSpn("abc").FindFirstNotOf(empty.rbegin(), empty.rend()), empty.rend()); + } +} diff --git a/util/string/ut/ya.make b/util/string/ut/ya.make index 7243176bfe..d3377a40a1 100644 --- a/util/string/ut/ya.make +++ b/util/string/ut/ya.make @@ -11,6 +11,7 @@ SRCS( string/printf_ut.cpp string/split_ut.cpp string/strip_ut.cpp + string/strspn_ut.cpp string/subst_ut.cpp string/type_ut.cpp string/util_ut.cpp diff --git a/util/thread/pool.cpp b/util/thread/pool.cpp index e312b6df47..a0d11e49fd 100644 --- a/util/thread/pool.cpp +++ b/util/thread/pool.cpp @@ -431,7 +431,7 @@ public: , Free_(0) , IdleTime_(TDuration::Max()) { - sprintf(Name_, "[mtp queue %ld]", ++MtpQueueCounter); + snprintf(Name_, sizeof(Name_), "[mtp queue %ld]", ++MtpQueueCounter); } inline ~TImpl() { diff --git a/ydb/core/driver_lib/run/export.cpp b/ydb/core/driver_lib/run/export.cpp new file mode 100644 index 0000000000..75d4386b4e --- /dev/null +++ b/ydb/core/driver_lib/run/export.cpp @@ -0,0 +1,36 @@ +#include "export.h" + +#include <kikimr/yndx/yt/export_yt.h> +#include <kikimr/yndx/yt/yt_shutdown.h> + +#include <contrib/ydb/core/tx/datashard/export_s3.h> + +NKikimr::NDataShard::IExport* TDataShardExportFactory::CreateExportToYt( + const IExport::TTask& task, const IExport::TTableColumns& columns) const +{ +#ifndef KIKIMR_DISABLE_YT + return new NKikimr::NYndx::TYtExport(task, columns); +#else + Y_UNUSED(task); + Y_UNUSED(columns); + return nullptr; +#endif +} + +NKikimr::NDataShard::IExport* TDataShardExportFactory::CreateExportToS3( + const IExport::TTask& task, const IExport::TTableColumns& columns) const +{ +#ifndef KIKIMR_DISABLE_S3_OPS + return new NKikimr::NDataShard::TS3Export(task, columns); +#else + Y_UNUSED(task); + Y_UNUSED(columns); + return nullptr; +#endif +} + +void TDataShardExportFactory::Shutdown() { +#ifndef KIKIMR_DISABLE_YT + ShutdownYT(); +#endif +} diff --git a/ydb/core/driver_lib/run/export.h b/ydb/core/driver_lib/run/export.h new file mode 100644 index 0000000000..5f422b5f40 --- /dev/null +++ b/ydb/core/driver_lib/run/export.h @@ -0,0 +1,12 @@ +#pragma once + +#include <contrib/ydb/core/tx/datashard/export_iface.h> + +class TDataShardExportFactory : public NKikimr::NDataShard::IExportFactory { + using IExport = NKikimr::NDataShard::IExport; + +public: + IExport* CreateExportToYt(const IExport::TTask& task, const IExport::TTableColumns& columns) const override; + IExport* CreateExportToS3(const IExport::TTask& task, const IExport::TTableColumns& columns) const override; + void Shutdown() override; +}; diff --git a/ydb/core/yt/export_yt.cpp b/ydb/core/yt/export_yt.cpp new file mode 100644 index 0000000000..bf64c5c4b4 --- /dev/null +++ b/ydb/core/yt/export_yt.cpp @@ -0,0 +1,647 @@ +#ifndef KIKIMR_DISABLE_YT + +#include "export_yt.h" +#include "yt_wrapper.h" + +#include <contrib/ydb/core/protos/flat_scheme_op.pb.h> +#include <contrib/ydb/library/services/services.pb.h> +#include <contrib/ydb/core/tablet_flat/flat_row_state.h> +#include <contrib/ydb/core/tx/datashard/export_common.h> +#include <contrib/ydb/library/binary_json/read.h> +#include <contrib/ydb/library/actors/core/actor_bootstrapped.h> +#include <contrib/ydb/library/actors/core/hfunc.h> + +#include <yt/yt/client/table_client/config.h> +#include <yt/yt/client/table_client/name_table.h> +#include <yt/yt/core/misc/shutdown.h> + +#include <util/generic/maybe.h> +#include <util/generic/vector.h> +#include <util/string/builder.h> +#include <util/string/join.h> + +namespace NKikimr { +namespace NYndx { + +using namespace NDataShard::NExportScan; +using namespace NWrappers; + +using namespace NYT::NApi; +using namespace NYT::NObjectClient; +using namespace NYT::NTableClient; +using namespace NYT::NYPath; +using namespace NYT::NYTree; +using namespace NYT::NYson; + +using TTableColumns = TYtExport::TTableColumns; +using TEvExportScan = NDataShard::TEvExportScan; +using TEvBuffer = TEvExportScan::TEvBuffer<TVector<TUnversionedOwningRow>>; + +namespace NCustomErrorCodes { + + constexpr auto InvalidNodeType = NYT::TErrorCode(100000); + +} // NCustomErrorCodes + +class TYtUploader: public TActorBootstrapped<TYtUploader> { + static ESimpleLogicalValueType ConvertType(NScheme::TTypeId type) { + switch (type) { + case NScheme::NTypeIds::Int32: + return ESimpleLogicalValueType::Int32; + case NScheme::NTypeIds::Uint32: + return ESimpleLogicalValueType::Uint32; + case NScheme::NTypeIds::Int64: + return ESimpleLogicalValueType::Int64; + case NScheme::NTypeIds::Uint64: + return ESimpleLogicalValueType::Uint64; + case NScheme::NTypeIds::Uint8: + return ESimpleLogicalValueType::Uint8; + case NScheme::NTypeIds::Int8: + return ESimpleLogicalValueType::Int8; + case NScheme::NTypeIds::Int16: + return ESimpleLogicalValueType::Int16; + case NScheme::NTypeIds::Uint16: + return ESimpleLogicalValueType::Uint16; + case NScheme::NTypeIds::Bool: + return ESimpleLogicalValueType::Boolean; + case NScheme::NTypeIds::Double: + case NScheme::NTypeIds::Float: + return ESimpleLogicalValueType::Double; + case NScheme::NTypeIds::Date: + return ESimpleLogicalValueType::Date; + case NScheme::NTypeIds::Datetime: + return ESimpleLogicalValueType::Datetime; + case NScheme::NTypeIds::Timestamp: + return ESimpleLogicalValueType::Timestamp; + case NScheme::NTypeIds::Interval: + return ESimpleLogicalValueType::Interval; + case NScheme::NTypeIds::Utf8: + return ESimpleLogicalValueType::Utf8; + case NScheme::NTypeIds::Yson: + return ESimpleLogicalValueType::Any; + case NScheme::NTypeIds::Json: + case NScheme::NTypeIds::JsonDocument: + return ESimpleLogicalValueType::Utf8; + default: + return ESimpleLogicalValueType::String; + } + } + + static TVector<TColumnSchema> GenColumnsSchema(const TTableColumns& columns) { + TVector<TColumnSchema> schema; + + for (const auto& [_, column] : columns) { + // TODO: support pg types + Y_ABORT_UNLESS(column.Type.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + schema.emplace_back(column.Name, ConvertType(column.Type.GetTypeId())); + } + + return schema; + } + + static TTableSchema GenTableSchema(const TTableColumns& columns) { + return TTableSchema(GenColumnsSchema(columns), false /* strict */); + } + + static bool IsRetryableError(const NYT::TError& error) { + if (// Table errors + error.FindMatching(NYT::NTableClient::EErrorCode::SortOrderViolation) + || error.FindMatching(NYT::NTableClient::EErrorCode::InvalidDoubleValue) + || error.FindMatching(NYT::NTableClient::EErrorCode::IncomparableTypes) + || error.FindMatching(NYT::NTableClient::EErrorCode::UnhashableType) + || error.FindMatching(NYT::NTableClient::EErrorCode::CorruptedNameTable) + || error.FindMatching(NYT::NTableClient::EErrorCode::UniqueKeyViolation) + || error.FindMatching(NYT::NTableClient::EErrorCode::SchemaViolation) + || error.FindMatching(NYT::NTableClient::EErrorCode::RowWeightLimitExceeded) + || error.FindMatching(NYT::NTableClient::EErrorCode::InvalidColumnFilter) + || error.FindMatching(NYT::NTableClient::EErrorCode::InvalidColumnRenaming) + || error.FindMatching(NYT::NTableClient::EErrorCode::IncompatibleKeyColumns) + || error.FindMatching(NYT::NTableClient::EErrorCode::TimestampOutOfRange) + || error.FindMatching(NYT::NTableClient::EErrorCode::InvalidSchemaValue) + || error.FindMatching(NYT::NTableClient::EErrorCode::FormatCannotRepresentRow) + || error.FindMatching(NYT::NTableClient::EErrorCode::IncompatibleSchemas) + // Cypress errors + || error.FindMatching(NYT::NYTree::EErrorCode::MaxChildCountViolation) + || error.FindMatching(NYT::NYTree::EErrorCode::MaxStringLengthViolation) + || error.FindMatching(NYT::NYTree::EErrorCode::MaxAttributeSizeViolation) + || error.FindMatching(NYT::NYTree::EErrorCode::MaxKeyLengthViolation) + // Security errors + || error.FindMatching(NYT::NRpc::EErrorCode::AuthenticationError) + || error.FindMatching(NYT::NSecurityClient::EErrorCode::AuthenticationError) + || error.FindMatching(NYT::NSecurityClient::EErrorCode::AuthorizationError) + || error.FindMatching(NYT::NSecurityClient::EErrorCode::AccountLimitExceeded) + || error.FindMatching(NYT::NSecurityClient::EErrorCode::UserBanned) + || error.FindMatching(NYT::NSecurityClient::EErrorCode::NoSuchAccount) + // Custom errors + || error.FindMatching(NCustomErrorCodes::InvalidNodeType) + ) { + return false; + } + + return true; + } + + static NYT::TError CheckNodeType(const TString& path, const TYsonString& yson, const TString& expected) { + try { + auto node = ConvertToNode(yson); + const TString actual = node->Attributes().Get<TString>("type"); + + if (actual == expected) { + return {}; + } + + return NYT::TError(NCustomErrorCodes::InvalidNodeType, TStringBuilder() << "Invalid type of " << path + << ": expected \"" << expected << "\"" + << ", actual \"" << actual << "\""); + } catch (const yexception& ex) { + return NYT::TError(TStringBuilder() << "Error while checking type of " << path + << ": " << ex.what()); + } + } + + void Handle(TEvYtWrapper::TEvNodeExistsResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvYtWrapper::TEvNodeExistsResponse" + << ": self# " << SelfId() + << ", result# " << ToString(result)); + + if (!CheckResult(result, TStringBuf("NodeExists"))) { + return; + } + + if (!result.Value()) { + TCreateNodeOptions opts; + opts.Recursive = true; + opts.IgnoreExisting = true; + + if (UseTypeV3) { + auto attrs = CreateEphemeralAttributes(); + attrs->Set("schema", Schema); + opts.Attributes = std::move(attrs); + } + + Send(Client, new TEvYtWrapper::TEvCreateNodeRequest(DstPath.GetPath(), EObjectType::Table, opts)); + } else { + TGetNodeOptions opts; + opts.Attributes = TVector<TString>{"type"}; + + Send(Client, new TEvYtWrapper::TEvGetNodeRequest(DstPath.GetPath(), opts)); + } + } + + static TTableWriterOptions TableWriterOptions() { + auto opts = TTableWriterOptions(); + opts.Config = NYT::New<TTableWriterConfig>(); + opts.Config->MaxRowWeight = MaxRowWeightLimit; + opts.Config->MaxKeyWeight = MaxKeyWeightLimit; + return opts; + } + + void Handle(TEvYtWrapper::TEvCreateNodeResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvYtWrapper::TEvCreateNodeResponse" + << ": self# " << SelfId() + << ", result# " << ToString(result)); + + if (!CheckResult(result, TStringBuf("CreateNode"))) { + return; + } + + Send(Client, new TEvYtWrapper::TEvCreateTableWriterRequest(DstPath, TableWriterOptions())); + } + + void Handle(TEvYtWrapper::TEvGetNodeResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvYtWrapper::TEvGetNodeResponse" + << ": self# " << SelfId() + << ", result# " << ToString(result)); + + if (!CheckResult(result, TStringBuf("GetNode"))) { + return; + } + + if (!CheckResult(CheckNodeType(DstPath.GetPath(), result.Value(), "table"), TStringBuf("CheckNodeType"))) { + return; + } + + Send(Client, new TEvYtWrapper::TEvCreateTableWriterRequest(DstPath, TableWriterOptions())); + } + + void Handle(TEvYtWrapper::TEvCreateTableWriterResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvYtWrapper::TEvCreateTableWriterResponse" + << ": self# " << SelfId() + << ", result# " << ToString(result)); + + if (!CheckResult(result, TStringBuf("CreateTableWriter"))) { + return; + } + + Writer = result.Value(); + Send(Writer, new TEvYtWrapper::TEvGetNameTableRequest()); + } + + void Handle(TEvYtWrapper::TEvGetNameTableResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvYtWrapper::TEvGetNameTableResponse" + << ": self# " << SelfId() + << ", result# " << ToString(result)); + + if (!CheckResult(result, TStringBuf("GetNameTable"))) { + return; + } + + NameTable = result.Value(); + for (const auto& column : Schema.Columns()) { + NameTable->RegisterName(column.Name()); + } + + if (Scanner) { + Send(Scanner, new TEvExportScan::TEvFeed()); + } + } + + void Handle(TEvExportScan::TEvReady::TPtr& ev) { + EXPORT_LOG_D("Handle TEvExportScan::TEvReady" + << ": self# " << SelfId() + << ", sender# " << ev->Sender); + + Scanner = ev->Sender; + + if (Error) { + return PassAway(); + } + + if (Writer && NameTable) { + Send(Scanner, new TEvExportScan::TEvFeed()); + } + } + + void Handle(TEvBuffer::TPtr& ev) { + EXPORT_LOG_D("Handle TEvExportScan::TEvBuffer" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", msg# " << ev->Get()->ToString()); + + if (ev->Sender != Scanner) { + EXPORT_LOG_W("Received buffer from unknown scanner" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", scanner# " << Scanner); + return; + } + + Last = ev->Get()->Last; + Send(Writer, new TEvYtWrapper::TEvWriteTableRequest(std::move(ev->Get()->Buffer), Last)); + } + + void Handle(TEvYtWrapper::TEvWriteTableResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvYtWrapper::TEvWriteTableResponse" + << ": self# " << SelfId() + << ", result# " << ToString(result)); + + if (!CheckResult(result, TStringBuf("WriteTable"))) { + return; + } + + if (Last) { + return Finish(); + } + + Send(Scanner, new TEvExportScan::TEvFeed()); + } + + template <typename TResult> + bool CheckResult(const TResult& result, const TStringBuf marker) { + if (result.IsOK()) { + return true; + } + + EXPORT_LOG_E("Error at '" << marker << "'" + << ": self# " << SelfId() + << ", error# " << ToString(result)); + RetryOrFinish(result); + + return false; + } + + void RetryOrFinish(const NYT::TError& error) { + if (Attempt++ < Retries && IsRetryableError(error)) { + Schedule(TDuration::Minutes(1), new TEvents::TEvWakeup()); + } else { + Finish(false, TStringBuilder() << "YT error: " << ToString(error)); + } + } + + void Finish(bool success = true, const TString& error = TString()) { + EXPORT_LOG_I("Finish" + << ": self# " << SelfId() + << ", success# " << success + << ", error# " << error); + + if (!success) { + Error = error; + } + + if (!Scanner) { + return; + } + + PassAway(); + } + + void PassAway() override { + if (Scanner) { + Send(Scanner, new TEvExportScan::TEvFinish(Error.Empty(), Error.GetOrElse(TString()))); + } + + Send(Writer, new TEvents::TEvPoisonPill()); + Send(Client, new TEvents::TEvPoisonPill()); + + TActor::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::EXPORT_YT_UPLOADER_ACTOR; + } + + static constexpr TStringBuf LogPrefix() { + return "yt"sv; + } + + explicit TYtUploader( + const TTableColumns& columns, + const TString& serverName, + const TString& token, + const TString& dstPath, + bool useTypeV3, + ui32 retries) + : ServerName(serverName) + , Token(token) + , DstPath(TRichYPath::Parse(dstPath)) + , UseTypeV3(useTypeV3) + , Schema(GenTableSchema(columns)) + , Retries(retries) + , Attempt(0) + { + } + + void Bootstrap() { + EXPORT_LOG_D("Bootstrap" + << ": self# " << SelfId() + << ", attempt# " << Attempt); + + NameTable.Reset(); + Last = false; + + if (Attempt) { + Send(std::exchange(Scanner, TActorId()), new TEvExportScan::TEvReset()); + Send(std::exchange(Writer, TActorId()), new TEvents::TEvPoisonPill()); + Send(std::exchange(Client, TActorId()), new TEvents::TEvPoisonPill()); + } + + Client = RegisterWithSameMailbox(CreateYtWrapper(ServerName, Token)); + Send(Client, new TEvYtWrapper::TEvNodeExistsRequest(DstPath.GetPath(), TNodeExistsOptions())); + + Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvYtWrapper::TEvNodeExistsResponse, Handle); + hFunc(TEvYtWrapper::TEvCreateNodeResponse, Handle); + hFunc(TEvYtWrapper::TEvGetNodeResponse, Handle); + hFunc(TEvYtWrapper::TEvCreateTableWriterResponse, Handle); + hFunc(TEvYtWrapper::TEvGetNameTableResponse, Handle); + hFunc(TEvYtWrapper::TEvWriteTableResponse, Handle); + + hFunc(TEvExportScan::TEvReady, Handle); + hFunc(TEvBuffer, Handle); + + cFunc(TEvents::TEvWakeup::EventType, Bootstrap); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: + const TString ServerName; + const TString Token; + const TRichYPath DstPath; + const bool UseTypeV3; + const TTableSchema Schema; + const ui32 Retries; + + ui32 Attempt; + TActorId Client; + TActorId Writer; + TActorId Scanner; + + TNameTablePtr NameTable; + bool Last; + TMaybe<TString> Error; + +}; // TYtUploader + +class TYtBuffer: public IBuffer { + struct TColumn { + NScheme::TTypeId Type; + int Id; // in name table + }; + + using TColumns = TMap<ui32, TColumn>; + using TOrderedColumns = TVector<TColumns::const_iterator>; + + static TColumns MakeColumns(const TTableColumns& columns) { + TColumns result; + + int i = 0; + for (const auto& [tag, column] : columns) { + // TODO: support pg types + Y_ABORT_UNLESS(column.Type.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + result[tag] = {column.Type.GetTypeId(), i++}; + } + + return result; + } + + static TUnversionedValue ConvertValue(NScheme::TTypeId type, const TCell& cell, int id, bool useTypeV3, TString& buffer) { + if (cell.IsNull()) { + return MakeUnversionedNullValue(id); + } + + switch (type) { + case NScheme::NTypeIds::Int32: + return MakeUnversionedInt64Value(cell.AsValue<i32>(), id); + case NScheme::NTypeIds::Uint32: + return MakeUnversionedUint64Value(cell.AsValue<ui32>(), id); + case NScheme::NTypeIds::Int64: + return MakeUnversionedInt64Value(cell.AsValue<i64>(), id); + case NScheme::NTypeIds::Uint64: + return MakeUnversionedUint64Value(cell.AsValue<ui64>(), id); + case NScheme::NTypeIds::Uint8: + //case NScheme::NTypeIds::Byte: + return MakeUnversionedUint64Value(cell.AsValue<ui8>(), id); + case NScheme::NTypeIds::Int8: + return MakeUnversionedInt64Value(cell.AsValue<i8>(), id); + case NScheme::NTypeIds::Int16: + return MakeUnversionedInt64Value(cell.AsValue<i16>(), id); + case NScheme::NTypeIds::Uint16: + return MakeUnversionedUint64Value(cell.AsValue<ui16>(), id); + case NScheme::NTypeIds::Bool: + return MakeUnversionedBooleanValue(cell.AsValue<bool>(), id); + case NScheme::NTypeIds::Double: + return MakeUnversionedDoubleValue(cell.AsValue<double>(), id); + case NScheme::NTypeIds::Float: + return MakeUnversionedDoubleValue(cell.AsValue<float>(), id); + default: + if (useTypeV3) { + switch (type) { + case NScheme::NTypeIds::Date: + return MakeUnversionedUint64Value(cell.AsValue<ui16>(), id); + case NScheme::NTypeIds::Datetime: + return MakeUnversionedUint64Value(cell.AsValue<ui32>(), id); + case NScheme::NTypeIds::Timestamp: + return MakeUnversionedUint64Value(cell.AsValue<ui64>(), id); + case NScheme::NTypeIds::Interval: + return MakeUnversionedInt64Value(cell.AsValue<i64>(), id); + case NScheme::NTypeIds::Decimal: + buffer = NDataShard::DecimalToString(cell.AsValue<std::pair<ui64, i64>>()); + return MakeUnversionedStringValue(buffer, id); + case NScheme::NTypeIds::DyNumber: + buffer = NDataShard::DyNumberToString(cell.AsBuf()); + return MakeUnversionedStringValue(buffer, id); + case NScheme::NTypeIds::Yson: + return MakeUnversionedAnyValue(cell.AsBuf(), id); + case NScheme::NTypeIds::JsonDocument: + buffer = NBinaryJson::SerializeToJson(cell.AsBuf()); + return MakeUnversionedStringValue(buffer, id); + default: + return MakeUnversionedStringValue(cell.AsBuf(), id); + } + } else { + return MakeUnversionedStringValue(cell.AsBuf(), id); + } + } + } + +public: + explicit TYtBuffer(const TTableColumns& columns, ui64 rowsLimit, ui64 bytesLimit, bool useTypeV3) + : Columns(MakeColumns(columns)) + , RowsLimit(rowsLimit) + , BytesLimit(bytesLimit) + , UseTypeV3(useTypeV3) + , BytesRead(0) + , BytesSent(0) + { + } + + void ColumnsOrder(const TVector<ui32>& tags) override { + Y_ABORT_UNLESS(tags.size() == Columns.size()); + + OrderedColumns.clear(); + OrderedColumns.reserve(tags.size()); + + for (const auto& tag : tags) { + auto it = Columns.find(tag); + Y_ABORT_UNLESS(it != Columns.end()); + OrderedColumns.push_back(it); + } + } + + bool Collect(const NTable::IScan::TRow& row) override { + TUnversionedOwningRowBuilder rowBuilder; + + for (ui32 i = 0; i < (*row).size(); ++i) { + Y_ABORT_UNLESS(i < OrderedColumns.size()); + const auto& column = OrderedColumns[i]->second; + const auto& cell = (*row)[i]; + + TString buffer; + const auto value = ConvertValue(column.Type, cell, column.Id, UseTypeV3, buffer); + + rowBuilder.AddValue(value); + BytesRead += cell.Size(); + BytesSent += EstimateRowValueSize(value); + } + + Buffer.emplace_back(rowBuilder.FinishRow()); + return true; + } + + IEventBase* PrepareEvent(bool last, IBuffer::TStats& stats) override { + stats.Rows = Buffer.size(); + stats.BytesRead = BytesRead; + stats.BytesSent = BytesSent; + + return new TEvBuffer(Flush(), last); + } + + void Clear() override { + Flush(); + } + + TVector<TUnversionedOwningRow> Flush() { + BytesRead = 0; + BytesSent = 0; + return std::exchange(Buffer, TVector<TUnversionedOwningRow>()); + } + + bool IsFilled() const override { + return Buffer.size() >= RowsLimit || BytesSent >= BytesLimit; + } + + TString GetError() const override { + Y_ABORT("unreachable"); + } + +private: + const TColumns Columns; + const ui64 RowsLimit; + const ui64 BytesLimit; + const bool UseTypeV3; + + TOrderedColumns OrderedColumns; + + TVector<TUnversionedOwningRow> Buffer; + ui64 BytesRead; + ui64 BytesSent; + +}; // TYtBuffer + +IActor* TYtExport::CreateUploader(const TActorId&, ui64) const { + const auto& settings = Task.GetYTSettings(); + return new TYtUploader( + Columns, + Join(':', settings.GetHost(), settings.GetPort()), + settings.GetToken(), + settings.GetTablePattern(), + settings.GetUseTypeV3(), + Task.GetNumberOfRetries() + ); +} + +IBuffer* TYtExport::CreateBuffer() const { + const auto& settings = Task.GetYTSettings(); + const auto& scanSettings = Task.GetScanSettings(); + const ui64 maxRows = scanSettings.GetRowsBatchSize() ? scanSettings.GetRowsBatchSize() : Max<ui64>(); + const ui64 maxBytes = scanSettings.GetBytesBatchSize(); + + return new TYtBuffer(Columns, maxRows, maxBytes, settings.GetUseTypeV3()); +} + +void TYtExport::Shutdown() const { + NYT::Shutdown(); +} + +} // NYndx +} // NKikimr + +#endif // KIKIMR_DISABLE_YT diff --git a/ydb/core/yt/export_yt.h b/ydb/core/yt/export_yt.h new file mode 100644 index 0000000000..6397f376fd --- /dev/null +++ b/ydb/core/yt/export_yt.h @@ -0,0 +1,32 @@ +#pragma once + +#ifndef KIKIMR_DISABLE_YT + +#include <contrib/ydb/core/tx/datashard/export_iface.h> + +namespace NKikimr { +namespace NYndx { + +class TYtExport: public NDataShard::IExport { +public: + explicit TYtExport(const TTask& task, const TTableColumns& columns) + : Task(task) + , Columns(columns) + { + Y_ABORT_UNLESS(task.HasYTSettings()); + } + + IActor* CreateUploader(const TActorId& dataShard, ui64 txId) const override; + IBuffer* CreateBuffer() const override; + + void Shutdown() const override; + +private: + const TTask Task; + const TTableColumns Columns; +}; + +} // NYndx +} // NKikimr + +#endif // KIKIMR_DISABLE_YT diff --git a/ydb/core/yt/ya.make b/ydb/core/yt/ya.make new file mode 100644 index 0000000000..2bfc26cd95 --- /dev/null +++ b/ydb/core/yt/ya.make @@ -0,0 +1,30 @@ +LIBRARY() + +OWNER(g:kikimr) + +IF (OS_WINDOWS) + CFLAGS( + -DKIKIMR_DISABLE_YT + ) +ELSE() + SRCS( + export_yt.cpp + export_yt.h + yt_shutdown.cpp + yt_shutdown.h + yt_wrapper.cpp + yt_wrapper.h + ) + PEERDIR( + contrib/ydb/library/actors/core + contrib/ydb/core/base + contrib/ydb/core/protos + contrib/ydb/library/aclib + contrib/ydb/library/binary_json + yt/yt/client + ) +ENDIF() + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/yt/yt_shutdown.cpp b/ydb/core/yt/yt_shutdown.cpp new file mode 100644 index 0000000000..4d45127198 --- /dev/null +++ b/ydb/core/yt/yt_shutdown.cpp @@ -0,0 +1,11 @@ +#include "yt_shutdown.h" + +#ifndef KIKIMR_DISABLE_YT +#include <yt/yt/core/misc/shutdown.h> +#endif + +void ShutdownYT() { +#ifndef KIKIMR_DISABLE_YT + NYT::Shutdown(); +#endif +} diff --git a/ydb/core/yt/yt_shutdown.h b/ydb/core/yt/yt_shutdown.h new file mode 100644 index 0000000000..2a87480033 --- /dev/null +++ b/ydb/core/yt/yt_shutdown.h @@ -0,0 +1,3 @@ +#pragma once + +void ShutdownYT(); diff --git a/ydb/core/yt/yt_wrapper.cpp b/ydb/core/yt/yt_wrapper.cpp new file mode 100644 index 0000000000..675a877cfb --- /dev/null +++ b/ydb/core/yt/yt_wrapper.cpp @@ -0,0 +1,377 @@ +#if defined(KIKIMR_DISABLE_YT) +#error "yt wrapper is disabled" +#endif + +#include "yt_wrapper.h" + +#include <contrib/ydb/library/services/services.pb.h> + +#include <contrib/ydb/library/actors/core/actor.h> +#include <contrib/ydb/library/actors/core/hfunc.h> + +#include <yt/yt/client/api/rpc_proxy/connection.h> +#include <yt/yt/client/api/table_writer.h> + +#include <util/generic/hash_set.h> + +#include <mutex> + +namespace NKikimr { +namespace NWrappers { + +using namespace NYT; +using namespace NYT::NApi; + +class TRequest; +using TRequestPtr = NYT::TIntrusivePtr<TRequest>; + +struct TEvPrivate { + enum EEv { + EvComplete = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), + EvCreateTableWriter, + EvCloseTableWriter, + + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); + + struct TEvComplete: public TEventLocal<TEvComplete, EvComplete> { + const TRequestPtr Request; + + explicit TEvComplete(const TRequestPtr& request) + : Request(request) + { + } + }; + + #define DEFINE_PRIVATE_EVENT(name, result_t) \ + struct TEv##name: public TEvYtWrapper::TGenericResponse<TEv##name, Ev##name, result_t> { \ + TActorId Sender; \ + explicit TEv##name(const TResult& result, const TActorId& sender) \ + : TBase(result) \ + , Sender(sender) \ + {} \ + } + + DEFINE_PRIVATE_EVENT(CreateTableWriter, ITableWriterPtr); + DEFINE_PRIVATE_EVENT(CloseTableWriter, void); + + #undef DEFINE_PRIVATE_EVENT + +}; // TEvPrivate + +template <typename TDerived> class TBaseWrapperActor; + +class TRequest: public NYT::TRefCounted { + template <typename TDerived> friend class TBaseWrapperActor; + +public: + explicit TRequest(const TActorSystem* sys, const TActorId& self, const TActorId& sender) + : ActorSystem(sys) + , Self(self) + , Sender(sender) + { + } + + void Send(const TActorId& recipient, IEventBase* ev) { + std::lock_guard<std::mutex> lock(RWActorSystem); + + if (ActorSystem) { + ActorSystem->Send(recipient, ev); + } + } + + void Complete(const TActorId& recipient, IEventBase* ev) { + Send(recipient, ev); + Send(Self, new TEvPrivate::TEvComplete(NYT::MakeStrong(this))); + } + + void Complete(IEventBase* ev) { + Complete(Sender, ev); + } + + const TActorId& GetSelf() const { + return Self; + } + + const TActorId& GetSender() const { + return Sender; + } + +private: + void ClearActorSystem() { + std::lock_guard<std::mutex> lock(RWActorSystem); + ActorSystem = nullptr; + } + +private: + const TActorSystem* ActorSystem; + const TActorId Self; + const TActorId Sender; + + std::mutex RWActorSystem; + +}; // TRequest + +template <typename TDerived> +class TBaseWrapperActor: public TActor<TDerived> { + void Handle(TEvPrivate::TEvComplete::TPtr& ev) { + Requests.erase(ev->Get()->Request); + } + + void PassAway() override { + Requests.clear(); + IActor::PassAway(); + } + +protected: + using TActor<TDerived>::TActor; + using TBase = TBaseWrapperActor<TDerived>; + + virtual ~TBaseWrapperActor() { + for (auto& request : Requests) { + request->ClearActorSystem(); + } + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPrivate::TEvComplete, Handle); + cFunc(TEvents::TEvPoison::EventType, PassAway); + } + } + + TWeakPtr<TRequest> MakeRequest(const TActorId& sender) { + auto request = New<TRequest>(TlsActivationContext->ActorSystem(), this->SelfId(), sender); + Requests.emplace(request); + return NYT::MakeWeak(request); + } + +private: + THashSet<TRequestPtr> Requests; + +}; // TBaseWrapperActor + +class TYtTableWriter: public TBaseWrapperActor<TYtTableWriter> { + TVector<NTableClient::TUnversionedRow> RowsRef() const { + TVector<NTableClient::TUnversionedRow> result(Reserve(Rows.size())); + + for (const auto& row : Rows) { + result.push_back(row); + } + + return result; + } + + void Handle(TEvYtWrapper::TEvGetNameTableRequest::TPtr& ev) { + Send(ev->Sender, new TEvYtWrapper::TEvGetNameTableResponse(Writer->GetNameTable())); + } + + void Handle(TEvYtWrapper::TEvWriteTableRequest::TPtr& ev) { + Rows.swap(std::get<0>(*ev->Get())); + const bool last = std::get<1>(*ev->Get()); + + auto request = MakeRequest(ev->Sender); + + try { + if (!Writer->Write(RowsRef())) { + Writer->GetReadyEvent() + .Subscribe(BIND([request, last](const TErrorOr<void>& err) { + MaybeClose(request, last, err); + })); + } else { + MaybeClose(request, last, TErrorOr<void>()); + } + } catch (const yexception& ex) { + MaybeClose(request, last, ex); + } + } + + static void MaybeClose(TWeakPtr<TRequest> request, bool last, const TErrorOr<void>& err) { + if (auto r = request.Lock()) { + if (!last && err.IsOK()) { + r->Complete(new TEvYtWrapper::TEvWriteTableResponse(err)); + } else { + r->Complete(r->GetSelf(), new TEvPrivate::TEvCloseTableWriter(err, r->GetSender())); + } + } + } + + void Handle(TEvPrivate::TEvCloseTableWriter::TPtr& ev) { + auto request = MakeRequest(ev->Get()->Sender); + + try { + Writer->Close().Subscribe(BIND([request, result = std::move(ev->Get()->Result)](const TErrorOr<void>& err) { + if (auto r = request.Lock()) { + if (!result.IsOK()) { + r->Complete(new TEvYtWrapper::TEvWriteTableResponse(result)); + } else { + r->Complete(new TEvYtWrapper::TEvWriteTableResponse(err)); + } + r->Send(r->GetSelf(), new TEvents::TEvPoisonPill()); + } + })); + } catch (const yexception& ex) { + if (auto r = request.Lock()) { + r->Complete(new TEvYtWrapper::TEvWriteTableResponse(ex)); + r->Send(r->GetSelf(), new TEvents::TEvPoisonPill()); + } + } + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::YT_TABLE_WRITER_ACTOR; + } + + explicit TYtTableWriter(ITableWriterPtr writer) + : TBase(&TThis::StateWork) + , Writer(writer) + { + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvYtWrapper::TEvGetNameTableRequest, Handle); + hFunc(TEvYtWrapper::TEvWriteTableRequest, Handle); + hFunc(TEvPrivate::TEvCloseTableWriter, Handle); + + default: + return TBase::StateWork(ev); + } + } + +private: + ITableWriterPtr Writer; + TVector<NTableClient::TUnversionedOwningRow> Rows; + +}; // TYtTableWriter + +class TYtWrapper: public TBaseWrapperActor<TYtWrapper> { + static IClientPtr CreateClient(NRpcProxy::TConnectionConfigPtr config, const TClientOptions& options) { + auto connection = NRpcProxy::CreateConnection(config); + return connection->CreateClient(options); + } + + template <typename TEvResponse, typename TClient, typename... Args> + using TFunc = typename TEvResponse::TAsyncResult(TClient::*)(Args...); + + template <typename TEvResponse> + using TCompleteFunc = void(*)(TWeakPtr<TRequest>, const typename TEvResponse::TResult&); + + template <typename TEvResponse, typename TEvRequestPtr, typename TClient, typename... Args> + void CallImpl(TEvRequestPtr& ev, TFunc<TEvResponse, TClient, Args...> func, TCompleteFunc<TEvResponse> completer) { + auto request = MakeRequest(ev->Sender); + auto args = std::move(ev->Get()->GetArgs()); + auto cb = BIND([request, completer](const typename TEvResponse::TResult& result) { + completer(request, result); + }); + + try { + std::apply(func, std::tuple_cat(std::tie(Client), std::move(args))).Subscribe(std::move(cb)); + } catch (const yexception& ex) { + completer(request, ex); + } + } + + template <typename TEvResponse> + static void Complete(TWeakPtr<TRequest> request, const typename TEvResponse::TResult& result) { + if (auto r = request.Lock()) { + r->Complete(new TEvResponse(result)); + } + } + + template <typename TEvResponse, typename TEvRequestPtr, typename TClient, typename... Args> + void Call(TEvRequestPtr& ev, TFunc<TEvResponse, TClient, Args...> func) { + CallImpl<TEvResponse, TEvRequestPtr, TClient, Args...>(ev, func, &TThis::Complete<TEvResponse>); + } + + template <typename TEvPrivResponse> + static void CompletePriv(TWeakPtr<TRequest> request, const typename TEvPrivResponse::TResult& result) { + if (auto r = request.Lock()) { + r->Complete(r->GetSelf(), new TEvPrivResponse(result, r->GetSender())); + } + } + + template <typename TEvPrivResponse, typename TEvRequestPtr, typename TClient, typename... Args> + void CallPriv(TEvRequestPtr& ev, TFunc<TEvPrivResponse, TClient, Args...> func) { + CallImpl<TEvPrivResponse, TEvRequestPtr, TClient, Args...>(ev, func, &TThis::CompletePriv<TEvPrivResponse>); + } + + template <typename TEvResponse, typename TProcessor, typename TEvPrivResponsePtr> + void HandlePriv(TEvPrivResponsePtr& ev) { + const auto& result = ev->Get()->Result; + if (result.IsOK()) { + Send(ev->Get()->Sender, new TEvResponse(RegisterWithSameMailbox(new TProcessor(result.Value())))); + } else { + Send(ev->Get()->Sender, new TEvResponse(yexception() << ToString(result))); + } + } + + void Handle(TEvYtWrapper::TEvCreateTableWriterRequest::TPtr& ev) { + CallPriv<TEvPrivate::TEvCreateTableWriter>(ev, &IClient::CreateTableWriter); + } + + void Handle(TEvPrivate::TEvCreateTableWriter::TPtr& ev) { + HandlePriv<TEvYtWrapper::TEvCreateTableWriterResponse, TYtTableWriter>(ev); + } + + void Handle(TEvYtWrapper::TEvGetNodeRequest::TPtr& ev) { + Call<TEvYtWrapper::TEvGetNodeResponse>(ev, &IClient::GetNode); + } + + void Handle(TEvYtWrapper::TEvCreateNodeRequest::TPtr& ev) { + Call<TEvYtWrapper::TEvCreateNodeResponse>(ev, &IClient::CreateNode); + } + + void Handle(TEvYtWrapper::TEvNodeExistsRequest::TPtr& ev) { + Call<TEvYtWrapper::TEvNodeExistsResponse>(ev, &IClient::NodeExists); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::YT_WRAPPER_ACTOR; + } + + explicit TYtWrapper(NRpcProxy::TConnectionConfigPtr config, const TClientOptions& options) + : TBase(&TThis::StateWork) + , Client(CreateClient(config, options)) + { + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + // Tables + hFunc(TEvYtWrapper::TEvCreateTableWriterRequest, Handle); + hFunc(TEvPrivate::TEvCreateTableWriter, Handle); + + // Cypress + hFunc(TEvYtWrapper::TEvGetNodeRequest, Handle); + hFunc(TEvYtWrapper::TEvCreateNodeRequest, Handle); + hFunc(TEvYtWrapper::TEvNodeExistsRequest, Handle); + + default: + return TBase::StateWork(ev); + } + } + +private: + IClientPtr Client; + +}; // TYtWrapper + +IActor* CreateYtWrapper(NRpcProxy::TConnectionConfigPtr config, const TClientOptions& options) { + return new TYtWrapper(config, options); +} + +IActor* CreateYtWrapper(const TString& serverName, const TString& token) { + auto config = New<NRpcProxy::TConnectionConfig>(); + config->ClusterUrl = serverName; + config->ConnectionType = EConnectionType::Rpc; + config->DynamicChannelPool->MaxPeerCount = 5; + return CreateYtWrapper(config, TClientOptions::FromToken(token)); +} + +} // NWrappers +} // NKikimr diff --git a/ydb/core/yt/yt_wrapper.h b/ydb/core/yt/yt_wrapper.h new file mode 100644 index 0000000000..f3df18a8d4 --- /dev/null +++ b/ydb/core/yt/yt_wrapper.h @@ -0,0 +1,156 @@ +#pragma once + +#include <contrib/ydb/core/base/events.h> + +#include <yt/yt/client/api/client.h> +#include <yt/yt/client/api/rpc_proxy/config.h> +#include <yt/yt/client/table_client/unversioned_row.h> +#include <yt/yt/client/ypath/rich.h> + +#include <util/generic/vector.h> + +namespace NKikimr { +namespace NWrappers { + +struct TEvYtWrapper { + template <typename TDerived, ui32 EventType, typename... Args> + struct TGenericRequest: public std::tuple<Args...>, public TEventLocal<TDerived, EventType> { + using TTupleArgs = std::tuple<Args...>; + + explicit TGenericRequest(Args... args) + : std::tuple<Args...>(args...) + { + } + + const TTupleArgs& GetArgs() const { + return *this; + } + + TTupleArgs&& GetArgs() { + return std::move(*this); + } + + using TBase = TGenericRequest<TDerived, EventType, Args...>; + }; + + template <typename TDerived, ui32 EventType> + struct TGenericRequest<TDerived, EventType, void>: public TEventLocal<TDerived, EventType> { + using TBase = TGenericRequest<TDerived, EventType, void>; + }; + + template <typename TDerived, ui32 EventType, typename T> + struct TGenericResponse: public TEventLocal<TDerived, EventType> { + using TResult = NYT::TErrorOr<T>; + using TAsyncResult = NYT::TFuture<T>; + + TResult Result; + + explicit TGenericResponse(const TResult& result) + : Result(result) + { + } + + using TBase = TGenericResponse<TDerived, EventType, T>; + }; + + #define EV_REQUEST_RESPONSE(name) \ + Ev##name##Request, \ + Ev##name##Response + + enum EEv { + EvBegin = EventSpaceBegin(TKikimrEvents::ES_YT_WRAPPER), + + // Transactions + EvTransactions, + EV_REQUEST_RESPONSE(StartTransaction), + + // Tables + EvTables = EvBegin + 1 * 100, + EV_REQUEST_RESPONSE(LookupRows), + EV_REQUEST_RESPONSE(VersionedLookupRows), + EV_REQUEST_RESPONSE(SelectRows), + EV_REQUEST_RESPONSE(ExplainQuery), + + EvTableReader = EvBegin + 2 * 100, + EV_REQUEST_RESPONSE(CreateTableReader), + + EvTableWriter = EvBegin + 3 * 100, + EV_REQUEST_RESPONSE(CreateTableWriter), + EV_REQUEST_RESPONSE(WriteTable), + EV_REQUEST_RESPONSE(GetNameTable), + + // Cypress + EvCypress = EvBegin + 4 * 100, + EV_REQUEST_RESPONSE(GetNode), + EV_REQUEST_RESPONSE(SetNode), + EV_REQUEST_RESPONSE(MultisetAttributesNode), + EV_REQUEST_RESPONSE(RemoveNode), + EV_REQUEST_RESPONSE(ListNode), + EV_REQUEST_RESPONSE(CreateNode), + EV_REQUEST_RESPONSE(LockNode), + EV_REQUEST_RESPONSE(UnlockNode), + EV_REQUEST_RESPONSE(CopyNode), + EV_REQUEST_RESPONSE(MoveNode), + EV_REQUEST_RESPONSE(LinkNode), + EV_REQUEST_RESPONSE(ConcatenateNodes), + EV_REQUEST_RESPONSE(NodeExists), + EV_REQUEST_RESPONSE(ExternalizeNode), + EV_REQUEST_RESPONSE(InternalizeNode), + + // Objects + EvObjects = EvBegin + 5 * 100, + EV_REQUEST_RESPONSE(CreateObject), + + // Files + EvFileReader = EvBegin + 6 * 100, + EV_REQUEST_RESPONSE(CreateFileReader), + + EvFileWriter = EvBegin + 7 * 100, + EV_REQUEST_RESPONSE(CreateFileWriter), + + // Journals + EvJournalReader = EvBegin + 8 * 100, + EV_REQUEST_RESPONSE(CreateJournalReader), + + EvJournalWriter = EvBegin + 9 * 100, + EV_REQUEST_RESPONSE(CreateJournalWriter), + + EvEnd, + }; + + #undef EV_REQUEST_RESPONSE + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_YT_WRAPPER), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_YT_WRAPPER)"); + + #define DEFINE_GENERIC_REQUEST(name, ...) \ + struct TEv##name##Request: public TGenericRequest<TEv##name##Request, Ev##name##Request, __VA_ARGS__> { \ + using TBase::TBase; \ + } + + #define DEFINE_GENERIC_RESPONSE(name, result_t) \ + struct TEv##name##Response: public TGenericResponse<TEv##name##Response, Ev##name##Response, result_t> { \ + using TBase::TBase; \ + } + + #define DEFINE_GENERIC_REQUEST_RESPONSE(name, result_t, ...) \ + DEFINE_GENERIC_REQUEST(name, __VA_ARGS__); \ + DEFINE_GENERIC_RESPONSE(name, result_t) + + DEFINE_GENERIC_REQUEST_RESPONSE(CreateTableWriter, TActorId, NYT::NYPath::TRichYPath, NYT::NApi::TTableWriterOptions); + DEFINE_GENERIC_REQUEST_RESPONSE(WriteTable, void, TVector<NYT::NTableClient::TUnversionedOwningRow>, bool); + DEFINE_GENERIC_REQUEST_RESPONSE(GetNameTable, NYT::NTableClient::TNameTablePtr, void); + DEFINE_GENERIC_REQUEST_RESPONSE(GetNode, NYT::NYson::TYsonString, NYT::NYPath::TYPath, NYT::NApi::TGetNodeOptions); + DEFINE_GENERIC_REQUEST_RESPONSE(CreateNode, NYT::NCypressClient::TNodeId, NYT::NYPath::TYPath, NYT::NObjectClient::EObjectType, NYT::NApi::TCreateNodeOptions); + DEFINE_GENERIC_REQUEST_RESPONSE(NodeExists, bool, NYT::NYPath::TYPath, NYT::NApi::TNodeExistsOptions); + + #undef DEFINE_GENERIC_REQUEST_RESPONSE + #undef DEFINE_GENERIC_RESPONSE + #undef DEFINE_GENERIC_REQUEST + +}; // TEvYtWrapper + +IActor* CreateYtWrapper(NYT::NApi::NRpcProxy::TConnectionConfigPtr config, const NYT::NApi::TClientOptions& options = {}); +IActor* CreateYtWrapper(const TString& serverName, const TString& token); + +} // NWrappers +} // NKikimr diff --git a/yt/yt/client/api/accounting_client.h b/yt/yt/client/api/accounting_client.h index 80113dc8c8..62dc7145d4 100644 --- a/yt/yt/client/api/accounting_client.h +++ b/yt/yt/client/api/accounting_client.h @@ -20,6 +20,8 @@ struct TTransferPoolResourcesOptions struct IAccountingClient { + virtual ~IAccountingClient() = default; + virtual TFuture<void> TransferAccountResources( const TString& srcAccount, const TString& dstAccount, diff --git a/yt/yt/client/api/admin_client.h b/yt/yt/client/api/admin_client.h index 500033b8ea..f2bc941ca0 100644 --- a/yt/yt/client/api/admin_client.h +++ b/yt/yt/client/api/admin_client.h @@ -192,6 +192,8 @@ struct TRequestRestartResult struct IAdminClient { + virtual ~IAdminClient() = default; + virtual TFuture<int> BuildSnapshot( const TBuildSnapshotOptions& options = {}) = 0; diff --git a/yt/yt/client/api/connection.h b/yt/yt/client/api/connection.h index 8c59c70319..795cb55a59 100644 --- a/yt/yt/client/api/connection.h +++ b/yt/yt/client/api/connection.h @@ -58,7 +58,7 @@ struct IConnection virtual IClientPtr CreateClient(const TClientOptions& options = {}) = 0; virtual NHiveClient::ITransactionParticipantPtr CreateTransactionParticipant( NHiveClient::TCellId cellId, - const TTransactionParticipantOptions& options = TTransactionParticipantOptions()) = 0; + const TTransactionParticipantOptions& options = {}) = 0; virtual void ClearMetadataCaches() = 0; virtual void Terminate() = 0; diff --git a/yt/yt/client/api/cypress_client.h b/yt/yt/client/api/cypress_client.h index 141b02ffb4..1668033b55 100644 --- a/yt/yt/client/api/cypress_client.h +++ b/yt/yt/client/api/cypress_client.h @@ -188,6 +188,8 @@ struct TInternalizeNodeOptions struct ICypressClientBase { + virtual ~ICypressClientBase() = default; + virtual TFuture<NYson::TYsonString> GetNode( const NYPath::TYPath& path, const TGetNodeOptions& options = {}) = 0; diff --git a/yt/yt/client/api/delegating_client.cpp b/yt/yt/client/api/delegating_client.cpp index ac0bec3f0a..541232ebb1 100644 --- a/yt/yt/client/api/delegating_client.cpp +++ b/yt/yt/client/api/delegating_client.cpp @@ -8,1019 +8,740 @@ TDelegatingClient::TDelegatingClient(IClientPtr underlying) : Underlying_(std::move(underlying)) { } -IConnectionPtr TDelegatingClient::GetConnection() -{ - return Underlying_->GetConnection(); -} +#define DELEGATE_METHOD(returnType, method, signature, args) \ + returnType TDelegatingClient::method signature \ + { \ + return Underlying_->method args; \ + } -std::optional<TStringBuf> TDelegatingClient::GetClusterName(bool fetchIfNull) -{ - return Underlying_->GetClusterName(fetchIfNull); -} +DELEGATE_METHOD(IConnectionPtr, GetConnection, (), ()) + +DELEGATE_METHOD(std::optional<TStringBuf>, GetClusterName, + (bool fetchIfNull), + (fetchIfNull)) -TFuture<ITransactionPtr> TDelegatingClient::StartTransaction( +DELEGATE_METHOD(TFuture<ITransactionPtr>, StartTransaction, ( NTransactionClient::ETransactionType type, - const TTransactionStartOptions& options) -{ - return Underlying_->StartTransaction(type, options); -} + const TTransactionStartOptions& options), + (type, options)) -TFuture<TUnversionedLookupRowsResult> TDelegatingClient::LookupRows( +DELEGATE_METHOD(TFuture<TUnversionedLookupRowsResult>, LookupRows, ( const NYPath::TYPath& path, NTableClient::TNameTablePtr nameTable, const TSharedRange<NTableClient::TLegacyKey>& keys, - const TLookupRowsOptions& options) -{ - return Underlying_->LookupRows(path, std::move(nameTable), keys, options); -} + const TLookupRowsOptions& options), + (path, nameTable, keys, options)) -TFuture<TVersionedLookupRowsResult> TDelegatingClient::VersionedLookupRows( +DELEGATE_METHOD(TFuture<TVersionedLookupRowsResult>, VersionedLookupRows, ( const NYPath::TYPath& path, NTableClient::TNameTablePtr nameTable, const TSharedRange<NTableClient::TLegacyKey>& keys, - const TVersionedLookupRowsOptions& options) -{ - return Underlying_->VersionedLookupRows(path, std::move(nameTable), keys, options); -} + const TVersionedLookupRowsOptions& options), + (path, nameTable, keys, options)) -TFuture<std::vector<TUnversionedLookupRowsResult>> TDelegatingClient::MultiLookup( +DELEGATE_METHOD(TFuture<std::vector<TUnversionedLookupRowsResult>>, MultiLookupRows, ( const std::vector<TMultiLookupSubrequest>& subrequests, - const TMultiLookupOptions& options) -{ - return Underlying_->MultiLookup(subrequests, options); -} + const TMultiLookupOptions& options), + (subrequests, options)) -TFuture<TSelectRowsResult> TDelegatingClient::SelectRows( +DELEGATE_METHOD(TFuture<TSelectRowsResult>, SelectRows, ( const TString& query, - const TSelectRowsOptions& options) -{ - return Underlying_->SelectRows(query, options); -} + const TSelectRowsOptions& options), + (query, options)) -TFuture<NYson::TYsonString> TDelegatingClient::ExplainQuery( +DELEGATE_METHOD(TFuture<NYson::TYsonString>, ExplainQuery, ( const TString& query, - const TExplainQueryOptions& options) -{ - return Underlying_->ExplainQuery(query, options); -} + const TExplainQueryOptions& options), + (query, options)) -TFuture<TPullRowsResult> TDelegatingClient::PullRows( +DELEGATE_METHOD(TFuture<TPullRowsResult>, PullRows, ( const NYPath::TYPath& path, - const TPullRowsOptions& options) -{ - return Underlying_->PullRows(path, options); -} + const TPullRowsOptions& options), + (path, options)) -TFuture<ITableReaderPtr> TDelegatingClient::CreateTableReader( +DELEGATE_METHOD(TFuture<ITableReaderPtr>, CreateTableReader, ( const NYPath::TRichYPath& path, - const TTableReaderOptions& options) -{ - return Underlying_->CreateTableReader(path, options); -} + const TTableReaderOptions& options), + (path, options)) -TFuture<ITableWriterPtr> TDelegatingClient::CreateTableWriter( +DELEGATE_METHOD(TFuture<ITableWriterPtr>, CreateTableWriter, ( const NYPath::TRichYPath& path, - const TTableWriterOptions& options) -{ - return Underlying_->CreateTableWriter(path, options); -} + const TTableWriterOptions& options), + (path, options)) -TFuture<NQueueClient::IQueueRowsetPtr> TDelegatingClient::PullQueue( +DELEGATE_METHOD(TFuture<NQueueClient::IQueueRowsetPtr>, PullQueue, ( const NYPath::TRichYPath& queuePath, i64 offset, int partitionIndex, const NQueueClient::TQueueRowBatchReadOptions& rowBatchReadOptions, - const TPullQueueOptions& options) -{ - return Underlying_->PullQueue(queuePath, offset, partitionIndex, rowBatchReadOptions, options); -} + const TPullQueueOptions& options), + (queuePath, offset, partitionIndex, rowBatchReadOptions, options)) -TFuture<NQueueClient::IQueueRowsetPtr> TDelegatingClient::PullConsumer( +DELEGATE_METHOD(TFuture<NQueueClient::IQueueRowsetPtr>, PullConsumer, ( const NYPath::TRichYPath& consumerPath, const NYPath::TRichYPath& queuePath, std::optional<i64> offset, int partitionIndex, const NQueueClient::TQueueRowBatchReadOptions& rowBatchReadOptions, - const TPullConsumerOptions& options) -{ - return Underlying_->PullConsumer(consumerPath, queuePath, offset, partitionIndex, rowBatchReadOptions, options); -} + const TPullConsumerOptions& options), + (consumerPath, queuePath, offset, partitionIndex, rowBatchReadOptions, options)) -TFuture<void> TDelegatingClient::RegisterQueueConsumer( +DELEGATE_METHOD(TFuture<void>, RegisterQueueConsumer, ( const NYPath::TRichYPath& queuePath, const NYPath::TRichYPath& consumerPath, bool vital, - const TRegisterQueueConsumerOptions& options) -{ - return Underlying_->RegisterQueueConsumer(queuePath, consumerPath, vital, options); -} + const TRegisterQueueConsumerOptions& options), + (queuePath, consumerPath, vital, options)) -TFuture<void> TDelegatingClient::UnregisterQueueConsumer( +DELEGATE_METHOD(TFuture<void>, UnregisterQueueConsumer, ( const NYPath::TRichYPath& queuePath, const NYPath::TRichYPath& consumerPath, - const TUnregisterQueueConsumerOptions& options) -{ - return Underlying_->UnregisterQueueConsumer(queuePath, consumerPath, options); -} + const TUnregisterQueueConsumerOptions& options), + (queuePath, consumerPath, options)) -TFuture<std::vector<TListQueueConsumerRegistrationsResult>> TDelegatingClient::ListQueueConsumerRegistrations( +DELEGATE_METHOD(TFuture<std::vector<TListQueueConsumerRegistrationsResult>>, ListQueueConsumerRegistrations, ( const std::optional<NYPath::TRichYPath>& queuePath, const std::optional<NYPath::TRichYPath>& consumerPath, - const TListQueueConsumerRegistrationsOptions& options) -{ - return Underlying_->ListQueueConsumerRegistrations(queuePath, consumerPath, options); -} + const TListQueueConsumerRegistrationsOptions& options), + (queuePath, consumerPath, options)) -TFuture<NYson::TYsonString> TDelegatingClient::GetNode( +DELEGATE_METHOD(TFuture<NYson::TYsonString>, GetNode, ( const NYPath::TYPath& path, - const TGetNodeOptions& options) -{ - return Underlying_->GetNode(path, options); -} + const TGetNodeOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::SetNode( +DELEGATE_METHOD(TFuture<void>, SetNode, ( const NYPath::TYPath& path, const NYson::TYsonString& value, - const TSetNodeOptions& options) -{ - return Underlying_->SetNode(path, value, options); -} + const TSetNodeOptions& options), + (path, value, options)) -TFuture<void> TDelegatingClient::MultisetAttributesNode( +DELEGATE_METHOD(TFuture<void>, MultisetAttributesNode, ( const NYPath::TYPath& path, const NYTree::IMapNodePtr& attributes, - const TMultisetAttributesNodeOptions& options) -{ - return Underlying_->MultisetAttributesNode(path, attributes, options); -} + const TMultisetAttributesNodeOptions& options), + (path, attributes, options)) -TFuture<void> TDelegatingClient::RemoveNode( +DELEGATE_METHOD(TFuture<void>, RemoveNode, ( const NYPath::TYPath& path, - const TRemoveNodeOptions& options) -{ - return Underlying_->RemoveNode(path, options); -} + const TRemoveNodeOptions& options), + (path, options)) -TFuture<NYson::TYsonString> TDelegatingClient::ListNode( +DELEGATE_METHOD(TFuture<NYson::TYsonString>, ListNode, ( const NYPath::TYPath& path, - const TListNodeOptions& options) -{ - return Underlying_->ListNode(path, options); -} + const TListNodeOptions& options), + (path, options)) -TFuture<NCypressClient::TNodeId> TDelegatingClient::CreateNode( +DELEGATE_METHOD(TFuture<NCypressClient::TNodeId>, CreateNode, ( const NYPath::TYPath& path, NObjectClient::EObjectType type, - const TCreateNodeOptions& options) -{ - return Underlying_->CreateNode(path, type, options); -} + const TCreateNodeOptions& options), + (path, type, options)) -TFuture<TLockNodeResult> TDelegatingClient::LockNode( +DELEGATE_METHOD(TFuture<TLockNodeResult>, LockNode, ( const NYPath::TYPath& path, NCypressClient::ELockMode mode, - const TLockNodeOptions& options) -{ - return Underlying_->LockNode(path, mode, options); -} + const TLockNodeOptions& options), + (path, mode, options)) -TFuture<void> TDelegatingClient::UnlockNode( +DELEGATE_METHOD(TFuture<void>, UnlockNode, ( const NYPath::TYPath& path, - const TUnlockNodeOptions& options) -{ - return Underlying_->UnlockNode(path, options); -} + const TUnlockNodeOptions& options), + (path, options)) -TFuture<NCypressClient::TNodeId> TDelegatingClient::CopyNode( +DELEGATE_METHOD(TFuture<NCypressClient::TNodeId>, CopyNode, ( const NYPath::TYPath& srcPath, const NYPath::TYPath& dstPath, - const TCopyNodeOptions& options) -{ - return Underlying_->CopyNode(srcPath, dstPath, options); -} + const TCopyNodeOptions& options), + (srcPath, dstPath, options)) -TFuture<NCypressClient::TNodeId> TDelegatingClient::MoveNode( +DELEGATE_METHOD(TFuture<NCypressClient::TNodeId>, MoveNode, ( const NYPath::TYPath& srcPath, const NYPath::TYPath& dstPath, - const TMoveNodeOptions& options) -{ - return Underlying_->MoveNode(srcPath, dstPath, options); -} + const TMoveNodeOptions& options), + (srcPath, dstPath, options)) -TFuture<NCypressClient::TNodeId> TDelegatingClient::LinkNode( +DELEGATE_METHOD(TFuture<NCypressClient::TNodeId>, LinkNode, ( const NYPath::TYPath& srcPath, const NYPath::TYPath& dstPath, - const TLinkNodeOptions& options) -{ - return Underlying_->LinkNode(srcPath, dstPath, options); -} + const TLinkNodeOptions& options), + (srcPath, dstPath, options)) -TFuture<void> TDelegatingClient::ConcatenateNodes( +DELEGATE_METHOD(TFuture<void>, ConcatenateNodes, ( const std::vector<NYPath::TRichYPath>& srcPaths, const NYPath::TRichYPath& dstPath, - const TConcatenateNodesOptions& options) -{ - return Underlying_->ConcatenateNodes(srcPaths, dstPath, options); -} + const TConcatenateNodesOptions& options), + (srcPaths, dstPath, options)) -TFuture<bool> TDelegatingClient::NodeExists( +DELEGATE_METHOD(TFuture<bool>, NodeExists, ( const NYPath::TYPath& path, - const TNodeExistsOptions& options) -{ - return Underlying_->NodeExists(path, options); -} + const TNodeExistsOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::ExternalizeNode( +DELEGATE_METHOD(TFuture<void>, ExternalizeNode, ( const NYPath::TYPath& path, NObjectClient::TCellTag cellTag, - const TExternalizeNodeOptions& options) -{ - return Underlying_->ExternalizeNode(path, cellTag, options); -} + const TExternalizeNodeOptions& options), + (path, cellTag, options)) -TFuture<void> TDelegatingClient::InternalizeNode( +DELEGATE_METHOD(TFuture<void>, InternalizeNode, ( const NYPath::TYPath& path, - const TInternalizeNodeOptions& options) -{ - return Underlying_->InternalizeNode(path, options); -} + const TInternalizeNodeOptions& options), + (path, options)) -TFuture<NObjectClient::TObjectId> TDelegatingClient::CreateObject( +DELEGATE_METHOD(TFuture<NObjectClient::TObjectId>, CreateObject, ( NObjectClient::EObjectType type, - const TCreateObjectOptions& options) -{ - return Underlying_->CreateObject(type, options); -} + const TCreateObjectOptions& options), + (type, options)) -TFuture<IFileReaderPtr> TDelegatingClient::CreateFileReader( +DELEGATE_METHOD(TFuture<IFileReaderPtr>, CreateFileReader, ( const NYPath::TYPath& path, - const TFileReaderOptions& options) -{ - return Underlying_->CreateFileReader(path, options); -} + const TFileReaderOptions& options), + (path, options)) -IFileWriterPtr TDelegatingClient::CreateFileWriter( +DELEGATE_METHOD(IFileWriterPtr, CreateFileWriter, ( const NYPath::TRichYPath& path, - const TFileWriterOptions& options) -{ - return Underlying_->CreateFileWriter(path, options); -} + const TFileWriterOptions& options), + (path, options)) -IJournalReaderPtr TDelegatingClient::CreateJournalReader( +DELEGATE_METHOD(IJournalReaderPtr, CreateJournalReader, ( const NYPath::TYPath& path, - const TJournalReaderOptions& options) -{ - return Underlying_->CreateJournalReader(path, options); -} + const TJournalReaderOptions& options), + (path, options)) -IJournalWriterPtr TDelegatingClient::CreateJournalWriter( +DELEGATE_METHOD(IJournalWriterPtr, CreateJournalWriter, ( const NYPath::TYPath& path, - const TJournalWriterOptions& options) -{ - return Underlying_->CreateJournalWriter(path, options); -} + const TJournalWriterOptions& options), + (path, options)) -void TDelegatingClient::Terminate() -{ - return Underlying_->Terminate(); -} +DELEGATE_METHOD(void, Terminate, (), ()) -const NTabletClient::ITableMountCachePtr& TDelegatingClient::GetTableMountCache() -{ - return Underlying_->GetTableMountCache(); -} -const NChaosClient::IReplicationCardCachePtr& TDelegatingClient::GetReplicationCardCache() -{ - return Underlying_->GetReplicationCardCache(); -} -const NTransactionClient::ITimestampProviderPtr& TDelegatingClient::GetTimestampProvider() { - return Underlying_->GetTimestampProvider(); -} +DELEGATE_METHOD(const NTabletClient::ITableMountCachePtr&, GetTableMountCache, (), ()) + +DELEGATE_METHOD(const NChaosClient::IReplicationCardCachePtr&, GetReplicationCardCache, (), ()) -ITransactionPtr TDelegatingClient::AttachTransaction( +DELEGATE_METHOD(const NTransactionClient::ITimestampProviderPtr&, GetTimestampProvider, (), ()) + +DELEGATE_METHOD(ITransactionPtr, AttachTransaction, ( NTransactionClient::TTransactionId transactionId, - const TTransactionAttachOptions& options) -{ - return Underlying_->AttachTransaction(transactionId, options); -} + const TTransactionAttachOptions& options), + (transactionId, options)) -TFuture<void> TDelegatingClient::MountTable( +DELEGATE_METHOD(TFuture<void>, MountTable, ( const NYPath::TYPath& path, - const TMountTableOptions& options) -{ - return Underlying_->MountTable(path, options); -} + const TMountTableOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::UnmountTable( +DELEGATE_METHOD(TFuture<void>, UnmountTable, ( const NYPath::TYPath& path, - const TUnmountTableOptions& options) -{ - return Underlying_->UnmountTable(path, options); -} + const TUnmountTableOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::RemountTable( +DELEGATE_METHOD(TFuture<void>, RemountTable, ( const NYPath::TYPath& path, - const TRemountTableOptions& options) -{ - return Underlying_->RemountTable(path, options); -} + const TRemountTableOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::FreezeTable( +DELEGATE_METHOD(TFuture<void>, FreezeTable, ( const NYPath::TYPath& path, - const TFreezeTableOptions& options) -{ - return Underlying_->FreezeTable(path, options); -} + const TFreezeTableOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::UnfreezeTable( +DELEGATE_METHOD(TFuture<void>, UnfreezeTable, ( const NYPath::TYPath& path, - const TUnfreezeTableOptions& options) -{ - return Underlying_->UnfreezeTable(path, options); -} + const TUnfreezeTableOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::ReshardTable( +DELEGATE_METHOD(TFuture<void>, ReshardTable, ( const NYPath::TYPath& path, const std::vector<NTableClient::TLegacyOwningKey>& pivotKeys, - const TReshardTableOptions& options) -{ - return Underlying_->ReshardTable(path, pivotKeys, options); -} + const TReshardTableOptions& options), + (path, pivotKeys, options)) -TFuture<void> TDelegatingClient::ReshardTable( +DELEGATE_METHOD(TFuture<void>, ReshardTable, ( const NYPath::TYPath& path, int tabletCount, - const TReshardTableOptions& options) -{ - return Underlying_->ReshardTable(path, tabletCount, options); -} + const TReshardTableOptions& options), + (path, tabletCount, options)) -TFuture<std::vector<NTabletClient::TTabletActionId>> TDelegatingClient::ReshardTableAutomatic( +DELEGATE_METHOD(TFuture<std::vector<NTabletClient::TTabletActionId>>, ReshardTableAutomatic, ( const NYPath::TYPath& path, - const TReshardTableAutomaticOptions& options) -{ - return Underlying_->ReshardTableAutomatic(path, options); -} + const TReshardTableAutomaticOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::TrimTable( +DELEGATE_METHOD(TFuture<void>, TrimTable, ( const NYPath::TYPath& path, int tabletIndex, i64 trimmedRowCount, - const TTrimTableOptions& options) -{ - return Underlying_->TrimTable(path, tabletIndex, trimmedRowCount, options); -} + const TTrimTableOptions& options), + (path, tabletIndex, trimmedRowCount, options)) -TFuture<void> TDelegatingClient::AlterTable( +DELEGATE_METHOD(TFuture<void>, AlterTable, ( const NYPath::TYPath& path, - const TAlterTableOptions& options) -{ - return Underlying_->AlterTable(path, options); -} + const TAlterTableOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::AlterTableReplica( +DELEGATE_METHOD(TFuture<void>, AlterTableReplica, ( NTabletClient::TTableReplicaId replicaId, - const TAlterTableReplicaOptions& options) -{ - return Underlying_->AlterTableReplica(replicaId, options); -} + const TAlterTableReplicaOptions& options), + (replicaId, options)) -TFuture<void> TDelegatingClient::AlterReplicationCard( +DELEGATE_METHOD(TFuture<void>, AlterReplicationCard, ( NChaosClient::TReplicationCardId replicationCardId, - const TAlterReplicationCardOptions& options) -{ - return Underlying_->AlterReplicationCard(replicationCardId, options); -} + const TAlterReplicationCardOptions& options), + (replicationCardId, options)) -TFuture<NYson::TYsonString> TDelegatingClient::GetTablePivotKeys( +DELEGATE_METHOD(TFuture<NYson::TYsonString>, GetTablePivotKeys, ( const NYPath::TYPath& path, - const TGetTablePivotKeysOptions& options) -{ - return Underlying_->GetTablePivotKeys(path, options); -} + const TGetTablePivotKeysOptions& options), + (path, options)) -TFuture<void> TDelegatingClient::CreateTableBackup( +DELEGATE_METHOD(TFuture<void>, CreateTableBackup, ( const TBackupManifestPtr& manifest, - const TCreateTableBackupOptions& options) -{ - return Underlying_->CreateTableBackup(manifest, options); -} + const TCreateTableBackupOptions& options), + (manifest, options)) -TFuture<void> TDelegatingClient::RestoreTableBackup( +DELEGATE_METHOD(TFuture<void>, RestoreTableBackup, ( const TBackupManifestPtr& manifest, - const TRestoreTableBackupOptions& options) -{ - return Underlying_->RestoreTableBackup(manifest, options); -} + const TRestoreTableBackupOptions& options), + (manifest, options)) -TFuture<std::vector<NTabletClient::TTableReplicaId>> TDelegatingClient::GetInSyncReplicas( +DELEGATE_METHOD(TFuture<std::vector<NTabletClient::TTableReplicaId>>, GetInSyncReplicas, ( const NYPath::TYPath& path, const NTableClient::TNameTablePtr& nameTable, const TSharedRange<NTableClient::TLegacyKey>& keys, - const TGetInSyncReplicasOptions& options) -{ - return Underlying_->GetInSyncReplicas(path, nameTable, keys, options); -} + const TGetInSyncReplicasOptions& options), + (path, nameTable, keys, options)) -TFuture<std::vector<NTabletClient::TTableReplicaId>> TDelegatingClient::GetInSyncReplicas( +DELEGATE_METHOD(TFuture<std::vector<NTabletClient::TTableReplicaId>>, GetInSyncReplicas, ( const NYPath::TYPath& path, - const TGetInSyncReplicasOptions& options) -{ - return Underlying_->GetInSyncReplicas(path, options); -} + const TGetInSyncReplicasOptions& options), + (path, options)) -TFuture<std::vector<TTabletInfo>> TDelegatingClient::GetTabletInfos( +DELEGATE_METHOD(TFuture<std::vector<TTabletInfo>>, GetTabletInfos, ( const NYPath::TYPath& path, const std::vector<int>& tabletIndexes, - const TGetTabletInfosOptions& options) -{ - return Underlying_->GetTabletInfos(path, tabletIndexes, options); -} + const TGetTabletInfosOptions& options), + (path, tabletIndexes, options)) -TFuture<TGetTabletErrorsResult> TDelegatingClient::GetTabletErrors( +DELEGATE_METHOD(TFuture<TGetTabletErrorsResult>, GetTabletErrors, ( const NYPath::TYPath& path, - const TGetTabletErrorsOptions& options) -{ - return Underlying_->GetTabletErrors(path, options); -} + const TGetTabletErrorsOptions& options), + (path, options)) -TFuture<std::vector<NTabletClient::TTabletActionId>> TDelegatingClient::BalanceTabletCells( +DELEGATE_METHOD(TFuture<std::vector<NTabletClient::TTabletActionId>>, BalanceTabletCells, ( const TString& tabletCellBundle, const std::vector<NYPath::TYPath>& movableTables, - const TBalanceTabletCellsOptions& options) -{ - return Underlying_->BalanceTabletCells(tabletCellBundle, movableTables, options); -} + const TBalanceTabletCellsOptions& options), + (tabletCellBundle, movableTables, options)) -TFuture<NChaosClient::TReplicationCardPtr> TDelegatingClient::GetReplicationCard( +DELEGATE_METHOD(TFuture<NChaosClient::TReplicationCardPtr>, GetReplicationCard, ( NChaosClient::TReplicationCardId replicationCardId, - const TGetReplicationCardOptions& options) -{ - return Underlying_->GetReplicationCard(replicationCardId, options); -} + const TGetReplicationCardOptions& options), + (replicationCardId, options)) -TFuture<void> TDelegatingClient::UpdateChaosTableReplicaProgress( +DELEGATE_METHOD(TFuture<void>, UpdateChaosTableReplicaProgress, ( NChaosClient::TReplicaId replicaId, - const TUpdateChaosTableReplicaProgressOptions& options) -{ - return Underlying_->UpdateChaosTableReplicaProgress(replicaId, options); -} + const TUpdateChaosTableReplicaProgressOptions& options), + (replicaId, options)) -TFuture<TSkynetSharePartsLocationsPtr> TDelegatingClient::LocateSkynetShare( +DELEGATE_METHOD(TFuture<TSkynetSharePartsLocationsPtr>, LocateSkynetShare, ( const NYPath::TRichYPath& path, - const TLocateSkynetShareOptions& options) -{ - return Underlying_->LocateSkynetShare(path, options); -} + const TLocateSkynetShareOptions& options), + (path, options)) -TFuture<std::vector<NTableClient::TColumnarStatistics>> TDelegatingClient::GetColumnarStatistics( +DELEGATE_METHOD(TFuture<std::vector<NTableClient::TColumnarStatistics>>, GetColumnarStatistics, ( const std::vector<NYPath::TRichYPath>& path, - const TGetColumnarStatisticsOptions& options) -{ - return Underlying_->GetColumnarStatistics(path, options); -} + const TGetColumnarStatisticsOptions& options), + (path, options)) -TFuture<TMultiTablePartitions> TDelegatingClient::PartitionTables( +DELEGATE_METHOD(TFuture<TMultiTablePartitions>, PartitionTables, ( const std::vector<NYPath::TRichYPath>& paths, - const TPartitionTablesOptions& options) -{ - return Underlying_->PartitionTables(paths, options); -} + const TPartitionTablesOptions& options), + (paths, options)) -TFuture<void> TDelegatingClient::TruncateJournal( +DELEGATE_METHOD(TFuture<void>, TruncateJournal, ( const NYPath::TYPath& path, i64 rowCount, - const TTruncateJournalOptions& options) -{ - return Underlying_->TruncateJournal(path, rowCount, options); -} + const TTruncateJournalOptions& options), + (path, rowCount, options)) -TFuture<TGetFileFromCacheResult> TDelegatingClient::GetFileFromCache( +DELEGATE_METHOD(TFuture<TGetFileFromCacheResult>, GetFileFromCache, ( const TString& md5, - const TGetFileFromCacheOptions& options) -{ - return Underlying_->GetFileFromCache(md5, options); -} + const TGetFileFromCacheOptions& options), + (md5, options)) -TFuture<TPutFileToCacheResult> TDelegatingClient::PutFileToCache( +DELEGATE_METHOD(TFuture<TPutFileToCacheResult>, PutFileToCache, ( const NYPath::TYPath& path, const TString& expectedMD5, - const TPutFileToCacheOptions& options) -{ - return Underlying_->PutFileToCache(path, expectedMD5, options); -} + const TPutFileToCacheOptions& options), + (path, expectedMD5, options)) -TFuture<void> TDelegatingClient::AddMember( +DELEGATE_METHOD(TFuture<void>, AddMember, ( const TString& group, const TString& member, - const TAddMemberOptions& options) -{ - return Underlying_->AddMember(group, member, options); -} + const TAddMemberOptions& options), + (group, member, options)) -TFuture<void> TDelegatingClient::RemoveMember( +DELEGATE_METHOD(TFuture<void>, RemoveMember, ( const TString& group, const TString& member, - const TRemoveMemberOptions& options) -{ - return Underlying_->RemoveMember(group, member, options); -} + const TRemoveMemberOptions& options), + (group, member, options)) -TFuture<TCheckPermissionResponse> TDelegatingClient::CheckPermission( +DELEGATE_METHOD(TFuture<TCheckPermissionResponse>, CheckPermission, ( const TString& user, const NYPath::TYPath& path, NYTree::EPermission permission, - const TCheckPermissionOptions& options) -{ - return Underlying_->CheckPermission(user, path, permission, options); -} + const TCheckPermissionOptions& options), + (user, path, permission, options)) -TFuture<TCheckPermissionByAclResult> TDelegatingClient::CheckPermissionByAcl( +DELEGATE_METHOD(TFuture<TCheckPermissionByAclResult>, CheckPermissionByAcl, ( const std::optional<TString>& user, NYTree::EPermission permission, NYTree::INodePtr acl, - const TCheckPermissionByAclOptions& options) -{ - return Underlying_->CheckPermissionByAcl(user, permission, std::move(acl), options); -} + const TCheckPermissionByAclOptions& options), + (user, permission, acl, options)) -TFuture<void> TDelegatingClient::TransferAccountResources( +DELEGATE_METHOD(TFuture<void>, TransferAccountResources, ( const TString& srcAccount, const TString& dstAccount, NYTree::INodePtr resourceDelta, - const TTransferAccountResourcesOptions& options) -{ - return Underlying_->TransferAccountResources( - srcAccount, dstAccount, std::move(resourceDelta), options - ); -} + const TTransferAccountResourcesOptions& options), + (srcAccount, dstAccount, resourceDelta, options)) -TFuture<void> TDelegatingClient::TransferPoolResources( +DELEGATE_METHOD(TFuture<void>, TransferPoolResources, ( const TString& srcPool, const TString& dstPool, const TString& poolTree, NYTree::INodePtr resourceDelta, - const TTransferPoolResourcesOptions& options) -{ - return Underlying_->TransferPoolResources( - srcPool, dstPool, poolTree, std::move(resourceDelta), options - ); -} + const TTransferPoolResourcesOptions& options), + (srcPool, dstPool, poolTree, resourceDelta, options)) -TFuture<NScheduler::TOperationId> TDelegatingClient::StartOperation( +DELEGATE_METHOD(TFuture<NScheduler::TOperationId>, StartOperation, ( NScheduler::EOperationType type, const NYson::TYsonString& spec, - const TStartOperationOptions& options) -{ - return Underlying_->StartOperation(type, spec, options); -} + const TStartOperationOptions& options), + (type, spec, options)) -TFuture<void> TDelegatingClient::AbortOperation( +DELEGATE_METHOD(TFuture<void>, AbortOperation, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, - const TAbortOperationOptions& options) -{ - return Underlying_->AbortOperation(operationIdOrAlias, options); -} + const TAbortOperationOptions& options), + (operationIdOrAlias, options)) -TFuture<void> TDelegatingClient::SuspendOperation( +DELEGATE_METHOD(TFuture<void>, SuspendOperation, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, - const TSuspendOperationOptions& options) -{ - return Underlying_->SuspendOperation(operationIdOrAlias, options); -} + const TSuspendOperationOptions& options), + (operationIdOrAlias, options)) -TFuture<void> TDelegatingClient::ResumeOperation( +DELEGATE_METHOD(TFuture<void>, ResumeOperation, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, - const TResumeOperationOptions& options) -{ - return Underlying_->ResumeOperation(operationIdOrAlias, options); -} + const TResumeOperationOptions& options), + (operationIdOrAlias, options)) -TFuture<void> TDelegatingClient::CompleteOperation( +DELEGATE_METHOD(TFuture<void>, CompleteOperation, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, - const TCompleteOperationOptions& options) -{ - return Underlying_->CompleteOperation(operationIdOrAlias, options); -} + const TCompleteOperationOptions& options), + (operationIdOrAlias, options)) -TFuture<void> TDelegatingClient::UpdateOperationParameters( +DELEGATE_METHOD(TFuture<void>, UpdateOperationParameters, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, const NYson::TYsonString& parameters, - const TUpdateOperationParametersOptions& options) -{ - return Underlying_->UpdateOperationParameters(operationIdOrAlias, parameters, options); -} + const TUpdateOperationParametersOptions& options), + (operationIdOrAlias, parameters, options)) -TFuture<TOperation> TDelegatingClient::GetOperation( +DELEGATE_METHOD(TFuture<TOperation>, GetOperation, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, - const TGetOperationOptions& options) -{ - return Underlying_->GetOperation(operationIdOrAlias, options); -} + const TGetOperationOptions& options), + (operationIdOrAlias, options)) -TFuture<void> TDelegatingClient::DumpJobContext( +DELEGATE_METHOD(TFuture<void>, DumpJobContext, ( NJobTrackerClient::TJobId jobId, const NYPath::TYPath& path, - const TDumpJobContextOptions& options) -{ - return Underlying_->DumpJobContext(jobId, path, options); -} + const TDumpJobContextOptions& options), + (jobId, path, options)) -TFuture<NConcurrency::IAsyncZeroCopyInputStreamPtr> TDelegatingClient::GetJobInput( +DELEGATE_METHOD(TFuture<NConcurrency::IAsyncZeroCopyInputStreamPtr>, GetJobInput, ( NJobTrackerClient::TJobId jobId, - const TGetJobInputOptions& options) -{ - return Underlying_->GetJobInput(jobId, options); -} + const TGetJobInputOptions& options), + (jobId, options)) -TFuture<NYson::TYsonString> TDelegatingClient::GetJobInputPaths( +DELEGATE_METHOD(TFuture<NYson::TYsonString>, GetJobInputPaths, ( NJobTrackerClient::TJobId jobId, - const TGetJobInputPathsOptions& options) -{ - return Underlying_->GetJobInputPaths(jobId, options); -} + const TGetJobInputPathsOptions& options), + (jobId, options)) -TFuture<NYson::TYsonString> TDelegatingClient::GetJobSpec( +DELEGATE_METHOD(TFuture<NYson::TYsonString>, GetJobSpec, ( NJobTrackerClient::TJobId jobId, - const TGetJobSpecOptions& options) -{ - return Underlying_->GetJobSpec(jobId, options); -} + const TGetJobSpecOptions& options), + (jobId, options)) -TFuture<TSharedRef> TDelegatingClient::GetJobStderr( +DELEGATE_METHOD(TFuture<TSharedRef>, GetJobStderr, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, NJobTrackerClient::TJobId jobId, - const TGetJobStderrOptions& options) -{ - return Underlying_->GetJobStderr(operationIdOrAlias, jobId, options); -} + const TGetJobStderrOptions& options), + (operationIdOrAlias, jobId, options)) -TFuture<TSharedRef> TDelegatingClient::GetJobFailContext( +DELEGATE_METHOD(TFuture<TSharedRef>, GetJobFailContext, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, NJobTrackerClient::TJobId jobId, - const TGetJobFailContextOptions& options) -{ - return Underlying_->GetJobFailContext(operationIdOrAlias, jobId, options); -} + const TGetJobFailContextOptions& options), + (operationIdOrAlias, jobId, options)) -TFuture<TListOperationsResult> TDelegatingClient::ListOperations( - const TListOperationsOptions& options) -{ - return Underlying_->ListOperations(options); -} +DELEGATE_METHOD(TFuture<TListOperationsResult>, ListOperations, ( + const TListOperationsOptions& options), + (options)) -TFuture<TListJobsResult> TDelegatingClient::ListJobs( +DELEGATE_METHOD(TFuture<TListJobsResult>, ListJobs, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, - const TListJobsOptions& options) -{ - return Underlying_->ListJobs(operationIdOrAlias, options); -} + const TListJobsOptions& options), + (operationIdOrAlias, options)) -TFuture<NYson::TYsonString> TDelegatingClient::GetJob( +DELEGATE_METHOD(TFuture<NYson::TYsonString>, GetJob, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, NJobTrackerClient::TJobId jobId, - const TGetJobOptions& options) -{ - return Underlying_->GetJob(operationIdOrAlias, jobId, options); -} + const TGetJobOptions& options), + (operationIdOrAlias, jobId, options)) -TFuture<void> TDelegatingClient::AbandonJob( +DELEGATE_METHOD(TFuture<void>, AbandonJob, ( NJobTrackerClient::TJobId jobId, - const TAbandonJobOptions& options) -{ - return Underlying_->AbandonJob(jobId, options); -} + const TAbandonJobOptions& options), + (jobId, options)) -TFuture<TPollJobShellResponse> TDelegatingClient::PollJobShell( +DELEGATE_METHOD(TFuture<TPollJobShellResponse>, PollJobShell, ( NJobTrackerClient::TJobId jobId, const std::optional<TString>& shellName, const NYson::TYsonString& parameters, - const TPollJobShellOptions& options) -{ - return Underlying_->PollJobShell(jobId, shellName, parameters, options); -} + const TPollJobShellOptions& options), + (jobId, shellName, parameters, options)) -TFuture<void> TDelegatingClient::AbortJob( +DELEGATE_METHOD(TFuture<void>, AbortJob, ( NJobTrackerClient::TJobId jobId, - const TAbortJobOptions& options) -{ - return Underlying_->AbortJob(jobId, options); -} + const TAbortJobOptions& options), + (jobId, options)) -TFuture<TClusterMeta> TDelegatingClient::GetClusterMeta( - const TGetClusterMetaOptions& options) -{ - return Underlying_->GetClusterMeta(options); -} +DELEGATE_METHOD(TFuture<TClusterMeta>, GetClusterMeta, ( + const TGetClusterMetaOptions& options), + (options)) -TFuture<void> TDelegatingClient::CheckClusterLiveness( - const TCheckClusterLivenessOptions& options) -{ - return Underlying_->CheckClusterLiveness(options); -} +DELEGATE_METHOD(TFuture<void>, CheckClusterLiveness, ( + const TCheckClusterLivenessOptions& options), + (options)) -TFuture<int> TDelegatingClient::BuildSnapshot( - const TBuildSnapshotOptions& options) -{ - return Underlying_->BuildSnapshot(options); -} +DELEGATE_METHOD(TFuture<int>, BuildSnapshot, ( + const TBuildSnapshotOptions& options), + (options)) -TFuture<TCellIdToSnapshotIdMap> TDelegatingClient::BuildMasterSnapshots( - const TBuildMasterSnapshotsOptions& options) -{ - return Underlying_->BuildMasterSnapshots(options); -} +DELEGATE_METHOD(TFuture<TCellIdToSnapshotIdMap>, BuildMasterSnapshots, ( + const TBuildMasterSnapshotsOptions& options), + (options)) -TFuture<void> TDelegatingClient::ExitReadOnly( +DELEGATE_METHOD(TFuture<void>, ExitReadOnly, ( NHydra::TCellId cellId, - const TExitReadOnlyOptions& options) -{ - return Underlying_->ExitReadOnly(cellId, options); -} + const TExitReadOnlyOptions& options), + (cellId, options)) -TFuture<void> TDelegatingClient::MasterExitReadOnly( - const TMasterExitReadOnlyOptions& options) -{ - return Underlying_->MasterExitReadOnly(options); -} +DELEGATE_METHOD(TFuture<void>, MasterExitReadOnly, ( + const TMasterExitReadOnlyOptions& options), + (options)) -TFuture<void> TDelegatingClient::DiscombobulateNonvotingPeers( +DELEGATE_METHOD(TFuture<void>, DiscombobulateNonvotingPeers, ( NHydra::TCellId cellId, - const TDiscombobulateNonvotingPeersOptions& options) -{ - return Underlying_->DiscombobulateNonvotingPeers(cellId, options); -} + const TDiscombobulateNonvotingPeersOptions& options), + (cellId, options)) -TFuture<void> TDelegatingClient::SwitchLeader( +DELEGATE_METHOD(TFuture<void>, SwitchLeader, ( NHydra::TCellId cellId, const TString& newLeaderAddress, - const TSwitchLeaderOptions& options) -{ - return Underlying_->SwitchLeader(cellId, newLeaderAddress, options); -} + const TSwitchLeaderOptions& options), + (cellId, newLeaderAddress, options)) -TFuture<void> TDelegatingClient::ResetStateHash( +DELEGATE_METHOD(TFuture<void>, ResetStateHash, ( NHydra::TCellId cellId, - const TResetStateHashOptions& options) -{ - return Underlying_->ResetStateHash(cellId, options); -} + const TResetStateHashOptions& options), + (cellId, options)) -TFuture<void> TDelegatingClient::GCCollect( - const TGCCollectOptions& options) -{ - return Underlying_->GCCollect(options); -} +DELEGATE_METHOD(TFuture<void>, GCCollect, ( + const TGCCollectOptions& options), + (options)) -TFuture<void> TDelegatingClient::KillProcess( +DELEGATE_METHOD(TFuture<void>, KillProcess, ( const TString& address, - const TKillProcessOptions& options) -{ - return Underlying_->KillProcess(address, options); -} + const TKillProcessOptions& options), + (address, options)) -TFuture<TString> TDelegatingClient::WriteCoreDump( +DELEGATE_METHOD(TFuture<TString>, WriteCoreDump, ( const TString& address, - const TWriteCoreDumpOptions& options) -{ - return Underlying_->WriteCoreDump(address, options); -} + const TWriteCoreDumpOptions& options), + (address, options)) -TFuture<TGuid> TDelegatingClient::WriteLogBarrier( +DELEGATE_METHOD(TFuture<TGuid>, WriteLogBarrier, ( const TString& address, - const TWriteLogBarrierOptions& options) -{ - return Underlying_->WriteLogBarrier(address, options); -} + const TWriteLogBarrierOptions& options), + (address, options)) -TFuture<TString> TDelegatingClient::WriteOperationControllerCoreDump( +DELEGATE_METHOD(TFuture<TString>, WriteOperationControllerCoreDump, ( NJobTrackerClient::TOperationId operationId, - const TWriteOperationControllerCoreDumpOptions& options) -{ - return Underlying_->WriteOperationControllerCoreDump(operationId, options); -} + const TWriteOperationControllerCoreDumpOptions& options), + (operationId, options)) -TFuture<void> TDelegatingClient::HealExecNode( +DELEGATE_METHOD(TFuture<void>, HealExecNode, ( const TString& address, - const THealExecNodeOptions& options) -{ - return Underlying_->HealExecNode(address, options); -} + const THealExecNodeOptions& options), + (address, options)) -TFuture<void> TDelegatingClient::SuspendCoordinator( +DELEGATE_METHOD(TFuture<void>, SuspendCoordinator, ( NObjectClient::TCellId coordinatorCellId, - const TSuspendCoordinatorOptions& options) -{ - return Underlying_->SuspendCoordinator(coordinatorCellId, options); -} + const TSuspendCoordinatorOptions& options), + (coordinatorCellId, options)) -TFuture<void> TDelegatingClient::ResumeCoordinator( +DELEGATE_METHOD(TFuture<void>, ResumeCoordinator, ( NObjectClient::TCellId coordinatorCellId, - const TResumeCoordinatorOptions& options) -{ - return Underlying_->ResumeCoordinator(coordinatorCellId, options); -} + const TResumeCoordinatorOptions& options), + (coordinatorCellId, options)) -TFuture<void> TDelegatingClient::MigrateReplicationCards( +DELEGATE_METHOD(TFuture<void>, MigrateReplicationCards, ( NObjectClient::TCellId chaosCellId, - const TMigrateReplicationCardsOptions& options) -{ - return Underlying_->MigrateReplicationCards(chaosCellId, options); -} + const TMigrateReplicationCardsOptions& options), + (chaosCellId, options)) -TFuture<void> TDelegatingClient::SuspendChaosCells( +DELEGATE_METHOD(TFuture<void>, SuspendChaosCells, ( const std::vector<NObjectClient::TCellId>& cellIds, - const TSuspendChaosCellsOptions& options) -{ - return Underlying_->SuspendChaosCells(cellIds, options); -} + const TSuspendChaosCellsOptions& options), + (cellIds, options)) -TFuture<void> TDelegatingClient::ResumeChaosCells( +DELEGATE_METHOD(TFuture<void>, ResumeChaosCells, ( const std::vector<NObjectClient::TCellId>& cellIds, - const TResumeChaosCellsOptions& options) -{ - return Underlying_->ResumeChaosCells(cellIds, options); -} + const TResumeChaosCellsOptions& options), + (cellIds, options)) -TFuture<void> TDelegatingClient::SuspendTabletCells( +DELEGATE_METHOD(TFuture<void>, SuspendTabletCells, ( const std::vector<NObjectClient::TCellId>& cellIds, - const TSuspendTabletCellsOptions& options) -{ - return Underlying_->SuspendTabletCells(cellIds, options); -} + const TSuspendTabletCellsOptions& options), + (cellIds, options)) -TFuture<void> TDelegatingClient::ResumeTabletCells( +DELEGATE_METHOD(TFuture<void>, ResumeTabletCells, ( const std::vector<NObjectClient::TCellId>& cellIds, - const TResumeTabletCellsOptions& options) -{ - return Underlying_->ResumeTabletCells(cellIds, options); -} + const TResumeTabletCellsOptions& options), + (cellIds, options)) -TFuture<TMaintenanceId> TDelegatingClient::AddMaintenance( +DELEGATE_METHOD(TFuture<TMaintenanceId>, AddMaintenance, ( EMaintenanceComponent component, const TString& address, EMaintenanceType type, const TString& comment, - const TAddMaintenanceOptions& options) -{ - return Underlying_->AddMaintenance(component, address, type, comment, options); -} + const TAddMaintenanceOptions& options), + (component, address, type, comment, options)) -TFuture<TMaintenanceCounts> TDelegatingClient::RemoveMaintenance( +DELEGATE_METHOD(TFuture<TMaintenanceCounts>, RemoveMaintenance, ( EMaintenanceComponent component, const TString& address, const TMaintenanceFilter& filter, - const TRemoveMaintenanceOptions& options) -{ - return Underlying_->RemoveMaintenance(component, address, filter, options); -} + const TRemoveMaintenanceOptions& options), + (component, address, filter, options)) -TFuture<TDisableChunkLocationsResult> TDelegatingClient::DisableChunkLocations( +DELEGATE_METHOD(TFuture<TDisableChunkLocationsResult>, DisableChunkLocations, ( const TString& nodeAddress, const std::vector<TGuid>& locationUuids, - const TDisableChunkLocationsOptions& options) -{ - return Underlying_->DisableChunkLocations(nodeAddress, locationUuids, options); -} + const TDisableChunkLocationsOptions& options), + (nodeAddress, locationUuids, options)) -TFuture<TDestroyChunkLocationsResult> TDelegatingClient::DestroyChunkLocations( +DELEGATE_METHOD(TFuture<TDestroyChunkLocationsResult>, DestroyChunkLocations, ( const TString& nodeAddress, bool recoverUnlinkedDisks, const std::vector<TGuid>& locationUuids, - const TDestroyChunkLocationsOptions& options) -{ - return Underlying_->DestroyChunkLocations(nodeAddress, recoverUnlinkedDisks, locationUuids, options); -} + const TDestroyChunkLocationsOptions& options), + (nodeAddress, recoverUnlinkedDisks, locationUuids, options)) -TFuture<TResurrectChunkLocationsResult> TDelegatingClient::ResurrectChunkLocations( +DELEGATE_METHOD(TFuture<TResurrectChunkLocationsResult>, ResurrectChunkLocations, ( const TString& nodeAddress, const std::vector<TGuid>& locationUuids, - const TResurrectChunkLocationsOptions& options) -{ - return Underlying_->ResurrectChunkLocations(nodeAddress, locationUuids, options); -} + const TResurrectChunkLocationsOptions& options), + (nodeAddress, locationUuids, options)) -TFuture<TRequestRestartResult> TDelegatingClient::RequestRestart( +DELEGATE_METHOD(TFuture<TRequestRestartResult>, RequestRestart, ( const TString& nodeAddress, - const TRequestRestartOptions& options) -{ - return Underlying_->RequestRestart(nodeAddress, options); -} + const TRequestRestartOptions& options), + (nodeAddress, options)) -TFuture<void> TDelegatingClient::SetUserPassword( +DELEGATE_METHOD(TFuture<void>, SetUserPassword, ( const TString& user, const TString& currentPasswordSha256, const TString& newPasswordSha256, - const TSetUserPasswordOptions& options) -{ - return Underlying_->SetUserPassword( - user, - currentPasswordSha256, - newPasswordSha256, - options); -} + const TSetUserPasswordOptions& options), + (user, currentPasswordSha256, newPasswordSha256, options)) -TFuture<TIssueTokenResult> TDelegatingClient::IssueToken( +DELEGATE_METHOD(TFuture<TIssueTokenResult>, IssueToken, ( const TString& user, const TString& passwordSha256, - const TIssueTokenOptions& options) -{ - return Underlying_->IssueToken( - user, - passwordSha256, - options); -} + const TIssueTokenOptions& options), + (user, passwordSha256, options)) -TFuture<void> TDelegatingClient::RevokeToken( +DELEGATE_METHOD(TFuture<void>, RevokeToken, ( const TString& user, const TString& passwordSha256, const TString& tokenSha256, - const TRevokeTokenOptions& options) -{ - return Underlying_->RevokeToken( - user, - passwordSha256, - tokenSha256, - options); -} + const TRevokeTokenOptions& options), + (user, passwordSha256, tokenSha256, options)) -TFuture<TListUserTokensResult> TDelegatingClient::ListUserTokens( +DELEGATE_METHOD(TFuture<TListUserTokensResult>, ListUserTokens, ( const TString& user, const TString& passwordSha256, - const TListUserTokensOptions& options) -{ - return Underlying_->ListUserTokens( - user, - passwordSha256, - options); -} + const TListUserTokensOptions& options), + (user, passwordSha256, options)) -TFuture<NQueryTrackerClient::TQueryId> TDelegatingClient::StartQuery( +DELEGATE_METHOD(TFuture<NQueryTrackerClient::TQueryId>, StartQuery, ( NQueryTrackerClient::EQueryEngine engine, const TString& query, - const TStartQueryOptions& options) -{ - return Underlying_->StartQuery(engine, query, options); -} + const TStartQueryOptions& options), + (engine, query, options)) -TFuture<void> TDelegatingClient::AbortQuery( +DELEGATE_METHOD(TFuture<void>, AbortQuery, ( NQueryTrackerClient::TQueryId queryId, - const TAbortQueryOptions& options) -{ - return Underlying_->AbortQuery(queryId, options); -} + const TAbortQueryOptions& options), + (queryId, options)) -TFuture<TQueryResult> TDelegatingClient::GetQueryResult( +DELEGATE_METHOD(TFuture<TQueryResult>, GetQueryResult, ( NQueryTrackerClient::TQueryId queryId, i64 resultIndex, - const TGetQueryResultOptions& options) -{ - return Underlying_->GetQueryResult(queryId, resultIndex, options); -} + const TGetQueryResultOptions& options), + (queryId, resultIndex, options)) -TFuture<IUnversionedRowsetPtr> TDelegatingClient::ReadQueryResult( +DELEGATE_METHOD(TFuture<IUnversionedRowsetPtr>, ReadQueryResult, ( NQueryTrackerClient::TQueryId queryId, i64 resultIndex, - const TReadQueryResultOptions& options) -{ - return Underlying_->ReadQueryResult(queryId, resultIndex, options); -} + const TReadQueryResultOptions& options), + (queryId, resultIndex, options)) -TFuture<TQuery> TDelegatingClient::GetQuery( +DELEGATE_METHOD(TFuture<TQuery>, GetQuery, ( NQueryTrackerClient::TQueryId queryId, - const TGetQueryOptions& options) -{ - return Underlying_->GetQuery(queryId, options); -} + const TGetQueryOptions& options), + (queryId, options)) -TFuture<TListQueriesResult> TDelegatingClient::ListQueries(const TListQueriesOptions& options) -{ - return Underlying_->ListQueries(options); -} +DELEGATE_METHOD(TFuture<TListQueriesResult>, ListQueries, ( + const TListQueriesOptions& options), + (options)) -TFuture<void> TDelegatingClient::AlterQuery( +DELEGATE_METHOD(TFuture<void>, AlterQuery, ( NQueryTrackerClient::TQueryId queryId, - const TAlterQueryOptions& options) -{ - return Underlying_->AlterQuery(queryId, options); -} + const TAlterQueryOptions& options), + (queryId, options)) -TFuture<TBundleConfigDescriptorPtr> TDelegatingClient::GetBundleConfig( +DELEGATE_METHOD(TFuture<TBundleConfigDescriptorPtr>, GetBundleConfig, ( const TString& bundleName, - const TGetBundleConfigOptions& options) -{ - return Underlying_->GetBundleConfig(bundleName, options); -} + const TGetBundleConfigOptions& options), + (bundleName, options)) + +#undef DELEGATE_METHOD //////////////////////////////////////////////////////////////////////////////// -// Method below ensures that delegating client contains implementations for all +// Ensure that delegating client contains implementations for all // methods of IClient. Tthis reduces the number of PR iterations you need to // find that some out-of-yt/yt implementation of IClient does not compile. void InstantiateDelegatingClient() { - auto delegatingClient = New<TDelegatingClient>(/*client*/ nullptr); - Y_UNUSED(delegatingClient); + Y_UNUSED(New<TDelegatingClient>(/*client*/ nullptr)); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/api/delegating_client.h b/yt/yt/client/api/delegating_client.h index a8df97651a..f455b3339b 100644 --- a/yt/yt/client/api/delegating_client.h +++ b/yt/yt/client/api/delegating_client.h @@ -7,7 +7,7 @@ namespace NYT::NApi { //////////////////////////////////////////////////////////////////////////////// //! A simple base class that implements IClient and delegates -//! all calls to an underlying instance. +//! all calls to the underlying instance. class TDelegatingClient : public IClient { @@ -37,7 +37,7 @@ public: const TSharedRange<NTableClient::TLegacyKey>& keys, const TVersionedLookupRowsOptions& options = {}) override; - TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookup( + TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookupRows( const std::vector<TMultiLookupSubrequest>& subrequests, const TMultiLookupOptions& options = {}) override; diff --git a/yt/yt/client/api/delegating_transaction.cpp b/yt/yt/client/api/delegating_transaction.cpp new file mode 100644 index 0000000000..1e3c6a4ae7 --- /dev/null +++ b/yt/yt/client/api/delegating_transaction.cpp @@ -0,0 +1,306 @@ +#include "delegating_transaction.h" + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +TDelegatingTransaction::TDelegatingTransaction(ITransactionPtr underlying) + : Underlying_(std::move(underlying)) +{ } + +#define DELEGATE_METHOD(returnType, method, signature, args) \ + returnType TDelegatingTransaction::method signature \ + { \ + return Underlying_->method args; \ + } + +DELEGATE_METHOD(IConnectionPtr, GetConnection, (), ()) + +DELEGATE_METHOD(TFuture<ITransactionPtr>, StartTransaction, ( + NTransactionClient::ETransactionType type, + const TTransactionStartOptions& options), + (type, options)) + +DELEGATE_METHOD(TFuture<TUnversionedLookupRowsResult>, LookupRows, ( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + const TSharedRange<NTableClient::TLegacyKey>& keys, + const TLookupRowsOptions& options), + (path, nameTable, keys, options)) + +DELEGATE_METHOD(TFuture<TVersionedLookupRowsResult>, VersionedLookupRows, ( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + const TSharedRange<NTableClient::TLegacyKey>& keys, + const TVersionedLookupRowsOptions& options), + (path, std::move(nameTable), keys, options)) + +DELEGATE_METHOD(TFuture<std::vector<TUnversionedLookupRowsResult>>, MultiLookupRows, ( + const std::vector<TMultiLookupSubrequest>& subrequests, + const TMultiLookupOptions& options), + (subrequests, options)) + +DELEGATE_METHOD(TFuture<TSelectRowsResult>, SelectRows, ( + const TString& query, + const TSelectRowsOptions& options), + (query, options)) + +DELEGATE_METHOD(TFuture<NYson::TYsonString>, ExplainQuery, ( + const TString& query, + const TExplainQueryOptions& options), + (query, options)) + +DELEGATE_METHOD(TFuture<TPullRowsResult>, PullRows, ( + const NYPath::TYPath& path, + const TPullRowsOptions& options), + (path, options)) + +DELEGATE_METHOD(TFuture<ITableReaderPtr>, CreateTableReader, ( + const NYPath::TRichYPath& path, + const TTableReaderOptions& options), + (path, options)) + +DELEGATE_METHOD(TFuture<ITableWriterPtr>, CreateTableWriter, ( + const NYPath::TRichYPath& path, + const TTableWriterOptions& options), + (path, options)) + +DELEGATE_METHOD(TFuture<NYson::TYsonString>, GetNode, ( + const NYPath::TYPath& path, + const TGetNodeOptions& options), + (path, options)) + +DELEGATE_METHOD(TFuture<void>, SetNode, ( + const NYPath::TYPath& path, + const NYson::TYsonString& value, + const TSetNodeOptions& options), + (path, value, options)) + +DELEGATE_METHOD(TFuture<void>, MultisetAttributesNode, ( + const NYPath::TYPath& path, + const NYTree::IMapNodePtr& attributes, + const TMultisetAttributesNodeOptions& options), + (path, attributes, options)) + +DELEGATE_METHOD(TFuture<void>, RemoveNode, ( + const NYPath::TYPath& path, + const TRemoveNodeOptions& options), + (path, options)) + +DELEGATE_METHOD(TFuture<NYson::TYsonString>, ListNode, ( + const NYPath::TYPath& path, + const TListNodeOptions& options), + (path, options)) + +DELEGATE_METHOD(TFuture<NCypressClient::TNodeId>, CreateNode, ( + const NYPath::TYPath& path, + NObjectClient::EObjectType type, + const TCreateNodeOptions& options), + (path, type, options)) + +DELEGATE_METHOD(TFuture<TLockNodeResult>, LockNode, ( + const NYPath::TYPath& path, + NCypressClient::ELockMode mode, + const TLockNodeOptions& options), + (path, mode, options)) + +DELEGATE_METHOD(TFuture<void>, UnlockNode, ( + const NYPath::TYPath& path, + const TUnlockNodeOptions& options), + (path, options)) + +DELEGATE_METHOD(TFuture<NCypressClient::TNodeId>, CopyNode, ( + const NYPath::TYPath& srcPath, + const NYPath::TYPath& dstPath, + const TCopyNodeOptions& options), + (srcPath, dstPath, options)) + +DELEGATE_METHOD(TFuture<NCypressClient::TNodeId>, MoveNode, ( + const NYPath::TYPath& srcPath, + const NYPath::TYPath& dstPath, + const TMoveNodeOptions& options), + (srcPath, dstPath, options)) + +DELEGATE_METHOD(TFuture<NCypressClient::TNodeId>, LinkNode, ( + const NYPath::TYPath& srcPath, + const NYPath::TYPath& dstPath, + const TLinkNodeOptions& options), + (srcPath, dstPath, options)) + +DELEGATE_METHOD(TFuture<void>, ConcatenateNodes, ( + const std::vector<NYPath::TRichYPath>& srcPaths, + const NYPath::TRichYPath& dstPath, + const TConcatenateNodesOptions& options), + (srcPaths, dstPath, options)) + +DELEGATE_METHOD(TFuture<bool>, NodeExists, ( + const NYPath::TYPath& path, + const TNodeExistsOptions& options), + (path, options)) + +DELEGATE_METHOD(TFuture<void>, ExternalizeNode, ( + const NYPath::TYPath& path, + NObjectClient::TCellTag cellTag, + const TExternalizeNodeOptions& options), + (path, cellTag, options)) + +DELEGATE_METHOD(TFuture<void>, InternalizeNode, ( + const NYPath::TYPath& path, + const TInternalizeNodeOptions& options), + (path, options)) + +DELEGATE_METHOD(TFuture<NObjectClient::TObjectId>, CreateObject, ( + NObjectClient::EObjectType type, + const TCreateObjectOptions& options), + (type, options)) + +DELEGATE_METHOD(TFuture<IFileReaderPtr>, CreateFileReader, ( + const NYPath::TYPath& path, + const TFileReaderOptions& options), + (path, options)) + +DELEGATE_METHOD(IFileWriterPtr, CreateFileWriter, ( + const NYPath::TRichYPath& path, + const TFileWriterOptions& options), + (path, options)) + +DELEGATE_METHOD(IJournalReaderPtr, CreateJournalReader, ( + const NYPath::TYPath& path, + const TJournalReaderOptions& options), + (path, options)) + +DELEGATE_METHOD(IJournalWriterPtr, CreateJournalWriter, ( + const NYPath::TYPath& path, + const TJournalWriterOptions& options), + (path, options)) + +DELEGATE_METHOD(IClientPtr, GetClient, () const, ()) + +DELEGATE_METHOD(NTransactionClient::ETransactionType, GetType, () const, ()) + +DELEGATE_METHOD(NTransactionClient::TTransactionId, GetId, () const, ()) + +DELEGATE_METHOD(NTransactionClient::TTimestamp, GetStartTimestamp, () const, ()) + +DELEGATE_METHOD(NTransactionClient::EAtomicity, GetAtomicity, () const, ()) + +DELEGATE_METHOD(NTransactionClient::EDurability, GetDurability, () const, ()) + +DELEGATE_METHOD(TDuration, GetTimeout, () const, ()) + +DELEGATE_METHOD(TFuture<void>, Ping, ( + const NApi::TTransactionPingOptions& options), + (options)) + +DELEGATE_METHOD(TFuture<TTransactionCommitResult>, Commit, ( + const TTransactionCommitOptions& options), + (options)) + +DELEGATE_METHOD(TFuture<void>, Abort, ( + const TTransactionAbortOptions& options), + (options)) + +DELEGATE_METHOD(void, Detach, (), ()) + +DELEGATE_METHOD(TFuture<TTransactionFlushResult>, Flush, (), ()) + +DELEGATE_METHOD(void, RegisterAlienTransaction, ( + const ITransactionPtr& transaction), + (transaction)) + +DELEGATE_METHOD(void, SubscribeCommitted, (const + TCommittedHandler& handler), + (handler)) + +DELEGATE_METHOD(void, UnsubscribeCommitted, ( + const TCommittedHandler& handler), + (handler)) + +DELEGATE_METHOD(void, SubscribeAborted, ( + const TAbortedHandler& handler), + (handler)) + +DELEGATE_METHOD(void, UnsubscribeAborted, ( + const TAbortedHandler& handler), + (handler)) + +DELEGATE_METHOD(void, WriteRows, ( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TUnversionedRow> rows, + const TModifyRowsOptions& options, + NTableClient::ELockType lockType), + (path, nameTable, rows, options, lockType)) + +DELEGATE_METHOD(void, WriteRows, ( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TVersionedRow> rows, + const TModifyRowsOptions& options), + (path, nameTable, rows, options)) + +DELEGATE_METHOD(void, DeleteRows, ( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + const TModifyRowsOptions& options), + (path, nameTable, keys, options)) + +DELEGATE_METHOD(void, LockRows, ( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + NTableClient::TLockMask lockMask), + (path, nameTable, keys, lockMask)) + +DELEGATE_METHOD(void, LockRows, ( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + NTableClient::ELockType lockType), + (path, nameTable, keys, lockType)) + +DELEGATE_METHOD(void, LockRows, ( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + const std::vector<TString>& locks, + NTableClient::ELockType lockType), + (path, nameTable, keys, locks, lockType)) + +DELEGATE_METHOD(void, ModifyRows, ( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<TRowModification> modifications, + const TModifyRowsOptions& options), + (path, nameTable, modifications, options)) + +DELEGATE_METHOD(void, AdvanceConsumer, ( + const NYPath::TYPath& path, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset), + (path, partitionIndex, oldOffset, newOffset)) + +DELEGATE_METHOD(void, AdvanceConsumer, ( + const NYPath::TRichYPath& consumerPath, + const NYPath::TRichYPath& queuePath, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset), + (consumerPath, queuePath, partitionIndex, oldOffset, newOffset)) + +DELEGATE_METHOD(TFuture<void>, AdvanceConsumer, ( + const NYT::NYPath::TRichYPath& consumer, + const NYT::NYPath::TRichYPath& queue, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset, + const NYT::NApi::TAdvanceConsumerOptions& options), + (consumer, queue, partitionIndex, oldOffset, newOffset, options)) + +#undef DELEGATE_METHOD + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi diff --git a/yt/yt/client/api/delegating_transaction.h b/yt/yt/client/api/delegating_transaction.h new file mode 100644 index 0000000000..030b0b0c7e --- /dev/null +++ b/yt/yt/client/api/delegating_transaction.h @@ -0,0 +1,253 @@ +#pragma once + +#include "transaction.h" + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +//! A simple base class that implements ITransaction and delegates +//! all calls to the underlying instance. +class TDelegatingTransaction + : public virtual ITransaction +{ +public: + explicit TDelegatingTransaction(ITransactionPtr underlying); + + // IClientBase methods + + IConnectionPtr GetConnection() override; + + // Transactions + TFuture<ITransactionPtr> StartTransaction( + NTransactionClient::ETransactionType type, + const TTransactionStartOptions& options) override; + + // Tables + TFuture<TUnversionedLookupRowsResult> LookupRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + const TSharedRange<NTableClient::TLegacyKey>& keys, + const TLookupRowsOptions& options) override; + + TFuture<TVersionedLookupRowsResult> VersionedLookupRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + const TSharedRange<NTableClient::TLegacyKey>& keys, + const TVersionedLookupRowsOptions& options) override; + + TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookupRows( + const std::vector<TMultiLookupSubrequest>& subrequests, + const TMultiLookupOptions& options) override; + + TFuture<TSelectRowsResult> SelectRows( + const TString& query, + const TSelectRowsOptions& options) override; + + TFuture<NYson::TYsonString> ExplainQuery( + const TString& query, + const TExplainQueryOptions& options) override; + + TFuture<TPullRowsResult> PullRows( + const NYPath::TYPath& path, + const TPullRowsOptions& options) override; + + TFuture<ITableReaderPtr> CreateTableReader( + const NYPath::TRichYPath& path, + const TTableReaderOptions& options) override; + + TFuture<ITableWriterPtr> CreateTableWriter( + const NYPath::TRichYPath& path, + const TTableWriterOptions& options) override; + + // Cypress + TFuture<NYson::TYsonString> GetNode( + const NYPath::TYPath& path, + const TGetNodeOptions& options) override; + + TFuture<void> SetNode( + const NYPath::TYPath& path, + const NYson::TYsonString& value, + const TSetNodeOptions& options) override; + + TFuture<void> MultisetAttributesNode( + const NYPath::TYPath& path, + const NYTree::IMapNodePtr& attributes, + const TMultisetAttributesNodeOptions& options) override; + + TFuture<void> RemoveNode( + const NYPath::TYPath& path, + const TRemoveNodeOptions& options) override; + + TFuture<NYson::TYsonString> ListNode( + const NYPath::TYPath& path, + const TListNodeOptions& options) override; + + TFuture<NCypressClient::TNodeId> CreateNode( + const NYPath::TYPath& path, + NObjectClient::EObjectType type, + const TCreateNodeOptions& options) override; + + TFuture<TLockNodeResult> LockNode( + const NYPath::TYPath& path, + NCypressClient::ELockMode mode, + const TLockNodeOptions& options) override; + + TFuture<void> UnlockNode( + const NYPath::TYPath& path, + const TUnlockNodeOptions& options) override; + + TFuture<NCypressClient::TNodeId> CopyNode( + const NYPath::TYPath& srcPath, + const NYPath::TYPath& dstPath, + const TCopyNodeOptions& options) override; + + TFuture<NCypressClient::TNodeId> MoveNode( + const NYPath::TYPath& srcPath, + const NYPath::TYPath& dstPath, + const TMoveNodeOptions& options) override; + + TFuture<NCypressClient::TNodeId> LinkNode( + const NYPath::TYPath& srcPath, + const NYPath::TYPath& dstPath, + const TLinkNodeOptions& options) override; + + TFuture<void> ConcatenateNodes( + const std::vector<NYPath::TRichYPath>& srcPaths, + const NYPath::TRichYPath& dstPath, + const TConcatenateNodesOptions& options) override; + + TFuture<bool> NodeExists( + const NYPath::TYPath& path, + const TNodeExistsOptions& options) override; + + TFuture<void> ExternalizeNode( + const NYPath::TYPath& path, + NObjectClient::TCellTag cellTag, + const TExternalizeNodeOptions& options) override; + + TFuture<void> InternalizeNode( + const NYPath::TYPath& path, + const TInternalizeNodeOptions& options) override; + + // Objects + TFuture<NObjectClient::TObjectId> CreateObject( + NObjectClient::EObjectType type, + const TCreateObjectOptions& options) override; + + // Files + TFuture<IFileReaderPtr> CreateFileReader( + const NYPath::TYPath& path, + const TFileReaderOptions& options) override; + + IFileWriterPtr CreateFileWriter( + const NYPath::TRichYPath& path, + const TFileWriterOptions& options) override; + + // Journals + IJournalReaderPtr CreateJournalReader( + const NYPath::TYPath& path, + const TJournalReaderOptions& options) override; + + IJournalWriterPtr CreateJournalWriter( + const NYPath::TYPath& path, + const TJournalWriterOptions& options) override; + + // ITransaction methods + IClientPtr GetClient() const override; + NTransactionClient::ETransactionType GetType() const override; + NTransactionClient::TTransactionId GetId() const override; + NTransactionClient::TTimestamp GetStartTimestamp() const override; + NTransactionClient::EAtomicity GetAtomicity() const override; + NTransactionClient::EDurability GetDurability() const override; + TDuration GetTimeout() const override; + + TFuture<void> Ping( + const NApi::TTransactionPingOptions& options) override; + + TFuture<TTransactionCommitResult> Commit( + const TTransactionCommitOptions& options = TTransactionCommitOptions()) override; + + TFuture<void> Abort( + const TTransactionAbortOptions& options = TTransactionAbortOptions()) override; + + void Detach() override; + TFuture<TTransactionFlushResult> Flush() override; + void RegisterAlienTransaction(const ITransactionPtr& transaction) override; + + void SubscribeCommitted(const TCommittedHandler& handler) override; + void UnsubscribeCommitted(const TCommittedHandler& handler) override; + + void SubscribeAborted(const TAbortedHandler& handler) override; + void UnsubscribeAborted(const TAbortedHandler& handler) override; + + // Tables + void WriteRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TUnversionedRow> rows, + const TModifyRowsOptions& options, + NTableClient::ELockType lockType = NTableClient::ELockType::Exclusive) override; + void WriteRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TVersionedRow> rows, + const TModifyRowsOptions& options) override; + + void DeleteRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + const TModifyRowsOptions& options) override; + + void LockRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + NTableClient::TLockMask lockMask) override; + void LockRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + NTableClient::ELockType lockType) override; + void LockRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + const std::vector<TString>& locks, + NTableClient::ELockType lockType) override; + + void ModifyRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<TRowModification> modifications, + const TModifyRowsOptions& options) override; + + // Queues + void AdvanceConsumer( + const NYPath::TYPath& path, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset) override; + void AdvanceConsumer( + const NYPath::TRichYPath& consumerPath, + const NYPath::TRichYPath& queuePath, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset) override; + TFuture<void> AdvanceConsumer( + const NYT::NYPath::TRichYPath& consumer, + const NYT::NYPath::TRichYPath& queue, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset, + const NYT::NApi::TAdvanceConsumerOptions& options) override; + +protected: + const ITransactionPtr Underlying_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi + diff --git a/yt/yt/client/api/dynamic_table_client.h b/yt/yt/client/api/dynamic_table_client.h new file mode 100644 index 0000000000..1a12180a27 --- /dev/null +++ b/yt/yt/client/api/dynamic_table_client.h @@ -0,0 +1,108 @@ +#pragma once + +#include "client_common.h" + +#include <yt/yt/client/table_client/row_base.h> + +#include <yt/yt/client/query_client/query_statistics.h> + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +struct TLookupRequestOptions + : public TFallbackReplicaOptions +{ + NTableClient::TColumnFilter ColumnFilter; + bool KeepMissingRows = false; + bool EnablePartialResult = false; + std::optional<bool> UseLookupCache; + TDetailedProfilingInfoPtr DetailedProfilingInfo; +}; + +struct TLookupRowsOptionsBase + : public TTabletReadOptions + , public TLookupRequestOptions + , public TMultiplexingBandOptions +{ }; + +struct TLookupRowsOptions + : public TLookupRowsOptionsBase +{ }; + +struct TVersionedLookupRowsOptions + : public TLookupRowsOptionsBase +{ + NTableClient::TRetentionConfigPtr RetentionConfig; +}; + +struct TMultiLookupSubrequest +{ + NYPath::TYPath Path; + NTableClient::TNameTablePtr NameTable; + TSharedRange<NTableClient::TLegacyKey> Keys; + + // NB: Other options from TLookupRowsOptions that are absent from TLookupRequestOptions are + // common and included in TMultiLookupOptions. + TLookupRequestOptions Options; +}; + +struct TMultiLookupOptions + : public TTimeoutOptions + , public TTabletReadOptionsBase + , public TMultiplexingBandOptions +{ }; + +struct TExplainQueryOptions + : public TSelectRowsOptionsBase +{ + bool VerboseOutput = false; +}; + +struct TSelectRowsResult +{ + IUnversionedRowsetPtr Rowset; + NQueryClient::TQueryStatistics Statistics; +}; + +template <class IRowset> +struct TLookupRowsResult +{ + TIntrusivePtr<IRowset> Rowset; +}; + +using TUnversionedLookupRowsResult = TLookupRowsResult<IUnversionedRowset>; +using TVersionedLookupRowsResult = TLookupRowsResult<IVersionedRowset>; + +//////////////////////////////////////////////////////////////////////////////// + +struct IDynamicTableClientBase +{ + virtual ~IDynamicTableClientBase() = default; + + virtual TFuture<TUnversionedLookupRowsResult> LookupRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + const TSharedRange<NTableClient::TLegacyKey>& keys, + const TLookupRowsOptions& options = {}) = 0; + virtual TFuture<TVersionedLookupRowsResult> VersionedLookupRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + const TSharedRange<NTableClient::TLegacyKey>& keys, + const TVersionedLookupRowsOptions& options = {}) = 0; + virtual TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookupRows( + const std::vector<TMultiLookupSubrequest>& subrequests, + const TMultiLookupOptions& options = {}) = 0; + + virtual TFuture<TSelectRowsResult> SelectRows( + const TString& query, + const TSelectRowsOptions& options = {}) = 0; + + virtual TFuture<NYson::TYsonString> ExplainQuery( + const TString& query, + const TExplainQueryOptions& options = {}) = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi diff --git a/yt/yt/client/api/dynamic_table_transaction.h b/yt/yt/client/api/dynamic_table_transaction.h new file mode 100644 index 0000000000..2164cdeacd --- /dev/null +++ b/yt/yt/client/api/dynamic_table_transaction.h @@ -0,0 +1,118 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/client/chaos_client/public.h> + +#include <yt/yt/client/table_client/row_base.h> +#include <yt/yt/client/table_client/schema.h> + +#include <yt/yt/client/tablet_client/public.h> + +#include <yt/yt/core/ypath/public.h> + +#include <library/cpp/yt/memory/shared_range.h> + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +//! Either a write or delete. +struct TRowModification +{ + //! Discriminates between writes and deletes. + ERowModificationType Type; + //! Either a row (for write; versioned or unversioned) or a key (for delete; always unversioned). + NTableClient::TTypeErasedRow Row; + //! Locks. + NTableClient::TLockMask Locks; +}; + +struct TModifyRowsOptions +{ + //! If this happens to be a modification of a replicated table, + //! controls if at least one sync replica is required. + bool RequireSyncReplica = true; + + //! For chaos replicated tables indicates if it is necessary to explore other replicas. + bool TopmostTransaction = true; + + //! For chaos replicas pass replication card to ensure that all data is sent using same meta info. + NChaosClient::TReplicationCardPtr ReplicationCard; + + //! For writes to replicas, this is the id of the replica at the upstream cluster. + NTabletClient::TTableReplicaId UpstreamReplicaId; + + //! Modifications are sent asynchronously. Sequential numbering is + //! required to restore their order. + //! + //! This parameter is only used by native client (in particular within RPC proxy server). + std::optional<i64> SequenceNumber; + + //! Modifications can be sent from several sources in case of several clients + //! attached to the same transaction. + //! + //! Modifications within one source will be serialized by this source sequence numbers. + //! Modifications from different sources will be serialized arbitrarily, that is why + //! different sources must send independent modifications. + //! + //! If sequence number is missing, source id is ignored. + //! + //! This parameter is only used by native client (in particular within RPC proxy server). + i64 SequenceNumberSourceId = 0; + + //! If set treat missing key columns as null. + bool AllowMissingKeyColumns = false; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct IDynamicTableTransaction +{ + virtual ~IDynamicTableTransaction() = default; + + virtual void WriteRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TUnversionedRow> rows, + const TModifyRowsOptions& options = {}, + NTableClient::ELockType lockType = NTableClient::ELockType::Exclusive) = 0; + virtual void WriteRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TVersionedRow> rows, + const TModifyRowsOptions& options = {}) = 0; + + virtual void DeleteRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + const TModifyRowsOptions& options = {}) = 0; + + virtual void LockRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + NTableClient::TLockMask lockMask) = 0; + virtual void LockRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong) = 0; + virtual void LockRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + const std::vector<TString>& locks, + NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong) = 0; + + virtual void ModifyRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<TRowModification> modifications, + const TModifyRowsOptions& options = {}) = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi diff --git a/yt/yt/client/api/dynamic_table_transaction_mixin.cpp b/yt/yt/client/api/dynamic_table_transaction_mixin.cpp new file mode 100644 index 0000000000..240e4ea548 --- /dev/null +++ b/yt/yt/client/api/dynamic_table_transaction_mixin.cpp @@ -0,0 +1,173 @@ +#include "dynamic_table_transaction_mixin.h" + +#include <yt/yt/client/table_client/unversioned_row.h> +#include <yt/yt/client/table_client/versioned_row.h> + +#include <yt/yt/client/tablet_client/table_mount_cache.h> + +namespace NYT::NApi { + +using namespace NTableClient; +using namespace NTabletClient; +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +void TDynamicTableTransactionMixin::WriteRows( + const NYPath::TYPath& path, + TNameTablePtr nameTable, + TSharedRange<TUnversionedRow> rows, + const TModifyRowsOptions& options, + ELockType lockType) +{ + THROW_ERROR_EXCEPTION_UNLESS(IsWriteLock(lockType), "Inappropriate lock type %Qlv given for write modification", + lockType); + + std::vector<TRowModification> modifications; + modifications.reserve(rows.Size()); + + switch (lockType) { + case ELockType::Exclusive: { + for (auto row : rows) { + modifications.push_back({ERowModificationType::Write, row.ToTypeErasedRow(), TLockMask()}); + } + + break; + } + + case ELockType::SharedWrite: { + // NB: This mount revision could differ from the one will be sent to tablet node. + // However locks correctness will be checked in native transaction. + const auto& tableMountCache = GetClient()->GetTableMountCache(); + auto tableInfo = WaitFor(tableMountCache->GetTableInfo(path)) + .ValueOrThrow(); + + std::vector<int> columnIndexToLockIndex; + GetLocksMapping( + *tableInfo->Schemas[ETableSchemaKind::Write], + GetAtomicity() == NTransactionClient::EAtomicity::Full, + &columnIndexToLockIndex); + + for (auto row : rows) { + TLockMask lockMask; + for (const auto& value : row) { + auto lockIndex = columnIndexToLockIndex[value.Id]; + if (lockIndex != -1) { + lockMask.Set(lockIndex, lockType); + } + } + + modifications.push_back({ERowModificationType::WriteAndLock, row.ToTypeErasedRow(), lockMask}); + } + + break; + } + + default: + YT_ABORT(); + } + + ModifyRows( + path, + std::move(nameTable), + MakeSharedRange(std::move(modifications), std::move(rows.ReleaseHolder())), + options); +} + +void TDynamicTableTransactionMixin::WriteRows( + const NYPath::TYPath& path, + TNameTablePtr nameTable, + TSharedRange<TVersionedRow> rows, + const TModifyRowsOptions& options) +{ + std::vector<TRowModification> modifications; + modifications.reserve(rows.Size()); + + for (auto row : rows) { + modifications.push_back({ERowModificationType::VersionedWrite, row.ToTypeErasedRow(), TLockMask()}); + } + + ModifyRows( + path, + std::move(nameTable), + MakeSharedRange(std::move(modifications), std::move(rows.ReleaseHolder())), + options); +} + +void TDynamicTableTransactionMixin::DeleteRows( + const NYPath::TYPath& path, + TNameTablePtr nameTable, + TSharedRange<TLegacyKey> keys, + const TModifyRowsOptions& options) +{ + std::vector<TRowModification> modifications; + modifications.reserve(keys.Size()); + for (auto key : keys) { + modifications.push_back({ERowModificationType::Delete, key.ToTypeErasedRow(), TLockMask()}); + } + + ModifyRows( + path, + std::move(nameTable), + MakeSharedRange(std::move(modifications), std::move(keys.ReleaseHolder())), + options); +} + +void TDynamicTableTransactionMixin::LockRows( + const NYPath::TYPath& path, + TNameTablePtr nameTable, + TSharedRange<TLegacyKey> keys, + TLockMask lockMask) +{ + std::vector<TRowModification> modifications; + modifications.reserve(keys.Size()); + + for (auto key : keys) { + TRowModification modification; + modification.Type = ERowModificationType::WriteAndLock; + modification.Row = key.ToTypeErasedRow(); + modification.Locks = lockMask; + modifications.push_back(modification); + } + + ModifyRows( + path, + std::move(nameTable), + MakeSharedRange(std::move(modifications), std::move(keys)), + TModifyRowsOptions()); +} + +void TDynamicTableTransactionMixin::LockRows( + const NYPath::TYPath& path, + TNameTablePtr nameTable, + TSharedRange<TLegacyKey> keys, + ELockType lockType) +{ + TLockMask lockMask; + lockMask.Set(PrimaryLockIndex, lockType); + LockRows(path, nameTable, keys, lockMask); +} + +void TDynamicTableTransactionMixin::LockRows( + const NYPath::TYPath& path, + TNameTablePtr nameTable, + TSharedRange<TLegacyKey> keys, + const std::vector<TString>& locks, + ELockType lockType) +{ + const auto& tableMountCache = GetClient()->GetTableMountCache(); + auto tableInfo = WaitFor(tableMountCache->GetTableInfo(path)) + .ValueOrThrow(); + + auto lockMask = GetLockMask( + *tableInfo->Schemas[ETableSchemaKind::Write], + GetAtomicity() == NTransactionClient::EAtomicity::Full, + locks, + lockType); + + LockRows(path, nameTable, keys, lockMask); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi diff --git a/yt/yt/client/api/dynamic_table_transaction_mixin.h b/yt/yt/client/api/dynamic_table_transaction_mixin.h new file mode 100644 index 0000000000..c8492f1bf6 --- /dev/null +++ b/yt/yt/client/api/dynamic_table_transaction_mixin.h @@ -0,0 +1,52 @@ +#pragma once + +#include "transaction.h" + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +class TDynamicTableTransactionMixin + : public virtual ITransaction + , public virtual IDynamicTableTransaction +{ +public: + void WriteRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TUnversionedRow> rows, + const TModifyRowsOptions& options = {}, + NTableClient::ELockType lockType = NTableClient::ELockType::Exclusive) override; + void WriteRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TVersionedRow> rows, + const TModifyRowsOptions& options = {}) override; + + void DeleteRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + const TModifyRowsOptions& options = {}) override; + + void LockRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + NTableClient::TLockMask lockMask) override; + void LockRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong) override; + void LockRows( + const NYPath::TYPath& path, + NTableClient::TNameTablePtr nameTable, + TSharedRange<NTableClient::TLegacyKey> keys, + const std::vector<TString>& locks, + NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong) override; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi diff --git a/yt/yt/client/api/etc_client.h b/yt/yt/client/api/etc_client.h index 743cc3d955..a485c87ae1 100644 --- a/yt/yt/client/api/etc_client.h +++ b/yt/yt/client/api/etc_client.h @@ -55,6 +55,8 @@ struct TCheckClusterLivenessOptions struct IEtcClientBase { + virtual ~IEtcClientBase() = default; + virtual TFuture<NObjectClient::TObjectId> CreateObject( NObjectClient::EObjectType type, const TCreateObjectOptions& options = {}) = 0; @@ -64,6 +66,8 @@ struct IEtcClientBase struct IEtcClient { + virtual ~IEtcClient() = default; + virtual TFuture<TClusterMeta> GetClusterMeta( const TGetClusterMetaOptions& options = {}) = 0; diff --git a/yt/yt/client/api/file_client.h b/yt/yt/client/api/file_client.h index 351c81bc52..52c03c1f02 100644 --- a/yt/yt/client/api/file_client.h +++ b/yt/yt/client/api/file_client.h @@ -59,6 +59,8 @@ struct TPutFileToCacheResult struct IFileClientBase { + virtual ~IFileClientBase() = default; + virtual TFuture<IFileReaderPtr> CreateFileReader( const NYPath::TYPath& path, const TFileReaderOptions& options = {}) = 0; @@ -72,6 +74,8 @@ struct IFileClientBase struct IFileClient { + virtual ~IFileClient() = default; + virtual TFuture<TGetFileFromCacheResult> GetFileFromCache( const TString& md5, const TGetFileFromCacheOptions& options = {}) = 0; diff --git a/yt/yt/client/api/journal_client.h b/yt/yt/client/api/journal_client.h index f1d68f24e7..20650e26e5 100644 --- a/yt/yt/client/api/journal_client.h +++ b/yt/yt/client/api/journal_client.h @@ -81,6 +81,8 @@ struct TTruncateJournalOptions struct IJournalClientBase { + virtual ~IJournalClientBase() = default; + virtual IJournalReaderPtr CreateJournalReader( const NYPath::TYPath& path, const TJournalReaderOptions& options = {}) = 0; diff --git a/yt/yt/client/api/operation_client.h b/yt/yt/client/api/operation_client.h index 4630ef1e28..35d62da527 100644 --- a/yt/yt/client/api/operation_client.h +++ b/yt/yt/client/api/operation_client.h @@ -358,6 +358,8 @@ struct TListJobsResult struct IOperationClient { + virtual ~IOperationClient() = default; + virtual TFuture<NScheduler::TOperationId> StartOperation( NScheduler::EOperationType type, const NYson::TYsonString& spec, diff --git a/yt/yt/client/api/query_tracker_client.h b/yt/yt/client/api/query_tracker_client.h index a0a7a99216..9169ff9f5c 100644 --- a/yt/yt/client/api/query_tracker_client.h +++ b/yt/yt/client/api/query_tracker_client.h @@ -144,6 +144,8 @@ struct TAlterQueryOptions struct IQueryTrackerClient { + virtual ~IQueryTrackerClient() = default; + virtual TFuture<NQueryTrackerClient::TQueryId> StartQuery( NQueryTrackerClient::EQueryEngine engine, const TString& query, diff --git a/yt/yt/client/api/queue_client.h b/yt/yt/client/api/queue_client.h index c665386fcd..17849e0462 100644 --- a/yt/yt/client/api/queue_client.h +++ b/yt/yt/client/api/queue_client.h @@ -39,10 +39,6 @@ struct TPullRowsResult bool Versioned = true; }; -struct TAdvanceConsumerOptions - : public TTimeoutOptions -{ }; - struct TPullQueueOptions : public TSelectRowsOptions , public TFallbackReplicaOptions @@ -81,6 +77,8 @@ struct TListQueueConsumerRegistrationsResult struct IQueueClientBase { + virtual ~IQueueClientBase() = default; + virtual TFuture<TPullRowsResult> PullRows( const NYPath::TYPath& path, const TPullRowsOptions& options = {}) = 0; @@ -90,6 +88,8 @@ struct IQueueClientBase struct IQueueClient { + virtual ~IQueueClient() = default; + //! Reads a batch of rows from a given partition of a given queue, starting at (at least) the given offset. //! Requires the user to have read-access to the specified queue. virtual TFuture<NQueueClient::IQueueRowsetPtr> PullQueue( diff --git a/yt/yt/client/api/queue_transaction.h b/yt/yt/client/api/queue_transaction.h new file mode 100644 index 0000000000..50ef073360 --- /dev/null +++ b/yt/yt/client/api/queue_transaction.h @@ -0,0 +1,53 @@ +#pragma once + +#include "client_common.h" + +#include <yt/yt/core/ypath/public.h> + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +struct TAdvanceConsumerOptions + : public TTimeoutOptions +{ }; + +//////////////////////////////////////////////////////////////////////////////// + +struct IQueueTransaction +{ + virtual ~IQueueTransaction() = default; + + // TODO(nadya73): Remove it: YT-20712 + virtual void AdvanceConsumer( + const NYPath::TYPath& path, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset) = 0; + + // TODO(nadya73): Remove it: YT-20712 + virtual void AdvanceConsumer( + const NYPath::TRichYPath& consumerPath, + const NYPath::TRichYPath& queuePath, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset) = 0; + + //! Advances the consumer's offset for partition with index #partitionIndex, setting it to #newOffset. + /*! + * If #oldOffset is specified, the current offset is read inside this transaction and compared with #oldOffset. + * If they are equal, the new offset is written, otherwise an exception is thrown. + */ + virtual TFuture<void> AdvanceConsumer( + const NYPath::TRichYPath& consumerPath, + const NYPath::TRichYPath& queuePath, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset, + const TAdvanceConsumerOptions& options) = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi + diff --git a/yt/yt/client/api/queue_transaction_mixin.cpp b/yt/yt/client/api/queue_transaction_mixin.cpp new file mode 100644 index 0000000000..cac63774ae --- /dev/null +++ b/yt/yt/client/api/queue_transaction_mixin.cpp @@ -0,0 +1,49 @@ +#include "queue_transaction_mixin.h" + +#include <yt/yt/client/tablet_client/table_mount_cache.h> + +#include <yt/yt/client/queue_client/consumer_client.h> + +namespace NYT::NApi { + +using namespace NQueueClient; +using namespace NConcurrency; + +///////////////////////////////////////////////////////////////////////////// + +void TQueueTransactionMixin::AdvanceConsumer( + const NYPath::TYPath& path, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset) +{ + THROW_ERROR_EXCEPTION_IF(newOffset < 0, "Queue consumer offset %v cannot be negative", newOffset); + + auto consumerClient = CreateBigRTConsumerClient(GetClient(), path); + consumerClient->Advance(this, partitionIndex, oldOffset, newOffset); +} + +void TQueueTransactionMixin::AdvanceConsumer( + const NYPath::TRichYPath& consumerPath, + const NYPath::TRichYPath& queuePath, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset) +{ + THROW_ERROR_EXCEPTION_IF(newOffset < 0, "Queue consumer offset %v cannot be negative", newOffset); + + auto tableMountCache = GetClient()->GetTableMountCache(); + auto queuePhysicalPath = queuePath; + auto queueTableInfoOrError = WaitFor(tableMountCache->GetTableInfo(queuePath.GetPath())); + if (queueTableInfoOrError.IsOK()) { + queuePhysicalPath = NYPath::TRichYPath(queueTableInfoOrError.Value()->PhysicalPath, queuePath.Attributes()); + } + + // TODO(achulkov2): Support consumers from any cluster. + auto subConsumerClient = CreateSubConsumerClient(GetClient(), /*queueClient*/ nullptr, consumerPath.GetPath(), queuePhysicalPath); + return subConsumerClient->Advance(this, partitionIndex, oldOffset, newOffset); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi diff --git a/yt/yt/client/api/queue_transaction_mixin.h b/yt/yt/client/api/queue_transaction_mixin.h new file mode 100644 index 0000000000..36a5ed8014 --- /dev/null +++ b/yt/yt/client/api/queue_transaction_mixin.h @@ -0,0 +1,34 @@ +#pragma once + +#include "transaction.h" + +#include <yt/yt/core/ypath/public.h> + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +class TQueueTransactionMixin + : public virtual ITransaction +{ +public: + // TODO(nadya73): Remove it: YT-20712 + void AdvanceConsumer( + const NYPath::TYPath& path, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset) override; + + // TODO(nadya73): Remove it: YT-20712 + void AdvanceConsumer( + const NYPath::TRichYPath& consumerPath, + const NYPath::TRichYPath& queuePath, + int partitionIndex, + std::optional<i64> oldOffset, + i64 newOffset) override;; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi + diff --git a/yt/yt/client/api/rpc_proxy/api_service_proxy.h b/yt/yt/client/api/rpc_proxy/api_service_proxy.h index 3a28b830ca..e9d5d3fcea 100644 --- a/yt/yt/client/api/rpc_proxy/api_service_proxy.h +++ b/yt/yt/client/api/rpc_proxy/api_service_proxy.h @@ -67,6 +67,7 @@ public: DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, LookupRows); DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, VersionedLookupRows); + // TODO(babenko): rename to MultiLookupRows DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, MultiLookup); DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, SelectRows); DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, ExplainQuery); diff --git a/yt/yt/client/api/rpc_proxy/client_base.cpp b/yt/yt/client/api/rpc_proxy/client_base.cpp index 0c5bdf0e32..64fbb0de05 100644 --- a/yt/yt/client/api/rpc_proxy/client_base.cpp +++ b/yt/yt/client/api/rpc_proxy/client_base.cpp @@ -858,7 +858,7 @@ TFuture<TVersionedLookupRowsResult> TClientBase::VersionedLookupRows( })); } -TFuture<std::vector<TUnversionedLookupRowsResult>> TClientBase::MultiLookup( +TFuture<std::vector<TUnversionedLookupRowsResult>> TClientBase::MultiLookupRows( const std::vector<TMultiLookupSubrequest>& subrequests, const TMultiLookupOptions& options) { diff --git a/yt/yt/client/api/rpc_proxy/client_base.h b/yt/yt/client/api/rpc_proxy/client_base.h index 2a182eaa86..5a85160692 100644 --- a/yt/yt/client/api/rpc_proxy/client_base.h +++ b/yt/yt/client/api/rpc_proxy/client_base.h @@ -63,7 +63,7 @@ public: const TSharedRange<NTableClient::TLegacyKey>& keys, const NApi::TVersionedLookupRowsOptions& options) override; - TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookup( + TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookupRows( const std::vector<TMultiLookupSubrequest>& subrequests, const TMultiLookupOptions& options = {}) override; diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index 687ca42ec4..ed3b04b835 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -17,6 +17,7 @@ #include <yt/yt/client/scheduler/operation_id_or_alias.h> +#include <yt/yt/client/table_client/columnar_statistics.h> #include <yt/yt/client/table_client/name_table.h> #include <yt/yt/client/table_client/row_base.h> #include <yt/yt/client/table_client/row_buffer.h> diff --git a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp index 0514415b8f..7089f413e1 100644 --- a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp @@ -556,12 +556,12 @@ TFuture<TVersionedLookupRowsResult> TTransaction::VersionedLookupRows( PatchTransactionTimestamp(options)); } -TFuture<std::vector<TUnversionedLookupRowsResult>> TTransaction::MultiLookup( +TFuture<std::vector<TUnversionedLookupRowsResult>> TTransaction::MultiLookupRows( const std::vector<TMultiLookupSubrequest>& subrequests, const TMultiLookupOptions& options) { ValidateActive(); - return Client_->MultiLookup( + return Client_->MultiLookupRows( subrequests, PatchTransactionTimestamp(options)); } diff --git a/yt/yt/client/api/rpc_proxy/transaction_impl.h b/yt/yt/client/api/rpc_proxy/transaction_impl.h index 5286ea931b..57f7850b3e 100644 --- a/yt/yt/client/api/rpc_proxy/transaction_impl.h +++ b/yt/yt/client/api/rpc_proxy/transaction_impl.h @@ -3,6 +3,8 @@ #include "client_base.h" #include <yt/yt/client/api/transaction.h> +#include <yt/yt/client/api/dynamic_table_transaction_mixin.h> +#include <yt/yt/client/api/queue_transaction_mixin.h> #include <yt/yt/core/rpc/public.h> @@ -24,7 +26,9 @@ DEFINE_ENUM(ETransactionState, ); class TTransaction - : public NApi::ITransaction + : public virtual NApi::ITransaction + , public NApi::TDynamicTableTransactionMixin + , public NApi::TQueueTransactionMixin { public: TTransaction( @@ -73,6 +77,7 @@ public: TSharedRange<NApi::TRowModification> modifications, const NApi::TModifyRowsOptions& options) override; + using TQueueTransactionMixin::AdvanceConsumer; TFuture<void> AdvanceConsumer( const NYPath::TRichYPath& consumerPath, const NYPath::TRichYPath& queuePath, @@ -98,7 +103,7 @@ public: const TSharedRange<NTableClient::TLegacyKey>& keys, const NApi::TVersionedLookupRowsOptions& options) override; - TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookup( + TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookupRows( const std::vector<TMultiLookupSubrequest>& subrequests, const TMultiLookupOptions& options) override; diff --git a/yt/yt/client/api/security_client.h b/yt/yt/client/api/security_client.h index 2c979a147d..87ceab45f3 100644 --- a/yt/yt/client/api/security_client.h +++ b/yt/yt/client/api/security_client.h @@ -109,6 +109,8 @@ struct TListUserTokensResult struct ISecurityClient { + virtual ~ISecurityClient() = default; + virtual TFuture<void> AddMember( const TString& group, const TString& member, diff --git a/yt/yt/client/api/table_client.h b/yt/yt/client/api/table_client.h index 2b9ecc320e..f333788ad2 100644 --- a/yt/yt/client/api/table_client.h +++ b/yt/yt/client/api/table_client.h @@ -1,26 +1,14 @@ #pragma once #include "client_common.h" +#include "dynamic_table_client.h" -#include <yt/yt/client/hydra/public.h> - -#include <yt/yt/client/tablet_client/public.h> - -#include <yt/yt/client/table_client/row_base.h> -#include <yt/yt/client/table_client/schema.h> #include <yt/yt/client/table_client/chunk_stripe_statistics.h> #include <yt/yt/client/table_client/columnar_statistics.h> +#include <yt/yt/client/table_client/schema.h> #include <yt/yt/client/chaos_client/replication_card.h> -#include <yt/yt/client/query_client/query_statistics.h> - -#include <yt/yt/core/ypath/public.h> - -#include <yt/yt/core/ytree/yson_struct.h> - -#include <library/cpp/yt/memory/shared_range.h> - namespace NYT::NApi { //////////////////////////////////////////////////////////////////////////////// @@ -39,70 +27,6 @@ struct TDetailedProfilingInfo final DEFINE_REFCOUNTED_TYPE(TDetailedProfilingInfo) -struct TLookupRequestOptions - : public TFallbackReplicaOptions -{ - NTableClient::TColumnFilter ColumnFilter; - bool KeepMissingRows = false; - bool EnablePartialResult = false; - std::optional<bool> UseLookupCache; - TDetailedProfilingInfoPtr DetailedProfilingInfo; -}; - -struct TLookupRowsOptionsBase - : public TTabletReadOptions - , public TLookupRequestOptions - , public TMultiplexingBandOptions -{ }; - -struct TLookupRowsOptions - : public TLookupRowsOptionsBase -{ }; - -struct TVersionedLookupRowsOptions - : public TLookupRowsOptionsBase -{ - NTableClient::TRetentionConfigPtr RetentionConfig; -}; - -struct TMultiLookupSubrequest -{ - NYPath::TYPath Path; - NTableClient::TNameTablePtr NameTable; - TSharedRange<NTableClient::TLegacyKey> Keys; - - // NB: Other options from TLookupRowsOptions that are absent from TLookupRequestOptions are - // common and included in TMultiLookupOptions. - TLookupRequestOptions Options; -}; - -struct TMultiLookupOptions - : public TTimeoutOptions - , public TTabletReadOptionsBase - , public TMultiplexingBandOptions -{ }; - -struct TExplainQueryOptions - : public TSelectRowsOptionsBase -{ - bool VerboseOutput = false; -}; - -struct TSelectRowsResult -{ - IUnversionedRowsetPtr Rowset; - NQueryClient::TQueryStatistics Statistics; -}; - -template <class IRowset> -struct TLookupRowsResult -{ - TIntrusivePtr<IRowset> Rowset; -}; - -using TUnversionedLookupRowsResult = TLookupRowsResult<IUnversionedRowset>; -using TVersionedLookupRowsResult = TLookupRowsResult<IVersionedRowset>; - struct TTableReaderOptions : public TTransactionalOptions , public TSuppressableAccessTrackingOptions @@ -412,31 +336,8 @@ struct TLocateSkynetShareOptions //////////////////////////////////////////////////////////////////////////////// struct ITableClientBase + : public IDynamicTableClientBase { - virtual TFuture<TUnversionedLookupRowsResult> LookupRows( - const NYPath::TYPath& path, - NTableClient::TNameTablePtr nameTable, - const TSharedRange<NTableClient::TLegacyKey>& keys, - const TLookupRowsOptions& options = {}) = 0; - - virtual TFuture<TVersionedLookupRowsResult> VersionedLookupRows( - const NYPath::TYPath& path, - NTableClient::TNameTablePtr nameTable, - const TSharedRange<NTableClient::TLegacyKey>& keys, - const TVersionedLookupRowsOptions& options = {}) = 0; - - virtual TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookup( - const std::vector<TMultiLookupSubrequest>& subrequests, - const TMultiLookupOptions& options = {}) = 0; - - virtual TFuture<TSelectRowsResult> SelectRows( - const TString& query, - const TSelectRowsOptions& options = {}) = 0; - - virtual TFuture<NYson::TYsonString> ExplainQuery( - const TString& query, - const TExplainQueryOptions& options = TExplainQueryOptions()) = 0; - virtual TFuture<ITableReaderPtr> CreateTableReader( const NYPath::TRichYPath& path, const TTableReaderOptions& options = {}) = 0; @@ -450,6 +351,8 @@ struct ITableClientBase struct ITableClient { + virtual ~ITableClient() = default; + virtual TFuture<void> MountTable( const NYPath::TYPath& path, const TMountTableOptions& options = {}) = 0; diff --git a/yt/yt/client/api/transaction.cpp b/yt/yt/client/api/transaction.cpp index 1c2b7858b8..6287cd793c 100644 --- a/yt/yt/client/api/transaction.cpp +++ b/yt/yt/client/api/transaction.cpp @@ -1,214 +1,9 @@ #include "transaction.h" -#include <yt/yt/client/tablet_client/table_mount_cache.h> - -#include <yt/yt/client/queue_client/consumer_client.h> - -#include <yt/yt/client/ypath/rich.h> - namespace NYT::NApi { //////////////////////////////////////////////////////////////////////////////// -using namespace NTableClient; -using namespace NTabletClient; -using namespace NQueueClient; -using namespace NConcurrency; - -///////////////////////////////////////////////////////////////////////////// - -void ITransaction::WriteRows( - const NYPath::TYPath& path, - TNameTablePtr nameTable, - TSharedRange<TUnversionedRow> rows, - const TModifyRowsOptions& options, - ELockType lockType) -{ - THROW_ERROR_EXCEPTION_UNLESS(IsWriteLock(lockType), "Inappropriate lock type %Qlv given for write modification", - lockType); - - std::vector<TRowModification> modifications; - modifications.reserve(rows.Size()); - - switch (lockType) { - case ELockType::Exclusive: { - for (auto row : rows) { - modifications.push_back({ERowModificationType::Write, row.ToTypeErasedRow(), TLockMask()}); - } - - break; - } - - case ELockType::SharedWrite: { - // NB: This mount revision could differ from the one will be sent to tablet node. - // However locks correctness will be checked in native transaction. - const auto& tableMountCache = GetClient()->GetTableMountCache(); - auto tableInfo = WaitFor(tableMountCache->GetTableInfo(path)) - .ValueOrThrow(); - - std::vector<int> columnIndexToLockIndex; - GetLocksMapping( - *tableInfo->Schemas[ETableSchemaKind::Write], - GetAtomicity() == NTransactionClient::EAtomicity::Full, - &columnIndexToLockIndex); - - for (auto row : rows) { - TLockMask lockMask; - for (const auto& value : row) { - auto lockIndex = columnIndexToLockIndex[value.Id]; - if (lockIndex != -1) { - lockMask.Set(lockIndex, lockType); - } - } - - modifications.push_back({ERowModificationType::WriteAndLock, row.ToTypeErasedRow(), lockMask}); - } - - break; - } - - default: - YT_ABORT(); - } - - ModifyRows( - path, - std::move(nameTable), - MakeSharedRange(std::move(modifications), std::move(rows.ReleaseHolder())), - options); -} - -void ITransaction::WriteRows( - const NYPath::TYPath& path, - TNameTablePtr nameTable, - TSharedRange<TVersionedRow> rows, - const TModifyRowsOptions& options) -{ - std::vector<TRowModification> modifications; - modifications.reserve(rows.Size()); - - for (auto row : rows) { - modifications.push_back({ERowModificationType::VersionedWrite, row.ToTypeErasedRow(), TLockMask()}); - } - - ModifyRows( - path, - std::move(nameTable), - MakeSharedRange(std::move(modifications), std::move(rows.ReleaseHolder())), - options); -} - -void ITransaction::DeleteRows( - const NYPath::TYPath& path, - TNameTablePtr nameTable, - TSharedRange<TLegacyKey> keys, - const TModifyRowsOptions& options) -{ - std::vector<TRowModification> modifications; - modifications.reserve(keys.Size()); - for (auto key : keys) { - modifications.push_back({ERowModificationType::Delete, key.ToTypeErasedRow(), TLockMask()}); - } - - ModifyRows( - path, - std::move(nameTable), - MakeSharedRange(std::move(modifications), std::move(keys.ReleaseHolder())), - options); -} - -//////////////////////////////////////////////////////////////////////////////// - -void ITransaction::LockRows( - const NYPath::TYPath& path, - TNameTablePtr nameTable, - TSharedRange<TLegacyKey> keys, - TLockMask lockMask) -{ - std::vector<TRowModification> modifications; - modifications.reserve(keys.Size()); - - for (auto key : keys) { - TRowModification modification; - modification.Type = ERowModificationType::WriteAndLock; - modification.Row = key.ToTypeErasedRow(); - modification.Locks = lockMask; - modifications.push_back(modification); - } - - ModifyRows( - path, - std::move(nameTable), - MakeSharedRange(std::move(modifications), std::move(keys)), - TModifyRowsOptions()); -} - -void ITransaction::LockRows( - const NYPath::TYPath& path, - TNameTablePtr nameTable, - TSharedRange<TLegacyKey> keys, - ELockType lockType) -{ - TLockMask lockMask; - lockMask.Set(PrimaryLockIndex, lockType); - LockRows(path, nameTable, keys, lockMask); -} - -void ITransaction::LockRows( - const NYPath::TYPath& path, - TNameTablePtr nameTable, - TSharedRange<TLegacyKey> keys, - const std::vector<TString>& locks, - ELockType lockType) -{ - const auto& tableMountCache = GetClient()->GetTableMountCache(); - auto tableInfo = WaitFor(tableMountCache->GetTableInfo(path)) - .ValueOrThrow(); - - auto lockMask = GetLockMask( - *tableInfo->Schemas[ETableSchemaKind::Write], - GetAtomicity() == NTransactionClient::EAtomicity::Full, - locks, - lockType); - - LockRows(path, nameTable, keys, lockMask); -} - -void ITransaction::AdvanceConsumer( - const NYPath::TYPath& path, - int partitionIndex, - std::optional<i64> oldOffset, - i64 newOffset) -{ - THROW_ERROR_EXCEPTION_IF(newOffset < 0, "Queue consumer offset %v cannot be negative", newOffset); - - auto consumerClient = CreateBigRTConsumerClient(GetClient(), path); - consumerClient->Advance(MakeStrong(this), partitionIndex, oldOffset, newOffset); -} - -void ITransaction::AdvanceConsumer( - const NYPath::TRichYPath& consumerPath, - const NYPath::TRichYPath& queuePath, - int partitionIndex, - std::optional<i64> oldOffset, - i64 newOffset) -{ - THROW_ERROR_EXCEPTION_IF(newOffset < 0, "Queue consumer offset %v cannot be negative", newOffset); - - auto tableMountCache = GetClient()->GetTableMountCache(); - auto queuePhysicalPath = queuePath; - auto queueTableInfoOrError = WaitFor(tableMountCache->GetTableInfo(queuePath.GetPath())); - if (queueTableInfoOrError.IsOK()) { - queuePhysicalPath = NYPath::TRichYPath(queueTableInfoOrError.Value()->PhysicalPath, queuePath.Attributes()); - } - - // TODO(achulkov2): Support consumers from any cluster. - auto subConsumerClient = CreateSubConsumerClient(GetClient(), /*queueClient*/ nullptr, consumerPath.GetPath(), queuePhysicalPath); - return subConsumerClient->Advance(MakeStrong(this), partitionIndex, oldOffset, newOffset); -} - -//////////////////////////////////////////////////////////////////////////////// - TFuture<ITransactionPtr> StartAlienTransaction( const ITransactionPtr& localTransaction, const IClientPtr& alienClient, diff --git a/yt/yt/client/api/transaction.h b/yt/yt/client/api/transaction.h index 3fb39fa740..1f6eb795d1 100644 --- a/yt/yt/client/api/transaction.h +++ b/yt/yt/client/api/transaction.h @@ -1,6 +1,8 @@ #pragma once #include "client.h" +#include "dynamic_table_transaction.h" +#include "queue_transaction.h" #include <yt/yt/client/table_client/unversioned_row.h> #include <yt/yt/client/table_client/versioned_row.h> @@ -94,54 +96,6 @@ struct TTransactionFlushResult std::vector<NElection::TCellId> ParticipantCellIds; }; -//! Either a write or delete. -struct TRowModification -{ - //! Discriminates between writes and deletes. - ERowModificationType Type; - //! Either a row (for write; versioned or unversioned) or a key (for delete; always unversioned). - NTableClient::TTypeErasedRow Row; - //! Locks. - NTableClient::TLockMask Locks; -}; - -struct TModifyRowsOptions -{ - //! If this happens to be a modification of a replicated table, - //! controls if at least one sync replica is required. - bool RequireSyncReplica = true; - - //! For chaos replicated tables indicates if it is necessary to explore other replicas. - bool TopmostTransaction = true; - - //! For chaos replicas pass replication card to ensure that all data is sent using same meta info. - NChaosClient::TReplicationCardPtr ReplicationCard; - - //! For writes to replicas, this is the id of the replica at the upstream cluster. - NTabletClient::TTableReplicaId UpstreamReplicaId; - - //! Modifications are sent asynchronously. Sequential numbering is - //! required to restore their order. - //! - //! This parameter is only used by native client (in particular within RPC proxy server). - std::optional<i64> SequenceNumber; - - //! Modifications can be sent from several sources in case of several clients - //! attached to the same transaction. - //! - //! Modifications within one source will be serialized by this source sequence numbers. - //! Modifications from different sources will be serialized arbitrarily, that is why - //! different sources must send independent modifications. - //! - //! If sequence number is missing, source id is ignored. - //! - //! This parameter is only used by native client (in particular within RPC proxy server). - i64 SequenceNumberSourceId = 0; - - //! If set treat missing key columns as null. - bool AllowMissingKeyColumns = false; -}; - //////////////////////////////////////////////////////////////////////////////// //! Represents a client-controlled transaction. @@ -156,6 +110,8 @@ struct TModifyRowsOptions */ struct ITransaction : public virtual IClientBase + , public virtual IDynamicTableTransaction + , public virtual IQueueTransaction { virtual IClientPtr GetClient() const = 0; virtual NTransactionClient::ETransactionType GetType() const = 0; @@ -191,81 +147,6 @@ struct ITransaction const TDerivedTransaction* As() const; template <class TDerivedTransaction> const TDerivedTransaction* TryAs() const; - - // Tables. - - void WriteRows( - const NYPath::TYPath& path, - NTableClient::TNameTablePtr nameTable, - TSharedRange<NTableClient::TUnversionedRow> rows, - const TModifyRowsOptions& options = {}, - NTableClient::ELockType lockType = NTableClient::ELockType::Exclusive); - - void WriteRows( - const NYPath::TYPath& path, - NTableClient::TNameTablePtr nameTable, - TSharedRange<NTableClient::TVersionedRow> rows, - const TModifyRowsOptions& options = {}); - - void DeleteRows( - const NYPath::TYPath& path, - NTableClient::TNameTablePtr nameTable, - TSharedRange<NTableClient::TLegacyKey> keys, - const TModifyRowsOptions& options = {}); - - void LockRows( - const NYPath::TYPath& path, - NTableClient::TNameTablePtr nameTable, - TSharedRange<NTableClient::TLegacyKey> keys, - NTableClient::TLockMask lockMask); - - void LockRows( - const NYPath::TYPath& path, - NTableClient::TNameTablePtr nameTable, - TSharedRange<NTableClient::TLegacyKey> keys, - NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong); - - void LockRows( - const NYPath::TYPath& path, - NTableClient::TNameTablePtr nameTable, - TSharedRange<NTableClient::TLegacyKey> keys, - const std::vector<TString>& locks, - NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong); - - virtual void ModifyRows( - const NYPath::TYPath& path, - NTableClient::TNameTablePtr nameTable, - TSharedRange<TRowModification> modifications, - const TModifyRowsOptions& options = TModifyRowsOptions()) = 0; - - // Consumers. - - // TODO(nadya73): Remove it: YT-20712 - void AdvanceConsumer( - const NYPath::TYPath& path, - int partitionIndex, - std::optional<i64> oldOffset, - i64 newOffset); - - // TODO(nadya73): Remove it: YT-20712 - void AdvanceConsumer( - const NYPath::TRichYPath& consumerPath, - const NYPath::TRichYPath& queuePath, - int partitionIndex, - std::optional<i64> oldOffset, - i64 newOffset); - - //! Advance the consumer's offset for the given partition with index #partitionIndex, setting it to #newOffset. - //! - //! If #oldOffset is specified, the current offset is read inside this transaction and compared with #oldOffset. - //! If they are equal, the new offset is written, otherwise an exception is thrown. - virtual TFuture<void> AdvanceConsumer( - const NYPath::TRichYPath& consumerPath, - const NYPath::TRichYPath& queuePath, - int partitionIndex, - std::optional<i64> oldOffset, - i64 newOffset, - const TAdvanceConsumerOptions& options) = 0; }; DEFINE_REFCOUNTED_TYPE(ITransaction) diff --git a/yt/yt/client/api/transaction_client.h b/yt/yt/client/api/transaction_client.h index 272223886a..7b22f1e6d0 100644 --- a/yt/yt/client/api/transaction_client.h +++ b/yt/yt/client/api/transaction_client.h @@ -74,6 +74,8 @@ struct TTransactionAttachOptions struct ITransactionClientBase { + virtual ~ITransactionClientBase() = default; + virtual TFuture<ITransactionPtr> StartTransaction( NTransactionClient::ETransactionType type, const TTransactionStartOptions& options = {}) = 0; @@ -83,6 +85,8 @@ struct ITransactionClientBase struct ITransactionClient { + virtual ~ITransactionClient() = default; + virtual ITransactionPtr AttachTransaction( NTransactionClient::TTransactionId transactionId, const TTransactionAttachOptions& options = {}) = 0; diff --git a/yt/yt/client/driver/table_commands.cpp b/yt/yt/client/driver/table_commands.cpp index 5f474f6fbc..c4ba308c67 100644 --- a/yt/yt/client/driver/table_commands.cpp +++ b/yt/yt/client/driver/table_commands.cpp @@ -9,6 +9,7 @@ #include <yt/yt/client/table_client/adapters.h> #include <yt/yt/client/table_client/blob_reader.h> +#include <yt/yt/client/table_client/columnar_statistics.h> #include <yt/yt/client/table_client/row_buffer.h> #include <yt/yt/client/table_client/table_consumer.h> #include <yt/yt/client/table_client/table_output.h> diff --git a/yt/yt/client/federated/client.cpp b/yt/yt/client/federated/client.cpp index 992331b41e..3ec3fc38dd 100644 --- a/yt/yt/client/federated/client.cpp +++ b/yt/yt/client/federated/client.cpp @@ -5,6 +5,8 @@ #include <yt/yt/client/api/client.h> #include <yt/yt/client/api/transaction.h> +#include <yt/yt/client/api/dynamic_table_transaction_mixin.h> +#include <yt/yt/client/api/queue_transaction_mixin.h> #include <yt/yt/client/misc/method_helpers.h> @@ -52,7 +54,9 @@ std::optional<TString> GetDataCenterByClient(const IClientPtr& client) } class TTransaction - : public ITransaction + : public virtual ITransaction + , public TDynamicTableTransactionMixin + , public TQueueTransactionMixin { public: TTransaction(TClientPtr client, int clientIndex, ITransactionPtr underlying); @@ -77,6 +81,7 @@ public: TSharedRange<TRowModification> modifications, const TModifyRowsOptions& options) override; + using TQueueTransactionMixin::AdvanceConsumer; TFuture<void> AdvanceConsumer( const NYPath::TRichYPath& consumerPath, const NYPath::TRichYPath& queuePath, @@ -98,7 +103,7 @@ public: const TSharedRange<NTableClient::TUnversionedRow>&, const TVersionedLookupRowsOptions&) override; - TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookup( + TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookupRows( const std::vector<TMultiLookupSubrequest>&, const TMultiLookupOptions&) override; @@ -247,7 +252,7 @@ public: TFuture<TSelectRowsResult> SelectRows( const TString& query, const TSelectRowsOptions& options = {}) override; - TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookup( + TFuture<std::vector<TUnversionedLookupRowsResult>> MultiLookupRows( const std::vector<TMultiLookupSubrequest>&, const TMultiLookupOptions&) override; TFuture<TVersionedLookupRowsResult> VersionedLookupRows( @@ -262,7 +267,6 @@ public: int, const NQueueClient::TQueueRowBatchReadOptions&, const TPullQueueOptions&) override; - TFuture<NQueueClient::IQueueRowsetPtr> PullConsumer( const NYPath::TRichYPath&, const NYPath::TRichYPath&, @@ -484,7 +488,7 @@ TRANSACTION_METHOD_IMPL(void, Ping, (const NApi::TTransactionPingOptions&)); TRANSACTION_METHOD_IMPL(TTransactionCommitResult, Commit, (const TTransactionCommitOptions&)); TRANSACTION_METHOD_IMPL(void, Abort, (const TTransactionAbortOptions&)); TRANSACTION_METHOD_IMPL(TVersionedLookupRowsResult, VersionedLookupRows, (const NYPath::TYPath&, NTableClient::TNameTablePtr, const TSharedRange<NTableClient::TUnversionedRow>&, const TVersionedLookupRowsOptions&)); -TRANSACTION_METHOD_IMPL(std::vector<TUnversionedLookupRowsResult>, MultiLookup, (const std::vector<TMultiLookupSubrequest>&, const TMultiLookupOptions&)); +TRANSACTION_METHOD_IMPL(std::vector<TUnversionedLookupRowsResult>, MultiLookupRows, (const std::vector<TMultiLookupSubrequest>&, const TMultiLookupOptions&)); TRANSACTION_METHOD_IMPL(TPullRowsResult, PullRows, (const NYPath::TYPath&, const TPullRowsOptions&)); TRANSACTION_METHOD_IMPL(void, AdvanceConsumer, (const NYPath::TRichYPath&, const NYPath::TRichYPath&, int, std::optional<i64>, i64, const TAdvanceConsumerOptions&)); TRANSACTION_METHOD_IMPL(NYson::TYsonString, ExplainQuery, (const TString&, const TExplainQueryOptions&)); @@ -643,7 +647,7 @@ TFuture<ResultType> TClient::MethodName(Y_METHOD_USED_ARGS_DECLARATION(Args)) CLIENT_METHOD_IMPL(TUnversionedLookupRowsResult, LookupRows, (const NYPath::TYPath&, NTableClient::TNameTablePtr, const TSharedRange<NTableClient::TLegacyKey>&, const TLookupRowsOptions&)); CLIENT_METHOD_IMPL(TSelectRowsResult, SelectRows, (const TString&, const TSelectRowsOptions&)); -CLIENT_METHOD_IMPL(std::vector<TUnversionedLookupRowsResult>, MultiLookup, (const std::vector<TMultiLookupSubrequest>&, const TMultiLookupOptions&)); +CLIENT_METHOD_IMPL(std::vector<TUnversionedLookupRowsResult>, MultiLookupRows, (const std::vector<TMultiLookupSubrequest>&, const TMultiLookupOptions&)); CLIENT_METHOD_IMPL(TVersionedLookupRowsResult, VersionedLookupRows, (const NYPath::TYPath&, NTableClient::TNameTablePtr, const TSharedRange<NTableClient::TUnversionedRow>&, const TVersionedLookupRowsOptions&)); CLIENT_METHOD_IMPL(TPullRowsResult, PullRows, (const NYPath::TYPath&, const TPullRowsOptions&)); CLIENT_METHOD_IMPL(NQueueClient::IQueueRowsetPtr, PullQueue, (const NYPath::TRichYPath&, i64, int, const NQueueClient::TQueueRowBatchReadOptions&, const TPullQueueOptions&)); diff --git a/yt/yt/client/hedging/hedging.cpp b/yt/yt/client/hedging/hedging.cpp index c0866b962e..349545115e 100644 --- a/yt/yt/client/hedging/hedging.cpp +++ b/yt/yt/client/hedging/hedging.cpp @@ -88,7 +88,7 @@ public: RETRYABLE_METHOD(TFuture<NYson::TYsonString>, ListNode, (const NYPath::TYPath&, const TListNodeOptions&)); RETRYABLE_METHOD(TFuture<bool>, NodeExists, (const NYPath::TYPath&, const TNodeExistsOptions&)); RETRYABLE_METHOD(TFuture<IFileReaderPtr>, CreateFileReader, (const NYPath::TYPath&, const TFileReaderOptions&)); - RETRYABLE_METHOD(TFuture<std::vector<TUnversionedLookupRowsResult>>, MultiLookup, (const std::vector<TMultiLookupSubrequest>&, const TMultiLookupOptions&)); + RETRYABLE_METHOD(TFuture<std::vector<TUnversionedLookupRowsResult>>, MultiLookupRows, (const std::vector<TMultiLookupSubrequest>&, const TMultiLookupOptions&)); // Unsupported methods. UNSUPPORTED_METHOD(TFuture<ITransactionPtr>, StartTransaction, (NTransactionClient::ETransactionType, const TTransactionStartOptions&)); diff --git a/yt/yt/client/unittests/mock/client.h b/yt/yt/client/unittests/mock/client.h index 87d4752c03..f48882e4fa 100644 --- a/yt/yt/client/unittests/mock/client.h +++ b/yt/yt/client/unittests/mock/client.h @@ -47,7 +47,7 @@ public: const TSharedRange<NTableClient::TLegacyKey>& keys, const TVersionedLookupRowsOptions& options), (override)); - MOCK_METHOD(TFuture<std::vector<TUnversionedLookupRowsResult>>, MultiLookup, ( + MOCK_METHOD(TFuture<std::vector<TUnversionedLookupRowsResult>>, MultiLookupRows, ( const std::vector<TMultiLookupSubrequest>& subrequests, const TMultiLookupOptions& options), (override)); diff --git a/yt/yt/client/unittests/mock/transaction.h b/yt/yt/client/unittests/mock/transaction.h index d8380bdd61..6b3ba70b77 100644 --- a/yt/yt/client/unittests/mock/transaction.h +++ b/yt/yt/client/unittests/mock/transaction.h @@ -4,6 +4,8 @@ #include <yt/yt/client/api/journal_reader.h> #include <yt/yt/client/api/journal_writer.h> #include <yt/yt/client/api/transaction.h> +#include <yt/yt/client/api/dynamic_table_transaction_mixin.h> +#include <yt/yt/client/api/queue_transaction_mixin.h> #include <library/cpp/testing/gtest_extensions/gtest_extensions.h> @@ -12,7 +14,9 @@ namespace NYT::NApi { //////////////////////////////////////////////////////////////////////////////// class TMockTransaction - : public ITransaction + : public virtual ITransaction + , public TDynamicTableTransactionMixin + , public TQueueTransactionMixin { public: MOCK_METHOD(IConnectionPtr, GetConnection, (), (override)); @@ -33,7 +37,7 @@ public: const TSharedRange<NTableClient::TLegacyKey>& keys, const TVersionedLookupRowsOptions& options), (override)); - MOCK_METHOD(TFuture<std::vector<TUnversionedLookupRowsResult>>, MultiLookup, ( + MOCK_METHOD(TFuture<std::vector<TUnversionedLookupRowsResult>>, MultiLookupRows, ( const std::vector<TMultiLookupSubrequest>& subrequests, const TMultiLookupOptions& options), (override)); @@ -195,6 +199,7 @@ public: MOCK_METHOD(void, SubscribeAborted, (const TAbortedHandler& callback), (override)); MOCK_METHOD(void, UnsubscribeAborted, (const TAbortedHandler& callback), (override)); + using TQueueTransactionMixin::AdvanceConsumer; MOCK_METHOD(TFuture<void>, AdvanceConsumer, ( const NYPath::TRichYPath& consumerPath, const NYPath::TRichYPath& queuePath, diff --git a/yt/yt/client/ya.make b/yt/yt/client/ya.make index 3acca6bfa2..641db7d7e4 100644 --- a/yt/yt/client/ya.make +++ b/yt/yt/client/ya.make @@ -12,6 +12,7 @@ SRCS( api/client_common.cpp api/client_cache.cpp api/delegating_client.cpp + api/delegating_transaction.cpp api/etc_client.cpp api/journal_client.cpp api/operation_client.cpp @@ -25,6 +26,8 @@ SRCS( api/rowset.cpp api/skynet.cpp api/transaction.cpp + api/queue_transaction_mixin.cpp + api/dynamic_table_transaction_mixin.cpp api/persistent_queue.cpp api/sticky_transaction_pool.cpp diff --git a/yt/yt/core/misc/atomic_ptr-inl.h b/yt/yt/core/misc/atomic_ptr-inl.h index f0d4477ef2..6d57b1e4c7 100644 --- a/yt/yt/core/misc/atomic_ptr-inl.h +++ b/yt/yt/core/misc/atomic_ptr-inl.h @@ -9,8 +9,10 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// +namespace NDetail { + template <class T> -TIntrusivePtr<T> MakeStrong(const THazardPtr<T>& ptr) +TIntrusivePtr<T> TryMakeStrongFromHazard(const THazardPtr<T>& ptr) { if (!ptr) { return nullptr; @@ -23,9 +25,11 @@ TIntrusivePtr<T> MakeStrong(const THazardPtr<T>& ptr) return nullptr; } - return TIntrusivePtr<T>(ptr.Get(), false); + return TIntrusivePtr<T>(ptr.Get(), /*addReference*/ false); } +} // namespace NDetail + //////////////////////////////////////////////////////////////////////////////// template <class T, bool EnableAcquireHazard> @@ -107,14 +111,14 @@ THazardPtr<T> TAtomicPtr<T, EnableAcquireHazard>::DoAcquireHazard() const template <class T, bool EnableAcquireHazard> TIntrusivePtr<T> TAtomicPtr<T, EnableAcquireHazard>::AcquireWeak() const { - return MakeStrong(DoAcquireHazard()); + return NYT::NDetail::TryMakeStrongFromHazard(DoAcquireHazard()); } template <class T, bool EnableAcquireHazard> TIntrusivePtr<T> TAtomicPtr<T, EnableAcquireHazard>::Acquire() const { while (auto hazardPtr = DoAcquireHazard()) { - if (auto ptr = MakeStrong(hazardPtr)) { + if (auto ptr = NYT::NDetail::TryMakeStrongFromHazard(hazardPtr)) { return ptr; } } diff --git a/yt/yt/core/misc/atomic_ptr.h b/yt/yt/core/misc/atomic_ptr.h index 4cd9375bbd..fe15898273 100644 --- a/yt/yt/core/misc/atomic_ptr.h +++ b/yt/yt/core/misc/atomic_ptr.h @@ -8,12 +8,6 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -// Overload for TIntrusivePtr<T> MakeStrong(T* p). -template <class T> -TIntrusivePtr<T> MakeStrong(const THazardPtr<T>& ptr); - -//////////////////////////////////////////////////////////////////////////////// - //! Holds an atomic pointer to an instance of ref-counted type |T| enabling concurrent //! read and write access. template <class T, bool EnableAcquireHazard = false> diff --git a/yt/yt/core/misc/unittests/hazard_ptr_ut.cpp b/yt/yt/core/misc/unittests/hazard_ptr_ut.cpp index de6cb2cd2a..388a754d8c 100644 --- a/yt/yt/core/misc/unittests/hazard_ptr_ut.cpp +++ b/yt/yt/core/misc/unittests/hazard_ptr_ut.cpp @@ -137,7 +137,7 @@ TEST_F(THazardPtrTest, DelayedDeallocation) EXPECT_STREQ("AC!D", output.Str().c_str()); EXPECT_TRUE(hazardPtr); - EXPECT_FALSE(MakeStrong(hazardPtr)); + EXPECT_FALSE(NDetail::TryMakeStrongFromHazard(hazardPtr)); ReclaimHazardPointers(/*flush*/ false); @@ -170,7 +170,7 @@ TEST_F(THazardPtrTest, DelayedDeallocationWithMultipleHPs) EXPECT_STREQ("AC!D", output.Str().c_str()); EXPECT_TRUE(hazardPtr1); - EXPECT_FALSE(MakeStrong(hazardPtr1)); + EXPECT_FALSE(NDetail::TryMakeStrongFromHazard(hazardPtr1)); ReclaimHazardPointers(/*flush*/ false); diff --git a/yt/yt/core/rpc/throttling_channel.cpp b/yt/yt/core/rpc/throttling_channel.cpp index 6333c0d4a1..faac72bd98 100644 --- a/yt/yt/core/rpc/throttling_channel.cpp +++ b/yt/yt/core/rpc/throttling_channel.cpp @@ -19,11 +19,14 @@ class TThrottlingChannel public: TThrottlingChannel( TThrottlingChannelConfigPtr config, - IChannelPtr underlyingChannel) + IChannelPtr underlyingChannel, + NProfiling::TProfiler profiler) : TChannelWrapper(std::move(underlyingChannel)) , Config_(std::move(config)) - , Throttler_(CreateReconfigurableThroughputThrottler(TThroughputThrottlerConfig::Create( - Config_->RateLimit))) + , Throttler_(CreateReconfigurableThroughputThrottler( + TThroughputThrottlerConfig::Create(Config_->RateLimit), + /*logger*/ {}, + std::move(profiler))) { } IClientRequestControlPtr Send( @@ -74,14 +77,16 @@ private: IThrottlingChannelPtr CreateThrottlingChannel( TThrottlingChannelConfigPtr config, - IChannelPtr underlyingChannel) + IChannelPtr underlyingChannel, + NProfiling::TProfiler profiler) { YT_VERIFY(config); YT_VERIFY(underlyingChannel); return New<TThrottlingChannel>( std::move(config), - std::move(underlyingChannel)); + std::move(underlyingChannel), + std::move(profiler)); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/rpc/throttling_channel.h b/yt/yt/core/rpc/throttling_channel.h index 153c3c31fe..7df85eb582 100644 --- a/yt/yt/core/rpc/throttling_channel.h +++ b/yt/yt/core/rpc/throttling_channel.h @@ -2,6 +2,8 @@ #include "channel.h" +#include <yt/yt/library/profiling/sensor.h> + namespace NYT::NRpc { //////////////////////////////////////////////////////////////////////////////// @@ -14,10 +16,13 @@ struct IThrottlingChannel DEFINE_REFCOUNTED_TYPE(IThrottlingChannel) +//////////////////////////////////////////////////////////////////////////////// + //! Constructs a channel that limits request rate to the underlying channel. IThrottlingChannelPtr CreateThrottlingChannel( TThrottlingChannelConfigPtr config, - IChannelPtr underlyingChannel); + IChannelPtr underlyingChannel, + NProfiling::TProfiler profiler = {}); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/yson/ypath_designated_consumer.cpp b/yt/yt/core/yson/ypath_designated_consumer.cpp index 7cb13fab1f..ae08adf83b 100644 --- a/yt/yt/core/yson/ypath_designated_consumer.cpp +++ b/yt/yt/core/yson/ypath_designated_consumer.cpp @@ -87,9 +87,10 @@ public: } else { SkipSubTree(); } - break; - default: - Tokenizer_.ThrowUnexpected(); + break; + + default: + Tokenizer_.ThrowUnexpected(); } } diff --git a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp index 2bebadb4b6..14d2ad172b 100644 --- a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp +++ b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp @@ -1662,7 +1662,6 @@ public: } }; - TEST(TYsonStructTest, TestOptionalNoInit) { TTestOptionalNoInit x; diff --git a/yt/yt/core/ytree/yson_struct_detail.cpp b/yt/yt/core/ytree/yson_struct_detail.cpp index c396e4d621..c340756dea 100644 --- a/yt/yt/core/ytree/yson_struct_detail.cpp +++ b/yt/yt/core/ytree/yson_struct_detail.cpp @@ -4,6 +4,8 @@ namespace NYT::NYTree { +using namespace NYPath; + //////////////////////////////////////////////////////////////////////////////// std::optional<EUnrecognizedStrategy> GetRecursiveUnrecognizedStrategy(EUnrecognizedStrategy strategy) @@ -94,7 +96,7 @@ void TYsonStructMeta::LoadParameter(TYsonStructBase* target, const TString& key, void TYsonStructMeta::Postprocess(TYsonStructBase* target, const TYPath& path) const { for (const auto& [name, parameter] : Parameters_) { - parameter->Postprocess(target, path + "/" + name); + parameter->Postprocess(target, path + "/" + ToYPathLiteral(name)); } try { @@ -140,7 +142,7 @@ void TYsonStructMeta::LoadStruct( } } auto loadOptions = TLoadParameterOptions{ - .Path = path + "/" + key, + .Path = path + "/" + ToYPathLiteral(key), .RecursiveUnrecognizedRecursively = GetRecursiveUnrecognizedStrategy(unrecognizedStrategy), }; parameter->Load(target, child, loadOptions); @@ -154,7 +156,7 @@ void TYsonStructMeta::LoadStruct( for (const auto& [key, child] : mapNode->GetChildren()) { if (!registeredKeys.contains(key)) { if (ShouldThrow(unrecognizedStrategy)) { - THROW_ERROR_EXCEPTION("Unrecognized field %Qv has been encountered", path + "/" + key) + THROW_ERROR_EXCEPTION("Unrecognized field %Qv has been encountered", path + "/" + ToYPathLiteral(key)) << TErrorAttribute("key", key) << TErrorAttribute("path", path); } @@ -187,7 +189,7 @@ void TYsonStructMeta::LoadStruct( auto createLoadOptions = [&] (TStringBuf key) { return TLoadParameterOptions{ - .Path = path + "/" + key, + .Path = path + "/" + ToYPathLiteral(key), .RecursiveUnrecognizedRecursively = GetRecursiveUnrecognizedStrategy(unrecognizedStrategy), }; }; @@ -242,7 +244,7 @@ void TYsonStructMeta::LoadStruct( return; } if (ShouldThrow(unrecognizedStrategy)) { - THROW_ERROR_EXCEPTION("Unrecognized field %Qv has been encountered", path + "/" + key) + THROW_ERROR_EXCEPTION("Unrecognized field %Qv has been encountered", path + "/" + ToYPathLiteral(key)) << TErrorAttribute("key", key) << TErrorAttribute("path", path); } diff --git a/yt/yt/library/query/base/public.h b/yt/yt/library/query/base/public.h deleted file mode 100644 index 81e384a4af..0000000000 --- a/yt/yt/library/query/base/public.h +++ /dev/null @@ -1,153 +0,0 @@ -#pragma once - -#include <yt/yt/client/query_client/public.h> - -#include <yt/yt/client/transaction_client/public.h> - -#include <yt/yt/client/table_client/public.h> - -#include <yt/yt/core/ypath/public.h> - -namespace NYT::NQueryClient { - -using NTransactionClient::TTimestamp; - -using NTableClient::TRowRange; - -using TReadSessionId = TGuid; - -struct TDataSplit; - -//////////////////////////////////////////////////////////////////////////////// - -namespace NProto { - -class TColumnDescriptor; -class TExpression; -class TGroupClause; -class TProjectClause; -class TWhenThenExpression; -class TJoinClause; -class TQuery; -class TQueryOptions; -class TDataSource; - -} // namespace NProto - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_STRUCT(TExpression) -using TConstExpressionPtr = TIntrusivePtr<const TExpression>; - -DECLARE_REFCOUNTED_STRUCT(TFunctionExpression) -using TConstFunctionExpressionPtr = TIntrusivePtr<const TFunctionExpression>; - -DECLARE_REFCOUNTED_STRUCT(TAggregateFunctionExpression) -using TConstAggregateFunctionExpressionPtr = TIntrusivePtr<const TAggregateFunctionExpression>; - -DECLARE_REFCOUNTED_STRUCT(TJoinClause) -using TConstJoinClausePtr = TIntrusivePtr<const TJoinClause>; - -DECLARE_REFCOUNTED_STRUCT(TGroupClause) -using TConstGroupClausePtr = TIntrusivePtr<const TGroupClause>; - -DECLARE_REFCOUNTED_STRUCT(TOrderClause) -using TConstOrderClausePtr = TIntrusivePtr<const TOrderClause>; - -DECLARE_REFCOUNTED_STRUCT(TProjectClause) -using TConstProjectClausePtr = TIntrusivePtr<const TProjectClause>; - -DECLARE_REFCOUNTED_STRUCT(TWhenThenExpression) -using TConstWhenThenExpressionPtr = TIntrusivePtr<const TWhenThenExpression>; - -DECLARE_REFCOUNTED_STRUCT(TBaseQuery) -using TConstBaseQueryPtr = TIntrusivePtr<const TBaseQuery>; - -DECLARE_REFCOUNTED_STRUCT(TFrontQuery) -using TConstFrontQueryPtr = TIntrusivePtr<const TFrontQuery>; - -DECLARE_REFCOUNTED_STRUCT(TQuery) -using TConstQueryPtr = TIntrusivePtr<const TQuery>; - -struct IPrepareCallbacks; - -struct TQueryStatistics; - -struct TQueryOptions; - -DECLARE_REFCOUNTED_STRUCT(IAggregateFunctionDescriptor) - -DECLARE_REFCOUNTED_STRUCT(ICallingConvention) - -DECLARE_REFCOUNTED_STRUCT(IExecutor) - -DECLARE_REFCOUNTED_STRUCT(IEvaluator) - -DECLARE_REFCOUNTED_CLASS(TExecutorConfig) - -DECLARE_REFCOUNTED_CLASS(TColumnEvaluator) - -DECLARE_REFCOUNTED_STRUCT(IColumnEvaluatorCache) - -DECLARE_REFCOUNTED_CLASS(TColumnEvaluatorCacheConfig) -DECLARE_REFCOUNTED_CLASS(TColumnEvaluatorCacheDynamicConfig) - -DECLARE_REFCOUNTED_STRUCT(TExternalCGInfo) -using TConstExternalCGInfoPtr = TIntrusivePtr<const TExternalCGInfo>; - -DECLARE_REFCOUNTED_STRUCT(TTypeInferrerMap) -using TConstTypeInferrerMapPtr = TIntrusivePtr<const TTypeInferrerMap>; - -const TConstTypeInferrerMapPtr GetBuiltinTypeInferrers(); - -DECLARE_REFCOUNTED_STRUCT(IFunctionRegistry) -DECLARE_REFCOUNTED_CLASS(ITypeInferrer) - -DECLARE_REFCOUNTED_CLASS(TFunctionImplCache) - -using NTableClient::ISchemafulUnversionedReader; -using NTableClient::ISchemafulUnversionedReaderPtr; -using NTableClient::ISchemalessUnversionedReader; -using NTableClient::ISchemalessUnversionedReaderPtr; -using NTableClient::IUnversionedRowsetWriter; -using NTableClient::IUnversionedRowsetWriterPtr; -using NTableClient::EValueType; -using NTableClient::TTableSchema; -using NTableClient::TTableSchemaPtr; -using NTableClient::TColumnSchema; -using NTableClient::TKeyColumns; -using NTableClient::TColumnFilter; -using NTableClient::TRowRange; - -using NTransactionClient::TTimestamp; -using NTransactionClient::NullTimestamp; - -using NTableClient::TRowBuffer; -using NTableClient::TRowBufferPtr; - -using TSchemaColumns = std::vector<NTableClient::TColumnSchema>; - -using TRow = NTableClient::TUnversionedRow; -using TMutableRow = NTableClient::TMutableUnversionedRow; -using TRowHeader = NTableClient::TUnversionedRowHeader; -using TRowBuilder = NTableClient::TUnversionedRowBuilder; -using TOwningRow = NTableClient::TUnversionedOwningRow; -using TOwningRowBuilder = NTableClient::TUnversionedOwningRowBuilder; -using TValue = NTableClient::TUnversionedValue; -using TValueData = NTableClient::TUnversionedValueData; -using TOwningValue = NTableClient::TUnversionedOwningValue; -using TLegacyOwningKey = NTableClient::TLegacyOwningKey; - -using TKeyRange = std::pair<TLegacyOwningKey, TLegacyOwningKey>; -using TMutableRowRange = std::pair<TMutableRow, TMutableRow>; -using TRowRanges = std::vector<TRowRange>; -using TMutableRowRanges = std::vector<TMutableRowRange>; - -//////////////////////////////////////////////////////////////////////////////// - -extern const NYPath::TYPath QueryPoolsPath; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NQueryClient - |