diff options
| author | Alexander Smirnov <alex@ydb.tech> | 2025-05-23 00:51:59 +0000 |
|---|---|---|
| committer | Alexander Smirnov <alex@ydb.tech> | 2025-05-23 00:51:59 +0000 |
| commit | 95cb2090f734bfe45593f1b7b7bb4423bf42acb8 (patch) | |
| tree | 11f326cd782858f25df2064e027c3fccd3db7f6a | |
| parent | 778fd6216f12e7ca0a9c16e83522da6c7e121d40 (diff) | |
| parent | bb919ff615bff1f8a16a321843cd843497ae83d1 (diff) | |
| download | ydb-95cb2090f734bfe45593f1b7b7bb4423bf42acb8.tar.gz | |
Merge branch 'rightlib' into merge-libs-250523-0050
103 files changed, 1376 insertions, 486 deletions
diff --git a/build/export_generators/ide-gradle/common_vars.jinja b/build/export_generators/ide-gradle/common_vars.jinja index bb3cbc63a1e..92ae2bf8d14 100644 --- a/build/export_generators/ide-gradle/common_vars.jinja +++ b/build/export_generators/ide-gradle/common_vars.jinja @@ -17,9 +17,6 @@ {#- KAPT -#} {%- set with_kapt = target.with_kapt -%} {%- set with_test_kapt = extra_targets|selectattr('with_kapt', 'eq', true)|map(attribute='with_kapt')|length -%} -{#- WAIT ya-bin release, disable just now -#} -{%- set with_kapt = false -%} -{%- set with_test_kapt = false -%} {%- endif -%} {#- ErrorProne -#} diff --git a/build/export_generators/ide-gradle/gradle/wrapper/gradle-wrapper.properties b/build/export_generators/ide-gradle/gradle/wrapper/gradle-wrapper.properties index f2df14bc09c..38b02ac9670 100644 --- a/build/export_generators/ide-gradle/gradle/wrapper/gradle-wrapper.properties +++ b/build/export_generators/ide-gradle/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://bucket.yandex-team.ru/v1/maven/gradle-distributions/distributions/gradle-8.6-bin.zip +distributionUrl=https\://bucket.yandex-team.ru/v1/maven/gradle-distributions/distributions/gradle-8.14-bin.zip networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/build/export_generators/ide-gradle/kotlin_plugins.jinja b/build/export_generators/ide-gradle/kotlin_plugins.jinja index 1735112c30c..0a2e7cd48d2 100644 --- a/build/export_generators/ide-gradle/kotlin_plugins.jinja +++ b/build/export_generators/ide-gradle/kotlin_plugins.jinja @@ -59,7 +59,6 @@ noArg { {%- if with_kotlin %} kotlin { - kotlinDaemonJvmArgs = listOf("-Xmx2G") jvmToolchain({{ jdk_version }}) } {% endif -%} diff --git a/build/export_generators/ide-gradle/proto_plugins.jinja b/build/export_generators/ide-gradle/proto_plugins.jinja index 28b9910492a..8bce751c965 100644 --- a/build/export_generators/ide-gradle/proto_plugins.jinja +++ b/build/export_generators/ide-gradle/proto_plugins.jinja @@ -1,7 +1,7 @@ {#- empty string #} plugins { id("java-library") - id("com.google.protobuf") version "0.9.4" + id("com.google.protobuf") version "0.9.5" {%- if publish %} `maven-publish` `signing` diff --git a/build/external_resources/gdb/resources.json b/build/external_resources/gdb/resources.json index fa013ee2de6..ea1833d4ee2 100644 --- a/build/external_resources/gdb/resources.json +++ b/build/external_resources/gdb/resources.json @@ -7,10 +7,10 @@ "uri": "sbr:3833498694" }, "linux-aarch64": { - "uri": "sbr:8509776935" + "uri": "sbr:8784096409" }, "linux-x86_64": { - "uri": "sbr:8509757921" + "uri": "sbr:8784094118" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index be35db7cc6e..5ecd508a105 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -1104,6 +1104,7 @@ "6164166258": "{registry_endpoint}/6164166258", "6682389529": "{registry_endpoint}/6682389529", "6478436039": "{registry_endpoint}/6478436039", + "8784096409": "{registry_endpoint}/8784096409", "6406540582": "{registry_endpoint}/6406540582", "6447362348": "{registry_endpoint}/6447362348", "6133337898": "{registry_endpoint}/6133337898", @@ -1111,6 +1112,7 @@ "6164128408": "{registry_endpoint}/6164128408", "6682380912": "{registry_endpoint}/6682380912", "6478494211": "{registry_endpoint}/6478494211", + "8784094118": "{registry_endpoint}/8784094118", "6406437536": "{registry_endpoint}/6406437536", "6447316775": "{registry_endpoint}/6447316775", "6133419349": "{registry_endpoint}/6133419349", @@ -2524,6 +2526,7 @@ "6164166258": "gdb-14-linux-aarch64-2f60b923acb68d45b101313954b70779c13a19b8", "6682389529": "gdb-14-linux-aarch64-533b7767fda1a4ec616af8b575d10c4694ae1f2f", "6478436039": "gdb-14-linux-aarch64-67ef8a9f1868bf62959ac3a14d552999a108ed38", + "8784096409": "gdb-14-linux-aarch64-6ea280bfa165555cc850c4ee3d49f07745e6f21b", "6406540582": "gdb-14-linux-aarch64-9d91f66a1caff272af94e2fc97e690a18d910204", "6447362348": "gdb-14-linux-aarch64-9db71d8b25a56ee316036c53fd941934a95831a3", "6133337898": "gdb-14-linux-aarch64-b1fa9be28bbf4ee845d6a39a049c7b60018a3695", @@ -2531,6 +2534,7 @@ "6164128408": "gdb-14-linux-x86_64-2f60b923acb68d45b101313954b70779c13a19b8", "6682380912": "gdb-14-linux-x86_64-533b7767fda1a4ec616af8b575d10c4694ae1f2f", "6478494211": "gdb-14-linux-x86_64-67ef8a9f1868bf62959ac3a14d552999a108ed38", + "8784094118": "gdb-14-linux-x86_64-6ea280bfa165555cc850c4ee3d49f07745e6f21b", "6406437536": "gdb-14-linux-x86_64-9d91f66a1caff272af94e2fc97e690a18d910204", "6447316775": "gdb-14-linux-x86_64-9db71d8b25a56ee316036c53fd941934a95831a3", "6133419349": "gdb-14-linux-x86_64-b1fa9be28bbf4ee845d6a39a049c7b60018a3695", diff --git a/build/platform/yfm/resources.json b/build/platform/yfm/resources.json index b9e9aedeb45..a27dd38d54e 100644 --- a/build/platform/yfm/resources.json +++ b/build/platform/yfm/resources.json @@ -1,16 +1,16 @@ { "by_platform": { "win32-x86_64": { - "uri": "sbr:8714329658" + "uri": "sbr:8778298308" }, "darwin-x86_64": { - "uri": "sbr:8714328247" + "uri": "sbr:8778297068" }, "linux-x86_64": { - "uri": "sbr:8714326826" + "uri": "sbr:8778295807" }, "darwin-arm64": { - "uri": "sbr:8714328247" + "uri": "sbr:8778297068" } } } diff --git a/build/plugins/nots.py b/build/plugins/nots.py index 3fc54dde0f2..5f9c4480ac4 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -654,7 +654,6 @@ def _setup_tsc_typecheck(unit: NotsUnitType) -> None: unit.on_peerdir_ts_resource("typescript") user_recipes = unit.get("TEST_RECIPES_VALUE") unit.on_setup_install_node_modules_recipe() - unit.on_setup_extract_output_tars_recipe([unit.get("MODDIR")]) test_type = TsTestType.TSC_TYPECHECK @@ -707,7 +706,6 @@ def _setup_stylelint(unit: NotsUnitType) -> None: recipes_value = unit.get("TEST_RECIPES_VALUE") unit.on_setup_install_node_modules_recipe() - unit.on_setup_extract_output_tars_recipe([unit.get("MODDIR")]) test_type = TsTestType.TS_STYLELINT diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 9e94e66b827..1c4d6ce2047 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -13,7 +13,7 @@ CPP_FAKEID=2024-01-23 GO_FAKEID=11100371 ANDROID_FAKEID=2023-05-17 CLANG_TIDY_FAKEID=2023-06-06 -CYTHON_FAKEID=16584065 +CYTHON_FAKEID=16586418 JAVA_FAKEID=14386852 PROTO_FAKEID=0 FBS_FAKEID=2024-03-13 diff --git a/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym b/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym index 149266d27fc..7b0fcdf701b 100644 --- a/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym +++ b/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym @@ -1,6 +1,6 @@ {% extends '//builtin/bag.ym' %} -{% block current_version %}20.1.4{% endblock %} +{% block current_version %}20.1.5{% endblock %} {% block current_url %} https://github.com/llvm/llvm-project/releases/download/llvmorg-{{self.version().strip()}}/compiler-rt-{{self.version().strip()}}.src.tar.xz diff --git a/contrib/libs/cxxsupp/builtins/ya.make b/contrib/libs/cxxsupp/builtins/ya.make index 5335f7cc9fc..3b86670e076 100644 --- a/contrib/libs/cxxsupp/builtins/ya.make +++ b/contrib/libs/cxxsupp/builtins/ya.make @@ -12,9 +12,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20.1.4) +VERSION(20.1.5) -ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.4/compiler-rt-20.1.4.src.tar.xz) +ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.5/compiler-rt-20.1.5.src.tar.xz) NO_COMPILER_WARNINGS() diff --git a/contrib/libs/libfuzzer/.yandex_meta/override.nix b/contrib/libs/libfuzzer/.yandex_meta/override.nix index afac6b61fbd..38c1a7c3091 100644 --- a/contrib/libs/libfuzzer/.yandex_meta/override.nix +++ b/contrib/libs/libfuzzer/.yandex_meta/override.nix @@ -1,11 +1,11 @@ pkgs: attrs: with pkgs; with attrs; rec { - version = "20.1.4"; + version = "20.1.5"; src = fetchFromGitHub { owner = "llvm"; repo = "llvm-project"; rev = "llvmorg-${version}"; - hash = "sha256-/WomqG2DdnUHwlVsMfpzaK/dhGV3zychfU0wLmihQac="; + hash = "sha256-WKfY+VvAsZEEc0xYgF6+MsXDXZz7haMU6bxqmUpaHuQ="; }; sourceRoot = "source/compiler-rt"; diff --git a/contrib/libs/libfuzzer/lib/fuzzer/afl/ya.make b/contrib/libs/libfuzzer/lib/fuzzer/afl/ya.make index 11a466f2bb7..c5b7a537747 100644 --- a/contrib/libs/libfuzzer/lib/fuzzer/afl/ya.make +++ b/contrib/libs/libfuzzer/lib/fuzzer/afl/ya.make @@ -8,7 +8,7 @@ LICENSE_TEXTS(.yandex_meta/licenses.list.txt) SUBSCRIBER(g:cpp-contrib) -VERSION(20.1.4) +VERSION(20.1.5) PEERDIR( contrib/libs/afl/llvm_mode diff --git a/contrib/libs/libfuzzer/ya.make b/contrib/libs/libfuzzer/ya.make index 6084aae3b5b..f838cd2693f 100644 --- a/contrib/libs/libfuzzer/ya.make +++ b/contrib/libs/libfuzzer/ya.make @@ -12,9 +12,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20.1.4) +VERSION(20.1.5) -ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.4.tar.gz) +ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.5.tar.gz) SET(SANITIZER_CFLAGS) diff --git a/contrib/libs/libunwind/.yandex_meta/override.nix b/contrib/libs/libunwind/.yandex_meta/override.nix index 0cd3ba022d9..150f6a04f70 100644 --- a/contrib/libs/libunwind/.yandex_meta/override.nix +++ b/contrib/libs/libunwind/.yandex_meta/override.nix @@ -1,11 +1,11 @@ pkgs: attrs: with pkgs; with attrs; rec { - version = "20.1.4"; + version = "20.1.5"; src = fetchFromGitHub { owner = "llvm"; repo = "llvm-project"; rev = "llvmorg-${version}"; - hash = "sha256-/WomqG2DdnUHwlVsMfpzaK/dhGV3zychfU0wLmihQac="; + hash = "sha256-WKfY+VvAsZEEc0xYgF6+MsXDXZz7haMU6bxqmUpaHuQ="; }; patches = []; diff --git a/contrib/libs/libunwind/ya.make b/contrib/libs/libunwind/ya.make index 0b2e66c37d0..036523dcb81 100644 --- a/contrib/libs/libunwind/ya.make +++ b/contrib/libs/libunwind/ya.make @@ -11,9 +11,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20.1.4) +VERSION(20.1.5) -ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.4.tar.gz) +ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.5.tar.gz) PEERDIR( library/cpp/sanitizer/include diff --git a/contrib/python/google-auth/py3/.dist-info/METADATA b/contrib/python/google-auth/py3/.dist-info/METADATA index ddba2bddaed..6d1bff83523 100644 --- a/contrib/python/google-auth/py3/.dist-info/METADATA +++ b/contrib/python/google-auth/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: google-auth -Version: 2.40.0 +Version: 2.40.1 Summary: Google Authentication Library Home-page: https://github.com/googleapis/google-auth-library-python Author: Google Cloud Platform diff --git a/contrib/python/google-auth/py3/google/auth/aio/_helpers.py b/contrib/python/google-auth/py3/google/auth/aio/_helpers.py index fd7d37a2f7b..1d5a4618cb6 100644 --- a/contrib/python/google-auth/py3/google/auth/aio/_helpers.py +++ b/contrib/python/google-auth/py3/google/auth/aio/_helpers.py @@ -53,5 +53,10 @@ async def response_log_async(logger: logging.Logger, response: Any) -> None: response: The HTTP response object to log. """ if _helpers.is_logging_enabled(logger): - json_response = await _parse_response_async(response) + # TODO(https://github.com/googleapis/google-auth-library-python/issues/1755): + # Parsing the response for async streaming logging results in + # the stream to be empty downstream. For now, we will not be logging + # the response for async responses until we investigate further. + # json_response = await _parse_response_async(response) + json_response = None _helpers._response_log_base(logger, json_response) diff --git a/contrib/python/google-auth/py3/google/auth/version.py b/contrib/python/google-auth/py3/google/auth/version.py index d1363c1ef0e..ee0bb12609d 100644 --- a/contrib/python/google-auth/py3/google/auth/version.py +++ b/contrib/python/google-auth/py3/google/auth/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.40.0" +__version__ = "2.40.1" diff --git a/contrib/python/google-auth/py3/tests/aio/test__helpers.py b/contrib/python/google-auth/py3/tests/aio/test__helpers.py index 7642431caec..829dc97ff33 100644 --- a/contrib/python/google-auth/py3/tests/aio/test__helpers.py +++ b/contrib/python/google-auth/py3/tests/aio/test__helpers.py @@ -65,17 +65,13 @@ async def test_response_log_debug_disabled(logger, caplog, base_logger): @pytest.mark.asyncio async def test_response_log_debug_enabled_response_json(logger, caplog, base_logger): - class MockResponse: - async def json(self): - return {"key1": "value1", "key2": "value2", "key3": "value3"} - - response = MockResponse() + response = None caplog.set_level(logging.DEBUG, logger=_MOCK_CHILD_LOGGER_NAME) await _helpers.response_log_async(logger, response) assert len(caplog.records) == 1 record = caplog.records[0] assert record.message == "Response received..." - assert record.httpResponse == {"key1": "value1", "key2": "value2", "key3": "value3"} + assert record.httpResponse == "<class 'NoneType'>" @pytest.mark.asyncio diff --git a/contrib/python/google-auth/py3/ya.make b/contrib/python/google-auth/py3/ya.make index 7ceec35306e..83c9200ad11 100644 --- a/contrib/python/google-auth/py3/ya.make +++ b/contrib/python/google-auth/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.40.0) +VERSION(2.40.1) LICENSE(Apache-2.0) diff --git a/contrib/restricted/google/benchmark/.yandex_meta/override.nix b/contrib/restricted/google/benchmark/.yandex_meta/override.nix index 48bdca47f1e..e36f8bcc55e 100644 --- a/contrib/restricted/google/benchmark/.yandex_meta/override.nix +++ b/contrib/restricted/google/benchmark/.yandex_meta/override.nix @@ -1,11 +1,11 @@ pkgs: attrs: with pkgs; with attrs; rec { - version = "1.9.3"; + version = "1.9.4"; src = fetchFromGitHub { owner = "google"; repo = "benchmark"; rev = "v${version}"; - hash = "sha256-iPK3qLrZL2L08XW1a7SGl7GAt5InQ5nY+Dn8hBuxSOg="; + hash = "sha256-P7wJcKkIBoWtN9FCRticpBzYbEZPq71a0iW/2oDTZRU="; }; buildInputs = [ gtest ]; diff --git a/contrib/restricted/google/benchmark/src/benchmark.cc b/contrib/restricted/google/benchmark/src/benchmark.cc index 48a2accdf90..8672c8a94f8 100644 --- a/contrib/restricted/google/benchmark/src/benchmark.cc +++ b/contrib/restricted/google/benchmark/src/benchmark.cc @@ -815,6 +815,12 @@ int InitializeStreams() { return 0; } +template <typename T> +std::make_unsigned_t<T> get_as_unsigned(T v) { + using UnsignedT = std::make_unsigned_t<T>; + return static_cast<UnsignedT>(v); +} + } // end namespace internal void MaybeReenterWithoutASLR(int /*argc*/, char** argv) { @@ -829,15 +835,23 @@ void MaybeReenterWithoutASLR(int /*argc*/, char** argv) { if (curr_personality == -1) return; // If ASLR is already disabled, we have nothing more to do. - if (curr_personality & ADDR_NO_RANDOMIZE) return; + if (internal::get_as_unsigned(curr_personality) & ADDR_NO_RANDOMIZE) return; // Try to change the personality to disable ASLR. - const auto prev_personality = personality( - static_cast<unsigned long>(curr_personality) | ADDR_NO_RANDOMIZE); + const auto proposed_personality = + internal::get_as_unsigned(curr_personality) | ADDR_NO_RANDOMIZE; + const auto prev_personality = personality(proposed_personality); // Have we failed to change the personality? That may happen. if (prev_personality == -1) return; + // Make sure the parsona has been updated with the no-ASLR flag, + // otherwise we will try to reenter infinitely. + // This seems impossible, but can happen in some docker configurations. + const auto new_personality = personality(0xffffffff); + if ((internal::get_as_unsigned(new_personality) & ADDR_NO_RANDOMIZE) == 0) + return; + execv(argv[0], argv); // The exec() functions return only if an error has occurred, // in which case we want to just continue as-is. diff --git a/contrib/restricted/google/benchmark/src/sysinfo.cc b/contrib/restricted/google/benchmark/src/sysinfo.cc index ea1dae40f14..4ca249da4fd 100644 --- a/contrib/restricted/google/benchmark/src/sysinfo.cc +++ b/contrib/restricted/google/benchmark/src/sysinfo.cc @@ -439,7 +439,7 @@ std::vector<CPUInfo::CacheInfo> GetCacheSizes() { return GetCacheSizesWindows(); #elif defined(BENCHMARK_OS_QNX) return GetCacheSizesQNX(); -#elif defined(BENCHMARK_OS_QURT) +#elif defined(BENCHMARK_OS_QURT) || defined(__EMSCRIPTEN__) return std::vector<CPUInfo::CacheInfo>(); #else return GetCacheSizesFromKVFS(); diff --git a/contrib/restricted/google/benchmark/test/ya.make b/contrib/restricted/google/benchmark/test/ya.make index 27a3a0fb0fc..3e2b7047516 100644 --- a/contrib/restricted/google/benchmark/test/ya.make +++ b/contrib/restricted/google/benchmark/test/ya.make @@ -4,7 +4,7 @@ GTEST(benchmark_gtest) WITHOUT_LICENSE_TEXTS() -VERSION(1.9.3) +VERSION(1.9.4) LICENSE(Apache-2.0) diff --git a/contrib/restricted/google/benchmark/tools/compare/ya.make b/contrib/restricted/google/benchmark/tools/compare/ya.make index 39eafaddf36..8bfa31414ad 100644 --- a/contrib/restricted/google/benchmark/tools/compare/ya.make +++ b/contrib/restricted/google/benchmark/tools/compare/ya.make @@ -4,9 +4,9 @@ PY3_PROGRAM() WITHOUT_LICENSE_TEXTS() -VERSION(1.9.3) +VERSION(1.9.4) -ORIGINAL_SOURCE(https://github.com/google/benchmark/archive/v1.9.3.tar.gz) +ORIGINAL_SOURCE(https://github.com/google/benchmark/archive/v1.9.4.tar.gz) LICENSE(Apache-2.0) diff --git a/contrib/restricted/google/benchmark/tools/ya.make b/contrib/restricted/google/benchmark/tools/ya.make index 90fb103e020..c49a52e605b 100644 --- a/contrib/restricted/google/benchmark/tools/ya.make +++ b/contrib/restricted/google/benchmark/tools/ya.make @@ -1,6 +1,6 @@ # Generated by devtools/yamaker. -VERSION(1.9.3) +VERSION(1.9.4) IF (NOT USE_STL_SYSTEM) IF (NOT USE_SYSTEM_PYTHON OR NOT _SYSTEM_PYTHON27) diff --git a/contrib/restricted/google/benchmark/ya.make b/contrib/restricted/google/benchmark/ya.make index 5218377accd..88a14ddd1eb 100644 --- a/contrib/restricted/google/benchmark/ya.make +++ b/contrib/restricted/google/benchmark/ya.make @@ -2,9 +2,9 @@ LIBRARY() -VERSION(1.9.3) +VERSION(1.9.4) -ORIGINAL_SOURCE(https://github.com/google/benchmark/archive/v1.9.3.tar.gz) +ORIGINAL_SOURCE(https://github.com/google/benchmark/archive/v1.9.4.tar.gz) LICENSE(Apache-2.0) @@ -21,7 +21,7 @@ NO_UTIL() CFLAGS( GLOBAL -DBENCHMARK_STATIC_DEFINE - -DBENCHMARK_VERSION=\"v1.9.3\" + -DBENCHMARK_VERSION=\"v1.9.4\" -DHAVE_POSIX_REGEX -DHAVE_PTHREAD_AFFINITY -DHAVE_STD_REGEX diff --git a/contrib/tools/cython/.dist-info/METADATA b/contrib/tools/cython/.dist-info/METADATA index 237a1673400..197a6e14edc 100644 --- a/contrib/tools/cython/.dist-info/METADATA +++ b/contrib/tools/cython/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: Cython -Version: 3.0.10 +Version: 3.0.11 Summary: The Cython compiler for writing C extensions in the Python language. Home-page: https://cython.org/ Author: Robert Bradshaw, Stefan Behnel, Dag Seljebotn, Greg Ewing, et al. diff --git a/contrib/tools/cython/.yandex_meta/override.nix b/contrib/tools/cython/.yandex_meta/override.nix index 1c3e41ff9d3..68cfcf5e2ba 100644 --- a/contrib/tools/cython/.yandex_meta/override.nix +++ b/contrib/tools/cython/.yandex_meta/override.nix @@ -1,10 +1,10 @@ pkgs: attrs: with pkgs; with pkgs.python311.pkgs; with attrs; rec { - version = "3.0.10"; + version = "3.0.11"; src = fetchPypi { - pname = "Cython"; + pname = "cython"; inherit version; - hash = "sha256-3MlnOTMfuFTc9QP5RgdXbP6EiAZsYcpQ39VYNvEy3pk="; + hash = "sha256-cUbdKvhoK0ymEzGFHmrrzp/lFY51MAND+AwHyoCx+v8="; }; patches = []; diff --git a/contrib/tools/cython/CHANGES.rst b/contrib/tools/cython/CHANGES.rst index e4a744da721..895bc335486 100644 --- a/contrib/tools/cython/CHANGES.rst +++ b/contrib/tools/cython/CHANGES.rst @@ -2,6 +2,49 @@ Cython Changelog ================ +3.0.11 (2024-08-05) +=================== + +Features added +-------------- + +* The C++11 ``emplace*`` methods were added to ``libcpp.deque``. + Patch by Somin An. (Github issue :issue:`6159`) + +Bugs fixed +---------- + +* The exception check value of functions declared in pxd files was not always applied in 3.0.10. + (Github issue :issue:`6122`) + +* A crash on exception deallocations was fixed. + (Github issue :issue:`6022`) + +* A crash was fixed when assigning a zero-length slice to a memoryview. + Patch by Michael Man. (Github issue :issue:`6227`) + +* ``libcpp.optional.value()`` could crash if it raised a C++ exception. + Patch by Alexander Condello. (Github issue :issue:`6190`) + +* The return type of ``str()`` was mishandled, leading to crashes with ``language_level=3``. + (Github issue :issue:`6166`) + +* ``bytes.startswith/endswith()`` failed for non-bytes substrings (e.g. ``bytearray``). + (Github issue :issue:`6168`) + +* Fused ctuples crashed Cython. + (Github issue :issue:`6068`) + +* A compiler crash was fixed when using extension types in fused types. + (Github issue :issue:`6204`) + +* The module cleanup code was incorrect for globally defined memory view slices. + (Github issue :issue:`6276`) + +* Some adaptations were made to enable compilation in Python 3.13. + (Github issues :issue:`5997`, :issue:`6182`, :issue:`6251`) + + 3.0.10 (2024-03-30) =================== @@ -540,12 +583,12 @@ Related changes * The Python ``int`` handling code was adapted to make use of the new ``PyLong`` internals in CPython 3.12. (Github issue :issue:`5353`) - + * A compile error when using ``__debug__`` was resolved. - + * The deprecated ``_PyGC_FINALIZED()`` C-API macro is no longer used. Patch by Thomas Caswell and Matúš Valo. (Github issue :issue:`5481`) - + * A crash in Python 2.7 was fixed when cleaning up extension type instances at program end. @@ -592,7 +635,7 @@ without recompilation. There is initial support for this in Cython. By defining the ``CYTHON_LIMITED_API`` macro, Cython cuts down its C-API usage and tries to adhere to the Limited C-API, probably at the cost of a bit of performance. -In order to get full benefit from the limited API you will also need to define the +In order to get full benefit from the limited API you will also need to define the CPython macro ``Py_LIMITED_API`` to a specific CPython compatibility version, which additionally restricts the C-API during the C compilation, thus enforcing the forward compatibility of the extension module. @@ -647,7 +690,7 @@ Related changes (Github issue :issue:`5419`) * Custom buffer slot methods are now supported in the Limited C-API of Python 3.9+. - Patch by Lisandro Dalcin. (Github issue :issue:`5422`) + Patch by Lisandro Dalcin. (Github issue :issue:`5422`) Improved fidelity to Python semantics @@ -705,24 +748,24 @@ Related changes * Binding staticmethods of Cython functions were not behaving like Python methods. Patch by Jeroen Demeyer. (Github issue :issue:`3106`, :issue:`3102`) - + * Compiling package ``__init__`` files could fail under Windows due to an - undefined export symbol. (Github issue :issue:`2968`) - + undefined export symbol. (Github issue :issue:`2968`) + * ``__init__.pyx`` files were not always considered as package indicators. (Github issue :issue:`2665`) - + * Setting ``language_level=2`` in a file did not work if ``language_level=3`` was enabled globally before. Patch by Jeroen Demeyer. (Github issue :issue:`2791`) - + * ``__doc__`` was not available inside of the class body during class creation. (Github issue :issue:`1635`) * The first function line number of functions with decorators pointed to the signature line and not the first decorator line, as in Python. Patch by Felix Kohlgrüber. (Github issue :issue:`2536`) - + * Pickling unbound methods of Python classes failed. Patch by Pierre Glaser. (Github issue :issue:`2972`) @@ -763,7 +806,7 @@ Related changes * ``__arg`` argument names in methods were not mangled with the class name. Patch by David Woods. (Github issue :issue:`1382`) - + * With ``language_level=3/3str``, Python classes without explicit base class are now new-style (type) classes also in Py2. Previously, they were created as old-style (non-type) classes. @@ -778,7 +821,7 @@ Related changes globals, since double-underscore names are not uncommon in C. Unmangled Python names are also still found as a legacy fallback but produce a warning. Patch by David Woods. (Github issue :issue:`3548`) - + * The ``print`` statement (not the ``print()`` function) is allowed in ``nogil`` code without an explicit ``with gil`` section. @@ -843,7 +886,7 @@ Related changes * Default values for memory views arguments were not properly supported. Patch by Corentin Cadiou. (Github issue :issue:`4313`) - + * Python object types were not allowed as ``->`` return type annotations. Patch by Matúš Valo. (Github issue :issue:`4433`) @@ -859,7 +902,7 @@ Related changes * Decorators on inner functions were not evaluated in the right scope. Patch by David Woods. (Github issue :issue:`4367`) - + * Cython did not type the ``self`` argument in special binary methods. Patch by David Woods. (Github issue :issue:`4434`) @@ -869,7 +912,7 @@ Related changes * ``__del__(self)`` on extension types now maps to ``tp_finalize`` in Python 3. Original patch by ax487. (Github issue :issue:`3612`) - + * Reusing an extension type attribute name as a method name is now an error. Patch by 0dminnimda. (Github issue :issue:`4661`) @@ -922,14 +965,14 @@ Related changes * ``int(Py_UCS4)`` returned the code point instead of the parsed digit value. (Github issue :issue:`5216`) - + * Calling bound classmethods of builtin types could fail trying to call the unbound method. (Github issue :issue:`5051`) * Generator expressions and comprehensions now look up their outer-most iterable on creation, as Python does, and not later on start, as they did previously. (Github issue :issue:`1159`) - + * Bound C methods can now coerce to Python objects. (Github issues :issue:`4890`, :issue:`5062`) @@ -942,7 +985,7 @@ Related changes * The special ``__*pow__`` methods now support the 2- and 3-argument variants. (Github issue :issue:`5160`) - + * The ``**`` power operator now behaves more like in Python by returning the correct complex result if required by math. A new ``cpow`` directive was added to turn on the previous C-like behaviour. @@ -997,7 +1040,7 @@ Related changes * ``nogil`` functions now avoid acquiring the GIL on function exit if possible even if they contain ``with gil`` blocks. (Github issue :issue:`3554`) - + * The ``@returns()`` decorator propagates exceptions by default for suitable C return types when no ``@exceptval()`` is defined. (Github issues :issue:`3625`, :issue:`3664`) @@ -1005,13 +1048,13 @@ Related changes * Extension types inheriting from Python classes could not safely be exposed in ``.pxd`` files. (Github issue :issue:`4106`) - + * Default arguments of methods were not exposed for introspection. Patch by Vladimir Matveev. (Github issue :issue:`4061`) * Literal list assignments to pointer variables declared in PEP-526 notation failed to compile. - + * The type ``cython.Py_hash_t`` is available in Python mode. * A ``cimport`` is now supported in pure Python code by prefixing the @@ -1100,12 +1143,12 @@ Related changes * Generated NumPy ufuncs could crash for large arrays due to incorrect GIL handling. (Github issue :issue:`5328`) - + * Some invalid directive usages are now detected and rejected, e.g. using ``@ccall`` together with ``@cfunc``, and applying ``@cfunc`` to a ``@ufunc``. Cython also warns now when a directive is applied needlessly. (Github issue :issue:`5399` et al.) - + * The normal ``@dataclasses.dataclass`` and ``@functools.total_ordering`` decorators can now be used on extension types. Using the corresponding ``@cython.*`` decorator will automatically turn a Python class into an extension type (no need for ``@cclass``). @@ -1209,7 +1252,7 @@ Related changes (Github issue :issue:`5430`) * Handling freshly raised exceptions that didn't have a traceback yet could crash. - (Github issue :issue:`5495`) + (Github issue :issue:`5495`) Optimizations @@ -1240,11 +1283,11 @@ Related changes * String concatenation can now happen in place if possible, by extending the existing string rather than always creating a new one. Patch by David Woods. (Github issue :issue:`3453`) - + * The ``str()`` builtin now calls ``PyObject_Str()`` instead of going through a Python call. Patch by William Ayd. (Github issue :issue:`3279`) - + * Reimports of already imported modules are substantially faster. (Github issue :issue:`2854`) @@ -1255,17 +1298,17 @@ Related changes * The fastcall/vectorcall protocols are used for several internal Python calls. (Github issue :issue:`3540`) - + * ``nogil`` functions now avoid acquiring the GIL on function exit if possible even if they contain ``with gil`` blocks. (Github issue :issue:`3554`) - + * Type inference now works for memory views and slices. Patch by David Woods. (Github issue :issue:`2227`) * For-in-loop iteration over ``bytearray`` and memory views is optimised. Patch by David Woods. (Github issue :issue:`2227`) - + * For-in-loop iteration over ``bytearray`` and memory views is optimised. Patch by David Woods. (Github issue :issue:`2227`) @@ -1383,7 +1426,7 @@ Related changes * ``cdef public`` functions declared in .pxd files could use an incorrectly mangled C name. Patch by EpigeneMax. (Github issue :issue:`2940`) - + * ``const`` types could not be returned from functions. Patch by Mike Graham. (Github issue :issue:`5135`) @@ -1398,11 +1441,11 @@ Related changes * Very long Python integer constants could exceed the maximum C name length of MSVC. Patch by 0dminnimda. (Github issue :issue:`5290`) - + * Some C compiler warnings were resolved. Patches by Matt Tyson, Lisandro Dalcin, Philipp Wagner, Matti Picus et al. (Github issues :issue:`5417`, :issue:`5418`, :issue:`5421`, :issue:`5437`, :issue:`5438`, :issue:`5443`) - + * Some typedef declarations for libc function types were fixed. (Github issue :issue:`5498`) @@ -1566,7 +1609,7 @@ Related changes * C++ containers of item type ``bint`` could conflict with those of item type ``int``. (Github issue :issue:`5516`) - + * Reverse iteration in C++ no longer removes the ``const`` qualifier from the item type. Patch by Isuru Fernando. (Github issue :issue:`5478`) @@ -1615,7 +1658,7 @@ Related changes * ``cython --version`` now prints the version to both stdout and stderr (unless that is a TTY). (Github issue :issue:`5504`) - + Build integration ----------------- @@ -1633,7 +1676,7 @@ Related changes * Binary Linux wheels now follow the manylinux2010 standard. Patch by Alexey Stepanov. (Github issue :issue:`3355`) - + * The search order for include files was changed. Previously it was ``include_directories``, ``Cython/Includes``, ``sys.path``. Now it is ``include_directories``, ``sys.path``, ``Cython/Includes``. This was done to @@ -1701,7 +1744,7 @@ Related changes * A new Cython build option ``--cython-compile-minimal`` was added to compile only a smaller set of Cython's own modules, which can be used to reduce the package and install size. - + * The environment variable ``CYTHON_FORCE_REGEN=1`` can be used to force ``cythonize`` to regenerate the output files regardless of modification times and changes. @@ -1713,15 +1756,15 @@ Related changes * Wheels now include a compiled parser again, which increases their size a little but gives about a 10% speed-up when running Cython. - + * The wheel building process was migrated to use the ``cibuildwheel`` tool. Patch by Thomas Li. (Github issue :issue:`4736`) - + * ``setup.cfg`` was missing from the source distribution. (Github issue :issue:`5199`) * Extended glob paths with ``/**/`` and ``\**\`` for finding source files failed on Windows. - + * Coverage analysis failed in projects with a separate source subdirectory. Patch by Sviatoslav Sydorenko and Ruben Vorderman. (Github issue :issue:`3636`) @@ -1789,12 +1832,12 @@ Other changes * Memoryviews failed to compile when the ``cache_builtins`` feature was disabled. Patch by David Woods. (Github issue :issue:`3406`) - + * Broadcast assignments to a multi-dimensional memory view slice could end up in the wrong places when the underlying memory view is known to be contiguous but the slice is not. (Github issue :issue:`2941`) - + * The Pythran ``shape`` attribute is supported. Patch by Serge Guelton. (Github issue :issue:`3307`) @@ -1824,7 +1867,7 @@ Other changes * Casting to ctuples is now allowed. Patch by David Woods. (Github issue :issue:`3808`) - + * Some issues were resolved that could lead to duplicated C names. Patch by David Woods. (Github issue :issue:`3716`, :issue:`3741`, :issue:`3734`) diff --git a/contrib/tools/cython/Cython/Build/Inline.py b/contrib/tools/cython/Cython/Build/Inline.py index fac6a79b9c3..fe62d4ca9ea 100644 --- a/contrib/tools/cython/Cython/Build/Inline.py +++ b/contrib/tools/cython/Cython/Build/Inline.py @@ -151,6 +151,11 @@ def _populate_unbound(kwds, unbound_symbols, locals=None, globals=None): locals = calling_frame.f_locals if globals is None: globals = calling_frame.f_globals + if not isinstance(locals, dict): + # FrameLocalsProxy is stricter than dict on how it looks up keys + # and this means our "EncodedStrings" don't match the keys in locals. + # Therefore copy to a dict. + locals = dict(locals) if symbol in locals: kwds[symbol] = locals[symbol] elif symbol in globals: diff --git a/contrib/tools/cython/Cython/Build/IpythonMagic.py b/contrib/tools/cython/Cython/Build/IpythonMagic.py index 3fa43c96d1e..315862e3660 100644 --- a/contrib/tools/cython/Cython/Build/IpythonMagic.py +++ b/contrib/tools/cython/Cython/Build/IpythonMagic.py @@ -565,7 +565,7 @@ class CythonMagics(Magics): __doc__ = __doc__.format( # rST doesn't see the -+ flag as part of an option list, so we # hide it from the module-level docstring. - CYTHON_DOC=dedent(CythonMagics.cython.__doc__\ + CYTHON_DOC=dedent(CythonMagics.cython.__doc__ .replace('-+, --cplus', '--cplus ')), CYTHON_INLINE_DOC=dedent(CythonMagics.cython_inline.__doc__), CYTHON_PYXIMPORT_DOC=dedent(CythonMagics.cython_pyximport.__doc__), diff --git a/contrib/tools/cython/Cython/Compiler/Code.py b/contrib/tools/cython/Cython/Compiler/Code.py index ac384d99fd3..cd7ca03443d 100644 --- a/contrib/tools/cython/Cython/Compiler/Code.py +++ b/contrib/tools/cython/Cython/Compiler/Code.py @@ -66,7 +66,7 @@ uncachable_builtins = [ # Global/builtin names that cannot be cached because they may or may not # be available at import time, for various reasons: ## Python 3.13+ - 'IncompleteInputError', + '_IncompleteInputError', 'PythonFinalizationError', ## Python 3.11+ 'BaseExceptionGroup', @@ -641,18 +641,23 @@ class UtilityCode(UtilityCodeBase): self.cleanup(writer, output.module_pos) -def sub_tempita(s, context, file=None, name=None): +def sub_tempita(s, context, file=None, name=None, __cache={}): "Run tempita on string s with given context." if not s: return None if file: - context['__name'] = "%s:%s" % (file, name) - elif name: + name = "%s:%s" % (file, name) + if name: context['__name'] = name - from ..Tempita import sub - return sub(s, **context) + try: + template = __cache[s] + except KeyError: + from ..Tempita import Template + template = __cache[s] = Template(s, name=name) + + return template.substitute(context) class TempitaUtilityCode(UtilityCode): @@ -1542,8 +1547,18 @@ class GlobalState(object): # which aren't necessarily PyObjects, so aren't appropriate # to clear. continue - self.parts['module_state_clear'].putln( - "Py_CLEAR(clear_module_state->%s);" % cname) + + self.parts['module_state_clear'].put_xdecref_clear( + "clear_module_state->%s" % cname, + c.type, + clear_before_decref=True, + nanny=False, + ) + + if c.type.is_memoryviewslice: + # TODO: Implement specific to type like CodeWriter.put_xdecref_clear() + cname += "->memview" + self.parts['module_state_traverse'].putln( "Py_VISIT(traverse_module_state->%s);" % cname) @@ -2084,14 +2099,6 @@ class CCodeWriter(object): elif fix_indent: self.level += 1 - def putln_tempita(self, code, **context): - from ..Tempita import sub - self.putln(sub(code, **context)) - - def put_tempita(self, code, **context): - from ..Tempita import sub - self.put(sub(code, **context)) - def increase_indent(self): self.level += 1 diff --git a/contrib/tools/cython/Cython/Compiler/ExprNodes.py b/contrib/tools/cython/Cython/Compiler/ExprNodes.py index 7876d09470f..952617e375d 100644 --- a/contrib/tools/cython/Cython/Compiler/ExprNodes.py +++ b/contrib/tools/cython/Cython/Compiler/ExprNodes.py @@ -481,7 +481,12 @@ class ExprNode(Node): constant_result = constant_value_not_set - child_attrs = property(fget=operator.attrgetter('subexprs')) + if getattr(getattr(sys, "implementation", None), "name", "cpython") == "cpython": + child_attrs = property(fget=operator.attrgetter('subexprs')) + else: + @property + def child_attrs(self): + return self.subexprs def analyse_annotations(self, env): pass @@ -6590,7 +6595,7 @@ class PyMethodCallNode(SimpleCallNode): self_arg = code.funcstate.allocate_temp(py_object_type, manage_ref=True) code.putln("%s = NULL;" % self_arg) - arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) + arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_uint_type, manage_ref=False) code.putln("%s = 0;" % arg_offset_cname) def attribute_is_likely_method(attr): diff --git a/contrib/tools/cython/Cython/Compiler/ModuleNode.py b/contrib/tools/cython/Cython/Compiler/ModuleNode.py index 493926637ad..469b968f6a8 100644 --- a/contrib/tools/cython/Cython/Compiler/ModuleNode.py +++ b/contrib/tools/cython/Cython/Compiler/ModuleNode.py @@ -972,7 +972,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): pass elif type.is_struct_or_union or type.is_cpp_class: self.generate_struct_union_predeclaration(entry, code) - elif type.is_ctuple and entry.used: + elif type.is_ctuple and not type.is_fused and entry.used: self.generate_struct_union_predeclaration(entry.type.struct_entry, code) elif type.is_extension_type: self.generate_objstruct_predeclaration(type, code) @@ -987,7 +987,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_enum_definition(entry, code) elif type.is_struct_or_union: self.generate_struct_union_definition(entry, code) - elif type.is_ctuple and entry.used: + elif type.is_ctuple and not type.is_fused and entry.used: self.generate_struct_union_definition(entry.type.struct_entry, code) elif type.is_cpp_class: self.generate_cpp_class_definition(entry, code) diff --git a/contrib/tools/cython/Cython/Compiler/Nodes.py b/contrib/tools/cython/Cython/Compiler/Nodes.py index c2fcd9d6727..e1fe6ee56c9 100644 --- a/contrib/tools/cython/Cython/Compiler/Nodes.py +++ b/contrib/tools/cython/Cython/Compiler/Nodes.py @@ -3126,6 +3126,7 @@ class DefNode(FuncDefNode): if scope is None: scope = cfunc.scope cfunc_type = cfunc.type + has_explicit_exc_clause=True if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs: error(self.pos, "wrong number of arguments") error(cfunc.pos, "previous declaration here") diff --git a/contrib/tools/cython/Cython/Compiler/Optimize.py b/contrib/tools/cython/Cython/Compiler/Optimize.py index da5f64c29b9..8dd48e951ac 100644 --- a/contrib/tools/cython/Cython/Compiler/Optimize.py +++ b/contrib/tools/cython/Cython/Compiler/Optimize.py @@ -2450,6 +2450,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, def _handle_simple_function_str(self, node, function, pos_args): """Optimize single argument calls to str(). """ + if node.type is Builtin.unicode_type: + # type already deduced as unicode (language_level=3) + return self._handle_simple_function_unicode(node, function, pos_args) if len(pos_args) != 1: if len(pos_args) == 0: return ExprNodes.StringNode(node.pos, value=EncodedString(), constant_result='') @@ -3516,7 +3519,8 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, if not result: return node func_cname, utility_code, extra_args, num_type = result - args = list(args)+extra_args + assert all([arg.type.is_pyobject for arg in args]) + args = list(args) + extra_args call_node = self._substitute_method_call( node, function, diff --git a/contrib/tools/cython/Cython/Compiler/PyrexTypes.py b/contrib/tools/cython/Cython/Compiler/PyrexTypes.py index e84fcb5f93a..b522a131751 100644 --- a/contrib/tools/cython/Cython/Compiler/PyrexTypes.py +++ b/contrib/tools/cython/Cython/Compiler/PyrexTypes.py @@ -1498,7 +1498,7 @@ class BuiltinObjectType(PyObjectType): # For backwards compatibility of (Py3) 'x: int' annotations in Py2, we also allow 'long' there. type_check = '__Pyx_Py3Int_Check' elif type_name == "memoryview": - # captialize doesn't catch the 'V' + # capitalize doesn't catch the 'V' type_check = "PyMemoryView_Check" else: type_check = 'Py%s_Check' % type_name.capitalize() @@ -4582,6 +4582,8 @@ class CTupleType(CType): is_ctuple = True + subtypes = ['components'] + def __init__(self, cname, components): self.cname = cname self.components = components @@ -4666,10 +4668,19 @@ class CTupleType(CType): def cast_code(self, expr_code): return expr_code + def specialize(self, values): + assert hasattr(self, "entry") + components = [c.specialize(values) for c in self.components] + new_entry = self.entry.scope.declare_tuple_type(self.entry.pos, components) + return new_entry.type + def c_tuple_type(components): components = tuple(components) - cname = Naming.ctuple_type_prefix + type_list_identifier(components) + if any(c.is_fused for c in components): + cname = "<dummy fused ctuple>" # should never end up in code + else: + cname = Naming.ctuple_type_prefix + type_list_identifier(components) tuple_type = CTupleType(cname, components) return tuple_type @@ -5051,7 +5062,7 @@ def best_match(arg_types, functions, pos=None, env=None, args=None): if len(candidates) == 1: return candidates[0][0] elif len(candidates) == 0: - if pos is not None: + if pos is not None and errors: func, errmsg = errors[0] if len(errors) == 1 or [1 for func, e in errors if e == errmsg]: error(pos, errmsg) diff --git a/contrib/tools/cython/Cython/Compiler/TypeInference.py b/contrib/tools/cython/Cython/Compiler/TypeInference.py index 70d64fcf525..bb68db8976a 100644 --- a/contrib/tools/cython/Cython/Compiler/TypeInference.py +++ b/contrib/tools/cython/Cython/Compiler/TypeInference.py @@ -233,10 +233,7 @@ class MarkParallelAssignments(EnvTransform): self.parallel_errors = True if node.is_prange: - child_attrs = node.child_attrs - node.child_attrs = ['body', 'target', 'args'] - self.visitchildren(node) - node.child_attrs = child_attrs + self.visitchildren(node, attrs=('body', 'target', 'args')) self.parallel_block_stack.pop() if node.else_clause: diff --git a/contrib/tools/cython/Cython/Compiler/TypeSlots.py b/contrib/tools/cython/Cython/Compiler/TypeSlots.py index 2c891f64810..ad27e0d6eaa 100644 --- a/contrib/tools/cython/Cython/Compiler/TypeSlots.py +++ b/contrib/tools/cython/Cython/Compiler/TypeSlots.py @@ -1102,6 +1102,7 @@ class SlotTable(object): EmptySlot("tp_vectorcall", ifdef="PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800)"), EmptySlot("tp_print", ifdef="__PYX_NEED_TP_PRINT_SLOT == 1"), EmptySlot("tp_watched", ifdef="PY_VERSION_HEX >= 0x030C0000"), + EmptySlot("tp_versions_used", ifdef="PY_VERSION_HEX >= 0x030d00A4"), # PyPy specific extension - only here to avoid C compiler warnings. EmptySlot("tp_pypy_flags", ifdef="CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000"), ) diff --git a/contrib/tools/cython/Cython/Debugger/libpython.py b/contrib/tools/cython/Cython/Debugger/libpython.py index 6e548800e25..0a680f9cf8e 100644 --- a/contrib/tools/cython/Cython/Debugger/libpython.py +++ b/contrib/tools/cython/Cython/Debugger/libpython.py @@ -1312,7 +1312,7 @@ class PyUnicodeObjectPtr(PyObjectPtr): ucs = ch + ch2 i += 1 - # Unfortuately, Python 2's unicode type doesn't seem + # Unfortunately, Python 2's unicode type doesn't seem # to expose the "isprintable" method printable = _unichr_is_printable(ucs) if printable: diff --git a/contrib/tools/cython/Cython/Includes/cpython/time.pxd b/contrib/tools/cython/Cython/Includes/cpython/time.pxd index 3f333a72a9d..f1a9f2967cd 100644 --- a/contrib/tools/cython/Cython/Includes/cpython/time.pxd +++ b/contrib/tools/cython/Cython/Includes/cpython/time.pxd @@ -5,10 +5,38 @@ Cython implementation of (parts of) the standard library time module. from libc.stdint cimport int64_t from cpython.exc cimport PyErr_SetFromErrno -cdef extern from "Python.h": - ctypedef int64_t _PyTime_t - _PyTime_t _PyTime_GetSystemClock() nogil - double _PyTime_AsSecondsDouble(_PyTime_t t) nogil +cdef extern from *: + """ + #if PY_VERSION_HEX >= 0x030d00b1 || defined(PyTime_t) + #define __Pyx_PyTime_t PyTime_t + #else + #define __Pyx_PyTime_t _PyTime_t + #endif + + #if PY_VERSION_HEX >= 0x030d00b1 || defined(PyTime_TimeRaw) + static CYTHON_INLINE __Pyx_PyTime_t __Pyx_PyTime_TimeUnchecked(void) { + __Pyx_PyTime_t tic; + (void) PyTime_TimeRaw(&tic); + return tic; + } + #else + #define __Pyx_PyTime_TimeUnchecked() _PyTime_GetSystemClock() + #endif + + #if PY_VERSION_HEX >= 0x030d00b1 || defined(PyTime_AsSecondsDouble) + #define __Pyx_PyTime_AsSecondsDouble(t) PyTime_AsSecondsDouble(t) + #else + #define __Pyx_PyTime_AsSecondsDouble(t) _PyTime_AsSecondsDouble(t) + #endif + """ + ctypedef int64_t PyTime_t "__Pyx_PyTime_t" + ctypedef int64_t _PyTime_t "__Pyx_PyTime_t" + + _PyTime_t _PyTime_GetSystemClock "__Pyx_PyTime_TimeUnchecked" () nogil + _PyTime_t PyTime_TimeUnchecked "__Pyx_PyTime_TimeUnchecked" () nogil + + double _PyTime_AsSecondsDouble "__Pyx_PyTime_AsSecondsDouble" (_PyTime_t t) nogil + double PyTime_AsSecondsDouble "__Pyx_PyTime_AsSecondsDouble" (_PyTime_t t) nogil from libc.time cimport ( tm, @@ -21,8 +49,8 @@ cdef inline double time() noexcept nogil: cdef: _PyTime_t tic - tic = _PyTime_GetSystemClock() - return _PyTime_AsSecondsDouble(tic) + tic = PyTime_TimeUnchecked() + return PyTime_AsSecondsDouble(tic) cdef inline int _raise_from_errno() except -1 with gil: @@ -46,6 +74,6 @@ cdef inline tm localtime() except * nogil: # See tmtotuple() in https://github.com/python/cpython/blob/master/Modules/timemodule.c result.tm_year += 1900 result.tm_mon += 1 - result.tm_wday = (result.tm_wday + 6) % 7 + result.tm_wday = <int> ((result.tm_wday + 6) % 7) result.tm_yday += 1 return result[0] diff --git a/contrib/tools/cython/Cython/Includes/libcpp/deque.pxd b/contrib/tools/cython/Cython/Includes/libcpp/deque.pxd index 5f189ffd1c1..e625318bd00 100644 --- a/contrib/tools/cython/Cython/Includes/libcpp/deque.pxd +++ b/contrib/tools/cython/Cython/Includes/libcpp/deque.pxd @@ -161,3 +161,5 @@ cdef extern from "<deque>" namespace "std" nogil: # C++11 methods void shrink_to_fit() except + + T& emplace_front(...) except + + T& emplace_back(...) except + diff --git a/contrib/tools/cython/Cython/Includes/libcpp/optional.pxd b/contrib/tools/cython/Cython/Includes/libcpp/optional.pxd index 284dfcd6e50..9b0b07a6d2b 100644 --- a/contrib/tools/cython/Cython/Includes/libcpp/optional.pxd +++ b/contrib/tools/cython/Cython/Includes/libcpp/optional.pxd @@ -13,7 +13,7 @@ cdef extern from "<optional>" namespace "std" nogil: optional(optional&) except + optional(T&) except + bool has_value() - T& value() + T& value() except + T& value_or[U](U& default_value) void swap(optional&) void reset() diff --git a/contrib/tools/cython/Cython/Shadow.py b/contrib/tools/cython/Cython/Shadow.py index d75cd3379b3..1400f1657c8 100644 --- a/contrib/tools/cython/Cython/Shadow.py +++ b/contrib/tools/cython/Cython/Shadow.py @@ -2,7 +2,7 @@ from __future__ import absolute_import # Possible version formats: "3.1.0", "3.1.0a1", "3.1.0a1.dev0" -__version__ = "3.0.10" +__version__ = "3.0.11" try: from __builtin__ import basestring diff --git a/contrib/tools/cython/Cython/Utility/AsyncGen.c b/contrib/tools/cython/Cython/Utility/AsyncGen.c index 9bffedcfa9c..94b68f19f3d 100644 --- a/contrib/tools/cython/Cython/Utility/AsyncGen.c +++ b/contrib/tools/cython/Cython/Utility/AsyncGen.c @@ -478,6 +478,9 @@ static PyTypeObject __pyx_AsyncGenType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif @@ -753,6 +756,9 @@ static PyTypeObject __pyx__PyAsyncGenASendType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif @@ -891,6 +897,9 @@ static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif @@ -1235,6 +1244,9 @@ static PyTypeObject __pyx__PyAsyncGenAThrowType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif diff --git a/contrib/tools/cython/Cython/Utility/Builtins.c b/contrib/tools/cython/Cython/Utility/Builtins.c index 87609b5f13a..063d5e4bfd9 100644 --- a/contrib/tools/cython/Cython/Utility/Builtins.c +++ b/contrib/tools/cython/Cython/Utility/Builtins.c @@ -170,16 +170,11 @@ static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject //////////////////// HasAttr.proto //////////////////// -#if __PYX_LIMITED_VERSION_HEX >= 0x030d00A1 -#define __Pyx_HasAttr(o, n) PyObject_HasAttrWithError(o, n) -#else static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); /*proto*/ -#endif //////////////////// HasAttr //////////////////// //@requires: ObjectHandling.c::GetAttr -#if __PYX_LIMITED_VERSION_HEX < 0x030d00A1 static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { PyObject *r; if (unlikely(!__Pyx_PyBaseString_Check(n))) { @@ -196,7 +191,6 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { return 1; } } -#endif //////////////////// Intern.proto //////////////////// diff --git a/contrib/tools/cython/Cython/Utility/Coroutine.c b/contrib/tools/cython/Cython/Utility/Coroutine.c index 688a62d8efb..c7ec8ee9ba0 100644 --- a/contrib/tools/cython/Cython/Utility/Coroutine.c +++ b/contrib/tools/cython/Cython/Utility/Coroutine.c @@ -1683,6 +1683,9 @@ static PyTypeObject __pyx_CoroutineAwaitType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif @@ -1878,6 +1881,9 @@ static PyTypeObject __pyx_CoroutineType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif @@ -2029,6 +2035,9 @@ static PyTypeObject __pyx_IterableCoroutineType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif @@ -2178,6 +2187,9 @@ static PyTypeObject __pyx_GeneratorType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif diff --git a/contrib/tools/cython/Cython/Utility/CythonFunction.c b/contrib/tools/cython/Cython/Utility/CythonFunction.c index 2ca461f6654..023701c63ae 100644 --- a/contrib/tools/cython/Cython/Utility/CythonFunction.c +++ b/contrib/tools/cython/Cython/Utility/CythonFunction.c @@ -1198,6 +1198,9 @@ static PyTypeObject __pyx_CyFunctionType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif @@ -1725,6 +1728,9 @@ static PyTypeObject __pyx_FusedFunctionType_type = { #if PY_VERSION_HEX >= 0x030C0000 0, /*tp_watched*/ #endif +#if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ +#endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 0, /*tp_pypy_flags*/ #endif diff --git a/contrib/tools/cython/Cython/Utility/Exceptions.c b/contrib/tools/cython/Cython/Utility/Exceptions.c index daf6578eb96..46f7dd57812 100644 --- a/contrib/tools/cython/Cython/Utility/Exceptions.c +++ b/contrib/tools/cython/Cython/Utility/Exceptions.c @@ -819,7 +819,7 @@ static void __Pyx_WriteUnraisable(const char *name, int clineno, Py_XINCREF(old_val); Py_XINCREF(old_tb); __Pyx_ErrRestore(old_exc, old_val, old_tb); - PyErr_PrintEx(1); + PyErr_PrintEx(0); } #if PY_MAJOR_VERSION < 3 ctx = PyString_FromString(name); diff --git a/contrib/tools/cython/Cython/Utility/MemoryView.pyx b/contrib/tools/cython/Cython/Utility/MemoryView.pyx index 57ffaf92b48..b84c05d3360 100644 --- a/contrib/tools/cython/Cython/Utility/MemoryView.pyx +++ b/contrib/tools/cython/Cython/Utility/MemoryView.pyx @@ -420,7 +420,7 @@ cdef class memoryview: if have_slices: obj = self.is_slice(value) - if obj: + if obj is not None: self.setitem_slice_assignment(self[index], obj) else: self.setitem_slice_assign_scalar(self[index], value) diff --git a/contrib/tools/cython/Cython/Utility/StringTools.c b/contrib/tools/cython/Cython/Utility/StringTools.c index 2ec61d0a616..c49a56950e1 100644 --- a/contrib/tools/cython/Cython/Utility/StringTools.c +++ b/contrib/tools/cython/Cython/Utility/StringTools.c @@ -759,7 +759,7 @@ static int __Pyx_PyBytes_SingleTailmatch(PyObject* self, PyObject* arg, } #endif else { - if (unlikely(PyObject_GetBuffer(self, &view, PyBUF_SIMPLE) == -1)) + if (unlikely(PyObject_GetBuffer(arg, &view, PyBUF_SIMPLE) == -1)) return -1; sub_ptr = (const char*) view.buf; sub_len = view.len; diff --git a/contrib/tools/cython/Cython/Utility/TypeConversion.c b/contrib/tools/cython/Cython/Utility/TypeConversion.c index 2e881eb3b7c..e918d354633 100644 --- a/contrib/tools/cython/Cython/Utility/TypeConversion.c +++ b/contrib/tools/cython/Cython/Utility/TypeConversion.c @@ -826,13 +826,20 @@ static CYTHON_INLINE PyObject* {{TO_PY_FUNCTION}}({{TYPE}} value) { } } { - int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&value; -#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; return _PyLong_FromByteArray(bytes, sizeof({{TYPE}}), little, !is_unsigned); #else // call int.from_bytes() + int one = 1; int little = (int)*(unsigned char *)&one; PyObject *from_bytes, *result = NULL; PyObject *py_bytes = NULL, *arg_tuple = NULL, *kwds = NULL, *order_str = NULL; from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); @@ -1075,220 +1082,219 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { } return ({{TYPE}}) val; } - } else + } #endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { + + if (unlikely(!PyLong_Check(x))) { + {{TYPE}} val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return ({{TYPE}}) -1; + val = {{FROM_PY_FUNCTION}}(tmp); + Py_DECREF(tmp); + return val; + } + + if (is_unsigned) { #if CYTHON_USE_PYLONG_INTERNALS - if (unlikely(__Pyx_PyLong_IsNeg(x))) { - goto raise_neg_overflow; - //} else if (__Pyx_PyLong_IsZero(x)) { - // return ({{TYPE}}) 0; - } else if (__Pyx_PyLong_IsCompact(x)) { - __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) - } else { - const digit* digits = __Pyx_PyLong_Digits(x); - assert(__Pyx_PyLong_DigitCount(x) > 1); - switch (__Pyx_PyLong_DigitCount(x)) { - {{for _size in (2, 3, 4)}} - case {{_size}}: - if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) { - if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { - __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}}) - } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) { - return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}}; - } + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + //} else if (__Pyx_PyLong_IsZero(x)) { + // return ({{TYPE}}) 0; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + {{for _size in (2, 3, 4)}} + case {{_size}}: + if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}}) + } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) { + return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}}; } - break; - {{endfor}} - } + } + break; + {{endfor}} } + } #endif #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } #else - { - // misuse Py_False as a quick way to compare to a '0' int object in PyPy - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return ({{TYPE}}) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } + { + // misuse Py_False as a quick way to compare to a '0' int object in PyPy + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return ({{TYPE}}) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } #endif - if ((sizeof({{TYPE}}) <= sizeof(unsigned long))) { - __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned long, PyLong_AsUnsignedLong(x)) + if ((sizeof({{TYPE}}) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if ((sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG))) { - __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } else if ((sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif - } - } else { - // signed + } + + } else { + // signed #if CYTHON_USE_PYLONG_INTERNALS - if (__Pyx_PyLong_IsCompact(x)) { - __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) - } else { - const digit* digits = __Pyx_PyLong_Digits(x); - assert(__Pyx_PyLong_DigitCount(x) > 1); - switch (__Pyx_PyLong_SignedDigitCount(x)) { - {{for _size in (2, 3, 4)}} - {{for _case in (-_size, _size)}} - case {{_case}}: - if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) { - if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { - __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}}) - } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) { - return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}}); - } + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + {{for _size in (2, 3, 4)}} + {{for _case in (-_size, _size)}} + case {{_case}}: + if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}}) + } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) { + return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}}); } - break; - {{endfor}} - {{endfor}} - } + } + break; + {{endfor}} + {{endfor}} } + } #endif - if ((sizeof({{TYPE}}) <= sizeof(long))) { - __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, long, PyLong_AsLong(x)) + if ((sizeof({{TYPE}}) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if ((sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG))) { - __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, PY_LONG_LONG, PyLong_AsLongLong(x)) + } else if ((sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif - } } + } - {{if IS_ENUM}} - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available, cannot convert large enums"); - return ({{TYPE}}) -1; - {{else}} - // large integer type and no access to PyLong internals => allow for a more expensive conversion - { - {{TYPE}} val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); -#if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } -#endif - if (likely(v)) { - int ret = -1; -#if PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); + // large integer type and no access to PyLong internals => allow for a more expensive conversion + { + {{TYPE}} val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + // failed + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); #else +{{if IS_ENUM}} + // The fallback implementation uses math operations like shifting, which do not work well with enums. + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() or PyLong_AsNativeBytes() not available, cannot convert large enums"); + val = ({{TYPE}}) -1; +{{else}} // Inefficient copy of bit chunks through the C-API. Probably still better than a "cannot do this" exception. -// This is substantially faster in CPython (>30%) than calling "int.to_bytes()" - PyObject *stepval = NULL, *mask = NULL, *shift = NULL; - int bits, remaining_bits, is_negative = 0; - long idigit; - int chunk_size = (sizeof(long) < 8) ? 30 : 62; - - // use exact PyLong to prevent user defined &&/<</etc. implementations - if (unlikely(!PyLong_CheckExact(v))) { - PyObject *tmp = v; - v = PyNumber_Long(v); - assert(PyLong_CheckExact(v)); - Py_DECREF(tmp); - if (unlikely(!v)) return ({{TYPE}}) -1; - } - -#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030B0000 - if (Py_SIZE(x) == 0) - return ({{TYPE}}) 0; - is_negative = Py_SIZE(x) < 0; -#else - { - // misuse Py_False as a quick way to compare to a '0' int object - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return ({{TYPE}}) -1; - is_negative = result == 1; - } -#endif +// This is substantially faster in CPython (>30%) than calling "int.to_bytes()" through the C-API. + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + + // Use exact PyLong to prevent user defined &&/<</etc. implementations (and make Py_SIZE() work below). + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return ({{TYPE}}) -1; + assert(PyLong_CheckExact(v)); + } - if (is_unsigned && unlikely(is_negative)) { - goto raise_neg_overflow; - } else if (is_negative) { - // bit-invert to make sure we can safely convert it - stepval = PyNumber_Invert(v); - if (unlikely(!stepval)) - return ({{TYPE}}) -1; - } else { - stepval = __Pyx_NewRef(v); - } - - // unpack full chunks of bits - val = ({{TYPE}}) 0; - mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; - shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; - for (bits = 0; bits < (int) sizeof({{TYPE}}) * 8 - chunk_size; bits += chunk_size) { - PyObject *tmp, *digit; - - digit = PyNumber_And(stepval, mask); - if (unlikely(!digit)) goto done; - idigit = PyLong_AsLong(digit); - Py_DECREF(digit); - if (unlikely(idigit < 0)) goto done; - - tmp = PyNumber_Rshift(stepval, shift); - if (unlikely(!tmp)) goto done; - Py_DECREF(stepval); stepval = tmp; - - val |= (({{TYPE}}) idigit) << bits; - - #if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030B0000 - if (Py_SIZE(stepval) == 0) - goto unpacking_done; - #endif - } - - // detect overflow when adding the last bits - idigit = PyLong_AsLong(stepval); - if (unlikely(idigit < 0)) goto done; - remaining_bits = ((int) sizeof({{TYPE}}) * 8) - bits - (is_unsigned ? 0 : 1); - if (unlikely(idigit >= (1L << remaining_bits))) - goto raise_overflow; - val |= (({{TYPE}}) idigit) << bits; - - #if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030B0000 - unpacking_done: - #endif - // handle sign and overflow into sign bit - if (!is_unsigned) { - // gcc warns about unsigned (val < 0) => test sign bit instead - if (unlikely(val & ((({{TYPE}}) 1) << (sizeof({{TYPE}}) * 8 - 1)))) - goto raise_overflow; - // undo the PyNumber_Invert() above - if (is_negative) - val = ~val; - } - ret = 0; - done: - Py_XDECREF(shift); - Py_XDECREF(mask); - Py_XDECREF(stepval); -#endif + // Misuse Py_False as a quick way to compare to a '0' int object. + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { Py_DECREF(v); - if (likely(!ret)) - return val; + return ({{TYPE}}) -1; } - return ({{TYPE}}) -1; + is_negative = result == 1; } - {{endif}} - } else { - {{TYPE}} val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return ({{TYPE}}) -1; - val = {{FROM_PY_FUNCTION}}(tmp); - Py_DECREF(tmp); + + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + // bit-invert to make sure we can safely convert it + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return ({{TYPE}}) -1; + } else { + stepval = v; + } + v = NULL; + + // Unpack full chunks of bits. + val = ({{TYPE}}) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof({{TYPE}}) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= (({{TYPE}}) idigit) << bits; + + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + + // Add the last bits and detect overflow. + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof({{TYPE}}) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= (({{TYPE}}) idigit) << bits; + } + + // Handle sign and overflow into sign bit. + if (!is_unsigned) { + // gcc warns about unsigned (val < 0) => test sign bit instead + if (unlikely(val & ((({{TYPE}}) 1) << (sizeof({{TYPE}}) * 8 - 1)))) + goto raise_overflow; + // undo the PyNumber_Invert() above + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +{{endif}} +#endif + if (unlikely(ret)) + return ({{TYPE}}) -1; return val; } diff --git a/contrib/tools/cython/cython.py b/contrib/tools/cython/cython.py index 3923742256e..d8ab80706b9 100755 --- a/contrib/tools/cython/cython.py +++ b/contrib/tools/cython/cython.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Change content of this file to change uids for cython programs - cython 3.0.10 r0 +# Change content of this file to change uids for cython programs - cython 3.0.11 r0 # # Cython -- Main Program, generic diff --git a/contrib/tools/cython/patches/pr6343.patch b/contrib/tools/cython/patches/pr6343.patch new file mode 100644 index 00000000000..faffef9e0b8 --- /dev/null +++ b/contrib/tools/cython/patches/pr6343.patch @@ -0,0 +1,36 @@ +From 2e774fca5ad55371b21ba2b531d218888319c151 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Gonzalo=20Tornar=C3=ADa?= <tornaria@cmat.edu.uy> +Date: Sun, 18 Aug 2024 17:43:05 -0300 +Subject: [PATCH 1/2] Better fix for #6122 to avoid #6535 + +The change in #6124 introduces a regression with functions that are +implicit noexcept in a pxd file. +--- + Cython/Compiler/Nodes.py | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py +index d4737f7c373..b70d766ed4e 100644 +--- a/Cython/Compiler/Nodes.py ++++ b/Cython/Compiler/Nodes.py +@@ -710,10 +710,8 @@ def analyse(self, return_type, env, nonempty=0, directive_locals=None, visibilit + and not self.has_explicit_exc_clause + and self.exception_check + and visibility != 'extern'): +- # If function is already declared from pxd, the exception_check has already correct value. +- if not (self.declared_name() in env.entries and not in_pxd): +- self.exception_check = False + # implicit noexcept, with a warning ++ self.exception_check = False + warning(self.pos, + "Implicit noexcept declaration is deprecated." + " Function declaration should contain 'noexcept' keyword.", +@@ -3128,6 +3126,7 @@ def as_cfunction(self, cfunc=None, scope=None, overridable=True, returns=None, e + if scope is None: + scope = cfunc.scope + cfunc_type = cfunc.type ++ has_explicit_exc_clause=True + if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs: + error(self.pos, "wrong number of arguments") + error(cfunc.pos, "previous declaration here") + diff --git a/contrib/tools/cython/ya.make b/contrib/tools/cython/ya.make index ba485359605..38619e6d9b4 100644 --- a/contrib/tools/cython/ya.make +++ b/contrib/tools/cython/ya.make @@ -11,9 +11,9 @@ LICENSE_TEXTS(.yandex_meta/licenses.list.txt) SUBSCRIBER(g:python-contrib) -VERSION(3.0.10) +VERSION(3.0.11) -ORIGINAL_SOURCE(mirror://pypi/C/Cython/Cython-3.0.10.tar.gz) +ORIGINAL_SOURCE(mirror://pypi/c/cython/cython-3.0.11.tar.gz) NO_LINT() diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h index f63f259d5f0..4a5df49f30a 100644 --- a/library/cpp/yt/logging/logger-inl.h +++ b/library/cpp/yt/logging/logger-inl.h @@ -332,9 +332,11 @@ inline void LogEventImpl( event.SourceLine = sourceLocation.Line; event.Anchor = anchor; if (Y_UNLIKELY(event.Level >= ELogLevel::Alert)) { + logger.Write(TLogEvent(event)); OnCriticalLogEvent(logger, event); + } else { + logger.Write(std::move(event)); } - logger.Write(std::move(event)); } } // namespace NDetail diff --git a/util/thread/pool.cpp b/util/thread/pool.cpp index 2cc72e7f4cc..798a9cf9484 100644 --- a/util/thread/pool.cpp +++ b/util/thread/pool.cpp @@ -1,6 +1,7 @@ #include <atomic> #include <util/system/defaults.h> +#include <util/system/spinlock.h> #if defined(_unix_) #include <pthread.h> @@ -295,30 +296,36 @@ private: private: void ChildAction() { - TTryGuard guard{ActionMutex}; - // If you get an error here, it means you've used fork(2) in multi-threaded environment and probably created thread pools often. - // Don't use fork(2) in multi-threaded programs, don't create thread pools often. - // The mutex is locked after fork iff the fork(2) call was concurrent with RegisterObject / UnregisterObject in another thread. - Y_ABORT_UNLESS(guard.WasAcquired(), "Failed to acquire ActionMutex after fork"); + Y_ABORT_UNLESS(ActionMutex.IsLocked(), "ActionMutex must be locked after fork"); for (auto it = RegisteredObjects.Begin(); it != RegisteredObjects.End(); ++it) { it->AtforkAction(); } + + ActionMutex.Release(); } static void ProcessChildAction() { Get().ChildAction(); } + static void ProcessParentBeforeFork() { + Get().ActionMutex.Acquire(); + } + + static void ProcessParentAfterFork() { + Get().ActionMutex.Release(); + } + TIntrusiveList<TImpl> RegisteredObjects; - TMutex ActionMutex; + TAdaptiveLock ActionMutex; public: inline TAtforkQueueRestarter() { #if defined(_bionic_) // no pthread_atfork on android libc #elif defined(_unix_) - pthread_atfork(nullptr, nullptr, ProcessChildAction); + pthread_atfork(ProcessParentBeforeFork, ProcessParentAfterFork, ProcessChildAction); #endif } }; diff --git a/yql/essentials/core/cbo/cbo_interesting_orderings.cpp b/yql/essentials/core/cbo/cbo_interesting_orderings.cpp index a0bcd96f250..82237151690 100644 --- a/yql/essentials/core/cbo/cbo_interesting_orderings.cpp +++ b/yql/essentials/core/cbo/cbo_interesting_orderings.cpp @@ -355,8 +355,11 @@ bool TOrderingsStateMachine::TLogicalOrderings::HasState() const { } bool TOrderingsStateMachine::TLogicalOrderings::IsSubsetOf(const TLogicalOrderings& logicalOrderings) { - Y_ASSERT(DFSM == logicalOrderings.DFSM); - return HasState() && logicalOrderings.HasState() && IsSubset(DFSM->Nodes[State].NFSMNodesBitset, logicalOrderings.DFSM->Nodes[logicalOrderings.State].NFSMNodesBitset); + if (DFSM == nullptr || logicalOrderings.DFSM == nullptr) { + return false; + } + + return (DFSM == logicalOrderings.DFSM) && HasState() && logicalOrderings.HasState() && IsSubset(DFSM->Nodes[State].NFSMNodesBitset, logicalOrderings.DFSM->Nodes[logicalOrderings.State].NFSMNodesBitset); } i64 TOrderingsStateMachine::TLogicalOrderings::GetState() const { diff --git a/yt/python/yt/yson/convert.py b/yt/python/yt/yson/convert.py index 3868fa42441..947e708979e 100644 --- a/yt/python/yt/yson/convert.py +++ b/yt/python/yt/yson/convert.py @@ -119,44 +119,91 @@ def json_to_yson(json_tree, use_byte_strings=None): return result -def yson_to_json(yson_tree, print_attributes=True): +def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, annotate_with_types=False): + should_annotate_with_types = ( + annotate_with_types and + not isinstance(yson_tree, list) and + not isinstance(yson_tree, dict) and + not isinstance(yson_tree, YsonEntity) and + yson_tree is not None + ) + def encode_key(key): - if PY3 and isinstance(key, binary_type): + if isinstance(key, binary_type): key = key.decode("ascii") if key and key[0] == "$": return "$" + key return key def process_dict(d): - return dict((encode_key(k), yson_to_json(v)) for k, v in iteritems(d)) + return dict( + ( + encode_key(k), + _yson_to_json(v, print_attributes=print_attributes, annotate_with_types=annotate_with_types), + ) for k, v in iteritems(d) + ) + + def get_type_name(): + if isinstance(yson_tree, YsonType): + yson_type_str = yson_tree.get_yson_type_str() + + if yson_type_str is not None: + return yson_type_str + + if isinstance(yson_tree, bool): + return "bool" + elif isinstance(yson_tree, int): + return "int64" + elif isinstance(yson_tree, float): + return "double" + elif isinstance(yson_tree, str) or isinstance(yson_tree, binary_type): + return "string" + else: + raise RuntimeError("Failed to perform yson to json conversion of {!r}, unknown type {!r} to annotate with types".format( + yson_tree, + type(yson_tree) + )) + + def do_annotate_with_types(value): + return {"$type": get_type_name(), "$value": value} if should_annotate_with_types else value + + if hasattr(yson_tree, "attributes") and yson_tree.attributes and print_attributes and not attributes_printed: + # If value is primitive do not pass annotate with types. + value_annotate_with_types = False if should_annotate_with_types else annotate_with_types + + result = { + "$attributes": process_dict(yson_tree.attributes), + "$value": _yson_to_json(yson_tree, print_attributes=print_attributes, attributes_printed=True, annotate_with_types=value_annotate_with_types), + } + + if should_annotate_with_types: + result["$type"] = get_type_name() + + return result - if hasattr(yson_tree, "attributes") and yson_tree.attributes and print_attributes: - return {"$attributes": process_dict(yson_tree.attributes), - "$value": yson_to_json(yson_tree, print_attributes=False)} if isinstance(yson_tree, list): - return list(imap(yson_to_json, yson_tree)) + return [_yson_to_json(element, print_attributes=print_attributes, annotate_with_types=annotate_with_types) for element in yson_tree] elif isinstance(yson_tree, dict): return process_dict(yson_tree) elif isinstance(yson_tree, YsonEntity): return None - elif PY3 and (isinstance(yson_tree, YsonString) or isinstance(yson_tree, binary_type)): - return yson_tree.decode("utf-8") + elif isinstance(yson_tree, YsonString) or isinstance(yson_tree, binary_type): + return do_annotate_with_types(yson_tree.decode("utf-8")) elif isinstance(yson_tree, bool) or isinstance(yson_tree, YsonBoolean): - return True if yson_tree else False + return do_annotate_with_types(True if yson_tree else False) else: - if type(yson_tree) is YsonEntity: - return None - bases = type(yson_tree).__bases__ - iter = 0 while len(bases) == 1 and YsonType not in bases: bases = bases[0].__bases__ - iter += 1 if YsonType in bases: other_types = list(set(bases) - set([YsonType])) if not other_types: raise RuntimeError("Failed to perform yson to json conversion of {!r}".format(yson_tree)) other = other_types[0] - return other(yson_tree) - return yson_tree + return do_annotate_with_types(other(yson_tree)) + return do_annotate_with_types(yson_tree) + + +def yson_to_json(yson_tree, print_attributes=True, annotate_with_types=False): + return _yson_to_json(yson_tree, print_attributes=print_attributes, annotate_with_types=annotate_with_types) diff --git a/yt/python/yt/yson/yson_types.py b/yt/python/yt/yson/yson_types.py index 00dbcc55477..eaa6608876e 100644 --- a/yt/python/yt/yson/yson_types.py +++ b/yt/python/yt/yson/yson_types.py @@ -41,6 +41,9 @@ class YsonType(object): raise TypeError("unhashable type: YSON has non-trivial attributes") return hash(type_(self)) + def get_yson_type_str(selfself): + return None + class YsonString(binary_type, YsonType): def __eq__(self, other): @@ -59,6 +62,9 @@ class YsonString(binary_type, YsonType): def __repr__(self): return self.to_str(binary_type, repr) + def get_yson_type_str(self): + return "string" + class YsonUnicode(text_type, YsonType): def __eq__(self, other): @@ -75,6 +81,9 @@ class YsonUnicode(text_type, YsonType): def __repr__(self): return self.to_str(text_type, repr) + def get_yson_type_str(self): + return "string" + class NotUnicodeError(YtError, TypeError): pass @@ -165,6 +174,9 @@ class YsonStringProxy(YsonType): def __ne__(self, other): return not (self == other) + def get_yson_type_str(self): + return "string" + def is_unicode(x): return isinstance(x, text_type) @@ -214,11 +226,13 @@ class YsonIntegerBase(_YsonIntegerBase, YsonType): class YsonInt64(YsonIntegerBase): - pass + def get_yson_type_str(self): + return "int64" class YsonUint64(YsonIntegerBase): - pass + def get_yson_type_str(self): + return "uint64" class YsonDouble(float, YsonType): @@ -239,6 +253,9 @@ class YsonDouble(float, YsonType): def __str__(self): return self.to_str(float, str) + def get_yson_type_str(self): + return "double" + class YsonBoolean(int, YsonType): def __eq__(self, other): @@ -261,6 +278,9 @@ class YsonBoolean(int, YsonType): def __str__(self): return self.__repr__() + def get_yson_type_str(self): + return "bool" + class YsonList(list, YsonType): def __eq__(self, other): diff --git a/yt/yt/client/api/query_tracker_client.h b/yt/yt/client/api/query_tracker_client.h index 700f4fa8205..69990a38761 100644 --- a/yt/yt/client/api/query_tracker_client.h +++ b/yt/yt/client/api/query_tracker_client.h @@ -57,8 +57,8 @@ struct TStartQueryOptions bool Draft = false; NYTree::IMapNodePtr Annotations; std::vector<TQueryFilePtr> Files; - std::optional<TString> AccessControlObject; // COMPAT(mpereskokova) - std::optional<std::vector<TString>> AccessControlObjects; + std::optional<std::string> AccessControlObject; // COMPAT(mpereskokova) + std::optional<std::vector<std::string>> AccessControlObjects; std::vector<TQuerySecretPtr> Secrets; }; @@ -157,8 +157,8 @@ struct TAlterQueryOptions , public TQueryTrackerOptions { NYTree::IMapNodePtr Annotations; - std::optional<TString> AccessControlObject; // COMPAT(mpereskokova) - std::optional<std::vector<TString>> AccessControlObjects; + std::optional<std::string> AccessControlObject; // COMPAT(mpereskokova) + std::optional<std::vector<std::string>> AccessControlObjects; }; struct TGetQueryTrackerInfoOptions @@ -173,7 +173,7 @@ struct TGetQueryTrackerInfoResult TString QueryTrackerStage; std::string ClusterName; NYson::TYsonString SupportedFeatures; - std::vector<TString> AccessControlObjects; + std::vector<std::string> AccessControlObjects; std::vector<std::string> Clusters; }; diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index 14517c47cae..61cf0e45dcf 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -2536,7 +2536,7 @@ TFuture<TGetQueryTrackerInfoResult> TClient::GetQueryTrackerInfo( .QueryTrackerStage = FromProto<TString>(rsp->query_tracker_stage()), .ClusterName = FromProto<std::string>(rsp->cluster_name()), .SupportedFeatures = TYsonString(rsp->supported_features()), - .AccessControlObjects = FromProto<std::vector<TString>>(rsp->access_control_objects()), + .AccessControlObjects = FromProto<std::vector<std::string>>(rsp->access_control_objects()), .Clusters = FromProto<std::vector<std::string>>(rsp->clusters()) }; })); diff --git a/yt/yt/client/api/security_client.h b/yt/yt/client/api/security_client.h index 23a37b1a428..24c1831cf59 100644 --- a/yt/yt/client/api/security_client.h +++ b/yt/yt/client/api/security_client.h @@ -41,7 +41,7 @@ struct TCheckPermissionResult NObjectClient::TObjectId ObjectId; std::optional<TString> ObjectName; NSecurityClient::TSubjectId SubjectId; - std::optional<TString> SubjectName; + std::optional<std::string> SubjectName; }; struct TCheckPermissionResponse @@ -64,8 +64,8 @@ struct TCheckPermissionByAclResult NSecurityClient::ESecurityAction Action; NSecurityClient::TSubjectId SubjectId; - std::optional<TString> SubjectName; - std::vector<TString> MissingSubjects; + std::optional<std::string> SubjectName; + std::vector<std::string> MissingSubjects; }; struct TSetUserPasswordOptions diff --git a/yt/yt/client/complex_types/check_type_compatibility.cpp b/yt/yt/client/complex_types/check_type_compatibility.cpp index 5d5ecf70b21..18aa1a3eb28 100644 --- a/yt/yt/client/complex_types/check_type_compatibility.cpp +++ b/yt/yt/client/complex_types/check_type_compatibility.cpp @@ -229,6 +229,12 @@ static TCompatibilityPair CheckTypeCompatibilitySimple( case ESimpleLogicalValueType::Datetime64: case ESimpleLogicalValueType::Timestamp64: case ESimpleLogicalValueType::Interval64: + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: case ESimpleLogicalValueType::Any: { const auto compatibility = oldElement == newElement ? ESchemaCompatibility::FullyCompatible : ESchemaCompatibility::Incompatible; diff --git a/yt/yt/client/driver/config.cpp b/yt/yt/client/driver/config.cpp index ea20d3a1ce9..bb36b67605d 100644 --- a/yt/yt/client/driver/config.cpp +++ b/yt/yt/client/driver/config.cpp @@ -49,6 +49,9 @@ void TDriverConfig::Register(TRegistrar registrar) registrar.Parameter("token", &TThis::Token) .Optional(); + registrar.Parameter("multiproxy_target_cluster", &TThis::MultiproxyTargetCluster) + .Optional(); + registrar.Parameter("proxy_discovery_cache", &TThis::ProxyDiscoveryCache) .DefaultNew(); diff --git a/yt/yt/client/driver/config.h b/yt/yt/client/driver/config.h index a5ddf95bf50..603abbd0498 100644 --- a/yt/yt/client/driver/config.h +++ b/yt/yt/client/driver/config.h @@ -38,6 +38,9 @@ struct TDriverConfig std::optional<TString> Token; + //! Target cluster for multiproxy mode. + std::optional<std::string> MultiproxyTargetCluster; + TAsyncExpiringCacheConfigPtr ProxyDiscoveryCache; bool EnableInternalCommands; @@ -57,4 +60,3 @@ DEFINE_REFCOUNTED_TYPE(TDriverConfig) //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NDriver - diff --git a/yt/yt/client/driver/driver.cpp b/yt/yt/client/driver/driver.cpp index 1a2ee7a8399..a722af45c71 100644 --- a/yt/yt/client/driver/driver.cpp +++ b/yt/yt/client/driver/driver.cpp @@ -59,6 +59,13 @@ constinit const auto Logger = DriverLogger; //////////////////////////////////////////////////////////////////////////////// +static TClientOptions GetRootClientOptions(const TDriverConfigPtr& config) +{ + auto result = TClientOptions::Root(); + result.MultiproxyTargetCluster = config->MultiproxyTargetCluster; + return result; +} + void Serialize(const TCommandDescriptor& descriptor, NYson::IYsonConsumer* consumer) { BuildYsonFluently(consumer) @@ -107,7 +114,6 @@ TCommandDescriptor IDriver::GetCommandDescriptorOrThrow(const TString& commandNa //////////////////////////////////////////////////////////////////////////////// - class TDriver : public IDriver { @@ -121,7 +127,7 @@ public: , ClientCache_(New<TClientCache>(Config_->ClientCache, Connection_)) , RootClient_(ClientCache_->Get( GetRootAuthenticationIdentity(), - TClientOptions::FromAuthenticationIdentity(GetRootAuthenticationIdentity()))) + GetRootClientOptions(Config_))) , ProxyDiscoveryCache_(CreateProxyDiscoveryCache( Config_->ProxyDiscoveryCache, RootClient_)) @@ -451,6 +457,7 @@ public: options.ServiceTicketAuth = request.ServiceTicket ? std::make_optional(New<NAuth::TServiceTicketFixedAuth>(*request.ServiceTicket)) : std::nullopt; + options.MultiproxyTargetCluster = Config_->MultiproxyTargetCluster; auto client = ClientCache_->Get(identity, options); diff --git a/yt/yt/client/driver/query_commands.cpp b/yt/yt/client/driver/query_commands.cpp index 28ace5ea5b0..25aca01345f 100644 --- a/yt/yt/client/driver/query_commands.cpp +++ b/yt/yt/client/driver/query_commands.cpp @@ -63,14 +63,14 @@ void TStartQueryCommand::Register(TRegistrar registrar) }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<std::optional<TString>>( + registrar.ParameterWithUniversalAccessor<std::optional<std::string>>( "access_control_object", [] (TThis* command) -> auto& { return command->Options.AccessControlObject; }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<std::optional<std::vector<TString>>>( + registrar.ParameterWithUniversalAccessor<std::optional<std::vector<std::string>>>( "access_control_objects", [] (TThis* command) -> auto& { return command->Options.AccessControlObjects; @@ -339,14 +339,14 @@ void TAlterQueryCommand::Register(TRegistrar registrar) }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<std::optional<TString>>( + registrar.ParameterWithUniversalAccessor<std::optional<std::string>>( "access_control_object", [] (TThis* command) -> auto& { return command->Options.AccessControlObject; }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<std::optional<std::vector<TString>>>( + registrar.ParameterWithUniversalAccessor<std::optional<std::vector<std::string>>>( "access_control_objects", [] (TThis* command) -> auto& { return command->Options.AccessControlObjects; diff --git a/yt/yt/client/table_client/logical_type-inl.h b/yt/yt/client/table_client/logical_type-inl.h index 28bd5b17a6c..fe193e42f7f 100644 --- a/yt/yt/client/table_client/logical_type-inl.h +++ b/yt/yt/client/table_client/logical_type-inl.h @@ -142,4 +142,34 @@ const TLogicalTypePtr& TTaggedLogicalType::GetElement() const //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT::NTableClient
\ No newline at end of file +#define XX(cppType, ytType) \ + template <> \ + struct TUnderlyingTimestampIntegerTypeImpl<ESimpleLogicalValueType::ytType> \ + { \ + using TValue = cppType; \ + }; \ + \ + template <> \ + struct TUnderlyingTzTypeImpl<ESimpleLogicalValueType::Tz##ytType> \ + { \ + static constexpr ESimpleLogicalValueType TValue = ESimpleLogicalValueType::ytType; \ + }; + + XX(ui16, Date) + XX(ui32, Datetime) + XX(ui64, Timestamp) + XX(i32, Date32) + XX(i64, Datetime64) + XX(i64, Timestamp64) + +#undef XX + +template<ESimpleLogicalValueType type> +constexpr ESimpleLogicalValueType GetUnderlyingDateType() +{ + return TUnderlyingTzTypeImpl<type>::TValue; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NTableClient diff --git a/yt/yt/client/table_client/logical_type.cpp b/yt/yt/client/table_client/logical_type.cpp index 9c635f4405d..409cd82d5f3 100644 --- a/yt/yt/client/table_client/logical_type.cpp +++ b/yt/yt/client/table_client/logical_type.cpp @@ -1287,6 +1287,12 @@ void TDictLogicalType::ValidateNode(const TWalkContext&) const case ESimpleLogicalValueType::Datetime64: case ESimpleLogicalValueType::Timestamp64: case ESimpleLogicalValueType::Interval64: + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: return; } YT_ABORT(); @@ -1891,6 +1897,13 @@ bool IsComparable(const TLogicalTypePtr& type) case ESimpleLogicalValueType::Datetime64: case ESimpleLogicalValueType::Timestamp64: case ESimpleLogicalValueType::Interval64: + + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: return true; case ESimpleLogicalValueType::Any: @@ -1922,6 +1935,26 @@ bool IsComparable(const TLogicalTypePtr& type) } } +bool IsTzType(const TLogicalTypePtr& logicalType) +{ + switch (logicalType->GetMetatype()) { + case ELogicalMetatype::Simple: + switch (logicalType->AsSimpleTypeRef().GetElement()) { + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: + return true; + default: + return false; + } + default: + return false; + } +} + //////////////////////////////////////////////////////////////////////////////// struct TV3Variant @@ -1964,6 +1997,13 @@ static const std::pair<ESimpleLogicalValueType, TString> V3SimpleLogicalValueTyp {ESimpleLogicalValueType::Datetime64, "datetime64"}, {ESimpleLogicalValueType::Timestamp64, "timestamp64"}, {ESimpleLogicalValueType::Interval64, "interval64"}, + + {ESimpleLogicalValueType::TzDate, "tz_date"}, + {ESimpleLogicalValueType::TzDatetime, "tz_datetime"}, + {ESimpleLogicalValueType::TzTimestamp, "tz_timestamp"}, + {ESimpleLogicalValueType::TzDate32, "tz_date32"}, + {ESimpleLogicalValueType::TzDatetime64, "tz_datetime64"}, + {ESimpleLogicalValueType::TzTimestamp64, "tz_timestamp64"}, }; static_assert(std::size(V3SimpleLogicalValueTypeEncoding) == TEnumTraits<ESimpleLogicalValueType>::GetDomainSize()); diff --git a/yt/yt/client/table_client/logical_type.h b/yt/yt/client/table_client/logical_type.h index 80cfebd081d..bffc45616fe 100644 --- a/yt/yt/client/table_client/logical_type.h +++ b/yt/yt/client/table_client/logical_type.h @@ -146,6 +146,8 @@ void FromProto(TLogicalTypePtr* logicalType, const NProto::TLogicalType& protoLo bool IsComparable(const TLogicalTypePtr& type); +bool IsTzType(const TLogicalTypePtr& logicalType); + //////////////////////////////////////////////////////////////////////////////// // Special wrapper class that allows to serialize LogicalType in type_v3 format @@ -441,6 +443,11 @@ TLogicalTypePtr MakeLogicalType(ESimpleLogicalValueType type, bool required); //////////////////////////////////////////////////////////////////////////////// +template<ESimpleLogicalValueType type> +constexpr ESimpleLogicalValueType GetUnderlyingDateType(); + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NTableClient template <> diff --git a/yt/yt/client/table_client/public.h b/yt/yt/client/table_client/public.h index 5c6164a0466..d314028ccb7 100644 --- a/yt/yt/client/table_client/public.h +++ b/yt/yt/client/table_client/public.h @@ -466,6 +466,15 @@ using TUUComparerSignature = int(const TUnversionedValue*, const TUnversionedVal struct TVersionedReadOptions; struct TVersionedWriteOptions; +template <ESimpleLogicalValueType type> +struct TUnderlyingTzTypeImpl; + +template <ESimpleLogicalValueType type> +struct TUnderlyingTimestampIntegerTypeImpl; + +template <ESimpleLogicalValueType type> +using TUnderlyingTimestampIntegerType = TUnderlyingTimestampIntegerTypeImpl<type>::TValue; + //////////////////////////////////////////////////////////////////////////////// YT_DEFINE_STRONG_TYPEDEF(TSignedDistributedWriteSessionPtr, NSignature::TSignaturePtr); diff --git a/yt/yt/client/table_client/row_base.h b/yt/yt/client/table_client/row_base.h index cc4a34c4467..e66bb9a52e1 100644 --- a/yt/yt/client/table_client/row_base.h +++ b/yt/yt/client/table_client/row_base.h @@ -76,6 +76,13 @@ DEFINE_ENUM_WITH_UNDERLYING_TYPE(ESimpleLogicalValueType, ui32, ((Datetime64) (0x1011)) ((Timestamp64) (0x1012)) ((Interval64) (0x1013)) + + ((TzDate) (0x1014)) + ((TzDatetime) (0x1015)) + ((TzTimestamp) (0x1016)) + ((TzDate32) (0x1017)) + ((TzDatetime64) (0x1018)) + ((TzTimestamp64) (0x1019)) ); //! Debug printers for Gtest unittests. @@ -229,6 +236,14 @@ inline constexpr EValueType GetPhysicalType(ESimpleLogicalValueType type) case ESimpleLogicalValueType::Interval64: return EValueType::Int64; + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: + return EValueType::String; + default: YT_ABORT(); } diff --git a/yt/yt/client/table_client/unversioned_row.cpp b/yt/yt/client/table_client/unversioned_row.cpp index 5cff922a3cd..ed7b8fed7aa 100644 --- a/yt/yt/client/table_client/unversioned_row.cpp +++ b/yt/yt/client/table_client/unversioned_row.cpp @@ -1074,6 +1074,13 @@ void ValidateValueType( CASE(ESimpleLogicalValueType::Datetime64) CASE(ESimpleLogicalValueType::Timestamp64) CASE(ESimpleLogicalValueType::Interval64) + + CASE(ESimpleLogicalValueType::TzDate) + CASE(ESimpleLogicalValueType::TzDatetime) + CASE(ESimpleLogicalValueType::TzTimestamp) + CASE(ESimpleLogicalValueType::TzDate32) + CASE(ESimpleLogicalValueType::TzDatetime64) + CASE(ESimpleLogicalValueType::TzTimestamp64) #undef CASE } YT_ABORT(); diff --git a/yt/yt/client/table_client/validate_logical_type-inl.h b/yt/yt/client/table_client/validate_logical_type-inl.h index e4e30051e0a..fe51870f8f0 100644 --- a/yt/yt/client/table_client/validate_logical_type-inl.h +++ b/yt/yt/client/table_client/validate_logical_type-inl.h @@ -8,6 +8,8 @@ #include <yt/yt/core/misc/error.h> +#include <yt/yt/library/tz_types/tz_types.h> + #include <util/charset/utf8.h> #include <cmath> @@ -213,6 +215,40 @@ void ValidateSimpleLogicalType(TStringBuf value) THROW_ERROR_EXCEPTION(NTableClient::EErrorCode::SchemaViolation, "Not a valid Uuid"); } + } else if constexpr ( + type == ESimpleLogicalValueType::TzDate || + type == ESimpleLogicalValueType::TzDatetime || + type == ESimpleLogicalValueType::TzTimestamp || + type == ESimpleLogicalValueType::TzDate32 || + type == ESimpleLogicalValueType::TzDatetime64 || + type == ESimpleLogicalValueType::TzTimestamp64) + { + try { + constexpr ESimpleLogicalValueType underlyingDateType = GetUnderlyingDateType<type>(); + + using TInt = TUnderlyingTimestampIntegerType<underlyingDateType>; + + const auto& [timestamp, tzName] = NTzTypes::ParseTzValue<TInt>(value); + + try { + if constexpr (std::is_signed_v<TInt>) { + ValidateSimpleLogicalType<underlyingDateType>(static_cast<i64>(timestamp)); + } else { + ValidateSimpleLogicalType<underlyingDateType>(static_cast<ui64>(timestamp)); + } + } catch (const std::exception& ex) { + THROW_ERROR_EXCEPTION("Cannot validate underlying timestamp of %Qv type", type) + << ex; + } + + NTzTypes::ValidateTzName(tzName); + } catch (const std::exception& ex) { + THROW_ERROR_EXCEPTION( + NTableClient::EErrorCode::SchemaViolation, + "Not a valid timezone time") + << TErrorAttribute("value", value) + << ex; + } } else { static_assert(type == ESimpleLogicalValueType::String, "Bad logical type"); } diff --git a/yt/yt/client/table_client/validate_logical_type.cpp b/yt/yt/client/table_client/validate_logical_type.cpp index b5af53f7797..f3c99ac7d74 100644 --- a/yt/yt/client/table_client/validate_logical_type.cpp +++ b/yt/yt/client/table_client/validate_logical_type.cpp @@ -206,6 +206,12 @@ private: CASE(ESimpleLogicalValueType::Datetime64) CASE(ESimpleLogicalValueType::Timestamp64) CASE(ESimpleLogicalValueType::Interval64) + CASE(ESimpleLogicalValueType::TzDate) + CASE(ESimpleLogicalValueType::TzDatetime) + CASE(ESimpleLogicalValueType::TzTimestamp) + CASE(ESimpleLogicalValueType::TzDate32) + CASE(ESimpleLogicalValueType::TzDatetime64) + CASE(ESimpleLogicalValueType::TzTimestamp64) #undef CASE } YT_ABORT(); diff --git a/yt/yt/client/unittests/validate_logical_type_ut.cpp b/yt/yt/client/unittests/validate_logical_type_ut.cpp index 67ff680f635..b8fe177b5b3 100644 --- a/yt/yt/client/unittests/validate_logical_type_ut.cpp +++ b/yt/yt/client/unittests/validate_logical_type_ut.cpp @@ -9,6 +9,8 @@ namespace NYT::NTableClient { +using namespace NTzTypes; + //////////////////////////////////////////////////////////////////////////////// #define ERROR_ATTRS(logicalType, ysonString) \ @@ -182,6 +184,68 @@ TEST(TValidateLogicalTypeTest, TestAnyType) EXPECT_GOOD_TYPE(SimpleLogicalType(ESimpleLogicalValueType::Any), "[<>1]"); } +TEST(TValidateLogicalTypeTest, TestTimezoneType) +{ + // Simple example. + auto correctValue = MakeTzString<ui16>(42, "Europe/Moscow"); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate>(correctValue)); + + // Short buffer. + TString shortValue = "1"; + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate>(shortValue), "Not a valid timezone time"); + + // Wrong timezone. + auto wrongTimezoneValue = MakeTzString<ui16>(42, "Wrong/Timezone"); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate>(wrongTimezoneValue), "Not a valid timezone time"); + + // Empty timezone. + auto emptyTimezoneValue = MakeTzString<ui16>(42, ""); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate>(emptyTimezoneValue), "Not a valid timezone time"); + + // Validate max value. + auto maxDate = MakeTzString<ui16>(DateUpperBound - 1, "Europe/Moscow"); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate>(maxDate)); + auto maxDatetime = MakeTzString<ui32>(DatetimeUpperBound - 1, "Europe/Moscow"); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDatetime>(maxDatetime)); + auto maxTimestamp = MakeTzString<ui64>(TimestampUpperBound - 1, "Europe/Moscow"); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzTimestamp>(maxTimestamp)); + auto maxDate32 = MakeTzString<i32>(Date32UpperBound - 1, "Europe/Moscow"); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate32>(maxDate32)); + auto maxDatetime64 = MakeTzString<i64>(Datetime64UpperBound - 1, GetTzName(1)); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDatetime64>(maxDatetime64)); + auto maxTimestamp64 = MakeTzString<i64>(Timestamp64UpperBound - 1, GetTzName(1)); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzTimestamp64>(maxTimestamp64)); + + // Validate too big value. + auto tooBigDate = MakeTzString<ui16>(DateUpperBound, GetTzName(1)); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate>(tooBigDate), "Not a valid timezone time"); + auto tooBigDatetime = MakeTzString<ui32>(DatetimeUpperBound, GetTzName(1)); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDatetime>(tooBigDatetime), "Not a valid timezone time"); + auto tooBigTimestamp = MakeTzString<ui64>(TimestampUpperBound, GetTzName(1)); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzTimestamp>(tooBigTimestamp), "Not a valid timezone time"); + auto tooBigDate32 = MakeTzString<i32>(Date32UpperBound, GetTzName(1)); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate32>(tooBigDate32), "Not a valid timezone time"); + auto tooBigDatetime64 = MakeTzString<i64>(Datetime64UpperBound, GetTzName(1)); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDatetime64>(tooBigDatetime64), "Not a valid timezone time"); + auto tooBigTimestamp64 = MakeTzString<i64>(Timestamp64UpperBound, GetTzName(1)); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzTimestamp64>(tooBigTimestamp64), "Not a valid timezone time"); + + // Validate min value. + auto minDate32 = MakeTzString<i32>(Date32LowerBound, GetTzName(1)); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate32>(minDate32)); + auto minDatetime64 = MakeTzString<i64>(Datetime64LowerBound, GetTzName(1)); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDatetime64>(minDatetime64)); + auto minTimestamp64 = MakeTzString<i64>(Timestamp64LowerBound, GetTzName(1)); + EXPECT_NO_THROW(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzTimestamp64>(minTimestamp64)); + + // Validate too small value. + auto tooSmallDate32 = MakeTzString<i32>(Date32LowerBound - 1, GetTzName(1)); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDate32>(tooSmallDate32), "Not a valid timezone time"); + auto tooSmallDatetime64 = MakeTzString<i64>(Datetime64LowerBound - 1, GetTzName(1)); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzDatetime64>(tooSmallDatetime64), "Not a valid timezone time"); + auto tooSmallTimestamp64 = MakeTzString<i64>(Timestamp64LowerBound - 1, GetTzName(1)); + EXPECT_THROW_WITH_SUBSTRING(ValidateSimpleLogicalType<ESimpleLogicalValueType::TzTimestamp64>(tooSmallTimestamp64), "Not a valid timezone time"); +} TEST(TValidateLogicalTypeTest, TestJsonType) { diff --git a/yt/yt/client/ya.make b/yt/yt/client/ya.make index 49d8db4c084..224cc1ba304 100644 --- a/yt/yt/client/ya.make +++ b/yt/yt/client/ya.make @@ -219,10 +219,11 @@ PEERDIR( yt/yt/core/https yt/yt/library/auth yt/yt/library/decimal - yt/yt/library/re2 yt/yt/library/erasure yt/yt/library/numeric yt/yt/library/quantile_digest + yt/yt/library/re2 + yt/yt/library/tz_types yt/yt_proto/yt/client library/cpp/digest/crc32c library/cpp/json diff --git a/yt/yt/core/yson/lexer.cpp b/yt/yt/core/yson/lexer.cpp index e23e0bd5c16..3f1c6462b11 100644 --- a/yt/yt/core/yson/lexer.cpp +++ b/yt/yt/core/yson/lexer.cpp @@ -14,12 +14,4 @@ size_t TStatelessLexer::ParseToken(TStringBuf data, TToken* token) //////////////////////////////////////////////////////////////////////////////// -size_t ParseToken(TStringBuf data, TToken* token) -{ - TStatelessLexer lexer; - return lexer.ParseToken(data, token); -} - -//////////////////////////////////////////////////////////////////////////////// - } // namespace NYT::NYson diff --git a/yt/yt/core/yson/lexer.h b/yt/yt/core/yson/lexer.h index 1d38ca1327d..7f05e5e09f1 100644 --- a/yt/yt/core/yson/lexer.h +++ b/yt/yt/core/yson/lexer.h @@ -18,8 +18,4 @@ private: //////////////////////////////////////////////////////////////////////////////// -size_t ParseToken(TStringBuf data, TToken* token); - -//////////////////////////////////////////////////////////////////////////////// - } // namespace NYT::NYson diff --git a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h index d0a4f2b6a8e..b9ea6afaf45 100644 --- a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h +++ b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h @@ -65,7 +65,7 @@ void TPolymorphicYsonStruct<TMapping>::Load( TSource source, bool postprocess, bool setDefaults, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { using TTraits = NPrivate::TYsonSourceTraits<TSource>; @@ -101,7 +101,7 @@ void TPolymorphicYsonStruct<TMapping>::Load( // "type" must be unrecognized for the original struct // therefore we must delete it prior to |Load| call. map->RemoveChild("type"); - Storage_->Load(map, postprocess, setDefaults, path); + Storage_->Load(map, postprocess, setDefaults, pathGetter); // NB(arkady-e1ppa): We must not actually remove contents of the node as a postcondition // since it mutates serialized data which might be used for config validation. @@ -180,7 +180,7 @@ void TPolymorphicYsonStruct<TMapping>::MergeWith(const TPolymorphicYsonStruct& o SerializedStorage_, /*postprocess*/ true, /*setDefaults*/ true, - /*path*/ "", + /*path*/ {}, /*recursiveUnrecognizedStrategy*/ std::nullopt); } diff --git a/yt/yt/core/ytree/polymorphic_yson_struct.h b/yt/yt/core/ytree/polymorphic_yson_struct.h index 3c89ee758bd..ee4732fa8e7 100644 --- a/yt/yt/core/ytree/polymorphic_yson_struct.h +++ b/yt/yt/core/ytree/polymorphic_yson_struct.h @@ -153,7 +153,7 @@ public: TSource source, bool postprocess = true, bool setDefaults = true, - const NYPath::TYPath& path = {}, + const std::function<NYPath::TYPath()>& pathGetter = {}, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy = {}); void Save(NYson::IYsonConsumer* consumer) const; diff --git a/yt/yt/core/ytree/yson_struct.cpp b/yt/yt/core/ytree/yson_struct.cpp index e019fc22fac..e4a6b30df76 100644 --- a/yt/yt/core/ytree/yson_struct.cpp +++ b/yt/yt/core/ytree/yson_struct.cpp @@ -64,9 +64,29 @@ void TYsonStructBase::Load( INodePtr node, bool postprocess, bool setDefaults, + const std::function<NYPath::TYPath()>& pathGetter) +{ + Meta_->LoadStruct(this, std::move(node), postprocess, setDefaults, pathGetter); +} + +void TYsonStructBase::Load( + TYsonPullParserCursor* cursor, + bool postprocess, + bool setDefaults, + const std::function<NYPath::TYPath()>& pathGetter) +{ + Meta_->LoadStruct(this, cursor, postprocess, setDefaults, pathGetter); +} + +void TYsonStructBase::Load( + INodePtr node, + bool postprocess, + bool setDefaults, const NYPath::TYPath& path) { - Meta_->LoadStruct(this, std::move(node), postprocess, setDefaults, path); + Load(std::move(node), postprocess, setDefaults, [&] { + return path; + }); } void TYsonStructBase::Load( @@ -75,7 +95,9 @@ void TYsonStructBase::Load( bool setDefaults, const NYPath::TYPath& path) { - Meta_->LoadStruct(this, cursor, postprocess, setDefaults, path); + Load(cursor, postprocess, setDefaults, [&] { + return path; + }); } void TYsonStructBase::Load(IInputStream* input) @@ -154,9 +176,9 @@ void TYsonStructBase::Save(IOutputStream* output) const context.Finish(); } -void TYsonStructBase::Postprocess(const TYPath& path) +void TYsonStructBase::Postprocess(const std::function<NYPath::TYPath()>& pathGetter) { - Meta_->PostprocessStruct(this, path); + Meta_->PostprocessStruct(this, pathGetter); } void TYsonStructBase::SetDefaults() diff --git a/yt/yt/core/ytree/yson_struct.h b/yt/yt/core/ytree/yson_struct.h index cd93566f865..7cceb5a2288 100644 --- a/yt/yt/core/ytree/yson_struct.h +++ b/yt/yt/core/ytree/yson_struct.h @@ -68,17 +68,29 @@ public: INodePtr node, bool postprocess = true, bool setDefaults = true, - const NYPath::TYPath& path = {}); + const std::function<NYPath::TYPath()>& pathGetter = {}); void Load( NYson::TYsonPullParserCursor* cursor, bool postprocess = true, bool setDefaults = true, - const NYPath::TYPath& path = {}); + const std::function<NYPath::TYPath()>& pathGetter = {}); + + void Load( + INodePtr node, + bool postprocess, + bool setDefaults, + const NYPath::TYPath& path); + + void Load( + NYson::TYsonPullParserCursor* cursor, + bool postprocess, + bool setDefaults, + const NYPath::TYPath& path); void Load(IInputStream* input); - void Postprocess(const NYPath::TYPath& path = {}); + void Postprocess(const std::function<NYPath::TYPath()>& pathGetter = {}); void SetDefaults(); @@ -241,7 +253,7 @@ concept CYsonStructLoadableFieldFor = S source, bool postprocess, bool setDefaults, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { // For YsonStruct. @@ -249,7 +261,7 @@ concept CYsonStructLoadableFieldFor = source, postprocess, setDefaults, - path, + pathGetter, recursiveUnrecognizedStrategy); }; diff --git a/yt/yt/core/ytree/yson_struct_detail-inl.h b/yt/yt/core/ytree/yson_struct_detail-inl.h index adff17d5d72..4e7722381ad 100644 --- a/yt/yt/core/ytree/yson_struct_detail-inl.h +++ b/yt/yt/core/ytree/yson_struct_detail-inl.h @@ -208,7 +208,7 @@ template <CYsonStructSource TSource, class T> void LoadFromSource( std::optional<T>& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy); // std::vector @@ -216,7 +216,7 @@ template <CYsonStructSource TSource, CStdVector TVector> void LoadFromSource( TVector& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy); // any map. @@ -224,7 +224,7 @@ template <CYsonStructSource TSource, CAnyMap TMap> void LoadFromSource( TMap& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy); //////////////////////////////////////////////////////////////////////////////// @@ -234,7 +234,7 @@ template <CYsonStructSource TSource, class T> void LoadFromSource( T& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> /*ignored*/) { using TTraits = TYsonSourceTraits<TSource>; @@ -242,7 +242,7 @@ void LoadFromSource( try { Deserialize(parameter, TTraits::AsNode(source)); } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error reading parameter %v", path) + THROW_ERROR_EXCEPTION("Error reading parameter %v", pathGetter()) << ex; } } @@ -252,7 +252,7 @@ template <CYsonStructSource TSource> void LoadFromSource( ::NYT::NYson::TYsonString& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> /*ignored*/) { using TTraits = TYsonSourceTraits<TSource>; @@ -260,7 +260,7 @@ void LoadFromSource( try { parameter = NYson::ConvertToYsonString(TTraits::AsNode(source)); } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) + THROW_ERROR_EXCEPTION("Error loading parameter %v", pathGetter()) << ex; } } @@ -270,7 +270,7 @@ template <CYsonStructSource TSource> void LoadFromSource( INodePtr& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> /*ignored*/) { using TTraits = TYsonSourceTraits<TSource>; @@ -283,7 +283,7 @@ void LoadFromSource( parameter = PatchNode(parameter, std::move(node)); } } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) + THROW_ERROR_EXCEPTION("Error loading parameter %v", pathGetter()) << ex; } } @@ -293,7 +293,7 @@ template <CYsonStructSource TSource, CYsonStructDerived T> void LoadFromSource( TIntrusivePtr<T>& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy) { if (!parameter) { @@ -304,7 +304,7 @@ void LoadFromSource( parameter->SetUnrecognizedStrategy(*unrecognizedStrategy); } - parameter->Load(std::move(source), /*postprocess*/ false, /*setDefaults*/ false, path); + parameter->Load(std::move(source), /*postprocess*/ false, /*setDefaults*/ false, pathGetter); } // YsonStructLite @@ -312,16 +312,16 @@ template <CYsonStructSource TSource, std::derived_from<TYsonStructLite> T> void LoadFromSource( T& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy) { try { if (unrecognizedStrategy) { parameter.SetUnrecognizedStrategy(*unrecognizedStrategy); } - parameter.Load(std::move(source), /*postprocess*/ false, /*setDefaults*/ false, path); + parameter.Load(std::move(source), /*postprocess*/ false, /*setDefaults*/ false, pathGetter); } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error reading parameter %v", path) + THROW_ERROR_EXCEPTION("Error reading parameter %v", pathGetter()) << ex; } } @@ -331,13 +331,13 @@ template <CYsonStructSource TSource, CExternallySerializable T> void LoadFromSource( T& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy) { try { Deserialize(parameter, std::move(source), /*postprocess*/ false, /*setDefaults*/ false, unrecognizedStrategy); } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error reading parameter %v", path) + THROW_ERROR_EXCEPTION("Error reading parameter %v", pathGetter()) << ex; } } @@ -350,7 +350,7 @@ template <CYsonStructSource TSource, CYsonStructLoadableFieldFor<TSource> TExten void LoadFromSource( TExtension& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy) { static_assert(CYsonStructFieldFor<TExtension, TSource>, "You must add alias TImplementsYsonStructField"); @@ -360,10 +360,10 @@ void LoadFromSource( std::move(source), /*postprocess*/ false, /*setDefaults*/ false, - path, + pathGetter, unrecognizedStrategy); } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) + THROW_ERROR_EXCEPTION("Error loading parameter %v", pathGetter()) << ex; } } @@ -373,7 +373,7 @@ template <CYsonStructSource TSource, class T> void LoadFromSource( std::optional<T>& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy) { using TTraits = TYsonSourceTraits<TSource>; @@ -386,16 +386,16 @@ void LoadFromSource( } if (parameter.has_value()) { - LoadFromSource(*parameter, std::move(source), path, unrecognizedStrategy); + LoadFromSource(*parameter, std::move(source), pathGetter, unrecognizedStrategy); return; } T value; - LoadFromSource(value, std::move(source), path, unrecognizedStrategy); + LoadFromSource(value, std::move(source), pathGetter, unrecognizedStrategy); parameter = std::move(value); } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) + THROW_ERROR_EXCEPTION("Error loading parameter %v", pathGetter()) << ex; } } @@ -405,7 +405,7 @@ template <CYsonStructSource TSource, CStdVector TVector> void LoadFromSource( TVector& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy) { using TTraits = TYsonSourceTraits<TSource>; @@ -418,12 +418,14 @@ void LoadFromSource( LoadFromSource( vector.emplace_back(), elementSource, - path + "/" + NYPath::ToYPathLiteral(index), + [&] { + return pathGetter() + "/" + NYPath::ToYPathLiteral(index); + }, unrecognizedStrategy); ++index; }); } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) + THROW_ERROR_EXCEPTION("Error loading parameter %v", pathGetter()) << ex; } } @@ -433,7 +435,7 @@ template <CYsonStructSource TSource, CAnyMap TMap> void LoadFromSource( TMap& parameter, TSource source, - const NYPath::TYPath& path, + const std::function<NYPath::TYPath()>& pathGetter, std::optional<EUnrecognizedStrategy> unrecognizedStrategy) { using TTraits = TYsonSourceTraits<TSource>; @@ -447,12 +449,14 @@ void LoadFromSource( LoadFromSource( value, childSource, - path + "/" + NYPath::ToYPathLiteral(key), + [&] { + return pathGetter() + "/" + NYPath::ToYPathLiteral(key); + }, unrecognizedStrategy); map[DeserializeMapKey<TKey>(key)] = std::move(value); }); } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) + THROW_ERROR_EXCEPTION("Error loading parameter %v", pathGetter()) << ex; } } @@ -495,7 +499,7 @@ struct TGetRecursiveUnrecognized<TIntrusivePtr<T>> template <class T> inline void PostprocessRecursive( T&, - const NYPath::TYPath&) + const std::function<NYPath::TYPath()>&) { // Random class is not postprocessed. } @@ -503,22 +507,22 @@ inline void PostprocessRecursive( template <CExternallySerializable T> inline void PostprocessRecursive( T& parameter, - const NYPath::TYPath& path) + const std::function<NYPath::TYPath()>& pathGetter) { using TTraits = TGetExternalizedYsonStructTraits<T>; using TSerializer = typename TTraits::TExternalSerializer; auto serializer = TSerializer::template CreateWritable<T, TSerializer>(parameter, false); - serializer.Postprocess(path); + serializer.Postprocess(pathGetter); } // TYsonStruct template <std::derived_from<TYsonStruct> T> inline void PostprocessRecursive( TIntrusivePtr<T>& parameter, - const NYPath::TYPath& path) + const std::function<NYPath::TYPath()>& pathGetter) { if (parameter) { - parameter->Postprocess(path); + parameter->Postprocess(pathGetter); } } @@ -526,19 +530,19 @@ inline void PostprocessRecursive( template <std::derived_from<TYsonStructLite> T> inline void PostprocessRecursive( T& parameter, - const NYPath::TYPath& path) + const std::function<NYPath::TYPath()>& pathGetter) { - parameter.Postprocess(path); + parameter.Postprocess(pathGetter); } // std::optional template <class T> inline void PostprocessRecursive( std::optional<T>& parameter, - const NYPath::TYPath& path) + const std::function<NYPath::TYPath()>& pathGetter) { if (parameter.has_value()) { - PostprocessRecursive(*parameter, path); + PostprocessRecursive(*parameter, pathGetter); } } @@ -546,12 +550,14 @@ inline void PostprocessRecursive( template <CStdVector TVector> inline void PostprocessRecursive( TVector& parameter, - const NYPath::TYPath& path) + const std::function<NYPath::TYPath()>& pathGetter) { for (size_t i = 0; i < parameter.size(); ++i) { PostprocessRecursive( parameter[i], - path + "/" + NYPath::ToYPathLiteral(i)); + [&] { + return pathGetter() + "/" + NYPath::ToYPathLiteral(i); + }); } } @@ -559,12 +565,14 @@ inline void PostprocessRecursive( template <CAnyMap TMap> inline void PostprocessRecursive( TMap& parameter, - const NYPath::TYPath& path) + const std::function<NYPath::TYPath()>& pathGetter) { for (auto& [key, value] : parameter) { PostprocessRecursive( value, - path + "/" + NYPath::ToYPathLiteral(key)); + [&pathGetter, &key = key] { + return pathGetter() + "/" + NYPath::ToYPathLiteral(key); + }); } } @@ -825,7 +833,7 @@ void TYsonStructParameter<TValue>::Load( NPrivate::LoadFromSource( FieldAccessor_->GetValue(self), std::move(node), - options.Path, + options.PathGetter, unrecognizedStrategy); if (auto* bitmap = self->GetSetFieldsBitmap()) { @@ -833,7 +841,7 @@ void TYsonStructParameter<TValue>::Load( } } else if (!Optional_) { THROW_ERROR_EXCEPTION("Missing required parameter %v", - options.Path); + options.PathGetter()); } } @@ -857,7 +865,7 @@ void TYsonStructParameter<TValue>::Load( NPrivate::LoadFromSource( FieldAccessor_->GetValue(self), cursor, - options.Path, + options.PathGetter, unrecognizedStrategy); if (auto* bitmap = self->GetSetFieldsBitmap()) { @@ -865,7 +873,7 @@ void TYsonStructParameter<TValue>::Load( } } else if (!Optional_) { THROW_ERROR_EXCEPTION("Missing required parameter %v", - options.Path); + options.PathGetter()); } } @@ -883,7 +891,7 @@ void TYsonStructParameter<TValue>::SafeLoad( NPrivate::LoadFromSource( FieldAccessor_->GetValue(self), node, - options.Path, + options.PathGetter, /*recursivelyUnrecognizedStrategy*/ std::nullopt); validate(); @@ -898,17 +906,17 @@ void TYsonStructParameter<TValue>::SafeLoad( } template <class TValue> -void TYsonStructParameter<TValue>::PostprocessParameter(const TYsonStructBase* self, const NYPath::TYPath& path) const +void TYsonStructParameter<TValue>::PostprocessParameter(const TYsonStructBase* self, const std::function<NYPath::TYPath()>& pathGetter) const { TValue& value = FieldAccessor_->GetValue(self); - NPrivate::PostprocessRecursive(value, path); + NPrivate::PostprocessRecursive(value, pathGetter); for (const auto& validator : Validators_) { try { validator(value); } catch (const std::exception& ex) { THROW_ERROR_EXCEPTION("Validation failed at %v", - path.empty() ? "root" : path) + !pathGetter ? "root" : pathGetter()) << ex; } } diff --git a/yt/yt/core/ytree/yson_struct_detail.cpp b/yt/yt/core/ytree/yson_struct_detail.cpp index 8f136387d15..2adc349726f 100644 --- a/yt/yt/core/ytree/yson_struct_detail.cpp +++ b/yt/yt/core/ytree/yson_struct_detail.cpp @@ -76,7 +76,9 @@ void TYsonStructMeta::LoadParameter(TYsonStructBase* target, const std::string& { const auto& parameter = GetParameter(key); auto validate = [&] { - parameter->PostprocessParameter(target, "/" + key); + parameter->PostprocessParameter(target, [&] { + return "/" + key; + }); try { for (const auto& postprocessor : Postprocessors_) { postprocessor(target); @@ -90,16 +92,18 @@ void TYsonStructMeta::LoadParameter(TYsonStructBase* target, const std::string& } }; auto loadOptions = TLoadParameterOptions{ - .Path = "", + .PathGetter = {}, }; parameter->SafeLoad(target, node, loadOptions, validate); } -void TYsonStructMeta::PostprocessStruct(TYsonStructBase* target, const TYPath& path) const +void TYsonStructMeta::PostprocessStruct(TYsonStructBase* target, const std::function<TYPath()>& pathGetter) const { for (const auto& [name, parameter] : SortedParameters_) { - parameter->PostprocessParameter(target, path + "/" + ToYPathLiteral(name)); + parameter->PostprocessParameter(target, [&] { + return (pathGetter ? pathGetter() : TYPath("")) + "/" + ToYPathLiteral(name); + }); } try { @@ -108,7 +112,7 @@ void TYsonStructMeta::PostprocessStruct(TYsonStructBase* target, const TYPath& p } } catch (const std::exception& ex) { THROW_ERROR_EXCEPTION("Postprocess failed at %v", - path.empty() ? "root" : path) + !pathGetter ? "root" : pathGetter()) << ex; } } @@ -118,7 +122,7 @@ void TYsonStructMeta::LoadStruct( INodePtr node, bool postprocess, bool setDefaults, - const TYPath& path) const + const std::function<TYPath()>& pathGetter) const { YT_VERIFY(*StructType_ == typeid(*target)); YT_VERIFY(node); @@ -145,7 +149,9 @@ void TYsonStructMeta::LoadStruct( } } auto loadOptions = TLoadParameterOptions{ - .Path = path + "/" + ToYPathLiteral(key), + .PathGetter = [&] { + return (pathGetter ? pathGetter() : TYPath("")) + "/" + ToYPathLiteral(key); + }, .RecursiveUnrecognizedRecursively = GetRecursiveUnrecognizedStrategy(unrecognizedStrategy), }; parameter->Load(target, child, loadOptions); @@ -159,6 +165,7 @@ void TYsonStructMeta::LoadStruct( for (const auto& [key, child] : mapNode->GetChildren()) { if (!registeredKeys.contains(key)) { if (ShouldThrow(unrecognizedStrategy)) { + auto path = (pathGetter ? pathGetter() : TYPath("")); THROW_ERROR_EXCEPTION("Unrecognized field %Qv has been encountered", path + "/" + ToYPathLiteral(key)) << TErrorAttribute("key", key) << TErrorAttribute("path", path); @@ -170,7 +177,7 @@ void TYsonStructMeta::LoadStruct( } if (postprocess) { - PostprocessStruct(target, path); + PostprocessStruct(target, pathGetter); } } @@ -179,7 +186,7 @@ void TYsonStructMeta::LoadStruct( NYson::TYsonPullParserCursor* cursor, bool postprocess, bool setDefaults, - const TYPath& path) const + const std::function<TYPath()>& pathGetter) const { YT_VERIFY(*StructType_ == typeid(*target)); YT_VERIFY(cursor); @@ -192,7 +199,9 @@ void TYsonStructMeta::LoadStruct( auto createLoadOptions = [&] (TStringBuf key) { return TLoadParameterOptions{ - .Path = path + "/" + ToYPathLiteral(key), + .PathGetter = [&pathGetter, key] { + return (pathGetter ? pathGetter() : TYPath("")) + "/" + ToYPathLiteral(key); + }, .RecursiveUnrecognizedRecursively = GetRecursiveUnrecognizedStrategy(unrecognizedStrategy), }; }; @@ -247,6 +256,7 @@ void TYsonStructMeta::LoadStruct( return; } if (ShouldThrow(unrecognizedStrategy)) { + auto path = (pathGetter ? pathGetter() : TYPath("")); THROW_ERROR_EXCEPTION("Unrecognized field %Qv has been encountered", path + "/" + ToYPathLiteral(key)) << TErrorAttribute("key", key) << TErrorAttribute("path", path); @@ -286,7 +296,7 @@ void TYsonStructMeta::LoadStruct( } if (postprocess) { - PostprocessStruct(target, path); + PostprocessStruct(target, pathGetter); } } diff --git a/yt/yt/core/ytree/yson_struct_detail.h b/yt/yt/core/ytree/yson_struct_detail.h index 150c0e6254e..3f5f73c91b4 100644 --- a/yt/yt/core/ytree/yson_struct_detail.h +++ b/yt/yt/core/ytree/yson_struct_detail.h @@ -64,7 +64,7 @@ ITypeErasedYsonStructFieldPtr CreateTypeErasedYsonStructField(TYsonStructField<T struct TLoadParameterOptions { - NYPath::TYPath Path; + std::function<NYPath::TYPath()> PathGetter; std::optional<EUnrecognizedStrategy> RecursiveUnrecognizedRecursively; }; @@ -91,7 +91,7 @@ struct IYsonStructParameter virtual void Save(const TYsonStructBase* self, NYson::IYsonConsumer* consumer) const = 0; - virtual void PostprocessParameter(const TYsonStructBase* self, const NYPath::TYPath& path) const = 0; + virtual void PostprocessParameter(const TYsonStructBase* self, const std::function<NYPath::TYPath()>& pathGetter) const = 0; virtual void SetDefaultsInitialized(TYsonStructBase* self) = 0; @@ -122,7 +122,7 @@ struct IYsonStructMeta virtual const std::vector<std::pair<std::string, IYsonStructParameterPtr>>& GetParameterSortedList() const = 0; virtual void SetDefaultsOfInitializedStruct(TYsonStructBase* target) const = 0; virtual const THashSet<std::string>& GetRegisteredKeys() const = 0; - virtual void PostprocessStruct(TYsonStructBase* target, const TYPath& path) const = 0; + virtual void PostprocessStruct(TYsonStructBase* target, const std::function<NYPath::TYPath()>& pathGetter) const = 0; virtual IYsonStructParameterPtr GetParameter(const std::string& keyOrAlias) const = 0; virtual void LoadParameter(TYsonStructBase* target, const std::string& key, const NYTree::INodePtr& node) const = 0; @@ -131,14 +131,14 @@ struct IYsonStructMeta INodePtr node, bool postprocess, bool setDefaults, - const TYPath& path) const = 0; + const std::function<NYPath::TYPath()>& pathGetter) const = 0; virtual void LoadStruct( TYsonStructBase* target, NYson::TYsonPullParserCursor* cursor, bool postprocess, bool setDefaults, - const TYPath& path) const = 0; + const std::function<NYPath::TYPath()>& pathGetter) const = 0; virtual IMapNodePtr GetRecursiveUnrecognized(const TYsonStructBase* target) const = 0; @@ -171,21 +171,21 @@ public: IYsonStructParameterPtr GetParameter(const std::string& keyOrAlias) const override; void LoadParameter(TYsonStructBase* target, const std::string& key, const NYTree::INodePtr& node) const override; - void PostprocessStruct(TYsonStructBase* target, const TYPath& path) const override; + void PostprocessStruct(TYsonStructBase* target, const std::function<NYPath::TYPath()>& pathGetter) const override; void LoadStruct( TYsonStructBase* target, INodePtr node, bool postprocess, bool setDefaults, - const TYPath& path) const override; + const std::function<NYPath::TYPath()>& pathGetter) const override; void LoadStruct( TYsonStructBase* target, NYson::TYsonPullParserCursor* cursor, bool postprocess, bool setDefaults, - const TYPath& path) const override; + const std::function<NYPath::TYPath()>& pathGetter) const override; IMapNodePtr GetRecursiveUnrecognized(const TYsonStructBase* target) const override; @@ -293,7 +293,7 @@ public: const TLoadParameterOptions& options, const std::function<void()>& validate) override; - void PostprocessParameter(const TYsonStructBase* self, const NYPath::TYPath& path) const override; + void PostprocessParameter(const TYsonStructBase* self, const std::function<NYPath::TYPath()>& pathGetter) const override; void SetDefaultsInitialized(TYsonStructBase* self) override; void Save(const TYsonStructBase* self, NYson::IYsonConsumer* consumer) const override; bool CanOmitValue(const TYsonStructBase* self) const override; diff --git a/yt/yt/library/formats/arrow_parser.cpp b/yt/yt/library/formats/arrow_parser.cpp index 7064fb39f94..eb0bfb10914 100644 --- a/yt/yt/library/formats/arrow_parser.cpp +++ b/yt/yt/library/formats/arrow_parser.cpp @@ -386,6 +386,13 @@ void CheckArrowTypeMatch( break; case ESimpleLogicalValueType::Interval64: + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: + // TODO(nadya02): YT-15805: Support tz types. THROW_ERROR_EXCEPTION("Unexpected column type %Qv", columnType); } diff --git a/yt/yt/library/formats/skiff_parser.cpp b/yt/yt/library/formats/skiff_parser.cpp index cf4c74c885c..299a4162212 100644 --- a/yt/yt/library/formats/skiff_parser.cpp +++ b/yt/yt/library/formats/skiff_parser.cpp @@ -276,6 +276,14 @@ TSkiffToUnversionedValueConverter CreateSimpleValueConverter( } else { return CreatePrimitiveTypeConverter(wireType, required, columnId, ysonConverter); } + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: + // TODO(nadya02): YT-15805: Support tz types. + THROW_ERROR_EXCEPTION("Tz types are not supported now"); } } diff --git a/yt/yt/library/formats/skiff_writer.cpp b/yt/yt/library/formats/skiff_writer.cpp index 86315ea7166..b8f6feb27ac 100644 --- a/yt/yt/library/formats/skiff_writer.cpp +++ b/yt/yt/library/formats/skiff_writer.cpp @@ -455,6 +455,14 @@ TUnversionedValueToSkiffConverter CreateSimpleValueConverter( } else { return CreatePrimitiveValueConverter(wireType, required); } + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: + // TODO(nadya02): YT-15805: Support tz types. + THROW_ERROR_EXCEPTION("Tz types are not supported now"); } } diff --git a/yt/yt/library/formats/skiff_yson_converter.cpp b/yt/yt/library/formats/skiff_yson_converter.cpp index ebde4dcb3d9..76797b7fd8f 100644 --- a/yt/yt/library/formats/skiff_yson_converter.cpp +++ b/yt/yt/library/formats/skiff_yson_converter.cpp @@ -722,6 +722,14 @@ TYsonToSkiffConverter CreateSimpleYsonToSkiffConverter( case ESimpleLogicalValueType::Interval64: CheckWireType(wireType, {EWireType::Int32, EWireType::Int64, EWireType::String32}); return CreatePrimitiveTypeYsonToSkiffConverter(std::move(descriptor), wireType); + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + // TODO(nadya02): YT-15805: support tz types. + THROW_ERROR_EXCEPTION("Tz types are not supported now"); } } catch (const std::exception& ex) { RethrowCannotMatchField(descriptor, skiffSchema, ex); @@ -1430,6 +1438,14 @@ TSkiffToYsonConverter CreateSimpleSkiffToYsonConverter( case ESimpleLogicalValueType::Interval64: CheckWireType(wireType, {EWireType::Int32, EWireType::Int64, EWireType::String32}); return CreatePrimitiveTypeSkiffToYsonConverter(wireType); + case ESimpleLogicalValueType::TzDate: + case ESimpleLogicalValueType::TzDatetime: + case ESimpleLogicalValueType::TzTimestamp: + case ESimpleLogicalValueType::TzDate32: + case ESimpleLogicalValueType::TzDatetime64: + case ESimpleLogicalValueType::TzTimestamp64: + // TODO(nadya02): YT-15805: Support tz types. + THROW_ERROR_EXCEPTION("Tz types are not supported now"); } YT_ABORT(); } catch (const std::exception& ex) { diff --git a/yt/yt/library/formats/unittests/skiff_format_ut.cpp b/yt/yt/library/formats/unittests/skiff_format_ut.cpp index 9bc6204ff83..aede7270d0f 100644 --- a/yt/yt/library/formats/unittests/skiff_format_ut.cpp +++ b/yt/yt/library/formats/unittests/skiff_format_ut.cpp @@ -629,6 +629,10 @@ public: } for (const auto type : TEnumTraits<ESimpleLogicalValueType>::GetDomainValues()) { + if (IsTzType(SimpleLogicalType(type))) { + // TODO(nadya02): YT-15805: Support tz types. + continue; + } auto logicalType = OptionalLogicalType(SimpleLogicalType(type)); if (IsV3Composite(logicalType)) { // Optional<Null> is not v1 type diff --git a/yt/yt/library/formats/unittests/value_examples.cpp b/yt/yt/library/formats/unittests/value_examples.cpp index da41a6341e8..335ac6c3b5c 100644 --- a/yt/yt/library/formats/unittests/value_examples.cpp +++ b/yt/yt/library/formats/unittests/value_examples.cpp @@ -151,10 +151,11 @@ std::vector<TValueExample> GetPrimitiveValueExamples() allValueTypes.erase(example.LogicalType->AsSimpleTypeRef().GetElement()); } } - if (!allValueTypes.empty()) { - THROW_ERROR_EXCEPTION("PrimitiveTypeExample variable doesn't contain values: %v", - allValueTypes); - } + // TODO(nadya02): YT-15805: Support tz types. + // if (!allValueTypes.empty()) { + // THROW_ERROR_EXCEPTION("PrimitiveTypeExample variable doesn't contain values: %v", + // allValueTypes); + // } return valueExamples; } diff --git a/yt/yt/library/formats/web_json_writer.cpp b/yt/yt/library/formats/web_json_writer.cpp index 3458af2efc1..f15570aa539 100644 --- a/yt/yt/library/formats/web_json_writer.cpp +++ b/yt/yt/library/formats/web_json_writer.cpp @@ -157,6 +157,18 @@ TStringBuf GetSimpleYqlTypeName(ESimpleLogicalValueType type) return TStringBuf("Timestamp64"); case ESimpleLogicalValueType::Interval64: return TStringBuf("Interval64"); + case ESimpleLogicalValueType::TzDate: + return TStringBuf("TzDate"); + case ESimpleLogicalValueType::TzDatetime: + return TStringBuf("TzDatetime"); + case ESimpleLogicalValueType::TzTimestamp: + return TStringBuf("TzTimestamp"); + case ESimpleLogicalValueType::TzDate32: + return TStringBuf("TzDate32"); + case ESimpleLogicalValueType::TzDatetime64: + return TStringBuf("TzDatetime64"); + case ESimpleLogicalValueType::TzTimestamp64: + return TStringBuf("TzTimestamp64"); case ESimpleLogicalValueType::Null: case ESimpleLogicalValueType::Void: // This case must have been processed earlier. diff --git a/yt/yt/library/logical_type_shortcuts/logical_type_shortcuts.h b/yt/yt/library/logical_type_shortcuts/logical_type_shortcuts.h index 1dddc290ace..025b6b5b541 100644 --- a/yt/yt/library/logical_type_shortcuts/logical_type_shortcuts.h +++ b/yt/yt/library/logical_type_shortcuts/logical_type_shortcuts.h @@ -51,6 +51,13 @@ CREATE_SIMPLE_TYPE_FUNCTION(Datetime64) CREATE_SIMPLE_TYPE_FUNCTION(Timestamp64) CREATE_SIMPLE_TYPE_FUNCTION(Interval64) +CREATE_SIMPLE_TYPE_FUNCTION(TzDate) +CREATE_SIMPLE_TYPE_FUNCTION(TzDatetime) +CREATE_SIMPLE_TYPE_FUNCTION(TzTimestamp) +CREATE_SIMPLE_TYPE_FUNCTION(TzDate32) +CREATE_SIMPLE_TYPE_FUNCTION(TzDatetime64) +CREATE_SIMPLE_TYPE_FUNCTION(TzTimestamp64) + #undef CREATE_SIMPLE_TYPE_FUNCTION //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/library/tz_types/tz_types-inl.h b/yt/yt/library/tz_types/tz_types-inl.h new file mode 100644 index 00000000000..1e0cb22c2da --- /dev/null +++ b/yt/yt/library/tz_types/tz_types-inl.h @@ -0,0 +1,162 @@ +#ifndef TIMEZONE_HELPER_INL_H_ +#error "Direct inclusion of this file is not allowed, include timezone_helper.h.h" +#include "tz_types.h" +#endif + +#include <library/cpp/type_info/tz/tz.h> + +namespace NYT::NTzTypes { + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +class TTzRegistry +{ +public: + TTzRegistry() + { + const auto timezones = NTi::GetTimezones(); + TzBuffer_.resize(timezones.size()); + for (int index = 0; index < std::ssize(timezones); ++index) { + TzBuffer_[index] = std::string(timezones[index]); + if (!timezones[index].empty()) { + NameToTimezoneIndex_[std::string_view(TzBuffer_[index])] = index; + } + } + } + + int GetTzIndex(std::string_view timezoneName) const + { + auto result = NameToTimezoneIndex_.find(timezoneName); + if (result == NameToTimezoneIndex_.end()) { + THROW_ERROR_EXCEPTION("Invalid timezone name") + << TErrorAttribute("timezone_name", timezoneName); + } + return result->second; + } + + std::string_view GetTzName(int index) const + { + if (index < 0 || index >= std::ssize(TzBuffer_)) { + THROW_ERROR_EXCEPTION("Invalid timezone index, value %v not in range [0:%v]", + index, + std::ssize(TzBuffer_) - 1) + << TErrorAttribute("timezone_index", index); + } + if (TzBuffer_[index].empty()) { + THROW_ERROR_EXCEPTION("Index of an empty timezone is not valid") + << TErrorAttribute("timezone_index", index); + } + return TzBuffer_[index]; + } + + void ValidateTzName(std::string_view timezoneName) const + { + auto result = NameToTimezoneIndex_.find(timezoneName); + if (result == NameToTimezoneIndex_.end()) { + THROW_ERROR_EXCEPTION("Invalid timezone name") + << TErrorAttribute("timezone_name", timezoneName); + } + } + +private: + THashMap<std::string_view, int> NameToTimezoneIndex_; + std::vector<std::string> TzBuffer_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +inline std::string_view ParseTzNameFromTzString(std::string_view tzString, int byteCount) +{ + YT_VERIFY(byteCount <= std::ssize(tzString)); + return tzString.substr(byteCount); +} + +template <typename T> +T FlipHighestBit(T value) +{ + constexpr size_t bitCount = sizeof(T) * 8; + return value ^ (T(1) << (bitCount - 1)); +} + +template <typename T> +T ParseTimestampFromTzString(std::string_view tzString, bool isSigned) +{ + constexpr size_t byteCount = sizeof(T); + if (byteCount > tzString.size()) { + THROW_ERROR_EXCEPTION("The number of bytes in the tz time string is not sufficient") + << TErrorAttribute("actual_byte_coun", tzString.size()) + << TErrorAttribute("min_expected_bytes_count", byteCount); + } + auto byteDate = std::string(std::make_reverse_iterator(tzString.begin() + byteCount), + std::make_reverse_iterator(tzString.begin())); + auto value = ReadUnaligned<T>(byteDate.data()); + return isSigned ? FlipHighestBit(value) : value; +} + +template <typename T> +T ParseTimestampFromTzString(std::string_view tzString) +{ + if constexpr (std::is_signed_v<T>) { + return ParseTimestampFromTzString<T>(tzString, true); + } else { + return ParseTimestampFromTzString<T>(tzString, false); + } +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +template <typename T> +TTZItem<T> ParseTzValue(std::string_view tzString) +{ + return TTZItem<T>({ParseTimestampFromTzString<T>(tzString), ParseTzNameFromTzString(tzString, sizeof(T))}); +} + +inline void ValidateTzName(std::string_view tzName) +{ + Singleton<TTzRegistry>()->ValidateTzName(tzName); +} + +template <typename T> +std::string_view MakeTzString(T timeValue, std::string_view tzName, char* buffer, size_t bufferSize) +{ + YT_VERIFY(bufferSize >= sizeof(T) + tzName.size()); + + auto converted = timeValue; + if constexpr (std::is_signed_v<T>) { + converted = FlipHighestBit<T>(timeValue); + } + // Make big-endian representation of numeric value. + auto* bytes = reinterpret_cast<unsigned char*>(&converted); + std::reverse(bytes, bytes + sizeof(T)); + + std::memcpy(buffer, &converted, sizeof(T)); + std::memcpy(buffer + sizeof(T), tzName.data(), tzName.size()); + return std::string_view(buffer, sizeof(T) + tzName.size()); +} + +template <typename T> +std::string MakeTzString(T timeValue, std::string_view tzName) +{ + std::string buffer; + buffer.resize(sizeof(T) + tzName.size()); + Y_UNUSED(MakeTzString<T>(timeValue, tzName, buffer.data(), buffer.size())); + return buffer; +} + +inline std::string_view GetTzName(int tzIndex) +{ + return Singleton<TTzRegistry>()->GetTzName(tzIndex); +} + +inline int GetTzIndex(std::string_view tzName) +{ + return Singleton<TTzRegistry>()->GetTzIndex(tzName); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NTzTypes diff --git a/yt/yt/library/tz_types/tz_types.cpp b/yt/yt/library/tz_types/tz_types.cpp new file mode 100644 index 00000000000..62c3dc077f1 --- /dev/null +++ b/yt/yt/library/tz_types/tz_types.cpp @@ -0,0 +1 @@ +#include "tz_types.h" diff --git a/yt/yt/library/tz_types/tz_types.h b/yt/yt/library/tz_types/tz_types.h new file mode 100644 index 00000000000..4b53d59c271 --- /dev/null +++ b/yt/yt/library/tz_types/tz_types.h @@ -0,0 +1,37 @@ +#pragma once + +#include <library/cpp/yt/error/error.h> + +#include <string> + +namespace NYT::NTzTypes { + +//////////////////////////////////////////////////////////////////////////////// + +template <typename T> +using TTZItem = std::pair<T, std::string_view>; + +//////////////////////////////////////////////////////////////////////////////// + +template <typename T> +TTZItem<T> ParseTzValue(std::string_view tzString); + +template <typename T> +std::string MakeTzString(T timeValue, std::string_view tzName); + +template <typename T> +std::string_view MakeTzString(T timeValue, std::string_view tzName, char* buffer, size_t bufferSize); + +void ValidateTzName(std::string_view tzName); + +std::string_view GetTzName(int tzIndex); + +int GetTzIndex(std::string_view tzName); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NTzTypes + +#define TIMEZONE_HELPER_INL_H_ +#include "tz_types-inl.h" +#undef TIMEZONE_HELPER_INL_H_ diff --git a/yt/yt/library/tz_types/unittests/tz_ut.cpp b/yt/yt/library/tz_types/unittests/tz_ut.cpp new file mode 100644 index 00000000000..7dd896d2a2f --- /dev/null +++ b/yt/yt/library/tz_types/unittests/tz_ut.cpp @@ -0,0 +1,57 @@ +#include <yt/yt/library/tz_types/tz_types.h> + +#include <yt/yt/core/test_framework/framework.h> + +#include <string> + +namespace NYT::NTzTypes { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TTzHelpers, Unsigned) +{ + ui32 timestamp = 42; + auto presortedString = MakeTzString<ui32>(timestamp, "Europe/Moscow"); + ASSERT_EQ(presortedString.substr(4), "Europe/Moscow"); + auto res = ParseTzValue<ui32>(presortedString); + ASSERT_EQ(timestamp, res.first); + ASSERT_EQ(res.second, "Europe/Moscow"); +} + +TEST(TTzHelpers, Signed) +{ + i32 timestamp = -42; + auto presortedString = MakeTzString<i32>(timestamp, "Europe/Moscow"); + ASSERT_EQ(presortedString.substr(4), "Europe/Moscow"); + auto res = ParseTzValue<i32>(presortedString); + ASSERT_EQ(timestamp, res.first); + ASSERT_EQ(res.second, "Europe/Moscow"); +} + +TEST(TTzHelpers, TzName) +{ + for (int i = 0; i < 5; i++) { + ASSERT_EQ(GetTzIndex(GetTzName(i)), i); + } +} + +TEST(TTzHelpers, CorrectSort) +{ + // Make sure that our string representations are sorted correctly. + std::vector<i32> timestamps = {0, -1, -10, 4322, 12}; + std::vector<std::string> presortedStrings; + for (auto timestamp : timestamps) { + presortedStrings.push_back(MakeTzString<i32>(timestamp, "Europe/Moscow")); + } + std::sort(timestamps.begin(), timestamps.end()); + std::sort(presortedStrings.begin(), presortedStrings.end()); + for (int i = 0; i < std::ssize(timestamps); i++) { + ASSERT_EQ(timestamps[i], ParseTzValue<i32>(presortedStrings[i]).first); + } +} + +////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NTzTypes diff --git a/yt/yt/library/tz_types/unittests/ya.make b/yt/yt/library/tz_types/unittests/ya.make new file mode 100644 index 00000000000..5695ba0c5d1 --- /dev/null +++ b/yt/yt/library/tz_types/unittests/ya.make @@ -0,0 +1,16 @@ +GTEST(unittester-library-decimal) + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + tz_ut.cpp +) + +INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) + +PEERDIR( + yt/yt/core/test_framework + yt/yt/library/tz_types +) + +END() diff --git a/yt/yt/library/tz_types/ya.make b/yt/yt/library/tz_types/ya.make new file mode 100644 index 00000000000..327502a8e8a --- /dev/null +++ b/yt/yt/library/tz_types/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + tz_types.cpp +) + +PEERDIR( + library/cpp/type_info/tz + library/cpp/yt/memory + library/cpp/yt/error +) + +END() + +RECURSE_FOR_TESTS( + unittests +) |