diff options
64 files changed, 2143 insertions, 697 deletions
diff --git a/build/conf/compilers/gnu_compiler.conf b/build/conf/compilers/gnu_compiler.conf index 2b269831069..3382f99a79d 100644 --- a/build/conf/compilers/gnu_compiler.conf +++ b/build/conf/compilers/gnu_compiler.conf @@ -178,18 +178,6 @@ elsewhen ($CONSISTENT_DEBUG == "yes") { } YASM_DEBUG_INFO_DISABLE_CACHE__NO_UID__=--replace=${ARCADIA_BUILD_ROOT}=/-B --replace=${ARCADIA_ROOT}=/-S --replace=$(TOOL_ROOT)=/-T ${env:"YASM_TEST_SUITE=1"} } -elsewhen ($CONSISTENT_DEBUG_LIGHT == "yes") { - when ($CLANG == "yes") { - CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__=-fdebug-prefix-map=${ARCADIA_BUILD_ROOT}=/-B -Xclang -fdebug-compilation-dir -Xclang /tmp - } - otherwise { - CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__=-fdebug-prefix-map=${ARCADIA_BUILD_ROOT}=/-B - when ($ARCH_XTENSA == "yes") { - CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__= - } - } - YASM_DEBUG_INFO_DISABLE_CACHE__NO_UID__=--replace=${ARCADIA_BUILD_ROOT}=/-B ${env:"YASM_TEST_SUITE=1"} -} when ($OS_LINUX == "yes" || $OS_DARWIN == "yes" || $OS_WINDOWS == "yes") { # Some android toolchains using old compilers which didn't support some flags (f.e. -fmacro-prefix-map) diff --git a/contrib/libs/openssl/.yandex_meta/__init__.py b/contrib/libs/openssl/.yandex_meta/__init__.py index 912cdd7c01b..03d0dc5a5da 100644 --- a/contrib/libs/openssl/.yandex_meta/__init__.py +++ b/contrib/libs/openssl/.yandex_meta/__init__.py @@ -1,16 +1,27 @@ -import os.path as P +import os -from devtools.yamaker.fileutil import subcopy +from devtools.yamaker import fileutil +from devtools.yamaker import pathutil from devtools.yamaker.modules import Linkable, Switch from devtools.yamaker.project import NixProject +def post_build(self): + os.remove(f"{self.dstdir}/libssl.map") + for hdr in fileutil.files(f"{self.dstdir}/include", rel=False, test=pathutil.is_header): + with open(hdr, "rt") as hdr_file: + data = hdr_file.read() + with open(hdr, "wt") as hdr_file: + hdr_file.write("#include <contrib/libs/openssl/redef.h>\n") + hdr_file.write(data) + + def post_install(self): def d(s): return self.dstdir + "/" + s # Move asm sources to asm/linux. - subcopy(self.dstdir, d("asm/linux"), ["**/*.s"], move=True) + fileutil.subcopy(self.dstdir, d("asm/linux"), ["**/*.s"], move=True) with self.yamakes["crypto"] as m: asm = {s for s in m.SRCS if s.endswith(".s")} m.SRCS -= asm @@ -28,7 +39,7 @@ def post_install(self): ) # Shorten paths. m.SRCDIR = [] - m.SRCS = {P.relpath(s, "crypto") for s in m.SRCS} + m.SRCS = {os.path.relpath(s, "crypto") for s in m.SRCS} m.SRCS -= {"dso/dso_dlfcn.c", "rand/rand_vms.c"} # Add suppression for ubsan, see also https://github.com/openssl/openssl/issues/22896 @@ -137,6 +148,7 @@ openssl = NixProject( "include/openssl/opensslconf.h", ], keep_paths=[ + # This asm files were generated manually "asm/aarch64/", "asm/android/", "asm/darwin/", @@ -144,9 +156,13 @@ openssl = NixProject( "asm/ios/", "asm/ppc64le/", "asm/windows/", - "openssl.package.json", + # This code is written by us + "ar.pyplugin", "sanitizers.h", "crypto/ubsan.supp", + "redef.h", + "system_openssl.ya.inc", ], + post_build=post_build, post_install=post_install, ) diff --git a/contrib/libs/openssl/apps/progs-linux.h b/contrib/libs/openssl/apps/progs-linux.h index 2fce5cbc797..b7c2e31b284 100644 --- a/contrib/libs/openssl/apps/progs-linux.h +++ b/contrib/libs/openssl/apps/progs-linux.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by apps/progs.pl * - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2026 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the OpenSSL license (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/contrib/libs/openssl/include/crypto/bn_conf.h b/contrib/libs/openssl/include/crypto/bn_conf.h index eb25b5c6636..7b4d0a3b106 100644 --- a/contrib/libs/openssl/include/crypto/bn_conf.h +++ b/contrib/libs/openssl/include/crypto/bn_conf.h @@ -1,7 +1,8 @@ -#include <contrib/libs/openssl/redef.h> #pragma once -#if defined(__ANDROID__) && defined(__arm__) +#if defined(__wasm__) && !defined(__wasm64__) +# include "bn_conf-wasm32.h" +#elif defined(__ANDROID__) && defined(__arm__) # include "bn_conf-android-arm.h" #elif defined(__ANDROID__) && defined(__aarch64__) # include "bn_conf-android-arm64.h" @@ -21,8 +22,6 @@ # include "bn_conf-linux-arm.h" #elif defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) # include "bn_conf-linux-aarch64.h" -#elif defined(__wasm__) && !defined(__wasm64__) -# include "bn_conf-wasm32.h" #else # include "bn_conf-linux.h" #endif diff --git a/contrib/libs/openssl/include/crypto/dso_conf.h b/contrib/libs/openssl/include/crypto/dso_conf.h index fd60cdfe832..39695b0aaa8 100644 --- a/contrib/libs/openssl/include/crypto/dso_conf.h +++ b/contrib/libs/openssl/include/crypto/dso_conf.h @@ -1,4 +1,3 @@ -#include <contrib/libs/openssl/redef.h> #pragma once #if defined(__ANDROID__) diff --git a/contrib/libs/openssl/include/openssl/opensslconf-freebsd.h b/contrib/libs/openssl/include/openssl/opensslconf-freebsd-x86_64.h index bdeb5b9a471..bdeb5b9a471 100644 --- a/contrib/libs/openssl/include/openssl/opensslconf-freebsd.h +++ b/contrib/libs/openssl/include/openssl/opensslconf-freebsd-x86_64.h diff --git a/contrib/libs/openssl/include/openssl/opensslconf.h b/contrib/libs/openssl/include/openssl/opensslconf.h index 38164e3cb65..d7f65c6ab1e 100644 --- a/contrib/libs/openssl/include/openssl/opensslconf.h +++ b/contrib/libs/openssl/include/openssl/opensslconf.h @@ -1,7 +1,10 @@ -#include <contrib/libs/openssl/redef.h> #pragma once -#if defined(__ANDROID__) && defined(__arm__) +#if defined(__wasm__) && !defined(__wasm64__) +# include "opensslconf-wasm32.h" +#elif defined(__wasm64__) +# include "opensslconf-wasm64.h" +#elif defined(__ANDROID__) && defined(__arm__) # include "opensslconf-android-arm.h" #elif defined(__ANDROID__) && defined(__aarch64__) # include "opensslconf-android-arm64.h" @@ -21,16 +24,12 @@ # include "opensslconf-win-x86_64.h" #elif defined(_MSC_VER) && (defined(__i686__) || defined(_M_IX86)) # include "opensslconf-win-i686.h" +#elif defined(__FreeBSD__) && (defined(__x86_64__) || defined(_M_X64)) +# include "opensslconf-freebsd-x86_64.h" #elif defined(__linux__) && defined(__arm__) # include "opensslconf-linux-arm.h" #elif defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) # include "opensslconf-linux-aarch64.h" -#elif defined(__wasm__) && !defined(__wasm64__) -# include "opensslconf-wasm32.h" -#elif defined(__wasm64__) -# include "opensslconf-wasm64.h" -#elif defined(__FreeBSD__) -# include "opensslconf-freebsd.h" #else # include "opensslconf-linux.h" #endif diff --git a/contrib/libs/openssl/libssl.map b/contrib/libs/openssl/libssl.map deleted file mode 100644 index 153358459d1..00000000000 --- a/contrib/libs/openssl/libssl.map +++ /dev/null @@ -1,543 +0,0 @@ -OPENSSL_1_1_0 { - global: - BIO_f_ssl; - BIO_new_buffer_ssl_connect; - BIO_new_ssl; - BIO_new_ssl_connect; - BIO_ssl_copy_session_id; - BIO_ssl_shutdown; - DTLS_client_method; - DTLS_method; - DTLS_server_method; - DTLSv1_2_client_method; - DTLSv1_2_method; - DTLSv1_2_server_method; - DTLSv1_client_method; - DTLSv1_listen; - DTLSv1_method; - DTLSv1_server_method; - ERR_load_SSL_strings; - OPENSSL_init_ssl; - PEM_read_SSL_SESSION; - PEM_read_bio_SSL_SESSION; - PEM_write_SSL_SESSION; - PEM_write_bio_SSL_SESSION; - SRP_Calc_A_param; - SSL_CIPHER_description; - SSL_CIPHER_find; - SSL_CIPHER_get_auth_nid; - SSL_CIPHER_get_bits; - SSL_CIPHER_get_cipher_nid; - SSL_CIPHER_get_digest_nid; - SSL_CIPHER_get_id; - SSL_CIPHER_get_kx_nid; - SSL_CIPHER_get_name; - SSL_CIPHER_get_version; - SSL_CIPHER_is_aead; - SSL_CIPHER_standard_name; - SSL_COMP_add_compression_method; - SSL_COMP_get_compression_methods; - SSL_COMP_get_name; - SSL_COMP_set0_compression_methods; - SSL_CONF_CTX_clear_flags; - SSL_CONF_CTX_finish; - SSL_CONF_CTX_free; - SSL_CONF_CTX_new; - SSL_CONF_CTX_set1_prefix; - SSL_CONF_CTX_set_flags; - SSL_CONF_CTX_set_ssl; - SSL_CONF_CTX_set_ssl_ctx; - SSL_CONF_cmd; - SSL_CONF_cmd_argv; - SSL_CONF_cmd_value_type; - SSL_CTX_SRP_CTX_free; - SSL_CTX_SRP_CTX_init; - SSL_CTX_add_client_CA; - SSL_CTX_add_client_custom_ext; - SSL_CTX_add_server_custom_ext; - SSL_CTX_add_session; - SSL_CTX_callback_ctrl; - SSL_CTX_check_private_key; - SSL_CTX_clear_options; - SSL_CTX_config; - SSL_CTX_ct_is_enabled; - SSL_CTX_ctrl; - SSL_CTX_dane_clear_flags; - SSL_CTX_dane_enable; - SSL_CTX_dane_mtype_set; - SSL_CTX_dane_set_flags; - SSL_CTX_enable_ct; - SSL_CTX_flush_sessions; - SSL_CTX_free; - SSL_CTX_get0_certificate; - SSL_CTX_get0_ctlog_store; - SSL_CTX_get0_param; - SSL_CTX_get0_privatekey; - SSL_CTX_get0_security_ex_data; - SSL_CTX_get_cert_store; - SSL_CTX_get_ciphers; - SSL_CTX_get_client_CA_list; - SSL_CTX_get_client_cert_cb; - SSL_CTX_get_default_passwd_cb; - SSL_CTX_get_default_passwd_cb_userdata; - SSL_CTX_get_ex_data; - SSL_CTX_get_info_callback; - SSL_CTX_get_options; - SSL_CTX_get_quiet_shutdown; - SSL_CTX_get_security_callback; - SSL_CTX_get_security_level; - SSL_CTX_get_ssl_method; - SSL_CTX_get_timeout; - SSL_CTX_get_verify_callback; - SSL_CTX_get_verify_depth; - SSL_CTX_get_verify_mode; - SSL_CTX_has_client_custom_ext; - SSL_CTX_load_verify_locations; - SSL_CTX_new; - SSL_CTX_remove_session; - SSL_CTX_sess_get_get_cb; - SSL_CTX_sess_get_new_cb; - SSL_CTX_sess_get_remove_cb; - SSL_CTX_sess_set_get_cb; - SSL_CTX_sess_set_new_cb; - SSL_CTX_sess_set_remove_cb; - SSL_CTX_sessions; - SSL_CTX_set0_ctlog_store; - SSL_CTX_set0_security_ex_data; - SSL_CTX_set1_param; - SSL_CTX_set_alpn_protos; - SSL_CTX_set_alpn_select_cb; - SSL_CTX_set_cert_cb; - SSL_CTX_set_cert_store; - SSL_CTX_set_cert_verify_callback; - SSL_CTX_set_cipher_list; - SSL_CTX_set_client_CA_list; - SSL_CTX_set_client_cert_cb; - SSL_CTX_set_client_cert_engine; - SSL_CTX_set_cookie_generate_cb; - SSL_CTX_set_cookie_verify_cb; - SSL_CTX_set_ct_validation_callback; - SSL_CTX_set_ctlog_list_file; - SSL_CTX_set_default_ctlog_list_file; - SSL_CTX_set_default_passwd_cb; - SSL_CTX_set_default_passwd_cb_userdata; - SSL_CTX_set_default_read_buffer_len; - SSL_CTX_set_default_verify_dir; - SSL_CTX_set_default_verify_file; - SSL_CTX_set_default_verify_paths; - SSL_CTX_set_ex_data; - SSL_CTX_set_generate_session_id; - SSL_CTX_set_info_callback; - SSL_CTX_set_msg_callback; - SSL_CTX_set_next_proto_select_cb; - SSL_CTX_set_next_protos_advertised_cb; - SSL_CTX_set_not_resumable_session_callback; - SSL_CTX_set_options; - SSL_CTX_set_psk_client_callback; - SSL_CTX_set_psk_server_callback; - SSL_CTX_set_purpose; - SSL_CTX_set_quiet_shutdown; - SSL_CTX_set_security_callback; - SSL_CTX_set_security_level; - SSL_CTX_set_session_id_context; - SSL_CTX_set_srp_cb_arg; - SSL_CTX_set_srp_client_pwd_callback; - SSL_CTX_set_srp_password; - SSL_CTX_set_srp_strength; - SSL_CTX_set_srp_username; - SSL_CTX_set_srp_username_callback; - SSL_CTX_set_srp_verify_param_callback; - SSL_CTX_set_ssl_version; - SSL_CTX_set_timeout; - SSL_CTX_set_tlsext_use_srtp; - SSL_CTX_set_tmp_dh_callback; - SSL_CTX_set_trust; - SSL_CTX_set_verify; - SSL_CTX_set_verify_depth; - SSL_CTX_up_ref; - SSL_CTX_use_PrivateKey; - SSL_CTX_use_PrivateKey_ASN1; - SSL_CTX_use_PrivateKey_file; - SSL_CTX_use_RSAPrivateKey; - SSL_CTX_use_RSAPrivateKey_ASN1; - SSL_CTX_use_RSAPrivateKey_file; - SSL_CTX_use_certificate; - SSL_CTX_use_certificate_ASN1; - SSL_CTX_use_certificate_chain_file; - SSL_CTX_use_certificate_file; - SSL_CTX_use_psk_identity_hint; - SSL_CTX_use_serverinfo; - SSL_CTX_use_serverinfo_file; - SSL_SESSION_free; - SSL_SESSION_get0_cipher; - SSL_SESSION_get0_hostname; - SSL_SESSION_get0_id_context; - SSL_SESSION_get0_peer; - SSL_SESSION_get0_ticket; - SSL_SESSION_get_compress_id; - SSL_SESSION_get_ex_data; - SSL_SESSION_get_id; - SSL_SESSION_get_master_key; - SSL_SESSION_get_protocol_version; - SSL_SESSION_get_ticket_lifetime_hint; - SSL_SESSION_get_time; - SSL_SESSION_get_timeout; - SSL_SESSION_has_ticket; - SSL_SESSION_new; - SSL_SESSION_print; - SSL_SESSION_print_fp; - SSL_SESSION_print_keylog; - SSL_SESSION_set1_id; - SSL_SESSION_set1_id_context; - SSL_SESSION_set_ex_data; - SSL_SESSION_set_time; - SSL_SESSION_set_timeout; - SSL_SESSION_up_ref; - SSL_SRP_CTX_free; - SSL_SRP_CTX_init; - SSL_accept; - SSL_add1_host; - SSL_add_client_CA; - SSL_add_dir_cert_subjects_to_stack; - SSL_add_file_cert_subjects_to_stack; - SSL_add_ssl_module; - SSL_alert_desc_string; - SSL_alert_desc_string_long; - SSL_alert_type_string; - SSL_alert_type_string_long; - SSL_callback_ctrl; - SSL_certs_clear; - SSL_check_chain; - SSL_check_private_key; - SSL_clear; - SSL_clear_options; - SSL_client_version; - SSL_config; - SSL_connect; - SSL_copy_session_id; - SSL_ct_is_enabled; - SSL_ctrl; - SSL_dane_clear_flags; - SSL_dane_enable; - SSL_dane_set_flags; - SSL_dane_tlsa_add; - SSL_do_handshake; - SSL_dup; - SSL_dup_CA_list; - SSL_enable_ct; - SSL_export_keying_material; - SSL_extension_supported; - SSL_free; - SSL_get0_alpn_selected; - SSL_get0_dane; - SSL_get0_dane_authority; - SSL_get0_dane_tlsa; - SSL_get0_next_proto_negotiated; - SSL_get0_param; - SSL_get0_peer_scts; - SSL_get0_peername; - SSL_get0_security_ex_data; - SSL_get0_verified_chain; - SSL_get1_session; - SSL_get1_supported_ciphers; - SSL_get_SSL_CTX; - SSL_get_all_async_fds; - SSL_get_certificate; - SSL_get_changed_async_fds; - SSL_get_cipher_list; - SSL_get_ciphers; - SSL_get_client_CA_list; - SSL_get_client_ciphers; - SSL_get_client_random; - SSL_get_current_cipher; - SSL_get_current_compression; - SSL_get_current_expansion; - SSL_get_default_passwd_cb; - SSL_get_default_passwd_cb_userdata; - SSL_get_default_timeout; - SSL_get_error; - SSL_get_ex_data; - SSL_get_ex_data_X509_STORE_CTX_idx; - SSL_get_fd; - SSL_get_finished; - SSL_get_info_callback; - SSL_get_options; - SSL_get_peer_cert_chain; - SSL_get_peer_certificate; - SSL_get_peer_finished; - SSL_get_privatekey; - SSL_get_psk_identity; - SSL_get_psk_identity_hint; - SSL_get_quiet_shutdown; - SSL_get_rbio; - SSL_get_read_ahead; - SSL_get_rfd; - SSL_get_security_callback; - SSL_get_security_level; - SSL_get_selected_srtp_profile; - SSL_get_server_random; - SSL_get_servername; - SSL_get_servername_type; - SSL_get_session; - SSL_get_shared_ciphers; - SSL_get_shared_sigalgs; - SSL_get_shutdown; - SSL_get_sigalgs; - SSL_get_srp_N; - SSL_get_srp_g; - SSL_get_srp_userinfo; - SSL_get_srp_username; - SSL_get_srtp_profiles; - SSL_get_ssl_method; - SSL_get_state; - SSL_get_verify_callback; - SSL_get_verify_depth; - SSL_get_verify_mode; - SSL_get_verify_result; - SSL_get_version; - SSL_get_wbio; - SSL_get_wfd; - SSL_has_matching_session_id; - SSL_has_pending; - SSL_in_before; - SSL_in_init; - SSL_is_dtls; - SSL_is_init_finished; - SSL_is_server; - SSL_load_client_CA_file; - SSL_new; - SSL_peek; - SSL_pending; - SSL_read; - SSL_renegotiate; - SSL_renegotiate_abbreviated; - SSL_renegotiate_pending; - SSL_rstate_string; - SSL_rstate_string_long; - SSL_select_next_proto; - SSL_session_reused; - SSL_set0_rbio; - SSL_set0_security_ex_data; - SSL_set0_wbio; - SSL_set1_host; - SSL_set1_param; - SSL_set_SSL_CTX; - SSL_set_accept_state; - SSL_set_alpn_protos; - SSL_set_bio; - SSL_set_cert_cb; - SSL_set_cipher_list; - SSL_set_client_CA_list; - SSL_set_connect_state; - SSL_set_ct_validation_callback; - SSL_set_debug; - SSL_set_default_passwd_cb; - SSL_set_default_passwd_cb_userdata; - SSL_set_default_read_buffer_len; - SSL_set_ex_data; - SSL_set_fd; - SSL_set_generate_session_id; - SSL_set_hostflags; - SSL_set_info_callback; - SSL_set_msg_callback; - SSL_set_not_resumable_session_callback; - SSL_set_options; - SSL_set_psk_client_callback; - SSL_set_psk_server_callback; - SSL_set_purpose; - SSL_set_quiet_shutdown; - SSL_set_read_ahead; - SSL_set_rfd; - SSL_set_security_callback; - SSL_set_security_level; - SSL_set_session; - SSL_set_session_id_context; - SSL_set_session_secret_cb; - SSL_set_session_ticket_ext; - SSL_set_session_ticket_ext_cb; - SSL_set_shutdown; - SSL_set_srp_server_param; - SSL_set_srp_server_param_pw; - SSL_set_ssl_method; - SSL_set_tlsext_use_srtp; - SSL_set_tmp_dh_callback; - SSL_set_trust; - SSL_set_verify; - SSL_set_verify_depth; - SSL_set_verify_result; - SSL_set_wfd; - SSL_shutdown; - SSL_srp_server_param_with_username; - SSL_state_string; - SSL_state_string_long; - SSL_up_ref; - SSL_use_PrivateKey; - SSL_use_PrivateKey_ASN1; - SSL_use_PrivateKey_file; - SSL_use_RSAPrivateKey; - SSL_use_RSAPrivateKey_ASN1; - SSL_use_RSAPrivateKey_file; - SSL_use_certificate; - SSL_use_certificate_ASN1; - SSL_use_certificate_chain_file; - SSL_use_certificate_file; - SSL_use_psk_identity_hint; - SSL_version; - SSL_waiting_for_async; - SSL_want; - SSL_write; - TLS_client_method; - TLS_method; - TLS_server_method; - TLSv1_1_client_method; - TLSv1_1_method; - TLSv1_1_server_method; - TLSv1_2_client_method; - TLSv1_2_method; - TLSv1_2_server_method; - TLSv1_client_method; - TLSv1_method; - TLSv1_server_method; - d2i_SSL_SESSION; - i2d_SSL_SESSION; -}; - -OPENSSL_1_1_0d { - global: - SSL_COMP_get0_name; - SSL_COMP_get_id; -} OPENSSL_1_1_0; - -OPENSSL_1_1_1 { - global: - DTLS_get_data_mtu; - DTLS_set_timer_cb; - OPENSSL_cipher_name; - SSL_CIPHER_get_handshake_digest; - SSL_CIPHER_get_protocol_id; - SSL_CTX_add1_to_CA_list; - SSL_CTX_add_custom_ext; - SSL_CTX_get0_CA_list; - SSL_CTX_get_keylog_callback; - SSL_CTX_get_max_early_data; - SSL_CTX_get_num_tickets; - SSL_CTX_get_record_padding_callback_arg; - SSL_CTX_get_recv_max_early_data; - SSL_CTX_set0_CA_list; - SSL_CTX_set1_cert_store; - SSL_CTX_set_allow_early_data_cb; - SSL_CTX_set_block_padding; - SSL_CTX_set_ciphersuites; - SSL_CTX_set_client_hello_cb; - SSL_CTX_set_keylog_callback; - SSL_CTX_set_max_early_data; - SSL_CTX_set_num_tickets; - SSL_CTX_set_post_handshake_auth; - SSL_CTX_set_psk_find_session_callback; - SSL_CTX_set_psk_use_session_callback; - SSL_CTX_set_record_padding_callback; - SSL_CTX_set_record_padding_callback_arg; - SSL_CTX_set_recv_max_early_data; - SSL_CTX_set_session_ticket_cb; - SSL_CTX_set_stateless_cookie_generate_cb; - SSL_CTX_set_stateless_cookie_verify_cb; - SSL_CTX_set_tlsext_max_fragment_length; - SSL_CTX_use_cert_and_key; - SSL_CTX_use_serverinfo_ex; - SSL_SESSION_dup; - SSL_SESSION_get0_alpn_selected; - SSL_SESSION_get0_ticket_appdata; - SSL_SESSION_get_max_early_data; - SSL_SESSION_get_max_fragment_length; - SSL_SESSION_is_resumable; - SSL_SESSION_set1_alpn_selected; - SSL_SESSION_set1_hostname; - SSL_SESSION_set1_master_key; - SSL_SESSION_set1_ticket_appdata; - SSL_SESSION_set_cipher; - SSL_SESSION_set_max_early_data; - SSL_SESSION_set_protocol_version; - SSL_add1_to_CA_list; - SSL_alloc_buffers; - SSL_bytes_to_cipher_list; - SSL_client_hello_get0_ciphers; - SSL_client_hello_get0_compression_methods; - SSL_client_hello_get0_ext; - SSL_client_hello_get0_legacy_version; - SSL_client_hello_get0_random; - SSL_client_hello_get0_session_id; - SSL_client_hello_get1_extensions_present; - SSL_client_hello_isv2; - SSL_export_keying_material_early; - SSL_free_buffers; - SSL_get0_CA_list; - SSL_get0_peer_CA_list; - SSL_get_early_data_status; - SSL_get_key_update_type; - SSL_get_max_early_data; - SSL_get_num_tickets; - SSL_get_peer_signature_type_nid; - SSL_get_pending_cipher; - SSL_get_record_padding_callback_arg; - SSL_get_recv_max_early_data; - SSL_key_update; - SSL_peek_ex; - SSL_read_early_data; - SSL_read_ex; - SSL_set0_CA_list; - SSL_set_allow_early_data_cb; - SSL_set_block_padding; - SSL_set_ciphersuites; - SSL_set_max_early_data; - SSL_set_num_tickets; - SSL_set_post_handshake_auth; - SSL_set_psk_find_session_callback; - SSL_set_psk_use_session_callback; - SSL_set_record_padding_callback; - SSL_set_record_padding_callback_arg; - SSL_set_recv_max_early_data; - SSL_set_tlsext_max_fragment_length; - SSL_stateless; - SSL_use_cert_and_key; - SSL_verify_client_post_handshake; - SSL_write_early_data; - SSL_write_ex; -} OPENSSL_1_1_0d; - -OPENSSL_1_1_1a { - global: - SSL_get_signature_type_nid; - local: *; -} OPENSSL_1_1_1; - -OPENSSL_1_1_1i { - global: - SSL_CIPHER_get_prf_nid; - SSL_CTX_set_quic_method; - SSL_get_peer_quic_transport_params; - SSL_get_peer_quic_transport_version; - SSL_get_quic_transport_version; - SSL_is_quic; - SSL_process_quic_post_handshake; - SSL_provide_quic_data; - SSL_quic_max_handshake_flight_len; - SSL_quic_read_level; - SSL_quic_write_level; - SSL_set_quic_method; - SSL_set_quic_transport_params; - SSL_set_quic_transport_version; - SSL_set_quic_use_legacy_codepoint; -} OPENSSL_1_1_1a; - -OPENSSL_1_1_1j { - global: - SSL_set_quic_early_data_enabled; -} OPENSSL_1_1_1i; - -OPENSSL_1_1_1k { - global: - SSL_new_session_ticket; - local: *; -} OPENSSL_1_1_1j; - - diff --git a/contrib/libs/openssl/openssl.package.json b/contrib/libs/openssl/openssl.package.json deleted file mode 100644 index e98964e5067..00000000000 --- a/contrib/libs/openssl/openssl.package.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "meta": { - "name": "libyandex-openssl-dev", - "version": "1.0.2j-r{revision}", - "maintainer": "Sergey Mironov <[email protected]>", - "description": "Statically linked OpenSSL from arcadia, headers\nWith Yandex' patches\nApproved by security" - }, - "build": { - "targets": [ - "contrib/libs/openssl" - ] - }, - "data": [ - { - "source": { - "type": "BUILD_OUTPUT", - "path": "contrib/libs/openssl", - "files": [ - "libcontrib-libs-openssl.a" - ] - }, - "destination": { - "path": "/usr/lib/" - } - }, - { - "source": { - "path": "contrib/libs/openssl/include/openssl", - "scheme": "svn", - "type": "ARCADIA" - }, - "destination": { - "path": "/usr/include/yandex-newssl/" - } - } - ] -} diff --git a/util/ysafeptr.cpp b/util/ysafeptr.cpp index 96f25140673..db2ff21ec0c 100644 --- a/util/ysafeptr.cpp +++ b/util/ysafeptr.cpp @@ -1,5 +1,7 @@ #include "ysafeptr.h" +#include <util/system/sanitizers.h> + #ifdef CHECK_YPTR2 Y_POD_THREAD(bool) IObjectBase::DisableThreadCheck; @@ -24,6 +26,11 @@ void IObjectBase::ReleaseObjComplete(int nMask) { } } //////////////////////////////////////////////////////////////////////////////////////////////////// +void IObjectBase::UnpoisonCounters() noexcept { + NSan::Unpoison(&ObjData, sizeof(ObjData)); + NSan::Unpoison(&RefData, sizeof(RefData)); +} +//////////////////////////////////////////////////////////////////////////////////////////////////// void IObjectBase::ReleaseRefComplete() { assert(RefData == 0); if ((ObjData & 0x3fffffff) == 0) { diff --git a/util/ysafeptr.h b/util/ysafeptr.h index ab5f26fb1ba..807ed63b67e 100644 --- a/util/ysafeptr.h +++ b/util/ysafeptr.h @@ -91,6 +91,8 @@ protected: // function should clear contents of object, easy to implement via consequent calls to // destructor and constructor, this function should not be called directly, use Clear() virtual void DestroyContents() = 0; + // Unpoison ObjData/RefData to mute deliberate use-after-dtor + void UnpoisonCounters() noexcept; virtual ~IObjectBase() = default; inline void CopyValidFlag(const IObjectBase& a) { ObjData &= 0x7fffffff; @@ -207,6 +209,7 @@ public: \ protected: \ virtual void DestroyContents() override { \ this->~classname(); \ + this->UnpoisonCounters(); \ int nHoldRefs = this->RefData, nHoldObjs = this->ObjData; \ new (this) classname(); \ this->RefData += nHoldRefs; \ diff --git a/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp b/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp index 2c82652f573..1b74ca1e817 100644 --- a/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp +++ b/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp @@ -3143,6 +3143,30 @@ TExprNode::TPtr ExpandPartitionsByKeys(const TExprNode::TPtr& node, TExprContext return ctx.ReplaceNode(node->Tail().TailPtr(), node->Tail().Head().Head(), std::move(sort)); } +TExprNode::TPtr ExpandLPartitionsByKeys(const TExprNode::TPtr& node, TExprContext& ctx) { + YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content(); + return ctx.Builder(node->Pos()) + .Callable("Block") + .Lambda(0) + .Param("parent") + .Callable("Collect") + .Callable(0, "PartitionsByKeys") + .Callable(0, "Iterator") + .Add(0, node->HeadPtr()) + .Callable(1, "DependsOn") + .Arg(0, "parent") + .Seal() + .Seal() + .Add(1, node->ChildPtr(1)) + .Add(2, node->ChildPtr(2)) + .Add(3, node->ChildPtr(3)) + .Add(4, node->ChildPtr(4)) + .Seal() + .Seal() + .Seal() + .Seal().Build(); +} + TExprNode::TPtr ExpandIsKeySwitch(const TExprNode::TPtr& node, TExprContext& ctx) { YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content(); return ctx.Builder(node->Pos()) @@ -9349,6 +9373,7 @@ struct TPeepHoleRules { {"Fold1Map", &CleckClosureOnUpperLambdaOverList<1U, 2U>}, {"Chain1Map", &CleckClosureOnUpperLambdaOverList<1U, 2U>}, {"PartitionsByKeys", &ExpandPartitionsByKeys}, + {"LPartitionsByKeys", &ExpandLPartitionsByKeys}, {"DictItems", &MapForOptionalContainer}, {"DictKeys", &MapForOptionalContainer}, {"DictPayloads", &MapForOptionalContainer}, diff --git a/yql/essentials/core/type_ann/type_ann_blocks.cpp b/yql/essentials/core/type_ann/type_ann_blocks.cpp index eee3e6a8db3..dd5e3dae0a4 100644 --- a/yql/essentials/core/type_ann/type_ann_blocks.cpp +++ b/yql/essentials/core/type_ann/type_ann_blocks.cpp @@ -1377,10 +1377,24 @@ IGraphTransformer::TStatus BlockPgCallWrapper(const TExprNode::TPtr& input, TExp } auto content = setting->Head().Content(); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Unexpected setting " << content << " in function " << name)); + if (content == "collation_oid") { + // BlockPgResolvedCall is only ever built by rewriting an already-resolved + // PgResolvedCall (see the peephole rewrite in yql_opt_peephole_physical.cpp), + // so its settings never carry a raw "collation" name - only the already-resolved + // "collation_oid", trusted as-is here, same as the analogous case in PgCallWrapper. + if (!EnsureTupleSize(*setting, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } - return IGraphTransformer::TStatus::Error; + if (!EnsureAtom(setting->Tail(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + } else { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Unexpected setting " << content << " in function " << name)); + + return IGraphTransformer::TStatus::Error; + } } TVector<ui32> argTypes; diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp index 1fd05710857..74dca3df57a 100644 --- a/yql/essentials/core/type_ann/type_ann_core.cpp +++ b/yql/essentials/core/type_ann/type_ann_core.cpp @@ -16087,7 +16087,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["ToBytes"] = &ToBytesWrapper; Functions["GroupByKey"] = &GroupByKeyWrapper; Functions["PartitionByKey"] = &PartitionByKeyWrapper; - Functions["PartitionsByKeys"] = &PartitionsByKeysWrapper; + Functions["PartitionsByKeys"] = &PartitionsByKeysWrapper<false>; + Functions["LPartitionsByKeys"] = &PartitionsByKeysWrapper<true>; Functions["Reverse"] = &ReverseWrapper; ExtFunctions["Skip"] = &TakeWrapperEx; ExtFunctions["Take"] = &TakeWrapperEx; diff --git a/yql/essentials/core/type_ann/type_ann_list.cpp b/yql/essentials/core/type_ann/type_ann_list.cpp index bfdcb22552b..d2cd11fad47 100644 --- a/yql/essentials/core/type_ann/type_ann_list.cpp +++ b/yql/essentials/core/type_ann/type_ann_list.cpp @@ -4369,6 +4369,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } + template<bool LPartitionsByKeys> IGraphTransformer::TStatus PartitionsByKeysWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 5, ctx.Expr)) { @@ -4381,8 +4382,15 @@ namespace { } const TTypeAnnotationNode* itemType = nullptr; - if (!EnsureNewSeqType<false>(input->Head(), ctx.Expr, &itemType)) { - return IGraphTransformer::TStatus::Error; + if constexpr (LPartitionsByKeys) { + if (!EnsureListType(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + itemType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); + } else { + if (!EnsureNewSeqType<false>(input->Head(), ctx.Expr, &itemType)) { + return IGraphTransformer::TStatus::Error; + } } auto& lambdaKeySelector = input->ChildRef(1); @@ -4450,7 +4458,11 @@ namespace { return IGraphTransformer::TStatus::Error; } - if (!UpdateLambdaAllArgumentsTypes(lambdaFinalHandler, { input->Head().GetTypeAnn() }, ctx.Expr)) { + const auto lambdaFinalHandlerArgType = LPartitionsByKeys + ? ctx.Expr.MakeType<TStreamExprType>(itemType) + : input->Head().GetTypeAnn(); + + if (!UpdateLambdaAllArgumentsTypes(lambdaFinalHandler, { lambdaFinalHandlerArgType }, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -4467,10 +4479,17 @@ namespace { return IGraphTransformer::TStatus::Error; } - input->SetTypeAnn(lambdaFinalHandler->GetTypeAnn()); + if constexpr (LPartitionsByKeys) { + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(&GetSeqItemType(*lambdaFinalHandler->GetTypeAnn()))); + } else { + input->SetTypeAnn(lambdaFinalHandler->GetTypeAnn()); + } return IGraphTransformer::TStatus::Ok; } + template IGraphTransformer::TStatus PartitionsByKeysWrapper<true>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + template IGraphTransformer::TStatus PartitionsByKeysWrapper<false>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus ReverseWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; diff --git a/yql/essentials/core/type_ann/type_ann_list.h b/yql/essentials/core/type_ann/type_ann_list.h index 026a0e38a19..5a4d8038e74 100644 --- a/yql/essentials/core/type_ann/type_ann_list.h +++ b/yql/essentials/core/type_ann/type_ann_list.h @@ -89,6 +89,7 @@ namespace NYql::NTypeAnnImpl { IGraphTransformer::TStatus SessionWindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus GroupByKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus PartitionByKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + template<bool LPartitionsByKeys> IGraphTransformer::TStatus PartitionsByKeysWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ReverseWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus TakeWrapperEx(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); diff --git a/yql/essentials/core/type_ann/type_ann_pg.cpp b/yql/essentials/core/type_ann/type_ann_pg.cpp index accf9548635..18255f683b4 100644 --- a/yql/essentials/core/type_ann/type_ann_pg.cpp +++ b/yql/essentials/core/type_ann/type_ann_pg.cpp @@ -254,6 +254,34 @@ IGraphTransformer::TStatus PgCallWrapper(const TExprNode::TPtr& input, TExprNode } refinedType = NPg::LookupType(TString(setting->Tail().Content())).TypeId; + } else if (content == "collation") { + if (!EnsureTupleSize(*setting, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(setting->Tail(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + TString collationName(setting->Tail().Content()); + try { + NPg::LookupCollation(collationName); + } catch (const yexception& e) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Failed to resolve collation \"" << collationName << "\" for function " << name << ": " << e.what())); + return IGraphTransformer::TStatus::Error; + } + } else if (content == "collation_oid") { + // Resolved collation oid, computed once from the "collation" setting above + // when PgResolvedCall was built (see below) - trusted as-is, same as the + // resolved proc oid in input->Child(1) is trusted without re-resolving the name. + if (!EnsureTupleSize(*setting, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(setting->Tail(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } } else { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Unexpected setting " << content << " in function " << name)); @@ -373,6 +401,32 @@ IGraphTransformer::TStatus PgCallWrapper(const TExprNode::TPtr& input, TExprNode children.push_back(std::move(node)); } + // If the call had an explicit COLLATE, resolve it to a stable oid (see + // NPg::LookupCollation) and replace the "collation" (name) setting with the + // resolved "collation_oid" one - the name has done its job (validation) and + // the oid is now the single source of truth, same as how the function name + // atom stays but the unresolved settings entry itself doesn't need to persist. + auto& settingsNode = children[argStart - 1]; + for (const auto& setting : settingsNode->Children()) { + if (setting->Head().Content() != "collation") { + continue; + } + + auto collationOid = NPg::LookupCollation(TString(setting->Tail().Content())).Oid; + TExprNode::TListType newSettings; + for (const auto& other : settingsNode->Children()) { + if (other->Head().Content() != "collation") { + newSettings.push_back(other); + } + } + newSettings.push_back(ctx.Expr.NewList(input->Pos(), { + ctx.Expr.NewAtom(input->Pos(), "collation_oid"), + ctx.Expr.NewAtom(input->Pos(), ToString(collationOid)), + })); + settingsNode = ctx.Expr.NewList(input->Pos(), std::move(newSettings)); + break; + } + if (proc.Lang == NPg::LangSQL) { if (!proc.ExprNode) { throw yexception() << "Function " << proc.Name << " has no implementation"; diff --git a/yql/essentials/minikql/mkql_program_builder.cpp b/yql/essentials/minikql/mkql_program_builder.cpp index 7a902067c85..87ed9bbbeb7 100644 --- a/yql/essentials/minikql/mkql_program_builder.cpp +++ b/yql/essentials/minikql/mkql_program_builder.cpp @@ -6185,23 +6185,59 @@ TRuntimeNode TProgramBuilder::PgConst(TPgType* pgType, const std::string_view& v TRuntimeNode TProgramBuilder::PgResolvedCall(bool useContext, const std::string_view& name, ui32 id, const TArrayRef<const TRuntimeNode>& args, - TType* returnType, bool rangeFunction) { - TCallableBuilder callableBuilder(Env_, __func__, returnType); - callableBuilder.Add(NewDataLiteral(useContext)); - callableBuilder.Add(NewDataLiteral(rangeFunction)); - callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(name)); - callableBuilder.Add(NewDataLiteral(id)); - for (const auto& arg : args) { - callableBuilder.Add(arg); + TType* returnType, bool rangeFunction, ui32 collationOid) { + // On old runtimes, always use the "PgResolvedCall" callable exactly as before (and + // explicit COLLATE, which needs the collation oid slot, isn't representable at all). + // Once the runtime is new enough, ALWAYS use the distinct "PgResolvedCall2" callable - + // with an extra collation oid slot right after `id` - even when collationOid is 0, so + // that "PgResolvedCall" stops being emitted entirely once the whole fleet is upgraded + // past this version, and its read-side support can eventually be deleted. The two + // shapes are told apart by callable name rather than input count/type, since an arg's + // own static type can legitimately be non-Pg (e.g. a NULL literal). + if constexpr (RuntimeVersion < 81U) { + Y_ENSURE(collationOid == 0, "Explicit COLLATE requires minikql runtime version >= 81, current: " << RuntimeVersion); + TCallableBuilder callableBuilder(Env_, "PgResolvedCall", returnType); + callableBuilder.Add(NewDataLiteral(useContext)); + callableBuilder.Add(NewDataLiteral(rangeFunction)); + callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(name)); + callableBuilder.Add(NewDataLiteral(id)); + for (const auto& arg : args) { + callableBuilder.Add(arg); + } + return TRuntimeNode(callableBuilder.Build(), /*isImmediate=*/false); + } else { + TCallableBuilder callableBuilder(Env_, "PgResolvedCall2", returnType); + callableBuilder.Add(NewDataLiteral(useContext)); + callableBuilder.Add(NewDataLiteral(rangeFunction)); + callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(name)); + callableBuilder.Add(NewDataLiteral(id)); + callableBuilder.Add(NewDataLiteral(collationOid)); + for (const auto& arg : args) { + callableBuilder.Add(arg); + } + return TRuntimeNode(callableBuilder.Build(), /*isImmediate=*/false); } - return TRuntimeNode(callableBuilder.Build(), /*isImmediate=*/false); } TRuntimeNode TProgramBuilder::BlockPgResolvedCall(const std::string_view& name, ui32 id, - const TArrayRef<const TRuntimeNode>& args, TType* returnType) { - TCallableBuilder callableBuilder(Env_, __func__, returnType); + const TArrayRef<const TRuntimeNode>& args, TType* returnType, + ui32 collationOid) { + // Mirrors TProgramBuilder::PgResolvedCall - see the comment there. + if constexpr (RuntimeVersion < 81U) { + Y_ENSURE(collationOid == 0, "Explicit COLLATE requires minikql runtime version >= 81, current: " << RuntimeVersion); + TCallableBuilder callableBuilder(Env_, "BlockPgResolvedCall", returnType); + callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(name)); + callableBuilder.Add(NewDataLiteral(id)); + for (const auto& arg : args) { + callableBuilder.Add(arg); + } + return TRuntimeNode(callableBuilder.Build(), /*isImmediate=*/false); + } + + TCallableBuilder callableBuilder(Env_, "BlockPgResolvedCall2", returnType); callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(name)); callableBuilder.Add(NewDataLiteral(id)); + callableBuilder.Add(NewDataLiteral(collationOid)); for (const auto& arg : args) { callableBuilder.Add(arg); } diff --git a/yql/essentials/minikql/mkql_program_builder.h b/yql/essentials/minikql/mkql_program_builder.h index 7499782e162..32fccf7c7c8 100644 --- a/yql/essentials/minikql/mkql_program_builder.h +++ b/yql/essentials/minikql/mkql_program_builder.h @@ -288,7 +288,8 @@ public: TRuntimeNode BlockToPg(TRuntimeNode input, TType* returnType); TRuntimeNode BlockFromPg(TRuntimeNode input, TType* returnType); TRuntimeNode BlockPgResolvedCall(const std::string_view& name, ui32 id, - const TArrayRef<const TRuntimeNode>& args, TType* returnType); + const TArrayRef<const TRuntimeNode>& args, TType* returnType, + ui32 collationOid); TRuntimeNode BlockStorage(TRuntimeNode list, TType* returnType); TRuntimeNode BlockMapJoinIndex(TRuntimeNode blockStorage, TType* listItemType, const TArrayRef<const ui32>& keyColumns, bool any, TType* returnType); TRuntimeNode BlockMapJoinCore(TRuntimeNode leftStream, TRuntimeNode rightBlockStorage, TType* rightListItemType, EJoinKind joinKind, @@ -750,7 +751,8 @@ public: TRuntimeNode PgConst(TPgType* pgType, const std::string_view& value, TRuntimeNode typeMod = {}); TRuntimeNode PgResolvedCall(bool useContext, const std::string_view& name, ui32 id, - const TArrayRef<const TRuntimeNode>& args, TType* returnType, bool rangeFunction); + const TArrayRef<const TRuntimeNode>& args, TType* returnType, bool rangeFunction, + ui32 collationOid); TRuntimeNode PgCast(TRuntimeNode input, TType* returnType, TRuntimeNode typeMod = {}); TRuntimeNode FromPg(TRuntimeNode input, TType* returnType); TRuntimeNode ToPg(TRuntimeNode input, TType* returnType); diff --git a/yql/essentials/parser/pg_catalog/catalog.cpp b/yql/essentials/parser/pg_catalog/catalog.cpp index 88a0a8ac1fc..847bdb6f236 100644 --- a/yql/essentials/parser/pg_catalog/catalog.cpp +++ b/yql/essentials/parser/pg_catalog/catalog.cpp @@ -59,6 +59,8 @@ using TAggregations = THashMap<ui32, TAggregateDesc>; using TAms = THashMap<ui32, TAmDesc>; +using TCollations = THashMap<ui32, TCollationDesc>; + using TNamespaces = THashMap<decltype(TNamespaceDesc::Oid), TNamespaceDesc>; using TOpFamilies = THashMap<TString, TOpFamilyDesc>; @@ -1299,6 +1301,53 @@ private: TAms& Ams_; }; +class TCollationsParser: public TParser { +public: + explicit TCollationsParser(TCollations& collations) + : Collations_(collations) + { + } + + void OnKey(const TString& key, const TString& value) override { + if (key == "oid") { + CurrDesc_.Oid = FromString<ui32>(value); + } else if (key == "descr") { + CurrDesc_.Descr = value; + } else if (key == "collname") { + CurrDesc_.Name = value; + } else if (key == "collprovider") { + Y_ENSURE(value.size() == 1); + switch (value[0]) { + case (char)ECollationProvider::Default: + case (char)ECollationProvider::Icu: + case (char)ECollationProvider::Libc: + CurrDesc_.Provider = (ECollationProvider)value[0]; + break; + default: + throw yexception() << "Unknown collprovider value: " << value; + } + } else if (key == "collencoding") { + CurrDesc_.Encoding = FromString<i32>(value); + } else if (key == "collcollate") { + CurrDesc_.Collate = value; + } else if (key == "collctype") { + CurrDesc_.Ctype = value; + } else if (key == "colliculocale") { + CurrDesc_.IcuLocale = value; + } + } + + void OnFinish() override { + Y_ENSURE(!CurrDesc_.Name.empty()); + Collations_[CurrDesc_.Oid] = std::move(CurrDesc_); + CurrDesc_ = TCollationDesc(); + } + +private: + TCollationDesc CurrDesc_; + TCollations& Collations_; +}; + class TAmProcsParser: public TParser { public: TAmProcsParser(TAmProcs& amProcs, const THashMap<TString, ui32>& typeByName, @@ -1582,6 +1631,13 @@ TAms ParseAms(const TString& dat) { return ret; } +TCollations ParseCollations(const TString& dat) { + TCollations ret; + TCollationsParser parser(ret); + parser.Do(dat); + return ret; +} + TLanguages ParseLanguages(const TString& dat) { TLanguages ret; TLanguagesParser parser(ret); @@ -1634,6 +1690,14 @@ const char* AllowedProcsRaw[] = { #include "postgis_procs.h" }; +// Locale names known to the ICU library pg_collation_icu.generated.h was generated from +// (see gen_icu_collations). Append-only: a locale's position here becomes part of its +// collation oid (see IcuCollationOidBase below), so it must never change once assigned. +// NOLINTNEXTLINE(modernize-avoid-c-arrays) +const char* AllIcuCollationLocalesRaw[] = { +#include "pg_collation_icu.generated.h" +}; + struct TCatalog: public IExtensionSqlBuilder { TCatalog() { Init(); @@ -1733,6 +1797,8 @@ struct TCatalog: public IExtensionSqlBuilder { Y_ENSURE(NResource::FindExact("pg_conversion.dat", &conversionData)); TString amData; Y_ENSURE(NResource::FindExact("pg_am.dat", &amData)); + TString collationData; + Y_ENSURE(NResource::FindExact("pg_collation.dat", &collationData)); TString languagesData; Y_ENSURE(NResource::FindExact("pg_language.dat", &languagesData)); THashMap<ui32, TLazyTypeInfo> lazyTypeInfos; @@ -1860,6 +1926,22 @@ struct TCatalog: public IExtensionSqlBuilder { State->AmOps = ParseAmOps(amOpData, State->TypeByName, State->Types, State->OperatorsByName, State->Operators, State->OpFamilies); State->AmProcs = ParseAmProcs(amProcData, State->TypeByName, State->ProcByName, State->Procs, State->OpFamilies); State->Ams = ParseAms(amData); + State->Collations = ParseCollations(collationData); + for (const auto& [k, v] : State->Collations) { + Y_ENSURE(State->CollationByName.insert(std::make_pair(v.Name, k)).second); + } + + for (size_t i = 0; i < Y_ARRAY_SIZE(AllIcuCollationLocalesRaw); ++i) { + TCollationDesc desc; + desc.Oid = IcuCollationOidBase + i; + desc.IcuLocale = AllIcuCollationLocalesRaw[i]; + desc.Name = desc.IcuLocale + "-x-icu"; + desc.Provider = ECollationProvider::Icu; + desc.Descr = "ICU locale collation"; + Y_ENSURE(State->Collations.insert(std::make_pair(desc.Oid, desc)).second); + Y_ENSURE(State->CollationByName.insert(std::make_pair(desc.Name, desc.Oid)).second); + } + State->Namespaces = FillNamespaces(); for (auto& [k, v] : State->Types) { if (v.TypeId != v.ArrayTypeId) { @@ -2263,6 +2345,7 @@ struct TCatalog: public IExtensionSqlBuilder { TCasts Casts; TAggregations Aggregations; TAms Ams; + TCollations Collations; TNamespaces Namespaces; TOpFamilies OpFamilies; TOpClasses OpClasses; @@ -2272,6 +2355,7 @@ struct TCatalog: public IExtensionSqlBuilder { TLanguages Languages; THashMap<TString, TVector<ui32>> ProcByName; THashMap<TString, ui32> TypeByName; + THashMap<TString, ui32> CollationByName; THashMap<std::pair<ui32, ui32>, ui32> CastsByDir; THashMap<TString, TVector<ui32>> OperatorsByName; THashMap<TString, TVector<ui32>> AggregationsByName; @@ -2463,6 +2547,38 @@ void EnumAm(std::function<void(ui32, const TAmDesc&)> f) { } } +bool HasCollation(const TString& name) { + const auto& catalog = TCatalog::Instance(); + return catalog.State->CollationByName.contains(name); +} + +const TCollationDesc& LookupCollation(const TString& name) { + const auto& catalog = TCatalog::Instance(); + const auto oidPtr = catalog.State->CollationByName.FindPtr(name); + if (!oidPtr) { + throw yexception() << "No such collation: " << name; + } + + return LookupCollation(*oidPtr); +} + +const TCollationDesc& LookupCollation(ui32 oid) { + const auto& catalog = TCatalog::Instance(); + const auto descPtr = catalog.State->Collations.FindPtr(oid); + if (!descPtr) { + throw yexception() << "No such collation: " << oid; + } + + return *descPtr; +} + +void EnumCollation(std::function<void(ui32, const TCollationDesc&)> f) { + const auto& catalog = TCatalog::Instance(); + for (const auto& [oid, desc] : catalog.State->Collations) { + f(oid, desc); + } +} + void EnumConversions(std::function<void(const TConversionDesc&)> f) { const auto& catalog = TCatalog::Instance(); for (const auto& [_, desc] : catalog.State->Conversions) { diff --git a/yql/essentials/parser/pg_catalog/catalog.h b/yql/essentials/parser/pg_catalog/catalog.h index 1f30f692706..e06b7616d44 100644 --- a/yql/essentials/parser/pg_catalog/catalog.h +++ b/yql/essentials/parser/pg_catalog/catalog.h @@ -91,6 +91,32 @@ constexpr ui32 DefaultCollationOid = 100; constexpr ui32 C_CollationOid = 950; constexpr ui32 PosixCollationOid = 951; +// Oid space reserved for ICU locale collations (name "<locale>-x-icu"), kept clear of the +// oids used by pg_collation.dat (which top out in the low thousands). +// A locale's oid is IcuCollationOidBase plus its position in the checked-in, append-only +// pg_collation_icu.generated.h (see gen_icu_collations) - stable across processes/rebuilds +// regardless of which ICU library version is actually linked, since it is our own versioned +// data rather than something derived live from whatever ICU happens to be available. +constexpr ui32 IcuCollationOidBase = 1'000'000; + +// Copied from pg_collation.h, COLLPROVIDER_* constants +enum class ECollationProvider: char { + Default = 'd', + Icu = 'i', + Libc = 'c', +}; + +struct TCollationDesc { + ui32 Oid = 0; + TString Name; + TString Descr; + ECollationProvider Provider = ECollationProvider::Libc; + i32 Encoding = -1; + TString Collate; + TString Ctype; + TString IcuLocale; +}; + // Copied from pg_type_d.h, TYPTYPE_* constants enum class ETypType: char { Base = 'b', @@ -295,6 +321,11 @@ void EnumTypes(std::function<void(ui32, const TTypeDesc&)> f); const TAmDesc& LookupAm(ui32 oid); void EnumAm(std::function<void(ui32, const TAmDesc&)> f); +bool HasCollation(const TString& name); +const TCollationDesc& LookupCollation(const TString& name); +const TCollationDesc& LookupCollation(ui32 oid); +void EnumCollation(std::function<void(ui32, const TCollationDesc&)> f); + void EnumConversions(std::function<void(const TConversionDesc&)> f); const TNamespaceDesc& LookupNamespace(ui32 oid); @@ -497,6 +528,11 @@ inline void Out<NYql::NPg::ECoercionCode>(IOutputStream& out, NYql::NPg::ECoerci } template <> +inline void Out<NYql::NPg::ECollationProvider>(IOutputStream& out, NYql::NPg::ECollationProvider value) { + out.Write(static_cast<std::underlying_type<NYql::NPg::ECollationProvider>::type>(value)); +} + +template <> struct THash<NYql::NPg::TTableInfoKey> { size_t operator()(const NYql::NPg::TTableInfoKey& val) const { return val.Hash(); diff --git a/yql/essentials/parser/pg_catalog/gen_icu_collations/main.cpp b/yql/essentials/parser/pg_catalog/gen_icu_collations/main.cpp new file mode 100644 index 00000000000..9d9e4577806 --- /dev/null +++ b/yql/essentials/parser/pg_catalog/gen_icu_collations/main.cpp @@ -0,0 +1,107 @@ +// Regenerates pg_collation_icu.generated.h - the checked-in list of ICU collation locale +// names known to pg_catalog. Run manually and commit the result whenever the vendored ICU +// (contrib/libs/icu) is updated: +// +// ya make yql/essentials/parser/pg_catalog/gen_icu_collations +// yql/essentials/parser/pg_catalog/gen_icu_collations/gen_icu_collations \ +// yql/essentials/parser/pg_catalog/pg_collation_icu.generated.h +// +// The output is append-only: existing entries are read back first and kept in place, and +// only locales not already present are appended at the end. This keeps each locale's +// position in the file - and hence the collation oid pg_catalog derives from that position +// - stable across regenerations, even if the set of locales ICU knows about changes, +// exactly like AllStaticTablesRaw/pg_class.generated.h and library/cpp/type_info/tz do for +// their own generated tables. + +#include <unicode/uloc.h> + +#include <util/generic/algorithm.h> +#include <util/generic/hash_set.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> +#include <util/stream/file.h> +#include <util/system/fs.h> + +#include <cstdio> + +namespace { + +// Canonicalizes an ICU locale id (e.g. "de_DE") to its BCP47 language tag (e.g. "de-DE"). +// Returns an empty string if ICU can't parse it at all. +TString CanonicalizeIcuLocale(const TString& locale) { + UErrorCode status = U_ZERO_ERROR; + std::array<char, ULOC_FULLNAME_CAPACITY> tag; + auto len = uloc_toLanguageTag(locale.c_str(), tag.data(), tag.size(), /* strict */ true, &status); + if (U_FAILURE(status)) { + return {}; + } + + return TString(tag.data(), len); +} + +TVector<TString> ReadExistingLocales(const TString& path) { + TVector<TString> result; + if (!NFs::Exists(path)) { + return result; + } + + TFileInput input(path); + TString line; + while (input.ReadLine(line)) { + auto endQuote = line.find('"', 1); + Y_ENSURE(line.size() > 2 && line[0] == '"' && endQuote != TString::npos, "Malformed line in " << path << ": " << line); + result.push_back(line.substr(1, endQuote - 1)); + } + + return result; +} + +TVector<TString> DiscoverIcuLocales() { + TVector<TString> result; + if (auto root = CanonicalizeIcuLocale(""); !root.empty()) { + result.push_back(root); + } + + for (i32 i = 0; i < uloc_countAvailable(); ++i) { + if (auto tag = CanonicalizeIcuLocale(uloc_getAvailable(i)); !tag.empty()) { + result.push_back(tag); + } + } + + Sort(result.begin(), result.end()); + result.erase(Unique(result.begin(), result.end()), result.end()); + return result; +} + +} // namespace + +int main(int argc, char** argv) { + Y_ENSURE(argc == 2, "usage: gen_icu_collations <path-to-pg_collation_icu.generated.h>"); + TString outputPath = argv[1]; + + auto existing = ReadExistingLocales(outputPath); + THashSet<TString> seen(existing.begin(), existing.end()); + + TVector<TString> toAppend; + for (auto& tag : DiscoverIcuLocales()) { + if (seen.insert(tag).second) { + toAppend.push_back(tag); + } + } + + { + TFileOutput output(outputPath); + for (auto& tag : existing) { + output << "\"" << tag << "\",\n"; + } + + for (auto& tag : toAppend) { + output << "\"" << tag << "\",\n"; + } + } + + Cerr << "Wrote " << (existing.size() + toAppend.size()) << " ICU collation locales (" + << toAppend.size() << " new) to " << outputPath << Endl; + return 0; +} diff --git a/yql/essentials/parser/pg_catalog/gen_icu_collations/ya.make b/yql/essentials/parser/pg_catalog/gen_icu_collations/ya.make new file mode 100644 index 00000000000..b90c170c01a --- /dev/null +++ b/yql/essentials/parser/pg_catalog/gen_icu_collations/ya.make @@ -0,0 +1,11 @@ +PROGRAM(gen_icu_collations) + +SRCS( + main.cpp +) + +PEERDIR( + contrib/libs/icu +) + +END() diff --git a/yql/essentials/parser/pg_catalog/pg_collation_icu.generated.h b/yql/essentials/parser/pg_catalog/pg_collation_icu.generated.h new file mode 100644 index 00000000000..c1796fc4a7f --- /dev/null +++ b/yql/essentials/parser/pg_catalog/pg_collation_icu.generated.h @@ -0,0 +1,907 @@ +"af", +"af-NA", +"af-ZA", +"agq", +"agq-CM", +"ak", +"ak-GH", +"am", +"am-ET", +"ar", +"ar-001", +"ar-AE", +"ar-BH", +"ar-DJ", +"ar-DZ", +"ar-EG", +"ar-EH", +"ar-ER", +"ar-IL", +"ar-IQ", +"ar-JO", +"ar-KM", +"ar-KW", +"ar-LB", +"ar-LY", +"ar-MA", +"ar-MR", +"ar-OM", +"ar-PS", +"ar-QA", +"ar-SA", +"ar-SD", +"ar-SO", +"ar-SS", +"ar-SY", +"ar-TD", +"ar-TN", +"ar-YE", +"as", +"as-IN", +"asa", +"asa-TZ", +"ast", +"ast-ES", +"az", +"az-Cyrl", +"az-Cyrl-AZ", +"az-Latn", +"az-Latn-AZ", +"ba", +"ba-RU", +"bas", +"bas-CM", +"be", +"be-BY", +"bem", +"bem-ZM", +"bez", +"bez-TZ", +"bg", +"bg-BG", +"bgc", +"bgc-IN", +"bho", +"bho-IN", +"blo", +"blo-BJ", +"bm", +"bm-ML", +"bn", +"bn-BD", +"bn-IN", +"bo", +"bo-CN", +"bo-IN", +"br", +"br-FR", +"brx", +"brx-IN", +"bs", +"bs-Cyrl", +"bs-Cyrl-BA", +"bs-Latn", +"bs-Latn-BA", +"bua", +"bua-RU", +"ca", +"ca-AD", +"ca-ES", +"ca-FR", +"ca-IT", +"ccp", +"ccp-BD", +"ccp-IN", +"ce", +"ce-RU", +"ceb", +"ceb-PH", +"cgg", +"cgg-UG", +"chr", +"chr-US", +"ckb", +"ckb-IQ", +"ckb-IR", +"cs", +"cs-CZ", +"csw", +"csw-CA", +"cv", +"cv-RU", +"cy", +"cy-GB", +"da", +"da-DK", +"da-GL", +"dav", +"dav-KE", +"de", +"de-AT", +"de-BE", +"de-CH", +"de-DE", +"de-IT", +"de-LI", +"de-LU", +"dje", +"dje-NE", +"doi", +"doi-IN", +"dsb", +"dsb-DE", +"dua", +"dua-CM", +"dyo", +"dyo-SN", +"dz", +"dz-BT", +"ebu", +"ebu-KE", +"ee", +"ee-GH", +"ee-TG", +"el", +"el-CY", +"el-GR", +"en", +"en-001", +"en-150", +"en-AE", +"en-AG", +"en-AI", +"en-AS", +"en-AT", +"en-AU", +"en-BB", +"en-BE", +"en-BI", +"en-BM", +"en-BS", +"en-BW", +"en-BZ", +"en-CA", +"en-CC", +"en-CH", +"en-CK", +"en-CM", +"en-CX", +"en-CY", +"en-CZ", +"en-DE", +"en-DG", +"en-DK", +"en-DM", +"en-EE", +"en-ER", +"en-ES", +"en-FI", +"en-FJ", +"en-FK", +"en-FM", +"en-FR", +"en-GB", +"en-GD", +"en-GE", +"en-GG", +"en-GH", +"en-GI", +"en-GM", +"en-GS", +"en-GU", +"en-GY", +"en-HK", +"en-HU", +"en-ID", +"en-IE", +"en-IL", +"en-IM", +"en-IN", +"en-IO", +"en-IT", +"en-JE", +"en-JM", +"en-JP", +"en-KE", +"en-KI", +"en-KN", +"en-KY", +"en-LC", +"en-LR", +"en-LS", +"en-LT", +"en-LV", +"en-MG", +"en-MH", +"en-MO", +"en-MP", +"en-MS", +"en-MT", +"en-MU", +"en-MV", +"en-MW", +"en-MY", +"en-NA", +"en-NF", +"en-NG", +"en-NL", +"en-NO", +"en-NR", +"en-NU", +"en-NZ", +"en-PG", +"en-PH", +"en-PK", +"en-PL", +"en-PN", +"en-PR", +"en-PT", +"en-PW", +"en-RO", +"en-RW", +"en-SB", +"en-SC", +"en-SD", +"en-SE", +"en-SG", +"en-SH", +"en-SI", +"en-SK", +"en-SL", +"en-SS", +"en-SX", +"en-SZ", +"en-TC", +"en-TK", +"en-TO", +"en-TT", +"en-TV", +"en-TZ", +"en-UA", +"en-UG", +"en-UM", +"en-US", +"en-US-u-va-posix", +"en-VC", +"en-VG", +"en-VI", +"en-VU", +"en-WS", +"en-ZA", +"en-ZM", +"en-ZW", +"eo", +"eo-001", +"es", +"es-419", +"es-AR", +"es-BO", +"es-BR", +"es-BZ", +"es-CL", +"es-CO", +"es-CR", +"es-CU", +"es-DO", +"es-EA", +"es-EC", +"es-ES", +"es-GQ", +"es-GT", +"es-HN", +"es-IC", +"es-MX", +"es-NI", +"es-PA", +"es-PE", +"es-PH", +"es-PR", +"es-PY", +"es-SV", +"es-US", +"es-UY", +"es-VE", +"et", +"et-EE", +"eu", +"eu-ES", +"ewo", +"ewo-CM", +"fa", +"fa-AF", +"fa-IR", +"ff", +"ff-Adlm", +"ff-Adlm-BF", +"ff-Adlm-CM", +"ff-Adlm-GH", +"ff-Adlm-GM", +"ff-Adlm-GN", +"ff-Adlm-GW", +"ff-Adlm-LR", +"ff-Adlm-MR", +"ff-Adlm-NE", +"ff-Adlm-NG", +"ff-Adlm-SL", +"ff-Adlm-SN", +"ff-Latn", +"ff-Latn-BF", +"ff-Latn-CM", +"ff-Latn-GH", +"ff-Latn-GM", +"ff-Latn-GN", +"ff-Latn-GW", +"ff-Latn-LR", +"ff-Latn-MR", +"ff-Latn-NE", +"ff-Latn-NG", +"ff-Latn-SL", +"ff-Latn-SN", +"fi", +"fi-FI", +"fil", +"fil-PH", +"fo", +"fo-DK", +"fo-FO", +"fr", +"fr-BE", +"fr-BF", +"fr-BI", +"fr-BJ", +"fr-BL", +"fr-CA", +"fr-CD", +"fr-CF", +"fr-CG", +"fr-CH", +"fr-CI", +"fr-CM", +"fr-DJ", +"fr-DZ", +"fr-FR", +"fr-GA", +"fr-GF", +"fr-GN", +"fr-GP", +"fr-GQ", +"fr-HT", +"fr-KM", +"fr-LU", +"fr-MA", +"fr-MC", +"fr-MF", +"fr-MG", +"fr-ML", +"fr-MQ", +"fr-MR", +"fr-MU", +"fr-NC", +"fr-NE", +"fr-PF", +"fr-PM", +"fr-RE", +"fr-RW", +"fr-SC", +"fr-SN", +"fr-SY", +"fr-TD", +"fr-TG", +"fr-TN", +"fr-VU", +"fr-WF", +"fr-YT", +"fur", +"fur-IT", +"fy", +"fy-NL", +"ga", +"ga-GB", +"ga-IE", +"gaa", +"gaa-GH", +"gd", +"gd-GB", +"gl", +"gl-ES", +"gsw", +"gsw-CH", +"gsw-FR", +"gsw-LI", +"gu", +"gu-IN", +"guz", +"guz-KE", +"gv", +"gv-IM", +"ha", +"ha-GH", +"ha-NE", +"ha-NG", +"haw", +"haw-US", +"he", +"he-IL", +"hi", +"hi-IN", +"hi-Latn", +"hi-Latn-IN", +"hr", +"hr-BA", +"hr-HR", +"hsb", +"hsb-DE", +"hu", +"hu-HU", +"hy", +"hy-AM", +"ia", +"ia-001", +"id", +"id-ID", +"ie", +"ie-EE", +"ig", +"ig-NG", +"ii", +"ii-CN", +"is", +"is-IS", +"it", +"it-CH", +"it-IT", +"it-SM", +"it-VA", +"ja", +"ja-JP", +"jgo", +"jgo-CM", +"jmc", +"jmc-TZ", +"jv", +"jv-ID", +"ka", +"ka-GE", +"kab", +"kab-DZ", +"kam", +"kam-KE", +"kde", +"kde-TZ", +"kea", +"kea-CV", +"kgp", +"kgp-BR", +"khq", +"khq-ML", +"ki", +"ki-KE", +"kk", +"kk-Arab", +"kk-Arab-CN", +"kk-Cyrl", +"kk-Cyrl-KZ", +"kk-KZ", +"kkj", +"kkj-CM", +"kl", +"kl-GL", +"kln", +"kln-KE", +"km", +"km-KH", +"kn", +"kn-IN", +"ko", +"ko-CN", +"ko-KP", +"ko-KR", +"kok", +"kok-Deva", +"kok-Deva-IN", +"kok-Latn", +"kok-Latn-IN", +"ks", +"ks-Arab", +"ks-Arab-IN", +"ks-Deva", +"ks-Deva-IN", +"ksb", +"ksb-TZ", +"ksf", +"ksf-CM", +"ksh", +"ksh-DE", +"ku", +"ku-Latn", +"ku-Latn-IQ", +"ku-Latn-SY", +"ku-Latn-TR", +"ku-TR", +"kw", +"kw-GB", +"kxv", +"kxv-Deva", +"kxv-Deva-IN", +"kxv-Latn", +"kxv-Latn-IN", +"kxv-Orya", +"kxv-Orya-IN", +"kxv-Telu", +"kxv-Telu-IN", +"ky", +"ky-KG", +"lag", +"lag-TZ", +"lb", +"lb-LU", +"lg", +"lg-UG", +"lij", +"lij-IT", +"lkt", +"lkt-US", +"lmo", +"lmo-IT", +"ln", +"ln-AO", +"ln-CD", +"ln-CF", +"ln-CG", +"lo", +"lo-LA", +"lrc", +"lrc-IQ", +"lrc-IR", +"lt", +"lt-LT", +"lu", +"lu-CD", +"luo", +"luo-KE", +"luy", +"luy-KE", +"lv", +"lv-LV", +"mai", +"mai-IN", +"mas", +"mas-KE", +"mas-TZ", +"mer", +"mer-KE", +"mfe", +"mfe-MU", +"mg", +"mg-MG", +"mgh", +"mgh-MZ", +"mgo", +"mgo-CM", +"mi", +"mi-NZ", +"mk", +"mk-MK", +"ml", +"ml-IN", +"mn", +"mn-MN", +"mni", +"mni-Beng", +"mni-Beng-IN", +"mr", +"mr-IN", +"ms", +"ms-BN", +"ms-ID", +"ms-MY", +"ms-SG", +"mt", +"mt-MT", +"mua", +"mua-CM", +"my", +"my-MM", +"mzn", +"mzn-IR", +"naq", +"naq-NA", +"nb", +"nb-NO", +"nb-SJ", +"nd", +"nd-ZW", +"nds", +"nds-DE", +"nds-NL", +"ne", +"ne-IN", +"ne-NP", +"nl", +"nl-AW", +"nl-BE", +"nl-BQ", +"nl-CW", +"nl-NL", +"nl-SR", +"nl-SX", +"nmg", +"nmg-CM", +"nn", +"nn-NO", +"nnh", +"nnh-CM", +"no", +"nqo", +"nqo-GN", +"nso", +"nso-ZA", +"nus", +"nus-SS", +"nyn", +"nyn-UG", +"oc", +"oc-ES", +"oc-FR", +"om", +"om-ET", +"om-KE", +"or", +"or-IN", +"os", +"os-GE", +"os-RU", +"pa", +"pa-Arab", +"pa-Arab-PK", +"pa-Guru", +"pa-Guru-IN", +"pcm", +"pcm-NG", +"pl", +"pl-PL", +"pms", +"pms-IT", +"prg", +"prg-PL", +"ps", +"ps-AF", +"ps-PK", +"pt", +"pt-AO", +"pt-BR", +"pt-CH", +"pt-CV", +"pt-GQ", +"pt-GW", +"pt-LU", +"pt-MO", +"pt-MZ", +"pt-PT", +"pt-ST", +"pt-TL", +"qu", +"qu-BO", +"qu-EC", +"qu-PE", +"raj", +"raj-IN", +"rm", +"rm-CH", +"rn", +"rn-BI", +"ro", +"ro-MD", +"ro-RO", +"rof", +"rof-TZ", +"ru", +"ru-BY", +"ru-KG", +"ru-KZ", +"ru-MD", +"ru-RU", +"ru-UA", +"rw", +"rw-RW", +"rwk", +"rwk-TZ", +"sa", +"sa-IN", +"sah", +"sah-RU", +"saq", +"saq-KE", +"sat", +"sat-Olck", +"sat-Olck-IN", +"sbp", +"sbp-TZ", +"sc", +"sc-IT", +"scn", +"scn-IT", +"sd", +"sd-Arab", +"sd-Arab-PK", +"sd-Deva", +"sd-Deva-IN", +"se", +"se-FI", +"se-NO", +"se-SE", +"seh", +"seh-MZ", +"ses", +"ses-ML", +"sg", +"sg-CF", +"shi", +"shi-Latn", +"shi-Latn-MA", +"shi-Tfng", +"shi-Tfng-MA", +"shn", +"shn-MM", +"shn-TH", +"si", +"si-LK", +"sk", +"sk-SK", +"sl", +"sl-SI", +"smn", +"smn-FI", +"sn", +"sn-ZW", +"so", +"so-DJ", +"so-ET", +"so-KE", +"so-SO", +"sq", +"sq-AL", +"sq-MK", +"sq-XK", +"sr", +"sr-Cyrl", +"sr-Cyrl-BA", +"sr-Cyrl-ME", +"sr-Cyrl-RS", +"sr-Cyrl-XK", +"sr-Latn", +"sr-Latn-BA", +"sr-Latn-ME", +"sr-Latn-RS", +"sr-Latn-XK", +"st", +"st-LS", +"st-ZA", +"su", +"su-Latn", +"su-Latn-ID", +"sv", +"sv-AX", +"sv-FI", +"sv-SE", +"sw", +"sw-CD", +"sw-KE", +"sw-TZ", +"sw-UG", +"syr", +"syr-IQ", +"syr-SY", +"szl", +"szl-PL", +"ta", +"ta-IN", +"ta-LK", +"ta-MY", +"ta-SG", +"te", +"te-IN", +"teo", +"teo-KE", +"teo-UG", +"tg", +"tg-TJ", +"th", +"th-TH", +"ti", +"ti-ER", +"ti-ET", +"tk", +"tk-TM", +"tn", +"tn-BW", +"tn-ZA", +"to", +"to-TO", +"tok", +"tok-001", +"tr", +"tr-CY", +"tr-TR", +"tt", +"tt-RU", +"twq", +"twq-NE", +"tyv", +"tyv-RU", +"tzm", +"tzm-MA", +"ug", +"ug-CN", +"uk", +"uk-UA", +"und", +"ur", +"ur-IN", +"ur-PK", +"uz", +"uz-Arab", +"uz-Arab-AF", +"uz-Cyrl", +"uz-Cyrl-UZ", +"uz-Latn", +"uz-Latn-UZ", +"vai", +"vai-Latn", +"vai-Latn-LR", +"vai-Vaii", +"vai-Vaii-LR", +"vec", +"vec-IT", +"vi", +"vi-VN", +"vmw", +"vmw-MZ", +"vun", +"vun-TZ", +"wae", +"wae-CH", +"wo", +"wo-SN", +"xh", +"xh-ZA", +"xnr", +"xnr-IN", +"xog", +"xog-UG", +"yav", +"yav-CM", +"yi", +"yi-UA", +"yo", +"yo-BJ", +"yo-NG", +"yrl", +"yrl-BR", +"yrl-CO", +"yrl-VE", +"yue", +"yue-Hans", +"yue-Hans-CN", +"yue-Hant", +"yue-Hant-CN", +"yue-Hant-HK", +"yue-Hant-MO", +"za", +"za-CN", +"zgh", +"zgh-MA", +"zh", +"zh-Hans", +"zh-Hans-CN", +"zh-Hans-HK", +"zh-Hans-MO", +"zh-Hans-MY", +"zh-Hans-SG", +"zh-Hant", +"zh-Hant-HK", +"zh-Hant-MO", +"zh-Hant-MY", +"zh-Hant-TW", +"zu", +"zu-ZA", diff --git a/yql/essentials/parser/pg_catalog/ut/catalog_ut.cpp b/yql/essentials/parser/pg_catalog/ut/catalog_ut.cpp index 0dd8e623c14..1bc9f4bf3cd 100644 --- a/yql/essentials/parser/pg_catalog/ut/catalog_ut.cpp +++ b/yql/essentials/parser/pg_catalog/ut/catalog_ut.cpp @@ -207,3 +207,69 @@ Y_UNIT_TEST(TestOk) { UNIT_ASSERT_VALUES_EQUAL(LookupProc(procId).Name, "iso8859_1_to_utf8"); } } // Y_UNIT_TEST_SUITE(TConversionTests) + +Y_UNIT_TEST_SUITE(TCollationTests) { +Y_UNIT_TEST(TestMissing) { + UNIT_ASSERT(!HasCollation("_foo_bar_")); + UNIT_ASSERT_EXCEPTION(LookupCollation("_foo_bar_"), yexception); + UNIT_ASSERT_EXCEPTION(LookupCollation((ui32)123456), yexception); +} + +Y_UNIT_TEST(TestDefault) { + UNIT_ASSERT(HasCollation("default")); + const auto& ret = LookupCollation("default"); + UNIT_ASSERT_VALUES_EQUAL(ret.Oid, DefaultCollationOid); + UNIT_ASSERT_VALUES_EQUAL(ret.Provider, ECollationProvider::Default); + UNIT_ASSERT_VALUES_EQUAL(LookupCollation(ret.Oid).Name, "default"); +} + +Y_UNIT_TEST(TestC) { + const auto& ret = LookupCollation("C"); + UNIT_ASSERT_VALUES_EQUAL(ret.Oid, C_CollationOid); + UNIT_ASSERT_VALUES_EQUAL(ret.Provider, ECollationProvider::Libc); + UNIT_ASSERT_VALUES_EQUAL(ret.Collate, "C"); + UNIT_ASSERT_VALUES_EQUAL(ret.Ctype, "C"); +} + +Y_UNIT_TEST(TestPosix) { + const auto& ret = LookupCollation("POSIX"); + UNIT_ASSERT_VALUES_EQUAL(ret.Oid, PosixCollationOid); + UNIT_ASSERT_VALUES_EQUAL(ret.Provider, ECollationProvider::Libc); +} + +Y_UNIT_TEST(TestUnicodeIcu) { + const auto& ret = LookupCollation("unicode"); + UNIT_ASSERT_VALUES_EQUAL(ret.Provider, ECollationProvider::Icu); + UNIT_ASSERT_VALUES_EQUAL(ret.IcuLocale, "und"); +} + +Y_UNIT_TEST(TestEnum) { + size_t count = 0; + EnumCollation([&](ui32 oid, const TCollationDesc& desc) { + UNIT_ASSERT_VALUES_EQUAL(LookupCollation(desc.Name).Oid, oid); + ++count; + }); + + // 5 statically catalogued (pg_collation.dat) + generated ICU locale collations. + UNIT_ASSERT(count > 5); +} + +Y_UNIT_TEST(TestIcuLocale) { + const auto& ret = LookupCollation("de-DE-x-icu"); + UNIT_ASSERT_VALUES_EQUAL(ret.Provider, ECollationProvider::Icu); + UNIT_ASSERT_VALUES_EQUAL(ret.IcuLocale, "de-DE"); + UNIT_ASSERT(ret.Oid >= IcuCollationOidBase); + UNIT_ASSERT_VALUES_EQUAL(LookupCollation(ret.Oid).Name, "de-DE-x-icu"); + + // The same locale always resolves to the same oid. + UNIT_ASSERT_VALUES_EQUAL(ret.Oid, LookupCollation("de-DE-x-icu").Oid); + + // Different locales get different oids. + UNIT_ASSERT_UNEQUAL(ret.Oid, LookupCollation("en-US-x-icu").Oid); +} + +Y_UNIT_TEST(TestIcuLocaleMissing) { + UNIT_ASSERT(!HasCollation("xx-XX-x-icu")); + UNIT_ASSERT_EXCEPTION(LookupCollation("xx-XX-x-icu"), yexception); +} +} // Y_UNIT_TEST_SUITE(TCollationTests) diff --git a/yql/essentials/parser/pg_catalog/ya.make b/yql/essentials/parser/pg_catalog/ya.make index f7eecb3e7a8..ab657f751dc 100644 --- a/yql/essentials/parser/pg_catalog/ya.make +++ b/yql/essentials/parser/pg_catalog/ya.make @@ -13,6 +13,7 @@ RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_opclass.dat pg_opclass. RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_amproc.dat pg_amproc.dat) RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_amop.dat pg_amop.dat) RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_am.dat pg_am.dat) +RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_collation.dat pg_collation.dat) RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_conversion.dat pg_conversion.dat) RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_language.dat pg_language.dat) RESOURCE(../pg_wrapper/postgresql/src/backend/catalog/system_functions.sql system_functions.sql) @@ -31,6 +32,10 @@ PEERDIR( END() +RECURSE( + gen_icu_collations +) + RECURSE_FOR_TESTS( ut ) diff --git a/yql/essentials/parser/pg_wrapper/comp_factory.cpp b/yql/essentials/parser/pg_wrapper/comp_factory.cpp index 78b7e4c59fa..27601a078b3 100644 --- a/yql/essentials/parser/pg_wrapper/comp_factory.cpp +++ b/yql/essentials/parser/pg_wrapper/comp_factory.cpp @@ -463,6 +463,21 @@ public: }; ApplyFillers(AllPgExtensionFillers, Y_ARRAY_SIZE(AllPgExtensionFillers), PgExtensionFillers_); + } else if (Table_ == "pg_collation") { + static const std::pair<const char*, TPgCollationFiller> AllPgCollationFillers[] = { + {"oid", [](const NPg::TCollationDesc& desc) { return ScalarDatumToPod(ObjectIdGetDatum(desc.Oid)); }}, + {"collname", [](const NPg::TCollationDesc& desc) { return PointerDatumToPod((Datum)MakeFixedString(desc.Name, NAMEDATALEN)); }}, + {"collnamespace", [](const NPg::TCollationDesc&) { return ScalarDatumToPod(ObjectIdGetDatum(PG_CATALOG_NAMESPACE)); }}, + {"collowner", [](const NPg::TCollationDesc&) { return ScalarDatumToPod(ObjectIdGetDatum(1)); }}, + {"collprovider", [](const NPg::TCollationDesc& desc) { return ScalarDatumToPod(CharGetDatum((char)desc.Provider)); }}, + {"collisdeterministic", [](const NPg::TCollationDesc&) { return ScalarDatumToPod(BoolGetDatum(true)); }}, + {"collencoding", [](const NPg::TCollationDesc& desc) { return ScalarDatumToPod(Int32GetDatum(desc.Encoding)); }}, + {"collcollate", [](const NPg::TCollationDesc& desc) { return desc.Collate.empty() ? NUdf::TUnboxedValuePod() : PointerDatumToPod((Datum)MakeVar(desc.Collate)); }}, + {"collctype", [](const NPg::TCollationDesc& desc) { return desc.Ctype.empty() ? NUdf::TUnboxedValuePod() : PointerDatumToPod((Datum)MakeVar(desc.Ctype)); }}, + {"colliculocale", [](const NPg::TCollationDesc& desc) { return desc.IcuLocale.empty() ? NUdf::TUnboxedValuePod() : PointerDatumToPod((Datum)MakeVar(desc.IcuLocale)); }}, + }; + + ApplyFillers(AllPgCollationFillers, Y_ARRAY_SIZE(AllPgCollationFillers), PgCollationFillers_); } } else { if (Table_ == "tables") { @@ -864,6 +879,19 @@ public: sysFiller.Fill(items); rows.emplace_back(row); }); + } else if (Table_ == "pg_collation") { + NPg::EnumCollation([&](ui32, const NPg::TCollationDesc& desc) { + NUdf::TUnboxedValue* items; + auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgCollationFillers_.size(), items); + for (ui32 i = 0; i < PgCollationFillers_.size(); ++i) { + if (PgCollationFillers_[i]) { + items[i] = PgCollationFillers_[i](desc); + } + } + + sysFiller.Fill(items); + rows.emplace_back(row); + }); } } else { if (Table_ == "tables") { @@ -952,6 +980,9 @@ private: using TPgProcFiller = NUdf::TUnboxedValuePod(*)(const NPg::TProcDesc&); TVector<TPgProcFiller> PgProcFillers_; + using TPgCollationFiller = NUdf::TUnboxedValuePod(*)(const NPg::TCollationDesc&); + TVector<TPgCollationFiller> PgCollationFillers_; + using TPgAggregateFiller = NUdf::TUnboxedValuePod(*)(const NPg::TAggregateDesc&); TVector<TPgAggregateFiller> PgAggregateFillers_; @@ -1033,8 +1064,8 @@ private: }; struct TPgResolvedCallNodeState : public TPgResolvedCallState, public TComputationValue<TPgResolvedCallNodeState> { - TPgResolvedCallNodeState(TMemoryUsageInfo* memInfo, ui32 numArgs, const FmgrInfo* finfo) - : TPgResolvedCallState(numArgs, finfo) + TPgResolvedCallNodeState(TMemoryUsageInfo* memInfo, ui32 numArgs, const FmgrInfo* finfo, ui32 collationOid) + : TPgResolvedCallState(numArgs, finfo, collationOid) , TComputationValue(memInfo) {} }; @@ -1043,10 +1074,11 @@ class TPgResolvedCallNode : public TPgResolvedCall<UseContext>, public TMutableC typedef TMutableComputationNode<TPgResolvedCallNode<UseContext>> TBaseComputation; public: TPgResolvedCallNode(TComputationMutables& mutables, const std::string_view& name, ui32 id, - TComputationNodePtrVector&& argNodes, TVector<TType*>&& argTypes, TType* returnType) + TComputationNodePtrVector&& argNodes, TVector<TType*>&& argTypes, TType* returnType, ui32 collationOid) : TPgResolvedCall<UseContext>(name, id, std::move(argTypes), returnType) , TBaseComputation(mutables) , ArgNodes(std::move(argNodes)) + , CollationOid(collationOid) , StateIndex(mutables.CurValueIndex++) { Y_ENSURE(ArgNodes.size() == this->ArgDesc.size()); @@ -1068,13 +1100,14 @@ private: TPgResolvedCallNodeState& GetState(TComputationContext& compCtx) const { auto& result = compCtx.MutableValues[this->StateIndex]; if (!result.HasValue()) { - result = compCtx.HolderFactory.Create<TPgResolvedCallNodeState>(this->ArgNodes.size(), &this->FInfo); + result = compCtx.HolderFactory.Create<TPgResolvedCallNodeState>(this->ArgNodes.size(), &this->FInfo, CollationOid); } return *static_cast<TPgResolvedCallNodeState*>(result.AsBoxed().Get()); } const TComputationNodePtrVector ArgNodes; + const ui32 CollationOid; const ui32 StateIndex; }; @@ -1087,14 +1120,14 @@ private: public: TIterator(TMemoryUsageInfo* memInfo, const std::string_view& name, const TUnboxedValueVector& args, const TVector<NPg::TTypeDesc>& argDesc, const NPg::TTypeDesc& retTypeDesc, const NPg::TProcDesc& procDesc, - const FmgrInfo* fInfo, const TStructType* structType, const TVector<NPg::TTypeDesc>& structTypeDesc, const THolderFactory& holderFactory) + const FmgrInfo* fInfo, ui32 collationOid, const TStructType* structType, const TVector<NPg::TTypeDesc>& structTypeDesc, const THolderFactory& holderFactory) : TComputationValue<TIterator>(memInfo) , Name(name) , Args(args) , ArgDesc(argDesc) , RetTypeDesc(retTypeDesc) , ProcDesc(procDesc) - , CallInfo(argDesc.size(), fInfo) + , CallInfo(argDesc.size(), fInfo, collationOid) , StructType(structType) , StructTypeDesc(structTypeDesc) , HolderFactory(holderFactory) @@ -1302,7 +1335,7 @@ private: TListValue(TMemoryUsageInfo* memInfo, TComputationContext& compCtx, const std::string_view& name, TUnboxedValueVector&& args, const TVector<NPg::TTypeDesc>& argDesc, - const NPg::TTypeDesc& retTypeDesc, const NPg::TProcDesc& procDesc, const FmgrInfo* fInfo, + const NPg::TTypeDesc& retTypeDesc, const NPg::TProcDesc& procDesc, const FmgrInfo* fInfo, ui32 collationOid, const TStructType* structType, const TVector<NPg::TTypeDesc>& structTypeDesc, const THolderFactory& /*holderFactory*/) : TCustomListValue(memInfo) , CompCtx(compCtx) @@ -1312,6 +1345,7 @@ private: , RetTypeDesc(retTypeDesc) , ProcDesc(procDesc) , FInfo(fInfo) + , CollationOid(collationOid) , StructType(structType) , StructTypeDesc(structTypeDesc) { @@ -1319,7 +1353,7 @@ private: private: NUdf::TUnboxedValue GetListIterator() const final { - return CompCtx.HolderFactory.Create<TIterator>(Name, Args, ArgDesc, RetTypeDesc, ProcDesc, FInfo, StructType, StructTypeDesc, CompCtx.HolderFactory); + return CompCtx.HolderFactory.Create<TIterator>(Name, Args, ArgDesc, RetTypeDesc, ProcDesc, FInfo, CollationOid, StructType, StructTypeDesc, CompCtx.HolderFactory); } TComputationContext& CompCtx; @@ -1329,16 +1363,18 @@ private: const NPg::TTypeDesc& RetTypeDesc; const NPg::TProcDesc& ProcDesc; const FmgrInfo* FInfo; + const ui32 CollationOid; const TStructType* StructType; const TVector<NPg::TTypeDesc>& StructTypeDesc; }; public: TPgResolvedMultiCall(TComputationMutables& mutables, const std::string_view& name, ui32 id, - TComputationNodePtrVector&& argNodes, TVector<TType*>&& argTypes, TType* returnType, const TStructType* structType) + TComputationNodePtrVector&& argNodes, TVector<TType*>&& argTypes, TType* returnType, const TStructType* structType, ui32 collationOid) : TPgResolvedCallBase(name, id, std::move(argTypes), returnType, true, structType) , TBaseComputation(mutables) , ArgNodes(std::move(argNodes)) + , CollationOid(collationOid) , StructType(structType) { Y_ENSURE(ArgNodes.size() == ArgDesc.size()); @@ -1352,7 +1388,7 @@ public: args.push_back(value); } - return compCtx.HolderFactory.Create<TListValue>(compCtx, Name, std::move(args), ArgDesc, RetTypeDesc, ProcDesc, &FInfo, StructType, StructTypeDesc, compCtx.HolderFactory); + return compCtx.HolderFactory.Create<TListValue>(compCtx, Name, std::move(args), ArgDesc, RetTypeDesc, ProcDesc, &FInfo, CollationOid, StructType, StructTypeDesc, compCtx.HolderFactory); } private: @@ -1362,6 +1398,7 @@ private: } } const TComputationNodePtrVector ArgNodes; + const ui32 CollationOid; const TStructType* StructType; }; @@ -2757,7 +2794,7 @@ std::shared_ptr<arrow::compute::ScalarKernel> MakeToPgKernel(TType* inputType, T return kernel; } -std::shared_ptr<arrow::compute::ScalarKernel> MakePgKernel(TVector<TType*> argTypes, TType* resultType, TExecFunc execFunc, ui32 procId) { +std::shared_ptr<arrow::compute::ScalarKernel> MakePgKernel(TVector<TType*> argTypes, TType* resultType, TExecFunc execFunc, ui32 procId, ui32 collationOid) { std::shared_ptr<arrow::DataType> returnArrowType; MKQL_ENSURE(ConvertArrowType(AS_TYPE(TBlockType, resultType)->GetItemType(), returnArrowType), "Unsupported arrow type"); auto kernel = std::make_shared<arrow::compute::ScalarKernel>(ConvertToInputTypes(argTypes), ConvertToOutputType(resultType), @@ -2778,14 +2815,14 @@ std::shared_ptr<arrow::compute::ScalarKernel> MakePgKernel(TVector<TType*> argTy } kernel->null_handling = arrow::compute::NullHandling::COMPUTED_NO_PREALLOCATE; - kernel->init = [procId, pgArgTypes](arrow::compute::KernelContext*, const arrow::compute::KernelInitArgs&) { + kernel->init = [procId, pgArgTypes, collationOid](arrow::compute::KernelContext*, const arrow::compute::KernelInitArgs&) { auto state = std::make_unique<TPgKernelState>(); Zero(state->flinfo); GetPgFuncAddr(procId, state->flinfo); YQL_ENSURE(state->flinfo.fn_addr); state->resultinfo = nullptr; state->context = nullptr; - state->fncollation = DEFAULT_COLLATION_OID; + state->fncollation = collationOid ? collationOid : DEFAULT_COLLATION_OID; const auto& procDesc = NPg::LookupProc(procId); const auto& retTypeDesc = NPg::LookupType(procDesc.ResultType); state->Name = procDesc.Name; @@ -2855,7 +2892,8 @@ TComputationNodeFactory GetPgFactory() { return new TPgToRecord(ctx.Mutables, input, structType, std::move(members)); } - if (name == "PgResolvedCall") { + if (name == "PgResolvedCall" || name == "PgResolvedCall2") { + const bool hasCollation = (name == "PgResolvedCall2"); const auto useContextData = AS_VALUE(TDataLiteral, callable.GetInput(0)); const auto rangeFunctionData = AS_VALUE(TDataLiteral, callable.GetInput(1)); const auto nameData = AS_VALUE(TDataLiteral, callable.GetInput(2)); @@ -2864,9 +2902,18 @@ TComputationNodeFactory GetPgFactory() { auto rangeFunction = rangeFunctionData->AsValue().Get<bool>(); auto name = nameData->AsValue().AsStringRef(); auto id = idData->AsValue().Get<ui32>(); + + ui32 collationOid = 0; + ui32 argsStart = 4; + if (hasCollation) { + const auto collationOidData = AS_VALUE(TDataLiteral, callable.GetInput(4)); + collationOid = collationOidData->AsValue().Get<ui32>(); + argsStart = 5; + } + TComputationNodePtrVector argNodes; TVector<TType*> argTypes; - for (ui32 i = 4; i < callable.GetInputsCount(); ++i) { + for (ui32 i = argsStart; i < callable.GetInputsCount(); ++i) { argNodes.emplace_back(LocateNode(ctx.NodeLocator, callable, i)); argTypes.emplace_back(callable.GetInput(i).GetStaticType()); } @@ -2883,25 +2930,35 @@ TComputationNodeFactory GetPgFactory() { if (isList) { YQL_ENSURE(!useContext); - return new TPgResolvedMultiCall(ctx.Mutables, name, id, std::move(argNodes), std::move(argTypes), itemType, structType); + return new TPgResolvedMultiCall(ctx.Mutables, name, id, std::move(argNodes), std::move(argTypes), itemType, structType, collationOid); } else { YQL_ENSURE(!structType); if (useContext) { - return new TPgResolvedCallNode<true>(ctx.Mutables, name, id, std::move(argNodes), std::move(argTypes), returnType); + return new TPgResolvedCallNode<true>(ctx.Mutables, name, id, std::move(argNodes), std::move(argTypes), returnType, collationOid); } else { - return new TPgResolvedCallNode<false>(ctx.Mutables, name, id, std::move(argNodes), std::move(argTypes), returnType); + return new TPgResolvedCallNode<false>(ctx.Mutables, name, id, std::move(argNodes), std::move(argTypes), returnType, collationOid); } } } - if (name == "BlockPgResolvedCall") { + if (name == "BlockPgResolvedCall" || name == "BlockPgResolvedCall2") { + const bool hasCollation = (name == "BlockPgResolvedCall2"); const auto nameData = AS_VALUE(TDataLiteral, callable.GetInput(0)); const auto idData = AS_VALUE(TDataLiteral, callable.GetInput(1)); auto name = nameData->AsValue().AsStringRef(); auto id = idData->AsValue().Get<ui32>(); + + ui32 collationOid = 0; + ui32 argsStart = 2; + if (hasCollation) { + const auto collationOidData = AS_VALUE(TDataLiteral, callable.GetInput(2)); + collationOid = collationOidData->AsValue().Get<ui32>(); + argsStart = 3; + } + TComputationNodePtrVector argNodes; TVector<TType*> argTypes; - for (ui32 i = 2; i < callable.GetInputsCount(); ++i) { + for (ui32 i = argsStart; i < callable.GetInputsCount(); ++i) { argNodes.emplace_back(LocateNode(ctx.NodeLocator, callable, i)); argTypes.emplace_back(callable.GetInput(i).GetStaticType()); } @@ -2909,7 +2966,7 @@ TComputationNodeFactory GetPgFactory() { auto returnType = callable.GetType()->GetReturnType(); auto execFunc = FindExec(id); YQL_ENSURE(execFunc); - auto kernel = MakePgKernel(argTypes, returnType, execFunc, id); + auto kernel = MakePgKernel(argTypes, returnType, execFunc, id, collationOid); return new TBlockFuncNode(ctx.Mutables, ctx.RuntimeSettings->DatumValidation.Get(), callable.GetType()->GetName(), std::move(argNodes), argTypes, returnType, *kernel, kernel); } diff --git a/yql/essentials/parser/pg_wrapper/comp_factory_utils.cpp b/yql/essentials/parser/pg_wrapper/comp_factory_utils.cpp index 06b75438043..309e03621ea 100644 --- a/yql/essentials/parser/pg_wrapper/comp_factory_utils.cpp +++ b/yql/essentials/parser/pg_wrapper/comp_factory_utils.cpp @@ -169,7 +169,7 @@ NUdf::TUnboxedValue TPgConst::ExtractConst(i32 typeMod) const { return AnyDatumToPod(ret, TypeDesc.PassByValue); } -TFunctionCallInfo::TFunctionCallInfo(ui32 numArgs, const FmgrInfo* finfo) +TFunctionCallInfo::TFunctionCallInfo(ui32 numArgs, const FmgrInfo* finfo, ui32 collationOid) : NumArgs(numArgs) , CopyFmgrInfo(*finfo) { @@ -183,7 +183,7 @@ TFunctionCallInfo::TFunctionCallInfo(ui32 numArgs, const FmgrInfo* finfo) Zero(callInfo); callInfo.flinfo = &CopyFmgrInfo; // client may mutate fn_extra callInfo.nargs = NumArgs; - callInfo.fncollation = DEFAULT_COLLATION_OID; + callInfo.fncollation = collationOid ? collationOid : DEFAULT_COLLATION_OID; } FunctionCallInfoBaseData& TFunctionCallInfo::Ref() { @@ -197,8 +197,8 @@ TFunctionCallInfo::~TFunctionCallInfo() { } } -TPgResolvedCallState::TPgResolvedCallState(ui32 numArgs, const FmgrInfo* finfo) - : CallInfo(numArgs, finfo) +TPgResolvedCallState::TPgResolvedCallState(ui32 numArgs, const FmgrInfo* finfo, ui32 collationOid) + : CallInfo(numArgs, finfo, collationOid) , Args(numArgs) { } @@ -248,7 +248,7 @@ NUdf::TUnboxedValue TPgResolvedCall<UseContext>::CallFunction(TPgResolvedCallSta template <bool UseContext> THolder<TPgResolvedCallState> TPgResolvedCall<UseContext>::CreateState() const { - return MakeHolder<TPgResolvedCallState>(GetArgCount(), &FInfo); + return MakeHolder<TPgResolvedCallState>(GetArgCount(), &FInfo, NPg::InvalidCollationOid); } TPgCast::TPgCast(ui32 sourceId, ui32 targetId, bool hasTypeMod) @@ -496,7 +496,7 @@ template class TPgResolvedCall<false>; TPgResolvedCallWithCastState::TPgResolvedCallWithCastState( const TPgResolvedCall<false>& caller, const TVector<TPgCast>& casters) - : CallState(caller.GetArgCount(), &caller.GetFInfo()) + : CallState(caller.GetArgCount(), &caller.GetFInfo(), NPg::InvalidCollationOid) { CastStates.reserve(casters.size()); for (const auto& caster : casters) { diff --git a/yql/essentials/parser/pg_wrapper/comp_factory_utils.h b/yql/essentials/parser/pg_wrapper/comp_factory_utils.h index f31ebedfa9f..804ea63ab2a 100644 --- a/yql/essentials/parser/pg_wrapper/comp_factory_utils.h +++ b/yql/essentials/parser/pg_wrapper/comp_factory_utils.h @@ -103,7 +103,7 @@ protected: class TFunctionCallInfo { public: - TFunctionCallInfo(ui32 numArgs, const FmgrInfo* finfo); + TFunctionCallInfo(ui32 numArgs, const FmgrInfo* finfo, ui32 collationOid); ~TFunctionCallInfo(); TFunctionCallInfo(const TFunctionCallInfo&) = delete; @@ -119,7 +119,7 @@ private: }; struct TPgResolvedCallState { - TPgResolvedCallState(ui32 numArgs, const FmgrInfo* finfo); + TPgResolvedCallState(ui32 numArgs, const FmgrInfo* finfo, ui32 collationOid); TFunctionCallInfo CallInfo; NKikimr::NMiniKQL::TUnboxedValueVector Args; @@ -205,8 +205,8 @@ extern template class TPgResolvedCall<false>; struct TPgCastState { TPgCastState(const FmgrInfo* finfo1, const FmgrInfo* finfo2) - : CallInfo1(3, finfo1) - , CallInfo2(1, finfo2) + : CallInfo1(3, finfo1, NPg::InvalidCollationOid) + , CallInfo2(1, finfo2, NPg::InvalidCollationOid) { } diff --git a/yql/essentials/parser/pg_wrapper/pg_ops.cpp b/yql/essentials/parser/pg_wrapper/pg_ops.cpp index af7205eeb32..b9133398c22 100644 --- a/yql/essentials/parser/pg_wrapper/pg_ops.cpp +++ b/yql/essentials/parser/pg_wrapper/pg_ops.cpp @@ -52,8 +52,8 @@ public: : ZeroInterval_(zeroInterval) , EqCall_(eqCall) , GtCall_(gtCall) - , EqState_(MakeHolder<TPgResolvedCallState>(eqCall.GetArgCount(), &eqCall.GetFInfo())) - , GtState_(MakeHolder<TPgResolvedCallState>(gtCall.GetArgCount(), >Call.GetFInfo())) + , EqState_(MakeHolder<TPgResolvedCallState>(eqCall.GetArgCount(), &eqCall.GetFInfo(), NPg::InvalidCollationOid)) + , GtState_(MakeHolder<TPgResolvedCallState>(gtCall.GetArgCount(), >Call.GetFInfo(), NPg::InvalidCollationOid)) { } diff --git a/yql/essentials/parser/pg_wrapper/syscache.cpp b/yql/essentials/parser/pg_wrapper/syscache.cpp index 6e476f45108..7e95e9e3dfe 100644 --- a/yql/essentials/parser/pg_wrapper/syscache.cpp +++ b/yql/essentials/parser/pg_wrapper/syscache.cpp @@ -12,6 +12,7 @@ extern "C" { #include "catalog/pg_namespace.h" #include "catalog/pg_type.h" #include "catalog/pg_type_d.h" +#include "catalog/pg_collation.h" #include "catalog/pg_authid.h" #include "access/htup_details.h" #include "utils/fmgroids.h" @@ -129,7 +130,7 @@ bool ByNameProcEquals3(const THeapTupleKey& key1, const THeapTupleKey& key2) { } struct TSysCacheItem { - TSysCacheItem(THeapTupleHasher hasher, THeapTupleEquals equals, TupleDesc desc, ui32 numKeys = 1, + TSysCacheItem(THeapTupleHasher hasher, THeapTupleEquals equals, TupleDesc desc, ui32 numKeys = 1, THeapTupleHasher hasherRange1 = {}, THeapTupleEquals equalsRange1 = {}) : NumKeys(numKeys) , LookupMap(0, hasher, equals) @@ -199,7 +200,7 @@ struct TSysCache { static TSysCache& MutableInstance() { return *Singleton<TSysCache>(); - } + } TSysCache() { @@ -219,6 +220,7 @@ struct TSysCache { InitializeProcs(); InitializeTypes(); + InitializeCollations(); InitializeDatabase(); InitializeAuthId(); InitializeNameAndOidNamespaces(); @@ -477,7 +479,7 @@ struct TSysCache { Y_ENSURE(row->typalign == desc.TypeAlign); Y_ENSURE(row->typstorage == storage); - auto key1 = THeapTupleKey(oid, 0, 0, 0); + auto key1 = THeapTupleKey(oid, 0, 0, 0); lookupMap1.emplace(key1, h); auto key2 = THeapTupleKey((Datum)desc.Name.c_str(), PG_CATALOG_NAMESPACE, 0, 0); lookupMap2.emplace(key2, h); @@ -485,6 +487,59 @@ struct TSysCache { } + void InitializeCollations() { + TupleDesc tupleDesc = CreateTemplateTupleDesc(Natts_pg_collation); + FillAttr(tupleDesc, Anum_pg_collation_oid, OIDOID); + FillAttr(tupleDesc, Anum_pg_collation_collname, NAMEOID); + FillAttr(tupleDesc, Anum_pg_collation_collnamespace, OIDOID); + FillAttr(tupleDesc, Anum_pg_collation_collowner, OIDOID); + FillAttr(tupleDesc, Anum_pg_collation_collprovider, CHAROID); + FillAttr(tupleDesc, Anum_pg_collation_collisdeterministic, BOOLOID); + FillAttr(tupleDesc, Anum_pg_collation_collencoding, INT4OID); + FillAttr(tupleDesc, Anum_pg_collation_collcollate, TEXTOID); + FillAttr(tupleDesc, Anum_pg_collation_collctype, TEXTOID); + FillAttr(tupleDesc, Anum_pg_collation_colliculocale, TEXTOID); + FillAttr(tupleDesc, Anum_pg_collation_collicurules, TEXTOID); + FillAttr(tupleDesc, Anum_pg_collation_collversion, TEXTOID); + + auto& cacheItem = Items[COLLOID] = std::make_unique<TSysCacheItem>(OidHasher1, OidEquals1, tupleDesc); + auto& lookupMap = cacheItem->LookupMap; + + NPg::EnumCollation([&](ui32 oid, const NPg::TCollationDesc& desc) { + Datum values[Natts_pg_collation]; + bool nulls[Natts_pg_collation]; + Zero(values); + std::fill_n(nulls, Natts_pg_collation, true); + std::fill_n(nulls, Anum_pg_collation_collencoding, false); // fixed part of Form_pg_collation + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_oid, oid); + auto name = MakeFixedString(desc.Name, NAMEDATALEN); + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_collname, (Datum)name); + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_collnamespace, PG_CATALOG_NAMESPACE); + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_collowner, 1); + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_collprovider, (char)desc.Provider); + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_collisdeterministic, true); + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_collencoding, desc.Encoding); + if (!desc.Collate.empty()) { + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_collcollate, (Datum)MakeVar(desc.Collate)); + } + if (!desc.Ctype.empty()) { + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_collctype, (Datum)MakeVar(desc.Ctype)); + } + if (!desc.IcuLocale.empty()) { + FillDatum(Natts_pg_collation, values, nulls, Anum_pg_collation_colliculocale, (Datum)MakeVar(desc.IcuLocale)); + } + + HeapTuple h = heap_form_tuple(tupleDesc, values, nulls); + auto row = (Form_pg_collation)GETSTRUCT(h); + Y_ENSURE(row->oid == oid); + Y_ENSURE(NameStr(row->collname) == desc.Name); + Y_ENSURE(row->collprovider == (char)desc.Provider); + + auto key = THeapTupleKey(oid, 0, 0, 0); + lookupMap.emplace(key, h); + }); + } + static HeapTuple MakePgDatabaseHeapTuple(ui32 oid, const char* name) { TupleDesc tupleDesc = CreateTemplateTupleDesc(Natts_pg_database); FillAttr(tupleDesc, Anum_pg_database_oid, OIDOID); diff --git a/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp b/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp index d3f2d071b18..21b1137e6ef 100644 --- a/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp +++ b/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp @@ -2819,14 +2819,17 @@ TMkqlCommonCallableCompiler::TShared::TShared() { } bool rangeFunction = false; + ui32 collationOid = 0; for (const auto& child : node.Child(2)->Children()) { if (child->Head().Content() == "range") { rangeFunction = true; + } else if (child->Head().Content() == "collation_oid") { + collationOid = FromString<ui32>(child->Tail().Content()); } } auto returnType = ctx.BuildType(node, *node.GetTypeAnn()); - return ctx.ProgramBuilder.PgResolvedCall(node.IsCallable("PgResolvedCallCtx"), name, id, args, returnType, rangeFunction); + return ctx.ProgramBuilder.PgResolvedCall(node.IsCallable("PgResolvedCallCtx"), name, id, args, returnType, rangeFunction, collationOid); }); AddCallable("PgResolvedOp", [](const TExprNode& node, TMkqlBuildContext& ctx) { @@ -2840,7 +2843,7 @@ TMkqlCommonCallableCompiler::TShared::TShared() { } auto returnType = ctx.BuildType(node, *node.GetTypeAnn()); - return ctx.ProgramBuilder.PgResolvedCall(/*useContext=*/false, procName, procId, args, returnType, /*rangeFunction=*/false); + return ctx.ProgramBuilder.PgResolvedCall(/*useContext=*/false, procName, procId, args, returnType, /*rangeFunction=*/false, NPg::InvalidCollationOid); }); AddCallable("BlockPgResolvedCall", [](const TExprNode& node, TMkqlBuildContext& ctx) { @@ -2852,8 +2855,15 @@ TMkqlCommonCallableCompiler::TShared::TShared() { args.push_back(MkqlBuildExpr(*node.Child(i), ctx)); } + ui32 collationOid = 0; + for (const auto& child : node.Child(2)->Children()) { + if (child->Head().Content() == "collation_oid") { + collationOid = FromString<ui32>(child->Tail().Content()); + } + } + auto returnType = ctx.BuildType(node, *node.GetTypeAnn()); - return ctx.ProgramBuilder.BlockPgResolvedCall(name, id, args, returnType); + return ctx.ProgramBuilder.BlockPgResolvedCall(name, id, args, returnType, collationOid); }); AddCallable("BlockPgResolvedOp", [](const TExprNode& node, TMkqlBuildContext& ctx) { @@ -2867,7 +2877,7 @@ TMkqlCommonCallableCompiler::TShared::TShared() { } auto returnType = ctx.BuildType(node, *node.GetTypeAnn()); - return ctx.ProgramBuilder.BlockPgResolvedCall(procName, procId, args, returnType); + return ctx.ProgramBuilder.BlockPgResolvedCall(procName, procId, args, returnType, NPg::InvalidCollationOid); }); AddCallable("PgCast", [](const TExprNode& node, TMkqlBuildContext& ctx) { diff --git a/yql/essentials/sql/pg/pg_sql.cpp b/yql/essentials/sql/pg/pg_sql.cpp index 6990cbfefa7..9ea2a53e936 100644 --- a/yql/essentials/sql/pg/pg_sql.cpp +++ b/yql/essentials/sql/pg/pg_sql.cpp @@ -3725,6 +3725,33 @@ public: return result; } + bool ExtractCollationName(const CollateClause* value, TString& name) { + auto len = ListLength(value->collname); + if (len == 0) { + AddError("CollateClause: missing collation name"); + return false; + } + + auto x = ListNodeNth(value->collname, len - 1); + if (NodeTag(x) != T_String) { + NodeNotImplemented(value, x); + return false; + } + + name = StrVal(x); + return true; + } + + TAstNode* ParseCollateClause(const CollateClause* value, const TExprSettings& settings) { + AT_LOCATION(value); + // Collation propagation through arbitrary expressions (COALESCE, CASE, plain + // projections, ...) is not implemented - only ParseFuncCall picks up an explicit + // COLLATE that directly wraps one of its arguments (see there). Elsewhere COLLATE + // only affects collation-sensitive semantics (comparison, case mapping), not the + // expression's runtime value, so we just parse through to the wrapped expression. + return ParseExpr(value->arg, settings); + } + TAstNode* ParseExpr(const Node* node, const TExprSettings& settings) { switch (NodeTag(node)) { case T_A_Const: { @@ -3742,6 +3769,9 @@ public: case T_TypeCast: { return ParseTypeCast(CAST_NODE(TypeCast, node), settings); } + case T_CollateClause: { + return ParseCollateClause(CAST_NODE(CollateClause, node), settings); + } case T_BoolExpr: { return ParseBoolExpr(CAST_NODE(BoolExpr, node), settings); } @@ -4267,6 +4297,32 @@ public: callSettings.push_back(QL(QA("range"))); } + if (!value->agg_star) { + TMaybe<TString> collation; + for (int i = 0; i < ListLength(value->args); ++i) { + auto x = ListNodeNth(value->args, i); + if (NodeTag(x) != T_CollateClause) { + continue; + } + + TString collationName; + if (!ExtractCollationName(CAST_NODE(CollateClause, x), collationName)) { + return nullptr; + } + + if (collation && *collation != collationName) { + AddError(TStringBuilder() << "FuncCall: conflicting explicit collations: " << *collation << " and " << collationName); + return nullptr; + } + + collation = collationName; + } + + if (collation) { + callSettings.push_back(QL(QA("collation"), QAX(*collation))); + } + } + args.push_back(QVL(callSettings.data(), callSettings.size())); if (value->agg_star) { if (name != "count") { diff --git a/yql/essentials/sql/v1/builtin.cpp b/yql/essentials/sql/v1/builtin.cpp index cb91f956f89..43eeb6af9b4 100644 --- a/yql/essentials/sql/v1/builtin.cpp +++ b/yql/essentials/sql/v1/builtin.cpp @@ -860,7 +860,35 @@ public: } Args_[0] = BuildQuotedAtom(Args_[0]->GetPos(), Args_[0]->GetLiteralValue()); - Args_.insert(Args_.begin() + 1, RangeFunction ? Q(Y(Q(Y(Q("range"))))) : Q(Y())); + + TNodePtr settingsList = Y(); + if (RangeFunction) { + settingsList = L(settingsList, Q(Y(Q("range")))); + } + + // Optional named argument: PgCall("upper", x, "de-DE-x-icu" as Collation) + for (size_t i = 1; i < Args_.size();) { + const auto& label = Args_[i]->GetLabel(); + if (label.empty()) { + ++i; + continue; + } + + if (label != "Collation") { + ctx.Error(Args_[i]->GetPos()) << "PgCall: unsupported named argument: " << label << "; expected: Collation"; + return false; + } + + auto arg = Args_[i]; + if (!arg->Init(ctx, src)) { + return false; + } + + settingsList = L(settingsList, Q(Y(Q("collation"), MakeAtomFromExpression(Pos_, ctx, arg).Build()))); + Args_.erase(Args_.begin() + i); + } + + Args_.insert(Args_.begin() + 1, Q(settingsList)); return TCallNode::DoInit(ctx, src); } @@ -4218,6 +4246,28 @@ TNodeResult BuildBuiltinFunc( return TNonNull(TNodePtr(new TInvalidBuiltin(pos, "DISTINCT can only be used in aggregation functions"))); } + if ((normalizedName == "pgcall" || normalizedName == "pgrangecall") && mustUseNamed && *mustUseNamed) { + // PgCall/PgRangeCall support one named argument (collation => ...); TYqlPgCall::DoInit + // finds it by label, so just flatten the (positional tuple, named struct) pair that + // named-arg calls get reshaped into back into a single labeled arg list. + *mustUseNamed = false; + YQL_ENSURE(args.size() == 2); + auto positional = args[0]->GetTupleNode(); + YQL_ENSURE(positional); + auto named = args[1]->GetStructNode(); + YQL_ENSURE(named); + + TVector<TNodePtr> flatArgs(positional->Elements()); + for (const auto& arg : named->GetExprs()) { + flatArgs.push_back(arg); + } + + const bool isRange = (normalizedName == "pgrangecall"); + return TNonNull(isRange + ? TNodePtr(new TYqlPgCall<true>(pos, flatArgs)) + : TNodePtr(new TYqlPgCall<false>(pos, flatArgs))); + } + auto builtinCallback = builtinFuncs.find(normalizedName); if (builtinCallback != builtinFuncs.end()) { const auto& funcInfo = builtinCallback->second; diff --git a/yql/essentials/tests/s-expressions/minirun/part4/canondata/result.json b/yql/essentials/tests/s-expressions/minirun/part4/canondata/result.json index df0cc6a07e6..60a9d0991af 100644 --- a/yql/essentials/tests/s-expressions/minirun/part4/canondata/result.json +++ b/yql/essentials/tests/s-expressions/minirun/part4/canondata/result.json @@ -251,6 +251,20 @@ "uri": "https://{canondata_backend}/1937424/d5a759bb3d448219023419bd81b51a83c51e7fb9/resource.tar.gz#test.test_InMem-Guess--Results_/results.txt" } ], + "test.test[InMem-LPartitionsByKeys-default.txt-Debug]": [ + { + "checksum": "c5970844f3cd290c9cd0f6da7c09a92e", + "size": 537, + "uri": "https://{canondata_backend}/1903885/04250d1ff8dfb75396270ac9221941404e83162d/resource.tar.gz#test.test_InMem-LPartitionsByKeys-default.txt-Debug_/opt.yql" + } + ], + "test.test[InMem-LPartitionsByKeys-default.txt-Results]": [ + { + "checksum": "3159d6b27773bf2c0b434a77a64b62b2", + "size": 337, + "uri": "https://{canondata_backend}/1903885/04250d1ff8dfb75396270ac9221941404e83162d/resource.tar.gz#test.test_InMem-LPartitionsByKeys-default.txt-Results_/results.txt" + } + ], "test.test[InMem-ListFromRangeForDates-default.txt-Debug]": [ { "checksum": "9eb52e1fd2ea6bda896e831fdf864f27", diff --git a/yql/essentials/tests/s-expressions/minirun/part9/canondata/result.json b/yql/essentials/tests/s-expressions/minirun/part9/canondata/result.json index d9ea295302a..2a3a50eef4e 100644 --- a/yql/essentials/tests/s-expressions/minirun/part9/canondata/result.json +++ b/yql/essentials/tests/s-expressions/minirun/part9/canondata/result.json @@ -209,6 +209,20 @@ "uri": "https://{canondata_backend}/1936842/97e0ab044a6a521639a649b6b781407443c8c0f8/resource.tar.gz#test.test_InMem-Iterator-default.txt-Results_/results.txt" } ], + "test.test[InMem-LPartitionsByKeysSorted-default.txt-Debug]": [ + { + "checksum": "344f7eae976694446bf07e0a90cd6f07", + "size": 566, + "uri": "https://{canondata_backend}/1814674/2df5cbfd5d5a594d9f476819b99ebb23ca848426/resource.tar.gz#test.test_InMem-LPartitionsByKeysSorted-default.txt-Debug_/opt.yql" + } + ], + "test.test[InMem-LPartitionsByKeysSorted-default.txt-Results]": [ + { + "checksum": "20b62fca5b0e72f0ca2df54d7d61a5e2", + "size": 337, + "uri": "https://{canondata_backend}/1814674/2df5cbfd5d5a594d9f476819b99ebb23ca848426/resource.tar.gz#test.test_InMem-LPartitionsByKeysSorted-default.txt-Results_/results.txt" + } + ], "test.test[InMem-ReplaceMember-default.txt-Debug]": [ { "checksum": "356532c5c41738b56f422acc79110cbc", diff --git a/yql/essentials/tests/s-expressions/suites/InMem/LPartitionsByKeys.yqls b/yql/essentials/tests/s-expressions/suites/InMem/LPartitionsByKeys.yqls new file mode 100644 index 00000000000..15e48a09766 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/InMem/LPartitionsByKeys.yqls @@ -0,0 +1,22 @@ +( +#comment +(let config (DataSource 'config)) + +(let res_sink (DataSink 'result)) +(let list (AsList +'((Uint32 '1) (String 'a)) +'((Uint32 '2) (String 'b)) +'((Uint32 '1) (String 'c)))) + +(let keyExtractor (lambda '(x) (Nth x '0))) +(let handler (lambda '(x) (Condense1 x + (lambda '(row) '((Nth row '0) (Concat (String '"values:") (ToString (Nth row '1))))) + (lambda '(row state) (AggrNotEquals (Nth row '0) (Nth state '0))) + (lambda '(row state) '((Nth state '0) (Concat (Concat (Nth state '1) (String '" ")) (Nth row '1)))) +))) +(let data (LPartitionsByKeys list keyExtractor (Void) (Void) handler)) + +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/InMem/LPartitionsByKeysSorted.yqls b/yql/essentials/tests/s-expressions/suites/InMem/LPartitionsByKeysSorted.yqls new file mode 100644 index 00000000000..3d20a56d06e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/InMem/LPartitionsByKeysSorted.yqls @@ -0,0 +1,23 @@ +( +#comment +(let config (DataSource 'config)) + +(let res_sink (DataSink 'result)) +(let list (AsList +'((Uint32 '1) (String 'a)) +'((Uint32 '2) (String 'b)) +'((Uint32 '1) (String 'c)))) + +(let keyExtractor (lambda '(x) (Nth x '0))) +(let subkeyExtractor (lambda '(x) (Nth x '1))) +(let handler (lambda '(x) (Condense1 x + (lambda '(row) '((Nth row '0) (Concat (String '"values:") (ToString (Nth row '1))))) + (lambda '(row state) (AggrNotEquals (Nth row '0) (Nth state '0))) + (lambda '(row state) '((Nth state '0) (Concat (Concat (Nth state '1) (String '" ")) (Nth row '1)))) +))) +(let data (LPartitionsByKeys list keyExtractor (Bool 'false) subkeyExtractor handler)) + +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/sql/minirun/part0/canondata/result.json b/yql/essentials/tests/sql/minirun/part0/canondata/result.json index aaf0e9ac659..90221a7ca8b 100644 --- a/yql/essentials/tests/sql/minirun/part0/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part0/canondata/result.json @@ -951,6 +951,20 @@ "uri": "https://{canondata_backend}/1937424/d02bb271c9ab67557c669f52bc80a0a698a8a91e/resource.tar.gz#test.test_pg-YQL_20895_having_depth_3-default.txt-Results_/results.txt" } ], + "test.test[pg-collate_icu-default.txt-Debug]": [ + { + "checksum": "1652b1b4cff4644c499e6dc69fee811c", + "size": 765, + "uri": "https://{canondata_backend}/1871102/3abca1c03e1dffe75d1d9477b6aa625e8d988fcc/resource.tar.gz#test.test_pg-collate_icu-default.txt-Debug_/opt.yql" + } + ], + "test.test[pg-collate_icu-default.txt-Results]": [ + { + "checksum": "b0b6a05f71c0ea34ff54854f875e6e0f", + "size": 1731, + "uri": "https://{canondata_backend}/1871102/3abca1c03e1dffe75d1d9477b6aa625e8d988fcc/resource.tar.gz#test.test_pg-collate_icu-default.txt-Results_/results.txt" + } + ], "test.test[pg-distinct_from_pg_null-default.txt-Debug]": [ { "checksum": "28eb3cf7598e0504999612fc78525b89", diff --git a/yql/essentials/tests/sql/minirun/part10/canondata/result.json b/yql/essentials/tests/sql/minirun/part10/canondata/result.json index eebfb9f633a..c8688325f56 100644 --- a/yql/essentials/tests/sql/minirun/part10/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part10/canondata/result.json @@ -349,6 +349,27 @@ "uri": "https://{canondata_backend}/1777230/52d24600ceada6f625f4161edfba1861d2eed32d/resource.tar.gz#test.test_blocks-pg_call-default.txt-Results_/results.txt" } ], + "test.test[blocks-pg_call_collation-default.txt-Debug]": [ + { + "checksum": "06af59fe488e80891c72af615fab34d8", + "size": 567, + "uri": "https://{canondata_backend}/1936842/42d71fee557fda1f6e31e9529eb8834c94fcba49/resource.tar.gz#test.test_blocks-pg_call_collation-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-pg_call_collation-default.txt-Peephole]": [ + { + "checksum": "451da6daf3dc9800234a3a74b45a874b", + "size": 732, + "uri": "https://{canondata_backend}/1936842/42d71fee557fda1f6e31e9529eb8834c94fcba49/resource.tar.gz#test.test_blocks-pg_call_collation-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-pg_call_collation-default.txt-Results]": [ + { + "checksum": "15b1272df7ce88bf65e3f49c899e5b02", + "size": 1182, + "uri": "https://{canondata_backend}/1936842/42d71fee557fda1f6e31e9529eb8834c94fcba49/resource.tar.gz#test.test_blocks-pg_call_collation-default.txt-Results_/results.txt" + } + ], "test.test[blocks-take_skip-default.txt-Debug]": [ { "checksum": "14f19770f1c340e0707f94dd4c9351f4", diff --git a/yql/essentials/tests/sql/minirun/part2/canondata/result.json b/yql/essentials/tests/sql/minirun/part2/canondata/result.json index 859cdcf18ee..46167e94552 100644 --- a/yql/essentials/tests/sql/minirun/part2/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part2/canondata/result.json @@ -966,6 +966,20 @@ "uri": "https://{canondata_backend}/1773845/5e187b5c8712eed0b26575ad1c84f3158af43d1d/resource.tar.gz#test.test_pg-with_rec_distinct-default.txt-Results_/results.txt" } ], + "test.test[pg_catalog-pg_collation-default.txt-Debug]": [ + { + "checksum": "8a748166f29bb30eab0aadaa56beb680", + "size": 1174, + "uri": "https://{canondata_backend}/212715/18867d69579bd1c3245bfeba1a9a3d827bfd47f3/resource.tar.gz#test.test_pg_catalog-pg_collation-default.txt-Debug_/opt.yql" + } + ], + "test.test[pg_catalog-pg_collation-default.txt-Results]": [ + { + "checksum": "520bb16015bf0ec808f3934ca9a32551", + "size": 733, + "uri": "https://{canondata_backend}/212715/18867d69579bd1c3245bfeba1a9a3d827bfd47f3/resource.tar.gz#test.test_pg_catalog-pg_collation-default.txt-Results_/results.txt" + } + ], "test.test[pg_catalog-pg_namespace_pg_syntax-default.txt-Debug]": [ { "checksum": "227d6e0c6d613058c88f34d5bb633d20", diff --git a/yql/essentials/tests/sql/minirun/part3/canondata/result.json b/yql/essentials/tests/sql/minirun/part3/canondata/result.json index 65982115910..f987dc88b49 100644 --- a/yql/essentials/tests/sql/minirun/part3/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part3/canondata/result.json @@ -744,6 +744,20 @@ "uri": "https://{canondata_backend}/1781765/e6df7efbe28b8ae403e3d9815e1579ecc8bb5445/resource.tar.gz#test.test_pg-aggregate_scalar_minus_zero-default.txt-Results_/results.txt" } ], + "test.test[pg-collate_std-default.txt-Debug]": [ + { + "checksum": "29b4db3f942db540ae3c350b5fffa10b", + "size": 1426, + "uri": "https://{canondata_backend}/1942173/f178108c603ab2223424f5ca3701e7ac0acc65cb/resource.tar.gz#test.test_pg-collate_std-default.txt-Debug_/opt.yql" + } + ], + "test.test[pg-collate_std-default.txt-Results]": [ + { + "checksum": "6756fa684ac9c665cdc9434bfb356e17", + "size": 3722, + "uri": "https://{canondata_backend}/1942173/f178108c603ab2223424f5ca3701e7ac0acc65cb/resource.tar.gz#test.test_pg-collate_std-default.txt-Results_/results.txt" + } + ], "test.test[pg-compare-default.txt-Debug]": [ { "checksum": "d25a02298d33a67403f42601ea688cbe", diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index 657f918cfad..b52bb5fe2f7 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -2526,6 +2526,13 @@ "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_blocks-pg_call_/sql.yql" } ], + "test_sql2yql.test[blocks-pg_call_collation]": [ + { + "checksum": "fc5bb37bca3be522317e2108bc7a25d7", + "size": 1384, + "uri": "https://{canondata_backend}/1946324/4da69cf65420cbbd1b36d493c4602a8dfff3583d/resource.tar.gz#test_sql2yql.test_blocks-pg_call_collation_/sql.yql" + } + ], "test_sql2yql.test[blocks-pg_op]": [ { "checksum": "6fde398b5c3fed0e3fa982b753fe6018", @@ -6929,6 +6936,20 @@ "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_pg-coalesce_/sql.yql" } ], + "test_sql2yql.test[pg-collate_icu]": [ + { + "checksum": "229206950b633a4de2d73820c449ace8", + "size": 851, + "uri": "https://{canondata_backend}/1809005/58875b1649369677cd109dd8100bb7a1c63ac607/resource.tar.gz#test_sql2yql.test_pg-collate_icu_/sql.yql" + } + ], + "test_sql2yql.test[pg-collate_std]": [ + { + "checksum": "9f81b5324fbfe2e1fc479ef1d64a87d3", + "size": 1647, + "uri": "https://{canondata_backend}/1809005/58875b1649369677cd109dd8100bb7a1c63ac607/resource.tar.gz#test_sql2yql.test_pg-collate_std_/sql.yql" + } + ], "test_sql2yql.test[pg-compare]": [ { "checksum": "a25ef2b8abd25e4b952c428bcd821b03", @@ -8819,6 +8840,13 @@ "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_class_/sql.yql" } ], + "test_sql2yql.test[pg_catalog-pg_collation]": [ + { + "checksum": "57863f2853565b02219bc45355998aad", + "size": 659, + "uri": "https://{canondata_backend}/1781765/b51b8ce704a88a316462b50b2ccb16b8e6aff2d3/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_collation_/sql.yql" + } + ], "test_sql2yql.test[pg_catalog-pg_database]": [ { "checksum": "277583c5b369c99209d290134ee3c60f", @@ -14425,6 +14453,11 @@ "uri": "file://test_sql_format.test_blocks-pg_call_/formatted.sql" } ], + "test_sql_format.test[blocks-pg_call_collation]": [ + { + "uri": "file://test_sql_format.test_blocks-pg_call_collation_/formatted.sql" + } + ], "test_sql_format.test[blocks-pg_op]": [ { "uri": "file://test_sql_format.test_blocks-pg_op_/formatted.sql" @@ -20407,6 +20440,13 @@ "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql_negative.test_match_recognize-measures_aggr_factory-_/err_file.out" } ], + "test_sql_negative.test[pg-fail_collate_mixed-]": [ + { + "checksum": "27098fda5bb566087d95068537a89604", + "size": 146, + "uri": "https://{canondata_backend}/1925821/a0711c4178a5e0568199e19991c318a3fb84b630/resource.tar.gz#test_sql_negative.test_pg-fail_collate_mixed-_/err_file.out" + } + ], "test_sql_negative.test[pragma-classic_division_sqlfail-]": [ { "checksum": "f1b771232b48beee3305efe6ac52b431", diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-pg_call_collation_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-pg_call_collation_/formatted.sql new file mode 100644 index 00000000000..e5b66e0f373 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-pg_call_collation_/formatted.sql @@ -0,0 +1,11 @@ +$data = [ + <|x: "straße"p|>, + <|x: "foo"p|>, +]; + +SELECT + PgCall('upper', x, 'de-DE-x-icu' AS Collation), + PgCall('upper', x) +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/suites/blocks/pg_call_collation.yql b/yql/essentials/tests/sql/suites/blocks/pg_call_collation.yql new file mode 100644 index 00000000000..028dcb31263 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/pg_call_collation.yql @@ -0,0 +1,6 @@ +$data = [ + <|x:"straße"p|>, + <|x:"foo"p|>, +]; + +select PgCall("upper", x, "de-DE-x-icu" as Collation), PgCall("upper", x) from as_table($data); diff --git a/yql/essentials/tests/sql/suites/pg/collate_icu.sql b/yql/essentials/tests/sql/suites/pg/collate_icu.sql new file mode 100644 index 00000000000..5939a2a52c3 --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg/collate_icu.sql @@ -0,0 +1,2 @@ +--!syntax_pg +select upper('straße' COLLATE "de-DE-x-icu"), upper('straße'), upper('i' COLLATE "tr-TR-x-icu"), upper('i'); diff --git a/yql/essentials/tests/sql/suites/pg/collate_std.sql b/yql/essentials/tests/sql/suites/pg/collate_std.sql new file mode 100644 index 00000000000..aaa80ce8759 --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg/collate_std.sql @@ -0,0 +1,12 @@ +--!syntax_pg +select + upper('straße' COLLATE "default"), + upper('straße' COLLATE "C"), + upper('straße' COLLATE "POSIX"), + upper('straße' COLLATE "ucs_basic"), + upper('straße' COLLATE "unicode"), + upper('i' COLLATE "default"), + upper('i' COLLATE "C"), + upper('i' COLLATE "POSIX"), + upper('i' COLLATE "ucs_basic"), + upper('i' COLLATE "unicode"); diff --git a/yql/essentials/tests/sql/suites/pg/fail_collate_mixed.cfg b/yql/essentials/tests/sql/suites/pg/fail_collate_mixed.cfg new file mode 100644 index 00000000000..b106572754f --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg/fail_collate_mixed.cfg @@ -0,0 +1 @@ +xsqlfail diff --git a/yql/essentials/tests/sql/suites/pg/fail_collate_mixed.sql b/yql/essentials/tests/sql/suites/pg/fail_collate_mixed.sql new file mode 100644 index 00000000000..90a4cc1b1b5 --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg/fail_collate_mixed.sql @@ -0,0 +1,4 @@ +--!syntax_pg +/* custom error:FuncCall: conflicting explicit collations: default and C*/ +select + substr('foo' COLLATE "default", 1 COLLATE "C") diff --git a/yql/essentials/tests/sql/suites/pg_catalog/pg_collation.sql b/yql/essentials/tests/sql/suites/pg_catalog/pg_collation.sql new file mode 100644 index 00000000000..066a6da9f00 --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg_catalog/pg_collation.sql @@ -0,0 +1,2 @@ +--!syntax_pg +select count(*) from pg_catalog.pg_collation; diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/ut/ya.make b/yt/yql/providers/yt/fmr/coordinator/impl/ut/ya.make index 8ca30f57288..6deb56f58d7 100644 --- a/yt/yql/providers/yt/fmr/coordinator/impl/ut/ya.make +++ b/yt/yql/providers/yt/fmr/coordinator/impl/ut/ya.make @@ -23,7 +23,6 @@ PEERDIR( yt/yql/providers/yt/fmr/test_tools/mock_time_provider yt/yql/providers/yt/fmr/test_tools/sorted_partitioner yt/yql/providers/yt/fmr/test_tools/table_data_service - yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl yt/yql/providers/yt/gateway/fmr yt/yql/providers/yt/gateway/file yt/yql/providers/yt/codec/codegen/llvm16 diff --git a/yt/yql/providers/yt/fmr/coordinator/operation_manager/impl/map_reduce/yql_yt_map_reduce_stage_operation_manager.cpp b/yt/yql/providers/yt/fmr/coordinator/operation_manager/impl/map_reduce/yql_yt_map_reduce_stage_operation_manager.cpp index b919e13dea8..bb0c896898a 100644 --- a/yt/yql/providers/yt/fmr/coordinator/operation_manager/impl/map_reduce/yql_yt_map_reduce_stage_operation_manager.cpp +++ b/yt/yql/providers/yt/fmr/coordinator/operation_manager/impl/map_reduce/yql_yt_map_reduce_stage_operation_manager.cpp @@ -255,6 +255,9 @@ private: reduceTaskParams.Input = taskInput; reduceTaskParams.SerializedReduceJobState = operationParams.SerializedReduceJobState; reduceTaskParams.ReduceOperationSpec = operationParams.ReduceOperationSpec; + // The map stage above always shuffles via a synthetic _yql_key_hash prefix (see + // GenerateIntermediateTable), unlike a plain Reduce operation's real YT-sorted input. + reduceTaskParams.SortByHasKeyHashPrefix = true; std::vector<TFmrTableOutputRef> outputRefs; std::transform( diff --git a/yt/yql/providers/yt/fmr/coordinator/partitioner/yql_yt_reduce_partitioner.cpp b/yt/yql/providers/yt/fmr/coordinator/partitioner/yql_yt_reduce_partitioner.cpp index 8fd368852d8..69b9a40dc92 100644 --- a/yt/yql/providers/yt/fmr/coordinator/partitioner/yql_yt_reduce_partitioner.cpp +++ b/yt/yql/providers/yt/fmr/coordinator/partitioner/yql_yt_reduce_partitioner.cpp @@ -3,9 +3,19 @@ #include "yql_yt_sorted_partitioner_base.h" #include <yt/yql/providers/yt/fmr/utils/comparator/yql_yt_binary_yson_compare_impl.h> +#include <library/cpp/yson/node/node_io.h> namespace NYql::NFmr { +namespace { + +// Binary YSON key rows may contain arbitrary non-UTF8 bytes; text YSON escapes them, keeping error messages valid UTF-8. +TString FormatKeyRow(const TString& binaryYsonRow) { + return NYT::NodeToYsonString(NYT::NodeFromYsonString(binaryYsonRow)); +} + +} // namespace + TReducePartitioner::TReducePartitioner( const std::unordered_map<TFmrTableId, std::vector<TString>>& partIdsForTables, const std::unordered_map<TString, std::vector<TChunkStats>>& partIdStats, @@ -47,7 +57,7 @@ TFmrTableKeysRange TReducePartitioner::GetReadRangeFromSlices(const std::vector< YQL_ENSURE(taskRangeLeftBorderInclusive, " task range left border should always be true for reduce paritioner"); if (taskRangeLeftBorder == taskRangeRightBorder && !taskRangeRightBorderInclusive) { - ythrow yexception() << "Key " << taskRangeLeftBorder.Row << " takes all slice and is too large, cannot partition"; + ythrow yexception() << "Key " << FormatKeyRow(taskRangeLeftBorder.Row) << " takes all slice and is too large, cannot partition"; } taskRange.SetFirstKeysBound(taskRangeLeftBorder, taskRangeLeftBorderInclusive); @@ -93,7 +103,7 @@ void TReducePartitioner::CheckMaxKeySizePerSlices(const std::vector<TSlice>& sli if (*curLeftBorder != curRightBorder) { if (currentFilledSlicesWeight > maxKeySizePerPart) { - ythrow yexception() << "Key " << curLeftBorder->Row << " weighs at least " << currentFilledSlicesWeight << " bytes which is larget than maxKeySizePerPart - " << maxKeySizePerPart << " - cannot partition"; + ythrow yexception() << "Key " << FormatKeyRow(curLeftBorder->Row) << " weighs at least " << currentFilledSlicesWeight << " bytes which is larget than maxKeySizePerPart - " << maxKeySizePerPart << " - cannot partition"; } curLeftBorder = Nothing(); currentFilledSlicesWeight = 0; @@ -102,7 +112,7 @@ void TReducePartitioner::CheckMaxKeySizePerSlices(const std::vector<TSlice>& sli } } if (currentFilledSlicesWeight > maxKeySizePerPart) { - ythrow yexception() << "Key " << curLeftBorder->Row << " weighs at least " << currentFilledSlicesWeight << " bytes which is larget than maxKeySizePerPart - " << maxKeySizePerPart; + ythrow yexception() << "Key " << FormatKeyRow(curLeftBorder->Row) << " weighs at least " << currentFilledSlicesWeight << " bytes which is larget than maxKeySizePerPart - " << maxKeySizePerPart; } CurrentLastKeyWeight_ = currentFilledSlicesWeight; } diff --git a/yt/yql/providers/yt/fmr/process/yql_yt_job_fmr.cpp b/yt/yql/providers/yt/fmr/process/yql_yt_job_fmr.cpp index 6f4bc3e7d50..7d9c7a57b7f 100644 --- a/yt/yql/providers/yt/fmr/process/yql_yt_job_fmr.cpp +++ b/yt/yql/providers/yt/fmr/process/yql_yt_job_fmr.cpp @@ -113,6 +113,7 @@ void TFmrUserJob::Save(IOutputStream& s) const { ReduceOperationSpec_, IsMapReduceReducer_, IsMapReduceMap_, + SortByHasKeyHashPrefix_, ForceTableIndexMarking_ ); } @@ -132,6 +133,7 @@ void TFmrUserJob::Load(IInputStream& s) { ReduceOperationSpec_, IsMapReduceReducer_, IsMapReduceMap_, + SortByHasKeyHashPrefix_, ForceTableIndexMarking_ ); } @@ -283,6 +285,15 @@ void TFmrUserJob::FillQueueFromReduceInput() { YQL_ENSURE(ReduceOperationSpec_.Defined()); auto reduceBy = ReduceOperationSpec_->ReduceBy; auto sortBy = ReduceOperationSpec_->SortBy; + // sortBy is [...reduceBy, ...tiebreaker] (e.g. a join's "_yql_sort"), with a leading + // _yql_key_hash only for MapReduce's hash-shuffled intermediate tables (SortByHasKeyHashPrefix_, + // set by the coordinator - see TReduceTaskParams::SortByHasKeyHashPrefix) - a plain Reduce + // reads its input pre-sorted by the real reduceBy columns, with no hash involved. Either + // way, task boundaries must only constrain on the reduce-key prefix (hash-if-present + + // reduceBy) - see the comment on CompareRowToBoundaryPrefix for why comparing the tiebreaker + // too would silently split a reduce group across tasks whenever a task cut lands between + // rows that share a reduce key but differ in tiebreaker. + const size_t numBoundaryKeyColumns = (SortByHasKeyHashPrefix_ ? 1 : 0) + reduceBy.Columns.size(); for (const auto& inputTableRef : InputTables_.Inputs) { if (auto fmrInput = std::get_if<TFmrTableInputRef>(&inputTableRef)) { blockIterators.push_back(MakeIntrusive<TTableDataServiceBlockIterator>( @@ -296,7 +307,9 @@ void TFmrUserJob::FillQueueFromReduceInput() { fmrInput->IsFirstRowInclusive, fmrInput->IsLastRowInclusive, fmrInput->FirstRowKeys, - fmrInput->LastRowKeys + fmrInput->LastRowKeys, + /*readAheadChunks*/ ui64{4}, + numBoundaryKeyColumns )); } else { ythrow TFmrNonRetryableJobException() << "YtTables unsupported inside Reduce task for now"; @@ -486,7 +499,8 @@ TStatistics TFmrUserJob::DoFmrJob(const TFmrUserJobOptions& options) { sortedWriter->NotifyRowEnd(); } sortedWriter->Flush(); - TStatistics result({{OutputTables_[0], sortedWriter->GetStats()}}); + auto sortedWriterStats = sortedWriter->GetStats(); + TStatistics result({{OutputTables_[0], sortedWriterStats}}); // Direct (map-bypass) outputs at indices 1..K were already written to the real TDS via // their normal writers (see InitializeFmrUserJob) and flushed generically by diff --git a/yt/yql/providers/yt/fmr/process/yql_yt_job_fmr.h b/yt/yql/providers/yt/fmr/process/yql_yt_job_fmr.h index 54a8ff4ffde..a0bf4c70c07 100644 --- a/yt/yql/providers/yt/fmr/process/yql_yt_job_fmr.h +++ b/yt/yql/providers/yt/fmr/process/yql_yt_job_fmr.h @@ -116,6 +116,14 @@ public: IsMapReduceMap_ = value; } + // Whether this Reduce job's input is sorted by [_yql_key_hash, ...ReduceOperationSpec's + // SortBy] (a MapReduce operation's reduce stage) or by SortBy alone with no hash column at + // all (a plain Reduce operation reading real YT-sorted input) - see + // TReduceTaskParams::SortByHasKeyHashPrefix, which this is set from. + void SetSortByHasKeyHashPrefix(bool value) { + SortByHasKeyHashPrefix_ = value; + } + // Set by the gateway (from execCtx->InputTables_, before this job's state is serialized to // workers) whenever the ORIGINAL OPERATION has more than one distinct input position (e.g. a // JOIN's mapper reading the same or different tables via multiple sections). This must be a @@ -180,6 +188,7 @@ private: TMaybe<TReduceOperationSpec> ReduceOperationSpec_; bool IsMapReduceReducer_ = false; bool IsMapReduceMap_ = false; + bool SortByHasKeyHashPrefix_ = false; bool ForceTableIndexMarking_ = false; // End of serializable part diff --git a/yt/yql/providers/yt/fmr/proto/request_options.proto b/yt/yql/providers/yt/fmr/proto/request_options.proto index aacb3822b15..aafb3f2a32a 100644 --- a/yt/yql/providers/yt/fmr/proto/request_options.proto +++ b/yt/yql/providers/yt/fmr/proto/request_options.proto @@ -300,6 +300,11 @@ message TReduceTaskParams { repeated TFmrTableOutputRef Output = 2; bytes SerializedReduceJobState = 3; TReduceOperationSpec ReduceOperationSpec = 4; + // Whether Input's rows are sorted by [_yql_key_hash, ...ReduceOperationSpec.SortBy] (true for + // a MapReduce operation's reduce stage, which shuffles via a synthetic hash column) or by + // ReduceOperationSpec.SortBy alone (a plain Reduce operation, which reads real YT-sorted + // input with no hash column at all). + bool SortByHasKeyHashPrefix = 5; } message TSortOperationParams { diff --git a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp index 855b572b486..c8f992255b6 100644 --- a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp +++ b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp @@ -717,6 +717,7 @@ NProto::TReduceTaskParams ReduceTaskParamsToProto(const TReduceTaskParams& reduc protoReduceTaskParams.SetSerializedReduceJobState(reduceTaskParams.SerializedReduceJobState); auto protoReduceOperationSpec = ReduceOperationSpecToProto(reduceTaskParams.ReduceOperationSpec); protoReduceTaskParams.MutableReduceOperationSpec()->Swap(&protoReduceOperationSpec); + protoReduceTaskParams.SetSortByHasKeyHashPrefix(reduceTaskParams.SortByHasKeyHashPrefix); return protoReduceTaskParams; } @@ -730,6 +731,7 @@ TReduceTaskParams ReduceTaskParamsFromProto(const NProto::TReduceTaskParams& pro reduceTaskParams.Output = outputTables; reduceTaskParams.SerializedReduceJobState = protoReduceTaskParams.GetSerializedReduceJobState(); reduceTaskParams.ReduceOperationSpec = ReduceOperationSpecFromProto(protoReduceTaskParams.GetReduceOperationSpec()); + reduceTaskParams.SortByHasKeyHashPrefix = protoReduceTaskParams.GetSortByHasKeyHashPrefix(); return reduceTaskParams; } diff --git a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h index a32e521ba36..a2694941ae3 100644 --- a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h +++ b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h @@ -538,6 +538,11 @@ struct TReduceTaskParams { std::vector<TFmrTableOutputRef> Output; TString SerializedReduceJobState; TReduceOperationSpec ReduceOperationSpec; + // Whether Input's rows are sorted by [_yql_key_hash, ...ReduceOperationSpec.SortBy] (true for + // a MapReduce operation's reduce stage, which shuffles via a synthetic hash column) or by + // ReduceOperationSpec.SortBy alone (a plain Reduce operation, which reads real YT-sorted + // input with no hash column at all). + bool SortByHasKeyHashPrefix = false; }; // Service column added by map stage to hash-route rows to the correct reducer. diff --git a/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ut/ya.make b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ut/ya.make new file mode 100644 index 00000000000..61a5d2ff9f6 --- /dev/null +++ b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ut/ya.make @@ -0,0 +1,17 @@ +UNITTEST() + +SRCS( + yql_yt_yson_tds_block_iterator_ut.cpp +) + +PEERDIR( + yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl + yt/yql/providers/yt/fmr/request_options + yt/yql/providers/yt/fmr/table_data_service/local/impl + yt/yql/providers/yt/fmr/test_tools/yson + yt/yql/providers/yt/fmr/utils +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ut/yql_yt_yson_tds_block_iterator_ut.cpp b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ut/yql_yt_yson_tds_block_iterator_ut.cpp new file mode 100644 index 00000000000..e2f92d30156 --- /dev/null +++ b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ut/yql_yt_yson_tds_block_iterator_ut.cpp @@ -0,0 +1,88 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include <yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/yql_yt_yson_tds_block_iterator.h> +#include <yt/yql/providers/yt/fmr/test_tools/yson/yql_yt_yson_helpers.h> +#include <yt/yql/providers/yt/fmr/table_data_service/local/impl/yql_yt_table_data_service_local.h> +#include <yt/yql/providers/yt/fmr/utils/yql_yt_table_data_service_key.h> + +namespace NYql::NFmr { + +namespace { + +// Two rows sharing the same reduce key (_yql_key_hash=1, k=5) but differing only in the +// tiebreaker column _yql_sort - e.g. a join's build-side row (sort=0) and one of its probe-side +// rows (sort=1). Mirrors the real production row shape from yt/yql/providers/yt/fmr/job/impl/ +// yql_yt_reduce_reader.cpp's dumps. +const TVector<TString> KeyColumns = {"_yql_key_hash", "k", "_yql_sort"}; + +std::vector<TIndexedBlock> ReadAllBlocks(TTableDataServiceBlockIterator::TPtr iter) { + std::vector<TIndexedBlock> blocks; + TIndexedBlock block; + while (iter->NextBlock(block)) { + blocks.push_back(block); + } + return blocks; +} + +ui64 CountRows(const std::vector<TIndexedBlock>& blocks) { + ui64 total = 0; + for (const auto& b : blocks) { + total += b.Rows.size(); + } + return total; +} + +} // namespace + +Y_UNIT_TEST_SUITE(TableDataServiceBlockIteratorBoundaryTests) { + Y_UNIT_TEST(TieBreakerColumnMustNotSplitReduceGroupAcrossBoundary) { + auto tds = MakeLocalTableDataService(); + const TString tableId = "t"; + const TString partId = "p0"; + const TString group = GetTableDataServiceGroup(tableId, partId); + const TString chunkId = GetTableDataServiceChunkId(0, TString()); + + const TString chunkText = + "{\"_yql_key_hash\"=1u;\"k\"=5u;\"_yql_sort\"=0u};\n" + "{\"_yql_key_hash\"=1u;\"k\"=5u;\"_yql_sort\"=1u};\n"; + tds->Put(group, chunkId, GetBinaryYson(chunkText)).GetValueSync(); + + // The task boundary itself is built from the sort=0 row's real key - exactly what + // happens in production when that row is the last physical row before a task cut. + const TString boundaryRow = GetBinaryYson( + "{\"_yql_key_hash\"=1u;\"k\"=5u;\"_yql_sort\"=0u}", + NYson::EYsonType::Node + ); + + std::vector<TTableRange> tableRanges = {TTableRange{.PartId = partId, .MinChunk = 0, .MaxChunk = 1}}; + std::vector<ESortOrder> sortOrders(KeyColumns.size(), ESortOrder::Ascending); + + // Without a cap (numBoundaryKeyColumns defaults to comparing every key column, including + // the tiebreaker): reproduces the bug - the sort=1 row looks like it's past the boundary + // and gets dropped, even though it shares the exact same reduce key. + { + auto iter = MakeIntrusive<TTableDataServiceBlockIterator>( + tableId, tableRanges, tds, KeyColumns, sortOrders, + /*neededColumns*/ TVector<TString>{}, /*serializedColumnGroupsSpec*/ TString(), + /*isFirstRowKeysInclusive*/ Nothing(), /*isLastRowKeysInclusive*/ true, + /*firstRowKeys*/ Nothing(), /*lastRowKeys*/ boundaryRow + ); + UNIT_ASSERT_VALUES_EQUAL(CountRows(ReadAllBlocks(iter)), 1); + } + + // With the fix - numBoundaryKeyColumns=2 caps comparison to [_yql_key_hash, k], excluding + // _yql_sort: both rows of the reduce group survive the same boundary. + { + auto iter = MakeIntrusive<TTableDataServiceBlockIterator>( + tableId, tableRanges, tds, KeyColumns, sortOrders, + /*neededColumns*/ TVector<TString>{}, /*serializedColumnGroupsSpec*/ TString(), + /*isFirstRowKeysInclusive*/ Nothing(), /*isLastRowKeysInclusive*/ true, + /*firstRowKeys*/ Nothing(), /*lastRowKeys*/ boundaryRow, + /*readAheadChunks*/ ui64{4}, /*numBoundaryKeyColumns*/ size_t{2} + ); + UNIT_ASSERT_VALUES_EQUAL(CountRows(ReadAllBlocks(iter)), 2); + } + } +} + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ya.make b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ya.make index afb3020f720..7f68fea4899 100644 --- a/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ya.make +++ b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/ya.make @@ -21,3 +21,7 @@ PEERDIR( YQL_LAST_ABI_VERSION() END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/yql_yt_yson_tds_block_iterator.cpp b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/yql_yt_yson_tds_block_iterator.cpp index 584614844d9..e87a9791cbb 100644 --- a/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/yql_yt_yson_tds_block_iterator.cpp +++ b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/yql_yt_yson_tds_block_iterator.cpp @@ -21,7 +21,8 @@ TTableDataServiceBlockIterator::TTableDataServiceBlockIterator( TMaybe<bool> isLastRowKeysInclusive, TMaybe<TString> firstRowKeys, TMaybe<TString> lastRowKeys, - ui64 readAheadChunks + ui64 readAheadChunks, + TMaybe<size_t> numBoundaryKeyColumns ) : TableId_(std::move(tableId)) , TableRanges_(std::move(tableRanges)) @@ -31,6 +32,7 @@ TTableDataServiceBlockIterator::TTableDataServiceBlockIterator( , NeededColumns_(std::move(neededColumns)) , SerializedColumnGroupsSpec_(std::move(serializedColumnGroupsSpec)) , ReadAheadChunks_(readAheadChunks) + , NumBoundaryKeyColumns_(numBoundaryKeyColumns.GetOrElse(KeyColumns_.size())) { if (SortOrders_.empty()) { SortOrders_.assign(KeyColumns_.size(), ESortOrder::Ascending); @@ -38,6 +40,7 @@ TTableDataServiceBlockIterator::TTableDataServiceBlockIterator( if (SortOrders_.size() != KeyColumns_.size()) { ythrow yexception() << "SortOrders and KeyColumns sizes are different"; } + Y_ENSURE(NumBoundaryKeyColumns_ <= KeyColumns_.size(), "numBoundaryKeyColumns must not exceed keyColumns size"); if (firstRowKeys) { FirstBoundary_ = TFmrTableKeysBoundary(*firstRowKeys, KeyColumns_, SortOrders_); @@ -144,24 +147,29 @@ bool IsNullColumnValue(TStringBuf blob, const TColumnOffsetRange& range) { return SliceRange(blob, range) == TStringBuf(&EntitySymbol, 1); } -// FirstRowKeys/LastRowKeys boundaries are built by the coordinator from a task's chunk-stats -// key columns (e.g. [_yql_key_hash, ...ReduceBy] for MapReduce reduce tasks), which can be a -// strict PREFIX of the fuller key column set (e.g. [_yql_key_hash, ...SortBy]) this iterator -// parses actual rows with. TFmrTableKeysBoundary parses its own Row bytes with the SAME -// (fuller) key columns as the row markup, so any column beyond the boundary's real prefix comes -// out with an IsValid()==false range - not because the value is legitimately null, but because -// the boundary blob simply never encoded it. Comparing that as "boundary < row" (the usual -// null-handling rule) would incorrectly exclude real rows that land exactly on the boundary, so -// this comparison stops at the boundary's own valid prefix instead of delegating to the -// generic null-aware CompareKeyRowsAcrossYsonBlocks. +// FirstRowKeys/LastRowKeys boundaries are built by the coordinator from a task's chunk-stats, +// i.e. from an actual physical row's real key - for MapReduce reduce tasks over joins that key +// is [_yql_key_hash, ...ReduceBy, ...SortBy tiebreaker] in full, tiebreaker included, because +// that's what TReducePartitioner sorts intermediate data by (see +// yql_yt_map_reduce_stage_operation_manager.cpp). It is NOT reliably a strict prefix that omits +// the tiebreaker: whichever physical row happens to land on a task cut contributes its own real +// tiebreaker value to the boundary blob. So this can't stop at "wherever the boundary blob runs +// out of valid columns" (that already-invalid range doesn't reliably appear here) - it must be +// told explicitly, via numBoundaryKeyColumns, how many leading columns actually define a reduce +// group, and ignore everything past that. Comparing the tiebreaker as a real bound column would +// silently split a reduce group in half whenever a task cut lands between two physical rows that +// share a reduce key but differ in tiebreaker (e.g. a join's build-side row vs. its probe-side +// rows) - each side would look like it's on the "wrong side" of the other's boundary. int CompareRowToBoundaryPrefix( TStringBuf rowBlob, const TRowIndexMarkup& row, TStringBuf boundaryBlob, const TRowIndexMarkup& boundaryMarkup, - const std::vector<ESortOrder>& sortOrders + const std::vector<ESortOrder>& sortOrders, + size_t numBoundaryKeyColumns ) { - for (ui64 colIdx = 0; colIdx + 1 < boundaryMarkup.size(); ++colIdx) { + const size_t limit = std::min<size_t>(numBoundaryKeyColumns, boundaryMarkup.empty() ? 0 : boundaryMarkup.size() - 1); + for (ui64 colIdx = 0; colIdx < limit; ++colIdx) { const auto& boundaryRange = boundaryMarkup[colIdx]; if (!boundaryRange.IsValid()) { // Boundary doesn't encode this column (or any further one, since it's a prefix) - @@ -204,7 +212,8 @@ bool TTableDataServiceBlockIterator::RowInKeyBounds(const TString& blob, const T row, FirstBoundary_->Row, FirstBoundary_->Markup, - SortOrders_ + SortOrders_, + NumBoundaryKeyColumns_ ); if (c < 0) { // if row < first boundary return false; @@ -218,7 +227,8 @@ bool TTableDataServiceBlockIterator::RowInKeyBounds(const TString& blob, const T row, LastBoundary_->Row, LastBoundary_->Markup, - SortOrders_ + SortOrders_, + NumBoundaryKeyColumns_ ); if (c > 0) { // if row > last boundary return false; diff --git a/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/yql_yt_yson_tds_block_iterator.h b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/yql_yt_yson_tds_block_iterator.h index 4ab202869fc..8f661e5ec01 100644 --- a/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/yql_yt_yson_tds_block_iterator.h +++ b/yt/yql/providers/yt/fmr/utils/yson_block_iterator/impl/yql_yt_yson_tds_block_iterator.h @@ -27,8 +27,14 @@ public: TMaybe<bool> isLastRowKeysInclusive = Nothing(), TMaybe<TString> firstRowKeys = Nothing(), TMaybe<TString> lastRowKeys = Nothing(), - ui64 readAheadChunks = 4 - + ui64 readAheadChunks = 4, + // How many leading keyColumns actually define a reduce group (e.g. _yql_key_hash + + // reduceBy, excluding a join's _yql_sort tiebreaker). Boundary comparisons stop after + // this many columns instead of running to the end of keyColumns - see the comment on + // CompareRowToBoundaryPrefix for why relying on the boundary blob happening to omit + // trailing columns is not safe. Defaults to comparing every column in keyColumns, which + // is correct whenever keyColumns has no trailing tiebreaker (e.g. non-reduce contexts). + TMaybe<size_t> numBoundaryKeyColumns = Nothing() ); ~TTableDataServiceBlockIterator() final; @@ -73,6 +79,7 @@ private: std::vector<TString> GroupNamesToRead_; ui64 ReadAheadChunks_ = 4; + const size_t NumBoundaryKeyColumns_; std::deque<TPrefetchEntry> PrefetchQueue_; }; diff --git a/yt/yt/client/unittests/mock/table_reader.cpp b/yt/yt/client/unittests/mock/table_reader.cpp index 1e314884241..ef3c44d0bde 100644 --- a/yt/yt/client/unittests/mock/table_reader.cpp +++ b/yt/yt/client/unittests/mock/table_reader.cpp @@ -10,7 +10,7 @@ namespace NYT::NApi { TMockTableReader::TMockTableReader(NTableClient::TTableSchemaPtr schema) : Schema_(std::move(schema)) , NameTable_(NTableClient::TNameTable::FromSchema(*Schema_)) -{} +{ } const NTableClient::TNameTablePtr& TMockTableReader::GetNameTable() const { |