diff options
author | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
---|---|---|
committer | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
commit | 73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch) | |
tree | 188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /library/cpp | |
parent | 528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff) | |
download | ydb-73b89de71748a21e102d27b9f3ed1bf658766cb5.tar.gz |
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build.
Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'library/cpp')
216 files changed, 21895 insertions, 0 deletions
diff --git a/library/cpp/CMakeLists.darwin-x86_64.txt b/library/cpp/CMakeLists.darwin-x86_64.txt index 81a8d715c0..772027a342 100644 --- a/library/cpp/CMakeLists.darwin-x86_64.txt +++ b/library/cpp/CMakeLists.darwin-x86_64.txt @@ -71,6 +71,7 @@ add_subdirectory(retry) add_subdirectory(sanitizer) add_subdirectory(scheme) add_subdirectory(sighandler) +add_subdirectory(skiff) add_subdirectory(sliding_window) add_subdirectory(sse) add_subdirectory(streams) @@ -83,6 +84,7 @@ add_subdirectory(threading) add_subdirectory(time_provider) add_subdirectory(timezone_conversion) add_subdirectory(tld) +add_subdirectory(type_info) add_subdirectory(unicode) add_subdirectory(unified_agent_client) add_subdirectory(uri) diff --git a/library/cpp/CMakeLists.linux-aarch64.txt b/library/cpp/CMakeLists.linux-aarch64.txt index 5e6834dea1..cd50b0e3a4 100644 --- a/library/cpp/CMakeLists.linux-aarch64.txt +++ b/library/cpp/CMakeLists.linux-aarch64.txt @@ -70,6 +70,7 @@ add_subdirectory(retry) add_subdirectory(sanitizer) add_subdirectory(scheme) add_subdirectory(sighandler) +add_subdirectory(skiff) add_subdirectory(sliding_window) add_subdirectory(sse) add_subdirectory(streams) @@ -82,6 +83,7 @@ add_subdirectory(threading) add_subdirectory(time_provider) add_subdirectory(timezone_conversion) add_subdirectory(tld) +add_subdirectory(type_info) add_subdirectory(unicode) add_subdirectory(unified_agent_client) add_subdirectory(uri) diff --git a/library/cpp/CMakeLists.linux-x86_64.txt b/library/cpp/CMakeLists.linux-x86_64.txt index 81a8d715c0..772027a342 100644 --- a/library/cpp/CMakeLists.linux-x86_64.txt +++ b/library/cpp/CMakeLists.linux-x86_64.txt @@ -71,6 +71,7 @@ add_subdirectory(retry) add_subdirectory(sanitizer) add_subdirectory(scheme) add_subdirectory(sighandler) +add_subdirectory(skiff) add_subdirectory(sliding_window) add_subdirectory(sse) add_subdirectory(streams) @@ -83,6 +84,7 @@ add_subdirectory(threading) add_subdirectory(time_provider) add_subdirectory(timezone_conversion) add_subdirectory(tld) +add_subdirectory(type_info) add_subdirectory(unicode) add_subdirectory(unified_agent_client) add_subdirectory(uri) diff --git a/library/cpp/CMakeLists.windows-x86_64.txt b/library/cpp/CMakeLists.windows-x86_64.txt index 81a8d715c0..772027a342 100644 --- a/library/cpp/CMakeLists.windows-x86_64.txt +++ b/library/cpp/CMakeLists.windows-x86_64.txt @@ -71,6 +71,7 @@ add_subdirectory(retry) add_subdirectory(sanitizer) add_subdirectory(scheme) add_subdirectory(sighandler) +add_subdirectory(skiff) add_subdirectory(sliding_window) add_subdirectory(sse) add_subdirectory(streams) @@ -83,6 +84,7 @@ add_subdirectory(threading) add_subdirectory(time_provider) add_subdirectory(timezone_conversion) add_subdirectory(tld) +add_subdirectory(type_info) add_subdirectory(unicode) add_subdirectory(unified_agent_client) add_subdirectory(uri) diff --git a/library/cpp/containers/concurrent_hash/concurrent_hash.h b/library/cpp/containers/concurrent_hash/concurrent_hash.h new file mode 100644 index 0000000000..f15a1c3d6e --- /dev/null +++ b/library/cpp/containers/concurrent_hash/concurrent_hash.h @@ -0,0 +1,128 @@ +#pragma once + +#include <util/generic/hash.h> +#include <util/system/spinlock.h> + +#include <array> + +template <typename K, typename V, size_t BucketCount = 64, typename L = TAdaptiveLock> +class TConcurrentHashMap { +public: + using TActualMap = THashMap<K, V>; + using TLock = L; + + struct TBucket { + friend class TConcurrentHashMap; + + private: + TActualMap Map; + mutable TLock Mutex; + + public: + TLock& GetMutex() const { + return Mutex; + } + + TActualMap& GetMap() { + return Map; + } + const TActualMap& GetMap() const { + return Map; + } + + const V& GetUnsafe(const K& key) const { + typename TActualMap::const_iterator it = Map.find(key); + Y_VERIFY(it != Map.end(), "not found by key"); + return it->second; + } + + V& GetUnsafe(const K& key) { + typename TActualMap::iterator it = Map.find(key); + Y_VERIFY(it != Map.end(), "not found by key"); + return it->second; + } + + V RemoveUnsafe(const K& key) { + typename TActualMap::iterator it = Map.find(key); + Y_VERIFY(it != Map.end(), "removing non-existent key"); + V r = std::move(it->second); + Map.erase(it); + return r; + } + + bool HasUnsafe(const K& key) const { + typename TActualMap::const_iterator it = Map.find(key); + return (it != Map.end()); + } + }; + + std::array<TBucket, BucketCount> Buckets; + +public: + TBucket& GetBucketForKey(const K& key) { + return Buckets[THash<K>()(key) % BucketCount]; + } + + const TBucket& GetBucketForKey(const K& key) const { + return Buckets[THash<K>()(key) % BucketCount]; + } + + void Insert(const K& key, const V& value) { + TBucket& bucket = GetBucketForKey(key); + TGuard<TLock> guard(bucket.Mutex); + bucket.Map[key] = value; + } + + void InsertUnique(const K& key, const V& value) { + TBucket& bucket = GetBucketForKey(key); + TGuard<TLock> guard(bucket.Mutex); + if (!bucket.Map.insert(std::make_pair(key, value)).second) { + Y_FAIL("non-unique key"); + } + } + + V& InsertIfAbsent(const K& key, const V& value) { + TBucket& bucket = GetBucketForKey(key); + TGuard<TLock> guard(bucket.Mutex); + return bucket.Map.insert(std::make_pair(key, value)).first->second; + } + + template <typename Callable> + V& InsertIfAbsentWithInit(const K& key, Callable initFunc) { + TBucket& bucket = GetBucketForKey(key); + TGuard<TLock> guard(bucket.Mutex); + if (bucket.HasUnsafe(key)) { + return bucket.GetUnsafe(key); + } + + return bucket.Map.insert(std::make_pair(key, initFunc())).first->second; + } + + V Get(const K& key) const { + const TBucket& bucket = GetBucketForKey(key); + TGuard<TLock> guard(bucket.Mutex); + return bucket.GetUnsafe(key); + } + + bool Get(const K& key, V& result) const { + const TBucket& bucket = GetBucketForKey(key); + TGuard<TLock> guard(bucket.Mutex); + if (bucket.HasUnsafe(key)) { + result = bucket.GetUnsafe(key); + return true; + } + return false; + } + + V Remove(const K& key) { + TBucket& bucket = GetBucketForKey(key); + TGuard<TLock> guard(bucket.Mutex); + return bucket.RemoveUnsafe(key); + } + + bool Has(const K& key) const { + const TBucket& bucket = GetBucketForKey(key); + TGuard<TLock> guard(bucket.Mutex); + return bucket.HasUnsafe(key); + } +}; diff --git a/library/cpp/dwarf_backtrace/backtrace.cpp b/library/cpp/dwarf_backtrace/backtrace.cpp new file mode 100644 index 0000000000..a955d07249 --- /dev/null +++ b/library/cpp/dwarf_backtrace/backtrace.cpp @@ -0,0 +1,62 @@ +#include "backtrace.h" + +#include <contrib/libs/backtrace/backtrace.h> + +#include <util/generic/yexception.h> +#include <util/system/type_name.h> +#include <util/system/execpath.h> + +namespace NDwarf { + namespace { + struct TContext { + TCallback& Callback; + int Counter = 0; + TMaybe<TError> Error; + }; + + void HandleLibBacktraceError(void* data, const char* msg, int errnum) { + auto* context = reinterpret_cast<TContext*>(data); + context->Error = TError{.Code = errnum, .Message=msg}; + } + + int HandleLibBacktraceFrame(void* data, uintptr_t pc, const char* filename, int lineno, const char* function) { + auto* context = reinterpret_cast<TContext*>(data); + TLineInfo lineInfo{ + .FileName = filename != nullptr ? filename : "???", + .Line = lineno, + .Col = 0, // libbacktrace doesn't provide column numbers, so fill this field with a dummy value. + .FunctionName = function != nullptr ? CppDemangle(function) : "???", + .Address = pc, + .Index = context->Counter++, + }; + return static_cast<int>(context->Callback(lineInfo)); + } + } + + TMaybe<TError> ResolveBacktrace(TArrayRef<const void* const> backtrace, TCallback callback) { + TContext context{.Callback = callback}; + // Intentionally never freed (see https://a.yandex-team.ru/arc/trunk/arcadia/contrib/libs/backtrace/backtrace.h?rev=6789902#L80). + static auto* state = backtrace_create_state( + GetPersistentExecPath().c_str(), + 1 /* threaded */, + HandleLibBacktraceError, + &context /* data for the error callback */ + ); + if (nullptr == state) { + static const auto initError = context.Error; + return initError; + } + for (const void* address : backtrace) { + int status = backtrace_pcinfo( + state, + reinterpret_cast<uintptr_t>(address) - 1, // last byte of the call instruction + HandleLibBacktraceFrame, + HandleLibBacktraceError, + &context /* data for both callbacks */); + if (0 != status) { + break; + } + } + return context.Error; + } +} diff --git a/library/cpp/dwarf_backtrace/backtrace.h b/library/cpp/dwarf_backtrace/backtrace.h new file mode 100644 index 0000000000..62ec36dba5 --- /dev/null +++ b/library/cpp/dwarf_backtrace/backtrace.h @@ -0,0 +1,36 @@ +#pragma once + +#include <util/generic/array_ref.h> +#include <util/generic/maybe.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +#include <functional> + +namespace NDwarf { + struct TLineInfo { + TString FileName; + int Line; + int Col; + TString FunctionName; + uintptr_t Address; + int Index; + }; + + struct TError { + int Code; + TString Message; + }; + + enum class EResolving { + Continue = 0, + Break = 1, + }; + + using TCallback = std::function<EResolving(const TLineInfo&)>; + + // Resolves backtrace addresses and calls the callback for all line infos of inlined functions there. + // Stops execution if the callback returns `EResolving::Break`. + [[nodiscard]] TMaybe<TError> ResolveBacktrace(TArrayRef<const void* const> backtrace, TCallback callback); + +} diff --git a/library/cpp/dwarf_backtrace/ya.make b/library/cpp/dwarf_backtrace/ya.make new file mode 100644 index 0000000000..e955f9aa92 --- /dev/null +++ b/library/cpp/dwarf_backtrace/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +NO_WSHADOW() + +SRCS( + backtrace.cpp +) + +PEERDIR( + contrib/libs/backtrace +) + +END() + +IF (NOT OS_WINDOWS) + RECURSE_FOR_TESTS( + ut + ) +ENDIF() diff --git a/library/cpp/skiff/CMakeLists.darwin-x86_64.txt b/library/cpp/skiff/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..d8d296df29 --- /dev/null +++ b/library/cpp/skiff/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(library-cpp-skiff) +target_link_libraries(library-cpp-skiff PUBLIC + contrib-libs-cxxsupp + yutil + tools-enum_parser-enum_serialization_runtime +) +target_sources(library-cpp-skiff PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff_schema.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff_validator.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/zerocopy_output_writer.cpp +) +generate_enum_serilization(library-cpp-skiff + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/public.h + INCLUDE_HEADERS + library/cpp/skiff/public.h +) diff --git a/library/cpp/skiff/CMakeLists.linux-aarch64.txt b/library/cpp/skiff/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..b0656a7bbe --- /dev/null +++ b/library/cpp/skiff/CMakeLists.linux-aarch64.txt @@ -0,0 +1,33 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(library-cpp-skiff) +target_link_libraries(library-cpp-skiff PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + tools-enum_parser-enum_serialization_runtime +) +target_sources(library-cpp-skiff PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff_schema.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff_validator.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/zerocopy_output_writer.cpp +) +generate_enum_serilization(library-cpp-skiff + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/public.h + INCLUDE_HEADERS + library/cpp/skiff/public.h +) diff --git a/library/cpp/skiff/CMakeLists.linux-x86_64.txt b/library/cpp/skiff/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..b0656a7bbe --- /dev/null +++ b/library/cpp/skiff/CMakeLists.linux-x86_64.txt @@ -0,0 +1,33 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(library-cpp-skiff) +target_link_libraries(library-cpp-skiff PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + tools-enum_parser-enum_serialization_runtime +) +target_sources(library-cpp-skiff PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff_schema.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff_validator.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/zerocopy_output_writer.cpp +) +generate_enum_serilization(library-cpp-skiff + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/public.h + INCLUDE_HEADERS + library/cpp/skiff/public.h +) diff --git a/library/cpp/skiff/CMakeLists.txt b/library/cpp/skiff/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/skiff/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/skiff/CMakeLists.windows-x86_64.txt b/library/cpp/skiff/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..d8d296df29 --- /dev/null +++ b/library/cpp/skiff/CMakeLists.windows-x86_64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(library-cpp-skiff) +target_link_libraries(library-cpp-skiff PUBLIC + contrib-libs-cxxsupp + yutil + tools-enum_parser-enum_serialization_runtime +) +target_sources(library-cpp-skiff PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff_schema.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/skiff_validator.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/zerocopy_output_writer.cpp +) +generate_enum_serilization(library-cpp-skiff + ${CMAKE_SOURCE_DIR}/library/cpp/skiff/public.h + INCLUDE_HEADERS + library/cpp/skiff/public.h +) diff --git a/library/cpp/skiff/public.h b/library/cpp/skiff/public.h new file mode 100644 index 0000000000..d67c6f26ee --- /dev/null +++ b/library/cpp/skiff/public.h @@ -0,0 +1,63 @@ +#pragma once + +#include <vector> +#include <memory> + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +enum class EWireType +{ + Nothing /* "nothing" */, + Int8 /* "int8" */, + Int16 /* "int16" */, + Int32 /* "int32" */, + Int64 /* "int64" */, + Int128 /* "int128" */, + Uint8 /* "uint8" */, + Uint16 /* "uint16" */, + Uint32 /* "uint32" */, + Uint64 /* "uint64" */, + Uint128 /* "uint128" */, + Double /* "double" */, + Boolean /* "boolean" */, + String32 /* "string32" */, + Yson32 /* "yson32" */, + + Tuple /* "tuple" */, + Variant8 /* "variant8" */, + Variant16 /* "variant16" */, + RepeatedVariant8 /* "repeated_variant8" */, + RepeatedVariant16 /* "repeated_variant16" */, +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TSkiffSchema; +using TSkiffSchemaPtr = std::shared_ptr<TSkiffSchema>; + +using TSkiffSchemaList = std::vector<TSkiffSchemaPtr>; + +class TSimpleTypeSchema; +using TSimpleTypeSchemaPtr = std::shared_ptr<TSimpleTypeSchema>; + +class TSkiffValidator; + +class TUncheckedSkiffParser; +class TCheckedSkiffParser; + +class TUncheckedSkiffWriter; +class TCheckedSkiffWriter; + +#ifdef DEBUG +using TCheckedInDebugSkiffParser = TCheckedSkiffParser; +using TCheckedInDebugSkiffWriter = TCheckedSkiffWriter; +#else +using TCheckedInDebugSkiffParser = TUncheckedSkiffParser; +using TCheckedInDebugSkiffWriter = TUncheckedSkiffWriter; +#endif + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff diff --git a/library/cpp/skiff/skiff-inl.h b/library/cpp/skiff/skiff-inl.h new file mode 100644 index 0000000000..a3f68a9374 --- /dev/null +++ b/library/cpp/skiff/skiff-inl.h @@ -0,0 +1,39 @@ +#pragma once + +#ifndef SKIFF_H +#error "Direct inclusion of this file is not allowed, include skiff.h" +// For the sake of sane code completion. +#include "skiff.h" +#endif +#undef SKIFF_H + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +template <EWireType wireType> +constexpr auto TUnderlyingIntegerType<wireType>::F() { + if constexpr (wireType == EWireType::Int8) { + return i8{}; + } else if constexpr (wireType == EWireType::Int16) { + return i16{}; + } else if constexpr (wireType == EWireType::Int32) { + return i32{}; + } else if constexpr (wireType == EWireType::Int64) { + return i64{}; + } else if constexpr (wireType == EWireType::Uint8) { + return ui8{}; + } else if constexpr (wireType == EWireType::Uint16) { + return ui16{}; + } else if constexpr (wireType == EWireType::Uint32) { + return ui32{}; + } else if constexpr (wireType == EWireType::Uint64) { + return ui64{}; + } else { + static_assert(wireType == EWireType::Int8, "expected integer wire type"); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff diff --git a/library/cpp/skiff/skiff.cpp b/library/cpp/skiff/skiff.cpp new file mode 100644 index 0000000000..cbdbdfe364 --- /dev/null +++ b/library/cpp/skiff/skiff.cpp @@ -0,0 +1,591 @@ +#include "skiff.h" + +#include "skiff_validator.h" + +#include <util/stream/buffered.h> +#include <util/system/byteorder.h> +#include <util/system/unaligned_mem.h> + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +bool operator==(TInt128 lhs, TInt128 rhs) +{ + return lhs.Low == rhs.Low && lhs.High == rhs.High; +} + +bool operator!=(TInt128 lhs, TInt128 rhs) +{ + return !(lhs == rhs); +} + +bool operator==(TUint128 lhs, TUint128 rhs) +{ + return lhs.Low == rhs.Low && lhs.High == rhs.High; +} + +bool operator!=(TUint128 lhs, TUint128 rhs) +{ + return !(lhs == rhs); +} + +//////////////////////////////////////////////////////////////////////////////// + +TUncheckedSkiffParser::TUncheckedSkiffParser(IZeroCopyInput* underlying) + : Underlying_(underlying) + , Buffer_(512 * 1024) +{ } + +TUncheckedSkiffParser::TUncheckedSkiffParser(const std::shared_ptr<TSkiffSchema>& /*schema*/, IZeroCopyInput* underlying) + : TUncheckedSkiffParser(underlying) +{ } + +i8 TUncheckedSkiffParser::ParseInt8() +{ + return ParseSimple<i8>(); +} + +i16 TUncheckedSkiffParser::ParseInt16() +{ + return ParseSimple<i16>(); +} + +i32 TUncheckedSkiffParser::ParseInt32() +{ + return ParseSimple<i32>(); +} + +i64 TUncheckedSkiffParser::ParseInt64() +{ + return ParseSimple<i64>(); +} + +ui8 TUncheckedSkiffParser::ParseUint8() +{ + return ParseSimple<ui8>(); +} + +ui16 TUncheckedSkiffParser::ParseUint16() +{ + return ParseSimple<ui16>(); +} + +ui32 TUncheckedSkiffParser::ParseUint32() +{ + return ParseSimple<ui32>(); +} + +ui64 TUncheckedSkiffParser::ParseUint64() +{ + return ParseSimple<ui64>(); +} + +TInt128 TUncheckedSkiffParser::ParseInt128() +{ + auto low = ParseSimple<ui64>(); + auto high = ParseSimple<i64>(); + return {low, high}; +} + +TUint128 TUncheckedSkiffParser::ParseUint128() +{ + auto low = ParseSimple<ui64>(); + auto high = ParseSimple<ui64>(); + return {low, high}; +} + +double TUncheckedSkiffParser::ParseDouble() +{ + return ParseSimple<double>(); +} + +bool TUncheckedSkiffParser::ParseBoolean() +{ + ui8 result = ParseSimple<ui8>(); + if (result > 1) { + ythrow TSkiffException() << "Invalid boolean value \"" << result << "\""; + } + return result; +} + +TStringBuf TUncheckedSkiffParser::ParseString32() +{ + ui32 len = ParseSimple<ui32>(); + const void* data = GetData(len); + return TStringBuf(static_cast<const char*>(data), len); +} + +TStringBuf TUncheckedSkiffParser::ParseYson32() +{ + return ParseString32(); +} + +ui8 TUncheckedSkiffParser::ParseVariant8Tag() +{ + return ParseSimple<ui8>(); +} + +ui16 TUncheckedSkiffParser::ParseVariant16Tag() +{ + return ParseSimple<ui16>(); +} + +template <typename T> +T TUncheckedSkiffParser::ParseSimple() +{ + return ReadUnaligned<T>(GetData(sizeof(T))); +} + +const void* TUncheckedSkiffParser::GetData(size_t size) +{ + if (RemainingBytes() >= size) { + const void* result = Position_; + Advance(size); + return result; + } + + return GetDataViaBuffer(size); +} + +const void* TUncheckedSkiffParser::GetDataViaBuffer(size_t size) +{ + Buffer_.Clear(); + Buffer_.Reserve(size); + while (Buffer_.Size() < size) { + size_t toCopy = Min(size - Buffer_.Size(), RemainingBytes()); + Buffer_.Append(Position_, toCopy); + Advance(toCopy); + + if (RemainingBytes() == 0) { + RefillBuffer(); + if (Exhausted_ && Buffer_.Size() < size) { + ythrow TSkiffException() << "Premature end of stream while parsing Skiff"; + } + } + } + return Buffer_.Data(); +} + +size_t TUncheckedSkiffParser::RemainingBytes() const +{ + Y_ASSERT(End_ >= Position_); + return End_ - Position_; +} + +void TUncheckedSkiffParser::Advance(size_t size) +{ + Y_ASSERT(size <= RemainingBytes()); + Position_ += size; + ReadBytesCount_ += size; +} + +void TUncheckedSkiffParser::RefillBuffer() +{ + size_t bufferSize = Underlying_->Next(&Position_); + End_ = Position_ + bufferSize; + if (bufferSize == 0) { + Exhausted_ = true; + } +} + +bool TUncheckedSkiffParser::HasMoreData() +{ + if (RemainingBytes() == 0 && !Exhausted_) { + RefillBuffer(); + } + return !(RemainingBytes() == 0 && Exhausted_); +} + +void TUncheckedSkiffParser::ValidateFinished() +{ } + +ui64 TUncheckedSkiffParser::GetReadBytesCount() const +{ + return ReadBytesCount_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TCheckedSkiffParser::TCheckedSkiffParser(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyInput* stream) + : Parser_(stream) + , Validator_(std::make_unique<TSkiffValidator>(schema)) +{ } + +TCheckedSkiffParser::~TCheckedSkiffParser() = default; + +i8 TCheckedSkiffParser::ParseInt8() +{ + Validator_->OnSimpleType(EWireType::Int8); + return Parser_.ParseInt8(); +} + +i16 TCheckedSkiffParser::ParseInt16() +{ + Validator_->OnSimpleType(EWireType::Int16); + return Parser_.ParseInt16(); +} + +i32 TCheckedSkiffParser::ParseInt32() +{ + Validator_->OnSimpleType(EWireType::Int32); + return Parser_.ParseInt32(); +} + +i64 TCheckedSkiffParser::ParseInt64() +{ + Validator_->OnSimpleType(EWireType::Int64); + return Parser_.ParseInt64(); +} + +ui8 TCheckedSkiffParser::ParseUint8() +{ + Validator_->OnSimpleType(EWireType::Uint8); + return Parser_.ParseUint8(); +} + +ui16 TCheckedSkiffParser::ParseUint16() +{ + Validator_->OnSimpleType(EWireType::Uint16); + return Parser_.ParseUint16(); +} + +ui32 TCheckedSkiffParser::ParseUint32() +{ + Validator_->OnSimpleType(EWireType::Uint32); + return Parser_.ParseUint32(); +} + +ui64 TCheckedSkiffParser::ParseUint64() +{ + Validator_->OnSimpleType(EWireType::Uint64); + return Parser_.ParseUint64(); +} + +TInt128 TCheckedSkiffParser::ParseInt128() +{ + Validator_->OnSimpleType(EWireType::Int128); + return Parser_.ParseInt128(); +} + +TUint128 TCheckedSkiffParser::ParseUint128() +{ + Validator_->OnSimpleType(EWireType::Uint128); + return Parser_.ParseUint128(); +} + +double TCheckedSkiffParser::ParseDouble() +{ + Validator_->OnSimpleType(EWireType::Double); + return Parser_.ParseDouble(); +} + +bool TCheckedSkiffParser::ParseBoolean() +{ + Validator_->OnSimpleType(EWireType::Boolean); + return Parser_.ParseBoolean(); +} + +TStringBuf TCheckedSkiffParser::ParseString32() +{ + Validator_->OnSimpleType(EWireType::String32); + return Parser_.ParseString32(); +} + +TStringBuf TCheckedSkiffParser::ParseYson32() +{ + Validator_->OnSimpleType(EWireType::Yson32); + return Parser_.ParseYson32(); +} + +ui8 TCheckedSkiffParser::ParseVariant8Tag() +{ + Validator_->BeforeVariant8Tag(); + auto result = Parser_.ParseVariant8Tag(); + Validator_->OnVariant8Tag(result); + return result; +} + +ui16 TCheckedSkiffParser::ParseVariant16Tag() +{ + Validator_->BeforeVariant16Tag(); + auto result = Parser_.ParseVariant16Tag(); + Validator_->OnVariant16Tag(result); + return result; +} + +bool TCheckedSkiffParser::HasMoreData() +{ + return Parser_.HasMoreData(); +} + +void TCheckedSkiffParser::ValidateFinished() +{ + Validator_->ValidateFinished(); + Parser_.ValidateFinished(); +} + +ui64 TCheckedSkiffParser::GetReadBytesCount() const +{ + return Parser_.GetReadBytesCount(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TUncheckedSkiffWriter::TUncheckedSkiffWriter(IZeroCopyOutput* underlying) + : Underlying_(underlying) +{ } + +TUncheckedSkiffWriter::TUncheckedSkiffWriter(IOutputStream* underlying) + : BufferedOutput_(MakeHolder<TBufferedOutput>(underlying)) + , Underlying_(BufferedOutput_.Get()) +{ } + +TUncheckedSkiffWriter::TUncheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& /*schema*/, IZeroCopyOutput* underlying) + : TUncheckedSkiffWriter(underlying) +{ } + +TUncheckedSkiffWriter::TUncheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& /*schema*/, IOutputStream* underlying) + : TUncheckedSkiffWriter(underlying) +{ } + +TUncheckedSkiffWriter::~TUncheckedSkiffWriter() +{ + try { + Flush(); + } catch (...) { + } +} + +void TUncheckedSkiffWriter::WriteInt8(i8 value) +{ + WriteSimple<i8>(value); +} + +void TUncheckedSkiffWriter::WriteInt16(i16 value) +{ + WriteSimple<i16>(value); +} + +void TUncheckedSkiffWriter::WriteInt32(i32 value) +{ + WriteSimple<i32>(value); +} + +void TUncheckedSkiffWriter::WriteInt64(i64 value) +{ + WriteSimple<i64>(value); +} + +void TUncheckedSkiffWriter::WriteInt128(TInt128 value) +{ + WriteSimple<ui64>(value.Low); + WriteSimple<i64>(value.High); +} + +void TUncheckedSkiffWriter::WriteUint128(TUint128 value) +{ + WriteSimple<ui64>(value.Low); + WriteSimple<ui64>(value.High); +} + +void TUncheckedSkiffWriter::WriteUint8(ui8 value) +{ + WriteSimple<ui8>(value); +} + +void TUncheckedSkiffWriter::WriteUint16(ui16 value) +{ + WriteSimple<ui16>(value); +} + +void TUncheckedSkiffWriter::WriteUint32(ui32 value) +{ + WriteSimple<ui32>(value); +} + +void TUncheckedSkiffWriter::WriteUint64(ui64 value) +{ + WriteSimple<ui64>(value); +} + +void TUncheckedSkiffWriter::WriteDouble(double value) +{ + return WriteSimple<double>(value); +} + +void TUncheckedSkiffWriter::WriteBoolean(bool value) +{ + return WriteSimple<ui8>(value ? 1 : 0); +} + +void TUncheckedSkiffWriter::WriteString32(TStringBuf value) +{ + WriteSimple<ui32>(value.size()); + Underlying_.Write(value.data(), value.size()); +} + +void TUncheckedSkiffWriter::WriteYson32(TStringBuf value) +{ + WriteSimple<ui32>(value.size()); + Underlying_.Write(value.data(), value.size()); +} + +void TUncheckedSkiffWriter::WriteVariant8Tag(ui8 tag) +{ + WriteSimple<ui8>(tag); +} + +void TUncheckedSkiffWriter::WriteVariant16Tag(ui16 tag) +{ + WriteSimple<ui16>(tag); +} + +void TUncheckedSkiffWriter::Flush() +{ + Underlying_.UndoRemaining(); + if (BufferedOutput_) { + BufferedOutput_->Flush(); + } +} + +template <typename T> +Y_FORCE_INLINE void TUncheckedSkiffWriter::WriteSimple(T value) +{ + if constexpr (std::is_integral_v<T>) { + value = HostToLittle(value); + Underlying_.Write(&value, sizeof(T)); + } else { + Underlying_.Write(&value, sizeof(T)); + } +} + +void TUncheckedSkiffWriter::Finish() +{ + Flush(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TCheckedSkiffWriter::TCheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyOutput* underlying) + : Writer_(underlying) + , Validator_(std::make_unique<TSkiffValidator>(schema)) +{ } + +TCheckedSkiffWriter::TCheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IOutputStream* underlying) + : Writer_(underlying) + , Validator_(std::make_unique<TSkiffValidator>(schema)) +{ } + +TCheckedSkiffWriter::~TCheckedSkiffWriter() = default; + +void TCheckedSkiffWriter::WriteDouble(double value) +{ + Validator_->OnSimpleType(EWireType::Double); + Writer_.WriteDouble(value); +} + +void TCheckedSkiffWriter::WriteBoolean(bool value) +{ + Validator_->OnSimpleType(EWireType::Boolean); + Writer_.WriteBoolean(value); +} + +void TCheckedSkiffWriter::WriteInt8(i8 value) +{ + Validator_->OnSimpleType(EWireType::Int8); + Writer_.WriteInt8(value); +} + +void TCheckedSkiffWriter::WriteInt16(i16 value) +{ + Validator_->OnSimpleType(EWireType::Int16); + Writer_.WriteInt16(value); +} + +void TCheckedSkiffWriter::WriteInt32(i32 value) +{ + Validator_->OnSimpleType(EWireType::Int32); + Writer_.WriteInt32(value); +} + +void TCheckedSkiffWriter::WriteInt64(i64 value) +{ + Validator_->OnSimpleType(EWireType::Int64); + Writer_.WriteInt64(value); +} + +void TCheckedSkiffWriter::WriteUint8(ui8 value) +{ + Validator_->OnSimpleType(EWireType::Uint8); + Writer_.WriteUint8(value); +} + +void TCheckedSkiffWriter::WriteUint16(ui16 value) +{ + Validator_->OnSimpleType(EWireType::Uint16); + Writer_.WriteUint16(value); +} + +void TCheckedSkiffWriter::WriteUint32(ui32 value) +{ + Validator_->OnSimpleType(EWireType::Uint32); + Writer_.WriteUint32(value); +} + +void TCheckedSkiffWriter::WriteUint64(ui64 value) +{ + Validator_->OnSimpleType(EWireType::Uint64); + Writer_.WriteUint64(value); +} + +void TCheckedSkiffWriter::WriteInt128(TInt128 value) +{ + Validator_->OnSimpleType(EWireType::Int128); + Writer_.WriteInt128(value); +} + +void TCheckedSkiffWriter::WriteUint128(TUint128 value) +{ + Validator_->OnSimpleType(EWireType::Uint128); + Writer_.WriteUint128(value); +} + +void TCheckedSkiffWriter::WriteString32(TStringBuf value) +{ + Validator_->OnSimpleType(EWireType::String32); + Writer_.WriteString32(value); +} + +void TCheckedSkiffWriter::WriteYson32(TStringBuf value) +{ + Validator_->OnSimpleType(EWireType::Yson32); + Writer_.WriteYson32(value); +} + +void TCheckedSkiffWriter::WriteVariant8Tag(ui8 tag) +{ + Validator_->OnVariant8Tag(tag); + Writer_.WriteVariant8Tag(tag); +} + +void TCheckedSkiffWriter::WriteVariant16Tag(ui16 tag) +{ + Validator_->OnVariant16Tag(tag); + Writer_.WriteVariant16Tag(tag); +} + +void TCheckedSkiffWriter::Flush() +{ + Writer_.Flush(); +} + +void TCheckedSkiffWriter::Finish() +{ + Validator_->ValidateFinished(); + Writer_.Finish(); +} + +//////////////////////////////////////////////////////////////////// + +} // namespace NSkiff diff --git a/library/cpp/skiff/skiff.h b/library/cpp/skiff/skiff.h new file mode 100644 index 0000000000..183c112700 --- /dev/null +++ b/library/cpp/skiff/skiff.h @@ -0,0 +1,259 @@ +#pragma once + +#include "public.h" + +#include "zerocopy_output_writer.h" + +#include <util/generic/buffer.h> +#include <util/generic/yexception.h> + +#include <util/stream/input.h> +#include <util/stream/output.h> + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +class TSkiffException + : public yexception +{ }; + +//////////////////////////////////////////////////////////////////////////////// + +template <typename T> +constexpr T EndOfSequenceTag() +{ + static_assert(std::is_integral<T>::value && std::is_unsigned<T>::value, "T must be unsigned integer"); + return T(-1); +} + +//////////////////////////////////////////////////////////////////////////////// + +struct TInt128 +{ + ui64 Low = 0; + i64 High = 0; +}; + +struct TUint128 +{ + ui64 Low = 0; + ui64 High = 0; +}; + +bool operator==(TInt128 lhs, TInt128 rhs); +bool operator!=(TInt128 lhs, TInt128 rhs); + +bool operator==(TUint128 lhs, TUint128 rhs); +bool operator!=(TUint128 lhs, TUint128 rhs); + +//////////////////////////////////////////////////////////////////////////////// + +class TUncheckedSkiffParser +{ +public: + explicit TUncheckedSkiffParser(IZeroCopyInput* stream); + TUncheckedSkiffParser(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyInput* stream); + + i8 ParseInt8(); + i16 ParseInt16(); + i32 ParseInt32(); + i64 ParseInt64(); + + ui8 ParseUint8(); + ui16 ParseUint16(); + ui32 ParseUint32(); + ui64 ParseUint64(); + + TInt128 ParseInt128(); + TUint128 ParseUint128(); + + double ParseDouble(); + + bool ParseBoolean(); + + TStringBuf ParseString32(); + + TStringBuf ParseYson32(); + + ui8 ParseVariant8Tag(); + ui16 ParseVariant16Tag(); + + bool HasMoreData(); + + void ValidateFinished(); + + ui64 GetReadBytesCount() const; + +private: + const void* GetData(size_t size); + const void* GetDataViaBuffer(size_t size); + + size_t RemainingBytes() const; + void Advance(size_t size); + void RefillBuffer(); + + template <typename T> + T ParseSimple(); + +private: + IZeroCopyInput* const Underlying_; + + TBuffer Buffer_; + ui64 ReadBytesCount_ = 0; + char* Position_ = nullptr; + char* End_ = nullptr; + bool Exhausted_ = false; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TCheckedSkiffParser +{ +public: + TCheckedSkiffParser(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyInput* stream); + ~TCheckedSkiffParser(); + + i8 ParseInt8(); + i16 ParseInt16(); + i32 ParseInt32(); + i64 ParseInt64(); + + ui8 ParseUint8(); + ui16 ParseUint16(); + ui32 ParseUint32(); + ui64 ParseUint64(); + + TInt128 ParseInt128(); + TUint128 ParseUint128(); + + double ParseDouble(); + + bool ParseBoolean(); + + TStringBuf ParseString32(); + + TStringBuf ParseYson32(); + + ui8 ParseVariant8Tag(); + ui16 ParseVariant16Tag(); + + bool HasMoreData(); + + void ValidateFinished(); + + ui64 GetReadBytesCount() const; + +private: + TUncheckedSkiffParser Parser_; + std::unique_ptr<TSkiffValidator> Validator_; +}; + +//////////////////////////////////////////////////////////////////// + +class TUncheckedSkiffWriter +{ +public: + explicit TUncheckedSkiffWriter(IZeroCopyOutput* underlying); + explicit TUncheckedSkiffWriter(IOutputStream* underlying); + TUncheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyOutput* underlying); + TUncheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IOutputStream* underlying); + + ~TUncheckedSkiffWriter(); + + void WriteDouble(double value); + void WriteBoolean(bool value); + + void WriteInt8(i8 value); + void WriteInt16(i16 value); + void WriteInt32(i32 value); + void WriteInt64(i64 value); + + void WriteUint8(ui8 value); + void WriteUint16(ui16 value); + void WriteUint32(ui32 value); + void WriteUint64(ui64 value); + + void WriteInt128(TInt128 value); + void WriteUint128(TUint128 value); + + void WriteString32(TStringBuf value); + + void WriteYson32(TStringBuf value); + + void WriteVariant8Tag(ui8 tag); + void WriteVariant16Tag(ui16 tag); + + void Flush(); + void Finish(); + +private: + + template <typename T> + void WriteSimple(T data); + +private: + THolder<TBufferedOutput> BufferedOutput_; + TZeroCopyOutputStreamWriter Underlying_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TCheckedSkiffWriter +{ +public: + TCheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyOutput* underlying); + TCheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IOutputStream* underlying); + + ~TCheckedSkiffWriter(); + + void WriteInt8(i8 value); + void WriteInt16(i16 value); + void WriteInt32(i32 value); + void WriteInt64(i64 value); + + void WriteUint8(ui8 value); + void WriteUint16(ui16 value); + void WriteUint32(ui32 value); + void WriteUint64(ui64 value); + + void WriteDouble(double value); + void WriteBoolean(bool value); + + void WriteInt128(TInt128 value); + void WriteUint128(TUint128 value); + + void WriteString32(TStringBuf value); + + void WriteYson32(TStringBuf value); + + void WriteVariant8Tag(ui8 tag); + void WriteVariant16Tag(ui16 tag); + + void Flush(); + void Finish(); + +private: + TUncheckedSkiffWriter Writer_; + std::unique_ptr<TSkiffValidator> Validator_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <EWireType wireType> +class TUnderlyingIntegerType { +private: + TUnderlyingIntegerType() = default; + static constexpr auto F(); + +public: + using TValue = decltype(TUnderlyingIntegerType::F()); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff + +#define SKIFF_H +#include "skiff-inl.h" +#undef SKIFF_H diff --git a/library/cpp/skiff/skiff_schema-inl.h b/library/cpp/skiff/skiff_schema-inl.h new file mode 100644 index 0000000000..d66325b222 --- /dev/null +++ b/library/cpp/skiff/skiff_schema-inl.h @@ -0,0 +1,61 @@ +#pragma once + +#ifndef SKIFF_SCHEMA_H +#error "Direct inclusion of this file is not allowed, include skiff_schema.h" +// For the sake of sane code completion. +#include "skiff_schema.h" +#endif +#undef SKIFF_SCHEMA_H + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +inline bool IsSimpleType(EWireType type) +{ + switch (type) { + case EWireType::Int8: + case EWireType::Int16: + case EWireType::Int32: + case EWireType::Int64: + case EWireType::Int128: + + case EWireType::Uint8: + case EWireType::Uint16: + case EWireType::Uint32: + case EWireType::Uint64: + case EWireType::Uint128: + + case EWireType::Double: + case EWireType::Boolean: + case EWireType::String32: + case EWireType::Yson32: + case EWireType::Nothing: + return true; + case EWireType::Tuple: + case EWireType::Variant8: + case EWireType::Variant16: + case EWireType::RepeatedVariant8: + case EWireType::RepeatedVariant16: + return false; + } + Y_FAIL(); +} + +//////////////////////////////////////////////////////////////////////////////// + +template <EWireType WireType> +TComplexSchema<WireType>::TComplexSchema(TSkiffSchemaList elements) + : TSkiffSchema(WireType) + , Elements_(std::move(elements)) +{ } + +template <EWireType WireType> +const TSkiffSchemaList& TComplexSchema<WireType>::GetChildren() const +{ + return Elements_; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff diff --git a/library/cpp/skiff/skiff_schema.cpp b/library/cpp/skiff/skiff_schema.cpp new file mode 100644 index 0000000000..c762896ad0 --- /dev/null +++ b/library/cpp/skiff/skiff_schema.cpp @@ -0,0 +1,164 @@ +#include "skiff_schema.h" + +#include "skiff.h" + +#include <util/generic/hash.h> + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +bool operator==(const TSkiffSchema& lhs, const TSkiffSchema& rhs) +{ + if (lhs.GetWireType() != rhs.GetWireType() || lhs.GetName() != rhs.GetName()) { + return false; + } + const auto& lhsChildren = lhs.GetChildren(); + const auto& rhsChildren = rhs.GetChildren(); + return std::equal( + std::begin(lhsChildren), + std::end(lhsChildren), + std::begin(rhsChildren), + std::end(rhsChildren), + TSkiffSchemaPtrEqual()); +} + +bool operator!=(const TSkiffSchema& lhs, const TSkiffSchema& rhs) +{ + return !(lhs == rhs); +} + +//////////////////////////////////////////////////////////////////////////////// + +void PrintShortDebugString(const std::shared_ptr<const TSkiffSchema>& schema, IOutputStream* out) +{ + (*out) << ToString(schema->GetWireType()); + if (!IsSimpleType(schema->GetWireType())) { + auto children = schema->GetChildren(); + if (!children.empty()) { + (*out) << '<'; + for (const auto& child : children) { + PrintShortDebugString(child, out); + (*out) << ';'; + } + (*out) << '>'; + } + } +} + +TString GetShortDebugString(const std::shared_ptr<const TSkiffSchema>& schema) +{ + TStringStream out; + PrintShortDebugString(schema, &out); + return out.Str(); +} + +std::shared_ptr<TSimpleTypeSchema> CreateSimpleTypeSchema(EWireType type) +{ + return std::make_shared<TSimpleTypeSchema>(type); +} + +static void VerifyNonemptyChildren(const TSkiffSchemaList& children, EWireType wireType) +{ + if (children.empty()) { + ythrow TSkiffException() << "\"" << ToString(wireType) << "\" must have at least one child"; + } +} + +std::shared_ptr<TTupleSchema> CreateTupleSchema(TSkiffSchemaList children) +{ + return std::make_shared<TTupleSchema>(std::move(children)); +} + +std::shared_ptr<TVariant8Schema> CreateVariant8Schema(TSkiffSchemaList children) +{ + VerifyNonemptyChildren(children, EWireType::Variant8); + return std::make_shared<TVariant8Schema>(std::move(children)); +} + +std::shared_ptr<TVariant16Schema> CreateVariant16Schema(TSkiffSchemaList children) +{ + VerifyNonemptyChildren(children, EWireType::Variant16); + return std::make_shared<TVariant16Schema>(std::move(children)); +} + +std::shared_ptr<TRepeatedVariant8Schema> CreateRepeatedVariant8Schema(TSkiffSchemaList children) +{ + VerifyNonemptyChildren(children, EWireType::RepeatedVariant8); + return std::make_shared<TRepeatedVariant8Schema>(std::move(children)); +} + +std::shared_ptr<TRepeatedVariant16Schema> CreateRepeatedVariant16Schema(TSkiffSchemaList children) +{ + VerifyNonemptyChildren(children, EWireType::RepeatedVariant16); + return std::make_shared<TRepeatedVariant16Schema>(std::move(children)); +} + +//////////////////////////////////////////////////////////////////////////////// + +TSkiffSchema::TSkiffSchema(EWireType type) + : Type_(type) +{ } + +EWireType TSkiffSchema::GetWireType() const +{ + return Type_; +} + +std::shared_ptr<TSkiffSchema> TSkiffSchema::SetName(TString name) +{ + Name_ = std::move(name); + return shared_from_this(); +} + +const TString& TSkiffSchema::GetName() const +{ + return Name_; +} + +const TSkiffSchemaList& TSkiffSchema::GetChildren() const +{ + static const TSkiffSchemaList children; + return children; +} + +//////////////////////////////////////////////////////////////////////////////// + +TSimpleTypeSchema::TSimpleTypeSchema(EWireType type) + : TSkiffSchema(type) +{ + Y_VERIFY(IsSimpleType(type)); +} + +//////////////////////////////////////////////////////////////////////////////// + +size_t TSkiffSchemaPtrHasher::operator()(const std::shared_ptr<TSkiffSchema>& schema) const +{ + return THash<NSkiff::TSkiffSchema>()(*schema); +} + +size_t TSkiffSchemaPtrEqual::operator()( + const std::shared_ptr<TSkiffSchema>& lhs, + const std::shared_ptr<TSkiffSchema>& rhs) const +{ + return *lhs == *rhs; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff + +//////////////////////////////////////////////////////////////////////////////// + +size_t THash<NSkiff::TSkiffSchema>::operator()(const NSkiff::TSkiffSchema &schema) const +{ + auto hash = CombineHashes( + THash<TString>()(schema.GetName()), + static_cast<size_t>(schema.GetWireType())); + for (const auto& child : schema.GetChildren()) { + hash = CombineHashes(hash, (*this)(*child)); + } + return hash; +} + +//////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/skiff/skiff_schema.h b/library/cpp/skiff/skiff_schema.h new file mode 100644 index 0000000000..8952a84bac --- /dev/null +++ b/library/cpp/skiff/skiff_schema.h @@ -0,0 +1,121 @@ +#pragma once + +#include "public.h" + +#include <util/generic/string.h> +#include <util/string/cast.h> + +#include <vector> + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +template <EWireType WireType> +class TComplexSchema; + +using TTupleSchema = TComplexSchema<EWireType::Tuple>; +using TVariant8Schema = TComplexSchema<EWireType::Variant8>; +using TVariant16Schema = TComplexSchema<EWireType::Variant16>; +using TRepeatedVariant8Schema = TComplexSchema<EWireType::RepeatedVariant8>; +using TRepeatedVariant16Schema = TComplexSchema<EWireType::RepeatedVariant16>; + +using TTupleSchemaPtr = std::shared_ptr<TTupleSchema>; +using TVariant8SchemaPtr = std::shared_ptr<TVariant8Schema>; +using TVariant16SchemaPtr = std::shared_ptr<TVariant16Schema>; +using TRepeatedVariant8SchemaPtr = std::shared_ptr<TRepeatedVariant8Schema>; +using TRepeatedVariant16SchemaPtr = std::shared_ptr<TRepeatedVariant16Schema>; + + + +//////////////////////////////////////////////////////////////////////////////// + +class TSkiffSchema + : public std::enable_shared_from_this<TSkiffSchema> +{ +public: + virtual ~TSkiffSchema() = default; + + EWireType GetWireType() const; + std::shared_ptr<TSkiffSchema> SetName(TString name); + const TString& GetName() const; + + virtual const TSkiffSchemaList& GetChildren() const; + +protected: + explicit TSkiffSchema(EWireType type); + +private: + const EWireType Type_; + TString Name_; +}; + +bool operator==(const TSkiffSchema& lhs, const TSkiffSchema& rhs); +bool operator!=(const TSkiffSchema& lhs, const TSkiffSchema& rhs); + +//////////////////////////////////////////////////////////////////////////////// + +class TSimpleTypeSchema + : public TSkiffSchema +{ +public: + explicit TSimpleTypeSchema(EWireType type); +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <EWireType WireType> +class TComplexSchema + : public TSkiffSchema +{ +public: + explicit TComplexSchema(TSkiffSchemaList elements); + + virtual const TSkiffSchemaList& GetChildren() const override; + +private: + const TSkiffSchemaList Elements_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +bool IsSimpleType(EWireType type); +TString GetShortDebugString(const std::shared_ptr<const TSkiffSchema>& schema); +void PrintShortDebugString(const std::shared_ptr<const TSkiffSchema>& schema, IOutputStream* out); + +std::shared_ptr<TSimpleTypeSchema> CreateSimpleTypeSchema(EWireType type); +std::shared_ptr<TTupleSchema> CreateTupleSchema(TSkiffSchemaList children); +std::shared_ptr<TVariant8Schema> CreateVariant8Schema(TSkiffSchemaList children); +std::shared_ptr<TVariant16Schema> CreateVariant16Schema(TSkiffSchemaList children); +std::shared_ptr<TRepeatedVariant8Schema> CreateRepeatedVariant8Schema(TSkiffSchemaList children); +std::shared_ptr<TRepeatedVariant16Schema> CreateRepeatedVariant16Schema(TSkiffSchemaList children); + +//////////////////////////////////////////////////////////////////////////////// + +struct TSkiffSchemaPtrHasher +{ + size_t operator()(const std::shared_ptr<TSkiffSchema>& schema) const; +}; + +struct TSkiffSchemaPtrEqual +{ + size_t operator()( + const std::shared_ptr<TSkiffSchema>& lhs, + const std::shared_ptr<TSkiffSchema>& rhs) const; +}; + +} // namespace NSkiff + +//////////////////////////////////////////////////////////////////////////////// + +template <> +struct THash<NSkiff::TSkiffSchema> +{ + size_t operator()(const NSkiff::TSkiffSchema& schema) const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +#define SKIFF_SCHEMA_H +#include "skiff_schema-inl.h" +#undef SKIFF_SCHEMA_H diff --git a/library/cpp/skiff/skiff_validator.cpp b/library/cpp/skiff/skiff_validator.cpp new file mode 100644 index 0000000000..1b1b98d5a6 --- /dev/null +++ b/library/cpp/skiff/skiff_validator.cpp @@ -0,0 +1,396 @@ +#include "skiff.h" +#include "skiff_validator.h" + +#include <vector> +#include <stack> + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +struct IValidatorNode; + +using TValidatorNodeList = std::vector<std::shared_ptr<IValidatorNode>>; +using TSkiffSchemaList = std::vector<std::shared_ptr<TSkiffSchema>>; + +static std::shared_ptr<IValidatorNode> CreateUsageValidatorNode(const std::shared_ptr<TSkiffSchema>& skiffSchema); +static TValidatorNodeList CreateUsageValidatorNodeList(const TSkiffSchemaList& skiffSchemaList); + +//////////////////////////////////////////////////////////////////////////////// + +template <typename T> +inline void ThrowUnexpectedParseWrite(T wireType) +{ + ythrow TSkiffException() << "Unexpected parse/write of \"" << ::ToString(wireType) << "\" token"; +} + +//////////////////////////////////////////////////////////////////////////////// + +struct IValidatorNode +{ + virtual ~IValidatorNode() = default; + + virtual void OnBegin(TValidatorNodeStack* /*validatorNodeStack*/) + { } + + virtual void OnChildDone(TValidatorNodeStack* /*validatorNodeStack*/) + { + Y_FAIL(); + } + + virtual void OnSimpleType(TValidatorNodeStack* /*validatorNodeStack*/, EWireType wireType) + { + ThrowUnexpectedParseWrite(wireType); + } + + virtual void BeforeVariant8Tag() + { + ThrowUnexpectedParseWrite(EWireType::Variant8); + } + + virtual void OnVariant8Tag(TValidatorNodeStack* /*validatorNodeStack*/, ui8 /*tag*/) + { + IValidatorNode::BeforeVariant8Tag(); + } + + virtual void BeforeVariant16Tag() + { + ThrowUnexpectedParseWrite(EWireType::Variant16); + } + + virtual void OnVariant16Tag(TValidatorNodeStack* /*validatorNodeStack*/, ui16 /*tag*/) + { + IValidatorNode::BeforeVariant16Tag(); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TValidatorNodeStack +{ +public: + explicit TValidatorNodeStack(std::shared_ptr<IValidatorNode> validator) + : RootValidator_(std::move(validator)) + { } + + void PushValidator(IValidatorNode* validator) + { + ValidatorStack_.push(validator); + validator->OnBegin(this); + } + + void PopValidator() + { + Y_VERIFY(!ValidatorStack_.empty()); + ValidatorStack_.pop(); + if (!ValidatorStack_.empty()) { + ValidatorStack_.top()->OnChildDone(this); + } + } + + void PushRootIfRequired() + { + if (ValidatorStack_.empty()) { + PushValidator(RootValidator_.get()); + } + } + + IValidatorNode* Top() const + { + Y_VERIFY(!ValidatorStack_.empty()); + return ValidatorStack_.top(); + } + + bool IsFinished() const + { + return ValidatorStack_.empty(); + } + +private: + const std::shared_ptr<IValidatorNode> RootValidator_; + std::stack<IValidatorNode*> ValidatorStack_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TNothingTypeValidator + : public IValidatorNode +{ +public: + void OnBegin(TValidatorNodeStack* validatorNodeStack) override + { + validatorNodeStack->PopValidator(); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TSimpleTypeUsageValidator + : public IValidatorNode +{ +public: + explicit TSimpleTypeUsageValidator(EWireType type) + : Type_(type) + { } + + void OnSimpleType(TValidatorNodeStack* validatorNodeStack, EWireType type) override + { + if (type != Type_) { + ThrowUnexpectedParseWrite(type); + } + validatorNodeStack->PopValidator(); + } + +private: + const EWireType Type_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <typename TTag> +void ValidateVariantTag(TValidatorNodeStack* validatorNodeStack, TTag tag, const TValidatorNodeList& children) +{ + if (tag == EndOfSequenceTag<TTag>()) { + // Root validator is pushed into the stack before variant tag + // if the stack is empty. + validatorNodeStack->PopValidator(); + } else if (tag >= children.size()) { + ythrow TSkiffException() << "Variant tag \"" << tag << "\" " + << "exceeds number of children \"" << children.size(); + } else { + validatorNodeStack->PushValidator(children[tag].get()); + } +} + +class TVariant8TypeUsageValidator + : public IValidatorNode +{ +public: + explicit TVariant8TypeUsageValidator(TValidatorNodeList children) + : Children_(std::move(children)) + { } + + void BeforeVariant8Tag() override + { } + + void OnVariant8Tag(TValidatorNodeStack* validatorNodeStack, ui8 tag) override + { + ValidateVariantTag(validatorNodeStack, tag, Children_); + } + + void OnChildDone(TValidatorNodeStack* validatorNodeStack) override + { + validatorNodeStack->PopValidator(); + } + +private: + const TValidatorNodeList Children_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TVariant16TypeUsageValidator + : public IValidatorNode +{ +public: + explicit TVariant16TypeUsageValidator(TValidatorNodeList children) + : Children_(std::move(children)) + { } + + void BeforeVariant16Tag() override + { } + + void OnVariant16Tag(TValidatorNodeStack* validatorNodeStack, ui16 tag) override + { + ValidateVariantTag(validatorNodeStack, tag, Children_); + } + + void OnChildDone(TValidatorNodeStack* validatorNodeStack) override + { + validatorNodeStack->PopValidator(); + } + +private: + const TValidatorNodeList Children_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TRepeatedVariant8TypeUsageValidator + : public IValidatorNode +{ +public: + explicit TRepeatedVariant8TypeUsageValidator(TValidatorNodeList children) + : Children_(std::move(children)) + { } + + void BeforeVariant8Tag() override + { } + + void OnVariant8Tag(TValidatorNodeStack* validatorNodeStack, ui8 tag) override + { + ValidateVariantTag(validatorNodeStack, tag, Children_); + } + + void OnChildDone(TValidatorNodeStack* /*validatorNodeStack*/) override + { } + +private: + const TValidatorNodeList Children_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TRepeatedVariant16TypeUsageValidator + : public IValidatorNode +{ +public: + explicit TRepeatedVariant16TypeUsageValidator(TValidatorNodeList children) + : Children_(std::move(children)) + { } + + void BeforeVariant16Tag() override + { } + + void OnVariant16Tag(TValidatorNodeStack* validatorNodeStack, ui16 tag) override + { + ValidateVariantTag(validatorNodeStack, tag, Children_); + } + + void OnChildDone(TValidatorNodeStack* /*validatorNodeStack*/) override + { } + +private: + const TValidatorNodeList Children_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TTupleTypeUsageValidator + : public IValidatorNode +{ +public: + explicit TTupleTypeUsageValidator(TValidatorNodeList children) + : Children_(std::move(children)) + { } + + void OnBegin(TValidatorNodeStack* validatorNodeStack) override + { + Position_ = 0; + if (!Children_.empty()) { + validatorNodeStack->PushValidator(Children_[0].get()); + } + } + + void OnChildDone(TValidatorNodeStack* validatorNodeStack) override + { + Position_++; + if (Position_ < Children_.size()) { + validatorNodeStack->PushValidator(Children_[Position_].get()); + } else { + validatorNodeStack->PopValidator(); + } + } + +private: + const TValidatorNodeList Children_; + ui32 Position_ = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TSkiffValidator::TSkiffValidator(std::shared_ptr<TSkiffSchema> skiffSchema) + : Context_(std::make_unique<TValidatorNodeStack>(CreateUsageValidatorNode(std::move(skiffSchema)))) +{ } + +TSkiffValidator::~TSkiffValidator() +{ } + +void TSkiffValidator::BeforeVariant8Tag() +{ + Context_->PushRootIfRequired(); + Context_->Top()->BeforeVariant8Tag(); +} + +void TSkiffValidator::OnVariant8Tag(ui8 tag) +{ + Context_->PushRootIfRequired(); + Context_->Top()->OnVariant8Tag(Context_.get(), tag); +} + +void TSkiffValidator::BeforeVariant16Tag() +{ + Context_->PushRootIfRequired(); + Context_->Top()->BeforeVariant16Tag(); +} + +void TSkiffValidator::OnVariant16Tag(ui16 tag) +{ + Context_->PushRootIfRequired(); + Context_->Top()->OnVariant16Tag(Context_.get(), tag); +} + +void TSkiffValidator::OnSimpleType(EWireType value) +{ + Context_->PushRootIfRequired(); + Context_->Top()->OnSimpleType(Context_.get(), value); +} + +void TSkiffValidator::ValidateFinished() +{ + if (!Context_->IsFinished()) { + ythrow TSkiffException() << "Parse/write is not finished"; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TValidatorNodeList CreateUsageValidatorNodeList(const TSkiffSchemaList& skiffSchemaList) +{ + TValidatorNodeList result; + result.reserve(skiffSchemaList.size()); + for (const auto& skiffSchema : skiffSchemaList) { + result.push_back(CreateUsageValidatorNode(skiffSchema)); + } + return result; +} + +std::shared_ptr<IValidatorNode> CreateUsageValidatorNode(const std::shared_ptr<TSkiffSchema>& skiffSchema) +{ + switch (skiffSchema->GetWireType()) { + case EWireType::Int8: + case EWireType::Int16: + case EWireType::Int32: + case EWireType::Int64: + case EWireType::Int128: + + case EWireType::Uint8: + case EWireType::Uint16: + case EWireType::Uint32: + case EWireType::Uint64: + case EWireType::Uint128: + + case EWireType::Double: + case EWireType::Boolean: + case EWireType::String32: + case EWireType::Yson32: + return std::make_shared<TSimpleTypeUsageValidator>(skiffSchema->GetWireType()); + case EWireType::Nothing: + return std::make_shared<TNothingTypeValidator>(); + case EWireType::Tuple: + return std::make_shared<TTupleTypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren())); + case EWireType::Variant8: + return std::make_shared<TVariant8TypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren())); + case EWireType::Variant16: + return std::make_shared<TVariant16TypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren())); + case EWireType::RepeatedVariant8: + return std::make_shared<TRepeatedVariant8TypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren())); + case EWireType::RepeatedVariant16: + return std::make_shared<TRepeatedVariant16TypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren())); + } + Y_FAIL(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff diff --git a/library/cpp/skiff/skiff_validator.h b/library/cpp/skiff/skiff_validator.h new file mode 100644 index 0000000000..522cc74db6 --- /dev/null +++ b/library/cpp/skiff/skiff_validator.h @@ -0,0 +1,39 @@ +#pragma once + +#include "public.h" + +#include "skiff_schema.h" + +#include <util/string/cast.h> + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +class TValidatorNodeStack; + +//////////////////////////////////////////////////////////////////////////////// + +class TSkiffValidator +{ +public: + explicit TSkiffValidator(std::shared_ptr<TSkiffSchema> skiffSchema); + ~TSkiffValidator(); + + void BeforeVariant8Tag(); + void OnVariant8Tag(ui8 tag); + + void BeforeVariant16Tag(); + void OnVariant16Tag(ui16 tag); + + void OnSimpleType(EWireType value); + + void ValidateFinished(); + +private: + const std::unique_ptr<TValidatorNodeStack> Context_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff diff --git a/library/cpp/skiff/unittests/skiff_schema_ut.cpp b/library/cpp/skiff/unittests/skiff_schema_ut.cpp new file mode 100644 index 0000000000..c20a560dfc --- /dev/null +++ b/library/cpp/skiff/unittests/skiff_schema_ut.cpp @@ -0,0 +1,148 @@ +#include <library/cpp/skiff/skiff.h> +#include <library/cpp/skiff/skiff_schema.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/buffer.h> + +using namespace NSkiff; + +//////////////////////////////////////////////////////////////////////////////// + +template<> +void Out<TSkiffSchema>(IOutputStream& s, const TSkiffSchema& schema) +{ + s << "TSkiffSchema:" << GetShortDebugString(schema.shared_from_this()); +} + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TSkiffSchemaTestSuite) { + Y_UNIT_TEST(TestIntEqual) + { + std::shared_ptr<TSkiffSchema> schema1 = CreateSimpleTypeSchema(EWireType::Uint64); + schema1->SetName("schema"); + + std::shared_ptr<TSkiffSchema> schema2 = CreateSimpleTypeSchema(EWireType::Uint64); + schema2->SetName("schema"); + + UNIT_ASSERT_VALUES_EQUAL(*schema1, *schema2); + } + + Y_UNIT_TEST(TestTupleEqual) + { + std::shared_ptr<TSkiffSchema> schema1 = CreateTupleSchema({ + CreateSimpleTypeSchema(EWireType::Int64), + CreateSimpleTypeSchema(EWireType::String32), + }); + + std::shared_ptr<TSkiffSchema> schema2 = CreateTupleSchema({ + CreateSimpleTypeSchema(EWireType::Int64), + CreateSimpleTypeSchema(EWireType::String32), + }); + + Cerr << *schema1 << Endl; + + schema1->SetName("schema"); + UNIT_ASSERT_VALUES_UNEQUAL(*schema1, *schema2); + + schema2->SetName("schema"); + UNIT_ASSERT_VALUES_EQUAL(*schema1, *schema2); + } + + Y_UNIT_TEST(TestHashes) + { + TSet<size_t> hashes; + + auto schema = CreateSimpleTypeSchema(EWireType::Uint64); + schema->SetName("schema"); + hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema)); + + schema = CreateSimpleTypeSchema(EWireType::Uint64); + hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema)); + + auto schema2 = CreateTupleSchema({ + CreateSimpleTypeSchema(EWireType::Int64), + CreateSimpleTypeSchema(EWireType::String32), + }); + schema2->SetName("s"); + hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema2)); + + schema2->SetName("s0"); + hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema2)); + + schema2->SetName("s"); + hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema2)); + + auto schema3 = CreateRepeatedVariant16Schema({ + CreateSimpleTypeSchema(EWireType::Int64), + schema2, + }); + hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema3)); + + schema3->SetName("kek"); + hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema3)); + + auto schema4 = CreateRepeatedVariant8Schema({ + CreateSimpleTypeSchema(EWireType::Int64), + schema2, + }); + hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema4)); + + schema4->SetName("kek"); + hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema4)); + + UNIT_ASSERT_VALUES_EQUAL(hashes.size(), 8); + } + + Y_UNIT_TEST(TestDifferent) + { + TVector<std::shared_ptr<TSkiffSchema>> schemas; + + auto schema = CreateSimpleTypeSchema(EWireType::Uint64); + schema->SetName("schema"); + schemas.push_back(schema); + schemas.push_back(CreateSimpleTypeSchema(EWireType::Uint64)); + + auto schema2 = CreateTupleSchema({ + CreateSimpleTypeSchema(EWireType::Int64), + CreateSimpleTypeSchema(EWireType::String32), + }); + schema2->SetName("s"); + schemas.push_back(schema2); + + auto schema3 = CreateTupleSchema({ + CreateSimpleTypeSchema(EWireType::Int64), + CreateSimpleTypeSchema(EWireType::String32), + }); + schema3->SetName("s0"); + schemas.push_back(schema3); + + auto schema4 = CreateRepeatedVariant16Schema({ + CreateSimpleTypeSchema(EWireType::Int64), + schema2, + }); + schemas.push_back(schema4); + + auto schema5 = CreateRepeatedVariant16Schema({ + CreateSimpleTypeSchema(EWireType::Int64), + schema2, + }); + schema5->SetName("kek"); + schemas.push_back(schema5); + + auto schema6 = CreateRepeatedVariant8Schema({ + CreateSimpleTypeSchema(EWireType::Int64), + schema2, + }); + schemas.push_back(schema6); + + for (size_t i = 0; i < schemas.size(); ++i) { + for (size_t j = i + 1; j < schemas.size(); ++j) { + UNIT_ASSERT_VALUES_UNEQUAL(*schemas[i], *schemas[j]); + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/skiff/unittests/skiff_ut.cpp b/library/cpp/skiff/unittests/skiff_ut.cpp new file mode 100644 index 0000000000..5e4c709611 --- /dev/null +++ b/library/cpp/skiff/unittests/skiff_ut.cpp @@ -0,0 +1,627 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include <library/cpp/skiff/skiff.h> +#include <library/cpp/skiff/skiff_schema.h> + +#include <util/stream/buffer.h> +#include <util/string/hex.h> + +using namespace NSkiff; + +//////////////////////////////////////////////////////////////////////////////// + +static TString HexEncode(const TBuffer& buffer) +{ + auto result = HexEncode(buffer.Data(), buffer.Size()); + result.to_lower(); + return result; +} + +Y_UNIT_TEST_SUITE(Skiff) +{ + Y_UNIT_TEST(TestInt8) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Int8); + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteInt8(42); + tokenWriter.WriteInt8(-42); + tokenWriter.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "2a" + "d6"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt8(), 42); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt8(), -42); + } + + Y_UNIT_TEST(TestInt16) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Int16); + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteInt16(0x1234); + tokenWriter.WriteInt16(-0x1234); + tokenWriter.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "3412" + "cced"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt16(), 0x1234); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt16(), -0x1234); + } + + Y_UNIT_TEST(TestInt32) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Int32); + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteInt32(0x12345678); + tokenWriter.WriteInt32(-0x12345678); + tokenWriter.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "78563412" + "88a9cbed"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt32(), 0x12345678); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt32(), -0x12345678); + } + + Y_UNIT_TEST(TestInt64) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Int64); + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteInt64(-42); + tokenWriter.WriteInt64(100500); + tokenWriter.WriteInt64(-0x123456789abcdef0); + tokenWriter.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "d6ffffffffffffff" + "9488010000000000" + "1021436587a9cbed"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), -42); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), 100500); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), -0x123456789abcdef0); + } + + Y_UNIT_TEST(TestUint8) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Uint8); + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteUint8(42); + tokenWriter.WriteUint8(200); + tokenWriter.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "2a" + "c8"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint8(), 42); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint8(), 200); + } + + Y_UNIT_TEST(TestUint16) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Uint16); + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteUint16(0x1234); + tokenWriter.WriteUint16(0xfedc); + tokenWriter.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "3412" + "dcfe"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint16(), 0x1234); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint16(), 0xfedc); + } + + Y_UNIT_TEST(TestUint32) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Uint32); + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteUint32(0x12345678); + tokenWriter.WriteUint32(0x87654321); + tokenWriter.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "78563412" + "21436587"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint32(), 0x12345678); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint32(), 0x87654321); + } + + + Y_UNIT_TEST(TestUint64) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Uint64); + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteUint64(42); + tokenWriter.WriteUint64(100500); + tokenWriter.WriteUint64(0x123456789abcdef0); + tokenWriter.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "2a00000000000000" + "9488010000000000" + "f0debc9a78563412"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 42); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 100500); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 0x123456789abcdef0); + } + + Y_UNIT_TEST(TestInt128) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Int128); + + const TInt128 val1 = {0x1924cd4aeb9ced82, 0x0885e83f456d6a7e}; + const TInt128 val2 = {0xe9ba36585eccae1a, -0x7854b6f9ce448be9}; + + TCheckedSkiffWriter writer(schema, &bufferStream); + writer.WriteInt128(val1); + writer.WriteInt128(val2); + writer.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "82ed9ceb4acd2419" "7e6a6d453fe88508" + "1aaecc5e5836bae9" "1774bb310649ab87"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_EQUAL(parser.ParseInt128(), val1); + UNIT_ASSERT_EQUAL(parser.ParseInt128(), val2); + } + + Y_UNIT_TEST(TestUint128) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Uint128); + + const auto val1 = TUint128{0x1924cd4aeb9ced82, 0x0885e83f456d6a7e}; + const auto val2 = TUint128{0xe9ba36585eccae1a, 0x8854b6f9ce448be9}; + + TCheckedSkiffWriter writer(schema, &bufferStream); + writer.WriteUint128(val1); + writer.WriteUint128(val2); + writer.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "82ed9ceb4acd2419" "7e6a6d453fe88508" + "1aaecc5e5836bae9" "e98b44cef9b65488"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_EQUAL(parser.ParseUint128(), val1); + UNIT_ASSERT_EQUAL(parser.ParseUint128(), val2); + } + + Y_UNIT_TEST(TestBoolean) + { + auto schema = CreateSimpleTypeSchema(EWireType::Boolean); + + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteBoolean(true); + tokenWriter.WriteBoolean(false); + tokenWriter.Finish(); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseBoolean(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseBoolean(), false); + + { + TBufferStream bufferStream; + bufferStream.Write('\x02'); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_EXCEPTION(parser.ParseBoolean(), std::exception); + } + } + + Y_UNIT_TEST(TestVariant8) + { + auto schema = CreateVariant8Schema({ + CreateSimpleTypeSchema(EWireType::Nothing), + CreateSimpleTypeSchema(EWireType::Uint64), + }); + + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + UNIT_ASSERT_EXCEPTION(tokenWriter.WriteUint64(42), std::exception); + } + + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteVariant8Tag(0); + UNIT_ASSERT_EXCEPTION(tokenWriter.WriteUint64(42), std::exception); + } + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteVariant8Tag(1); + UNIT_ASSERT_EXCEPTION(tokenWriter.WriteInt64(42), std::exception); + } + + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteVariant8Tag(0); + tokenWriter.WriteVariant8Tag(1); + tokenWriter.WriteUint64(42); + tokenWriter.Finish(); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 0); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 1); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 42); + + parser.ValidateFinished(); + } + + Y_UNIT_TEST(TestTuple) + { + + auto schema = CreateTupleSchema({ + CreateSimpleTypeSchema(EWireType::Int64), + CreateSimpleTypeSchema(EWireType::String32), + }); + + { + TBufferStream bufferStream; + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteInt64(42); + tokenWriter.WriteString32("foobar"); + tokenWriter.Finish(); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), 42); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseString32(), "foobar"); + parser.ValidateFinished(); + } + } + + Y_UNIT_TEST(TestString) + { + + auto schema = CreateSimpleTypeSchema(EWireType::String32); + + { + TBufferStream bufferStream; + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteString32("foo"); + tokenWriter.Finish(); + + TCheckedSkiffParser parser(schema, &bufferStream); + + UNIT_ASSERT_VALUES_EQUAL(parser.ParseString32(), "foo"); + + parser.ValidateFinished(); + } + + { + TBufferStream bufferStream; + + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + tokenWriter.WriteString32("foo"); + tokenWriter.Finish(); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_EXCEPTION(parser.ParseInt64(), std::exception); + } + } + + Y_UNIT_TEST(TestRepeatedVariant8) + { + + auto schema = CreateRepeatedVariant8Schema({ + CreateSimpleTypeSchema(EWireType::Int64), + CreateSimpleTypeSchema(EWireType::Uint64), + }); + + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + + // row 0 + tokenWriter.WriteVariant8Tag(0); + tokenWriter.WriteInt64(-8); + + // row 2 + tokenWriter.WriteVariant8Tag(1); + tokenWriter.WriteUint64(42); + + // end + tokenWriter.WriteVariant8Tag(EndOfSequenceTag<ui8>()); + + tokenWriter.Finish(); + + { + TBufferInput input(bufferStream.Buffer()); + TCheckedSkiffParser parser(schema, &input); + + // row 1 + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 0); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), -8); + + // row 2 + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 1); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 42); + + // end + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), EndOfSequenceTag<ui8>()); + + parser.ValidateFinished(); + } + + { + TBufferInput input(bufferStream.Buffer()); + TCheckedSkiffParser parser(schema, &input); + + UNIT_ASSERT_EXCEPTION(parser.ParseInt64(), std::exception); + } + + { + TBufferInput input(bufferStream.Buffer()); + TCheckedSkiffParser parser(schema, &input); + + parser.ParseVariant8Tag(); + UNIT_ASSERT_EXCEPTION(parser.ParseUint64(), std::exception); + } + + { + TBufferInput input(bufferStream.Buffer()); + TCheckedSkiffParser parser(schema, &input); + + parser.ParseVariant8Tag(); + parser.ParseInt64(); + + UNIT_ASSERT_EXCEPTION(parser.ValidateFinished(), std::exception); + } + } + + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + + tokenWriter.WriteVariant8Tag(0); + UNIT_ASSERT_EXCEPTION(tokenWriter.WriteUint64(5), std::exception); + } + + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + + tokenWriter.WriteVariant8Tag(1); + tokenWriter.WriteUint64(5); + + UNIT_ASSERT_EXCEPTION(tokenWriter.Finish(), std::exception); + } + } + + Y_UNIT_TEST(TestRepeatedVariant16) + { + + auto schema = CreateRepeatedVariant16Schema({ + CreateSimpleTypeSchema(EWireType::Int64), + CreateSimpleTypeSchema(EWireType::Uint64), + }); + + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + + // row 0 + tokenWriter.WriteVariant16Tag(0); + tokenWriter.WriteInt64(-8); + + // row 2 + tokenWriter.WriteVariant16Tag(1); + tokenWriter.WriteUint64(42); + + // end + tokenWriter.WriteVariant16Tag(EndOfSequenceTag<ui16>()); + + tokenWriter.Finish(); + + TCheckedSkiffParser parser(schema, &bufferStream); + + // row 1 + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 0); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), -8); + + // row 2 + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 1); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 42); + + // end + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), EndOfSequenceTag<ui16>()); + + parser.ValidateFinished(); + } + + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + + tokenWriter.WriteVariant16Tag(0); + UNIT_ASSERT_EXCEPTION(tokenWriter.WriteUint64(5), std::exception); + } + + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + + tokenWriter.WriteVariant16Tag(1); + tokenWriter.WriteUint64(5); + + UNIT_ASSERT_EXCEPTION(tokenWriter.Finish(), std::exception); + } + + { + TBufferStream bufferStream; + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + + // row 0 + tokenWriter.WriteVariant16Tag(0); + tokenWriter.WriteInt64(-8); + + // row 2 + tokenWriter.WriteVariant16Tag(1); + tokenWriter.WriteUint64(42); + + // end + tokenWriter.WriteVariant16Tag(EndOfSequenceTag<ui16>()); + + tokenWriter.Finish(); + + { + TBufferInput input(bufferStream.Buffer()); + TCheckedSkiffParser parser(schema, &input); + + UNIT_ASSERT_EXCEPTION(parser.ParseInt64(), std::exception); + } + + { + TBufferInput input(bufferStream.Buffer()); + TCheckedSkiffParser parser(schema, &input); + + parser.ParseVariant16Tag(); + UNIT_ASSERT_EXCEPTION(parser.ParseUint64(), std::exception); + } + + { + TBufferInput input(bufferStream.Buffer()); + TCheckedSkiffParser parser(schema, &input); + + parser.ParseVariant16Tag(); + parser.ParseInt64(); + + UNIT_ASSERT_EXCEPTION(parser.ValidateFinished(), std::exception); + } + } + } + + Y_UNIT_TEST(TestStruct) + { + TBufferStream bufferStream; + + auto schema = CreateRepeatedVariant16Schema( + { + CreateSimpleTypeSchema(EWireType::Nothing), + CreateTupleSchema({ + CreateVariant8Schema({ + CreateSimpleTypeSchema(EWireType::Nothing), + CreateSimpleTypeSchema(EWireType::Int64) + }), + CreateSimpleTypeSchema(EWireType::Uint64), + }) + } + ); + + { + TCheckedSkiffWriter tokenWriter(schema, &bufferStream); + + // row 0 + tokenWriter.WriteVariant16Tag(0); + + // row 1 + tokenWriter.WriteVariant16Tag(1); + tokenWriter.WriteVariant8Tag(0); + tokenWriter.WriteUint64(1); + + // row 2 + tokenWriter.WriteVariant16Tag(1); + tokenWriter.WriteVariant8Tag(1); + tokenWriter.WriteInt64(2); + tokenWriter.WriteUint64(3); + + // end + tokenWriter.WriteVariant16Tag(EndOfSequenceTag<ui16>()); + + tokenWriter.Finish(); + } + + TCheckedSkiffParser parser(schema, &bufferStream); + + // row 0 + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 0); + + // row 1 + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 1); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 0); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 1); + + // row 2 + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 1); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 1); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), 2); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 3); + + // end + UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), EndOfSequenceTag<ui16>()); + + parser.ValidateFinished(); + } + + Y_UNIT_TEST(TestSimpleOutputStream) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Int8); + + TCheckedSkiffWriter tokenWriter(schema, static_cast<IOutputStream*>(&bufferStream)); + tokenWriter.WriteInt8(42); + tokenWriter.WriteInt8(-42); + tokenWriter.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "2a" + "d6"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt8(), 42); + UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt8(), -42); + } +} + +//////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/skiff/unittests/ya.make b/library/cpp/skiff/unittests/ya.make new file mode 100644 index 0000000000..d67ca8c618 --- /dev/null +++ b/library/cpp/skiff/unittests/ya.make @@ -0,0 +1,12 @@ +UNITTEST() + +SRCS( + skiff_ut.cpp + skiff_schema_ut.cpp +) + +PEERDIR( + library/cpp/skiff +) + +END() diff --git a/library/cpp/skiff/ya.make b/library/cpp/skiff/ya.make new file mode 100644 index 0000000000..ff3eb55c9f --- /dev/null +++ b/library/cpp/skiff/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +SRCS( + skiff.cpp + skiff_schema.cpp + skiff_validator.cpp + zerocopy_output_writer.cpp +) + +GENERATE_ENUM_SERIALIZATION(public.h) + +END() + +RECURSE_FOR_TESTS( + unittests +) diff --git a/library/cpp/skiff/zerocopy_output_writer-inl.h b/library/cpp/skiff/zerocopy_output_writer-inl.h new file mode 100644 index 0000000000..6bd067c9fa --- /dev/null +++ b/library/cpp/skiff/zerocopy_output_writer-inl.h @@ -0,0 +1,51 @@ +#pragma once +#ifndef ZEROCOPY_OUTPUT_WRITER_INL_H_ +#error "Direct inclusion of this file is not allowed, include zerocopy_output_writer.h" +// For the sake of sane code completion. +#include "zerocopy_output_writer.h" +#endif + +#include <util/system/yassert.h> + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +char* TZeroCopyOutputStreamWriter::Current() const +{ + return Current_; +} + +ui64 TZeroCopyOutputStreamWriter::RemainingBytes() const +{ + return RemainingBytes_; +} + +void TZeroCopyOutputStreamWriter::Advance(size_t bytes) +{ + Y_VERIFY(bytes <= RemainingBytes_); + Current_ += bytes; + RemainingBytes_ -= bytes; +} + +void TZeroCopyOutputStreamWriter::Write(const void* buffer, size_t length) +{ + if (length > RemainingBytes_) { + UndoRemaining(); + Output_->Write(buffer, length); + TotalWrittenBlockSize_ += length; + ObtainNextBlock(); + } else { + memcpy(Current_, buffer, length); + Advance(length); + } +} + +ui64 TZeroCopyOutputStreamWriter::GetTotalWrittenSize() const +{ + return TotalWrittenBlockSize_ - RemainingBytes_; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff diff --git a/library/cpp/skiff/zerocopy_output_writer.cpp b/library/cpp/skiff/zerocopy_output_writer.cpp new file mode 100644 index 0000000000..49492b55a4 --- /dev/null +++ b/library/cpp/skiff/zerocopy_output_writer.cpp @@ -0,0 +1,38 @@ +#include "zerocopy_output_writer.h" + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +TZeroCopyOutputStreamWriter::TZeroCopyOutputStreamWriter(IZeroCopyOutput* output) + : Output_(output) +{ + ObtainNextBlock(); +} + +TZeroCopyOutputStreamWriter::~TZeroCopyOutputStreamWriter() +{ + if (RemainingBytes_ > 0) { + UndoRemaining(); + } +} + +void TZeroCopyOutputStreamWriter::ObtainNextBlock() +{ + if (RemainingBytes_ > 0) { + UndoRemaining(); + } + RemainingBytes_ = Output_->Next(&Current_); + TotalWrittenBlockSize_ += RemainingBytes_; +} + +void TZeroCopyOutputStreamWriter::UndoRemaining() +{ + Output_->Undo(RemainingBytes_); + TotalWrittenBlockSize_ -= RemainingBytes_; + RemainingBytes_ = 0; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff diff --git a/library/cpp/skiff/zerocopy_output_writer.h b/library/cpp/skiff/zerocopy_output_writer.h new file mode 100644 index 0000000000..b0bccc5a63 --- /dev/null +++ b/library/cpp/skiff/zerocopy_output_writer.h @@ -0,0 +1,41 @@ +#pragma once + +#include <util/stream/zerocopy_output.h> + +namespace NSkiff { + +//////////////////////////////////////////////////////////////////////////////// + +// Simple wrapper around +class TZeroCopyOutputStreamWriter + : private TNonCopyable +{ +public: + explicit TZeroCopyOutputStreamWriter(IZeroCopyOutput* output); + + ~TZeroCopyOutputStreamWriter(); + + Y_FORCE_INLINE char* Current() const; + Y_FORCE_INLINE ui64 RemainingBytes() const; + Y_FORCE_INLINE void Advance(size_t bytes); + void UndoRemaining(); + Y_FORCE_INLINE void Write(const void* buffer, size_t length); + Y_FORCE_INLINE ui64 GetTotalWrittenSize() const; + +private: + void ObtainNextBlock(); + +private: + IZeroCopyOutput* Output_; + char* Current_ = nullptr; + ui64 RemainingBytes_ = 0; + ui64 TotalWrittenBlockSize_ = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSkiff + +#define ZEROCOPY_OUTPUT_WRITER_INL_H_ +#include "zerocopy_output_writer-inl.h" +#undef ZEROCOPY_OUTPUT_WRITER_INL_H_ diff --git a/library/cpp/threading/CMakeLists.txt b/library/cpp/threading/CMakeLists.txt index 681ef6b24e..018e72c5cc 100644 --- a/library/cpp/threading/CMakeLists.txt +++ b/library/cpp/threading/CMakeLists.txt @@ -7,7 +7,9 @@ add_subdirectory(atomic) +add_subdirectory(blocking_queue) add_subdirectory(chunk_queue) +add_subdirectory(cron) add_subdirectory(equeue) add_subdirectory(future) add_subdirectory(hot_swap) @@ -17,3 +19,4 @@ add_subdirectory(poor_man_openmp) add_subdirectory(queue) add_subdirectory(skip_list) add_subdirectory(task_scheduler) +add_subdirectory(thread_local) diff --git a/library/cpp/threading/blocking_queue/CMakeLists.darwin-x86_64.txt b/library/cpp/threading/blocking_queue/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..0fcf5ce481 --- /dev/null +++ b/library/cpp/threading/blocking_queue/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-threading-blocking_queue) +target_link_libraries(cpp-threading-blocking_queue PUBLIC + contrib-libs-cxxsupp + yutil +) +target_sources(cpp-threading-blocking_queue PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/blocking_queue/blocking_queue.cpp +) diff --git a/library/cpp/threading/blocking_queue/CMakeLists.linux-aarch64.txt b/library/cpp/threading/blocking_queue/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..640f1e9144 --- /dev/null +++ b/library/cpp/threading/blocking_queue/CMakeLists.linux-aarch64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-threading-blocking_queue) +target_link_libraries(cpp-threading-blocking_queue PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_sources(cpp-threading-blocking_queue PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/blocking_queue/blocking_queue.cpp +) diff --git a/library/cpp/threading/blocking_queue/CMakeLists.linux-x86_64.txt b/library/cpp/threading/blocking_queue/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..640f1e9144 --- /dev/null +++ b/library/cpp/threading/blocking_queue/CMakeLists.linux-x86_64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-threading-blocking_queue) +target_link_libraries(cpp-threading-blocking_queue PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_sources(cpp-threading-blocking_queue PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/blocking_queue/blocking_queue.cpp +) diff --git a/library/cpp/threading/blocking_queue/CMakeLists.txt b/library/cpp/threading/blocking_queue/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/threading/blocking_queue/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/threading/blocking_queue/CMakeLists.windows-x86_64.txt b/library/cpp/threading/blocking_queue/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..0fcf5ce481 --- /dev/null +++ b/library/cpp/threading/blocking_queue/CMakeLists.windows-x86_64.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-threading-blocking_queue) +target_link_libraries(cpp-threading-blocking_queue PUBLIC + contrib-libs-cxxsupp + yutil +) +target_sources(cpp-threading-blocking_queue PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/blocking_queue/blocking_queue.cpp +) diff --git a/library/cpp/threading/blocking_queue/blocking_queue.cpp b/library/cpp/threading/blocking_queue/blocking_queue.cpp new file mode 100644 index 0000000000..db199c80be --- /dev/null +++ b/library/cpp/threading/blocking_queue/blocking_queue.cpp @@ -0,0 +1,3 @@ +#include "blocking_queue.h" + +// just check compilability diff --git a/library/cpp/threading/blocking_queue/blocking_queue.h b/library/cpp/threading/blocking_queue/blocking_queue.h new file mode 100644 index 0000000000..48d3762f68 --- /dev/null +++ b/library/cpp/threading/blocking_queue/blocking_queue.h @@ -0,0 +1,158 @@ +#pragma once + +#include <util/generic/deque.h> +#include <util/generic/maybe.h> +#include <util/generic/yexception.h> +#include <util/system/condvar.h> +#include <util/system/guard.h> +#include <util/system/mutex.h> + +#include <utility> + +namespace NThreading { + /// + /// TBlockingQueue is a queue of elements of limited or unlimited size. + /// Queue provides Push and Pop operations that block if operation can't be executed + /// (queue is empty or maximum size is reached). + /// + /// Queue can be stopped, in that case all blocked operation will return `Nothing` / false. + /// + /// All operations are thread safe. + /// + /// + /// Example of usage: + /// TBlockingQueue<int> queue; + /// + /// ... + /// + /// // thread 1 + /// queue.Push(42); + /// queue.Push(100500); + /// + /// ... + /// + /// // thread 2 + /// while (TMaybe<int> number = queue.Pop()) { + /// ProcessNumber(number.GetRef()); + /// } + template <class TElement> + class TBlockingQueue { + public: + /// + /// Creates blocking queue with given maxSize + /// if maxSize == 0 then queue is unlimited + TBlockingQueue(size_t maxSize) + : MaxSize(maxSize == 0 ? Max<size_t>() : maxSize) + , Stopped(false) + { + } + + /// + /// Blocks until queue has some elements or queue is stopped or deadline is reached. + /// Returns `Nothing` if queue is stopped or deadline is reached. + /// Returns element otherwise. + TMaybe<TElement> Pop(TInstant deadline = TInstant::Max()) { + TGuard<TMutex> g(Lock); + + const auto canPop = [this]() { return CanPop(); }; + if (!CanPopCV.WaitD(Lock, deadline, canPop)) { + return Nothing(); + } + + if (Stopped && Queue.empty()) { + return Nothing(); + } + TElement e = std::move(Queue.front()); + Queue.pop_front(); + CanPushCV.Signal(); + return std::move(e); + } + + TMaybe<TElement> Pop(TDuration duration) { + return Pop(TInstant::Now() + duration); + } + + /// + /// Blocks until queue has space for new elements or queue is stopped or deadline is reached. + /// Returns false exception if queue is stopped and push failed or deadline is reached. + /// Pushes element to queue and returns true otherwise. + bool Push(const TElement& e, TInstant deadline = TInstant::Max()) { + return PushRef(e, deadline); + } + + bool Push(TElement&& e, TInstant deadline = TInstant::Max()) { + return PushRef(std::move(e), deadline); + } + + bool Push(const TElement& e, TDuration duration) { + return Push(e, TInstant::Now() + duration); + } + + bool Push(TElement&& e, TDuration duration) { + return Push(std::move(e), TInstant::Now() + duration); + } + + /// + /// Stops the queue, all blocked operations will be aborted. + void Stop() { + TGuard<TMutex> g(Lock); + Stopped = true; + CanPopCV.BroadCast(); + CanPushCV.BroadCast(); + } + + /// + /// Checks whether queue is empty. + bool Empty() const { + TGuard<TMutex> g(Lock); + return Queue.empty(); + } + + /// + /// Returns size of the queue. + size_t Size() const { + TGuard<TMutex> g(Lock); + return Queue.size(); + } + + /// + /// Checks whether queue is stopped. + bool IsStopped() const { + TGuard<TMutex> g(Lock); + return Stopped; + } + + private: + bool CanPush() const { + return Queue.size() < MaxSize || Stopped; + } + + bool CanPop() const { + return !Queue.empty() || Stopped; + } + + template <typename Ref> + bool PushRef(Ref e, TInstant deadline) { + TGuard<TMutex> g(Lock); + const auto canPush = [this]() { return CanPush(); }; + if (!CanPushCV.WaitD(Lock, deadline, canPush)) { + return false; + } + if (Stopped) { + return false; + } + Queue.push_back(std::forward<TElement>(e)); + CanPopCV.Signal(); + return true; + } + + private: + TMutex Lock; + TCondVar CanPopCV; + TCondVar CanPushCV; + TDeque<TElement> Queue; + size_t MaxSize; + bool Stopped; + }; + +} diff --git a/library/cpp/threading/blocking_queue/blocking_queue_ut.cpp b/library/cpp/threading/blocking_queue/blocking_queue_ut.cpp new file mode 100644 index 0000000000..26e125279d --- /dev/null +++ b/library/cpp/threading/blocking_queue/blocking_queue_ut.cpp @@ -0,0 +1,211 @@ +#include "blocking_queue.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/string/builder.h> +#include <util/system/thread.h> + +namespace { + class TFunctionThread: public ISimpleThread { + public: + using TFunc = std::function<void()>; + + private: + TFunc Func; + + public: + TFunctionThread(const TFunc& func) + : Func(func) + { + } + + void* ThreadProc() noexcept override { + Func(); + return nullptr; + } + }; + +} + +IOutputStream& operator<<(IOutputStream& o, const TMaybe<int>& val) { + if (val) { + o << "TMaybe<int>(" << val.GetRef() << ')'; + } else { + o << "TMaybe<int>()"; + } + return o; +} + +Y_UNIT_TEST_SUITE(BlockingQueueTest) { + Y_UNIT_TEST(SimplePushPopTest) { + const size_t limit = 100; + + NThreading::TBlockingQueue<int> queue(100); + + for (int i = 0; i != limit; ++i) { + queue.Push(i); + } + + for (int i = 0; i != limit; ++i) { + UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), i); + } + + UNIT_ASSERT(queue.Empty()); + } + + Y_UNIT_TEST(SimpleStopTest) { + const size_t limit = 100; + + NThreading::TBlockingQueue<int> queue(100); + + for (int i = 0; i != limit; ++i) { + queue.Push(i); + } + queue.Stop(); + + bool ok = queue.Push(100500); + UNIT_ASSERT_VALUES_EQUAL(ok, false); + + for (int i = 0; i != limit; ++i) { + UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), i); + } + + UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), TMaybe<int>()); + } + + Y_UNIT_TEST(BigPushPop) { + const int limit = 100000; + + NThreading::TBlockingQueue<int> queue(10); + + TFunctionThread pusher([&] { + for (int i = 0; i != limit; ++i) { + if (!queue.Push(i)) { + break; + } + } + }); + + pusher.Start(); + + try { + for (int i = 0; i != limit; ++i) { + size_t size = queue.Size(); + UNIT_ASSERT_C(size <= 10, (TStringBuilder() << "Size exceeds 10: " << size).data()); + UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), i); + } + } catch (...) { + // gracefull shutdown of pusher thread if assertion fails + queue.Stop(); + throw; + } + + pusher.Join(); + } + + Y_UNIT_TEST(StopWhenMultiplePoppers) { + NThreading::TBlockingQueue<int> queue(10); + TFunctionThread popper1([&] { + UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), TMaybe<int>()); + }); + TFunctionThread popper2([&] { + UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), TMaybe<int>()); + }); + popper1.Start(); + popper2.Start(); + + queue.Stop(); + + popper1.Join(); + popper2.Join(); + } + + Y_UNIT_TEST(StopWhenMultiplePushers) { + NThreading::TBlockingQueue<int> queue(1); + queue.Push(1); + TFunctionThread pusher1([&] { + UNIT_ASSERT_VALUES_EQUAL(queue.Push(2), false); + }); + TFunctionThread pusher2([&] { + UNIT_ASSERT_VALUES_EQUAL(queue.Push(2), false); + }); + pusher1.Start(); + pusher2.Start(); + + queue.Stop(); + + pusher1.Join(); + pusher2.Join(); + } + + Y_UNIT_TEST(InterruptPopByDeadline) { + NThreading::TBlockingQueue<int> queue1(10); + NThreading::TBlockingQueue<int> queue2(10); + + const auto popper1DeadLine = TInstant::Now(); + const auto popper2DeadLine = TInstant::Now() + TDuration::Seconds(2); + + TFunctionThread popper1([&] { + UNIT_ASSERT_VALUES_EQUAL(queue1.Pop(popper1DeadLine), TMaybe<int>()); + UNIT_ASSERT_VALUES_EQUAL(queue1.IsStopped(), false); + }); + + TFunctionThread popper2([&] { + UNIT_ASSERT_VALUES_EQUAL(queue2.Pop(popper2DeadLine), 2); + UNIT_ASSERT_VALUES_EQUAL(queue2.IsStopped(), false); + }); + + popper1.Start(); + popper2.Start(); + + Sleep(TDuration::Seconds(1)); + + queue1.Push(1); + queue2.Push(2); + + Sleep(TDuration::Seconds(1)); + + queue1.Stop(); + queue2.Stop(); + + popper1.Join(); + popper2.Join(); + } + + Y_UNIT_TEST(InterruptPushByDeadline) { + NThreading::TBlockingQueue<int> queue1(1); + NThreading::TBlockingQueue<int> queue2(1); + + queue1.Push(0); + queue2.Push(0); + + const auto pusher1DeadLine = TInstant::Now(); + const auto pusher2DeadLine = TInstant::Now() + TDuration::Seconds(2); + + TFunctionThread pusher1([&] { + UNIT_ASSERT_VALUES_EQUAL(queue1.Push(1, pusher1DeadLine), false); + UNIT_ASSERT_VALUES_EQUAL(queue1.IsStopped(), false); + }); + + TFunctionThread pusher2([&] { + UNIT_ASSERT_VALUES_EQUAL(queue2.Push(2, pusher2DeadLine), true); + UNIT_ASSERT_VALUES_EQUAL(queue2.IsStopped(), false); + }); + + pusher1.Start(); + pusher2.Start(); + + Sleep(TDuration::Seconds(1)); + + queue1.Pop(); + queue2.Pop(); + + Sleep(TDuration::Seconds(1)); + + queue1.Stop(); + queue2.Stop(); + + pusher1.Join(); + pusher2.Join(); + } +} diff --git a/library/cpp/threading/blocking_queue/ut/ya.make b/library/cpp/threading/blocking_queue/ut/ya.make new file mode 100644 index 0000000000..50f220d552 --- /dev/null +++ b/library/cpp/threading/blocking_queue/ut/ya.make @@ -0,0 +1,7 @@ +UNITTEST_FOR(library/cpp/threading/blocking_queue) + +SRCS( + blocking_queue_ut.cpp +) + +END() diff --git a/library/cpp/threading/blocking_queue/ya.make b/library/cpp/threading/blocking_queue/ya.make new file mode 100644 index 0000000000..ce1104c1c9 --- /dev/null +++ b/library/cpp/threading/blocking_queue/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +SRCS( + blocking_queue.cpp +) + +END() + +RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/threading/cron/CMakeLists.darwin-x86_64.txt b/library/cpp/threading/cron/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..429d3b0b0d --- /dev/null +++ b/library/cpp/threading/cron/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-threading-cron) +target_link_libraries(cpp-threading-cron PUBLIC + contrib-libs-cxxsupp + yutil + cpp-deprecated-atomic +) +target_sources(cpp-threading-cron PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/cron/cron.cpp +) diff --git a/library/cpp/threading/cron/CMakeLists.linux-aarch64.txt b/library/cpp/threading/cron/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..d5bb429c63 --- /dev/null +++ b/library/cpp/threading/cron/CMakeLists.linux-aarch64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-threading-cron) +target_link_libraries(cpp-threading-cron PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-deprecated-atomic +) +target_sources(cpp-threading-cron PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/cron/cron.cpp +) diff --git a/library/cpp/threading/cron/CMakeLists.linux-x86_64.txt b/library/cpp/threading/cron/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..d5bb429c63 --- /dev/null +++ b/library/cpp/threading/cron/CMakeLists.linux-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-threading-cron) +target_link_libraries(cpp-threading-cron PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-deprecated-atomic +) +target_sources(cpp-threading-cron PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/cron/cron.cpp +) diff --git a/library/cpp/threading/cron/CMakeLists.txt b/library/cpp/threading/cron/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/threading/cron/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/threading/cron/CMakeLists.windows-x86_64.txt b/library/cpp/threading/cron/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..429d3b0b0d --- /dev/null +++ b/library/cpp/threading/cron/CMakeLists.windows-x86_64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-threading-cron) +target_link_libraries(cpp-threading-cron PUBLIC + contrib-libs-cxxsupp + yutil + cpp-deprecated-atomic +) +target_sources(cpp-threading-cron PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/cron/cron.cpp +) diff --git a/library/cpp/threading/cron/cron.cpp b/library/cpp/threading/cron/cron.cpp new file mode 100644 index 0000000000..e7c1c59735 --- /dev/null +++ b/library/cpp/threading/cron/cron.cpp @@ -0,0 +1,69 @@ +#include "cron.h" + +#include <library/cpp/deprecated/atomic/atomic.h> + +#include <util/system/thread.h> +#include <util/system/event.h> + +using namespace NCron; + +namespace { + struct TPeriodicHandle: public IHandle { + inline TPeriodicHandle(TJob job, TDuration interval, const TString& threadName) + : Job(job) + , Interval(interval) + , Done(false) + { + TThread::TParams params(DoRun, this); + if (!threadName.empty()) { + params.SetName(threadName); + } + Thread = MakeHolder<TThread>(params); + Thread->Start(); + } + + static inline void* DoRun(void* data) noexcept { + ((TPeriodicHandle*)data)->Run(); + + return nullptr; + } + + inline void Run() noexcept { + while (true) { + Job(); + + Event.WaitT(Interval); + + if (AtomicGet(Done)) { + return; + } + } + } + + ~TPeriodicHandle() override { + AtomicSet(Done, true); + Event.Signal(); + Thread->Join(); + } + + TJob Job; + TDuration Interval; + TManualEvent Event; + TAtomic Done; + THolder<TThread> Thread; + }; +} + +IHandlePtr NCron::StartPeriodicJob(TJob job) { + return NCron::StartPeriodicJob(job, TDuration::Seconds(0), ""); +} + +IHandlePtr NCron::StartPeriodicJob(TJob job, TDuration interval) { + return NCron::StartPeriodicJob(job, interval, ""); +} + +IHandlePtr NCron::StartPeriodicJob(TJob job, TDuration interval, const TString& threadName) { + return new TPeriodicHandle(job, interval, threadName); +} + +IHandle::~IHandle() = default; diff --git a/library/cpp/threading/cron/cron.h b/library/cpp/threading/cron/cron.h new file mode 100644 index 0000000000..77fa40c5e2 --- /dev/null +++ b/library/cpp/threading/cron/cron.h @@ -0,0 +1,18 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/function.h> +#include <util/datetime/base.h> + +namespace NCron { + struct IHandle { + virtual ~IHandle(); + }; + + using TJob = std::function<void()>; + using IHandlePtr = TAutoPtr<IHandle>; + + IHandlePtr StartPeriodicJob(TJob job); + IHandlePtr StartPeriodicJob(TJob job, TDuration interval); + IHandlePtr StartPeriodicJob(TJob job, TDuration interval, const TString& threadName); +} diff --git a/library/cpp/threading/cron/ya.make b/library/cpp/threading/cron/ya.make new file mode 100644 index 0000000000..ead272e837 --- /dev/null +++ b/library/cpp/threading/cron/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + cron.cpp +) + +PEERDIR( + library/cpp/deprecated/atomic +) + +END() diff --git a/library/cpp/threading/thread_local/CMakeLists.darwin-x86_64.txt b/library/cpp/threading/thread_local/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..2a282845d1 --- /dev/null +++ b/library/cpp/threading/thread_local/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,31 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(cpp-threading-thread_local) +target_link_libraries(cpp-threading-thread_local PUBLIC + contrib-libs-cxxsupp + yutil + cpp-threading-hot_swap + cpp-threading-skip_list + tools-enum_parser-enum_serialization_runtime +) +target_sources(cpp-threading-thread_local PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/thread_local/thread_local.cpp +) +generate_enum_serilization(cpp-threading-thread_local + ${CMAKE_SOURCE_DIR}/library/cpp/threading/thread_local/thread_local.h + INCLUDE_HEADERS + library/cpp/threading/thread_local/thread_local.h +) diff --git a/library/cpp/threading/thread_local/CMakeLists.linux-aarch64.txt b/library/cpp/threading/thread_local/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..69ae5b9e2e --- /dev/null +++ b/library/cpp/threading/thread_local/CMakeLists.linux-aarch64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(cpp-threading-thread_local) +target_link_libraries(cpp-threading-thread_local PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-threading-hot_swap + cpp-threading-skip_list + tools-enum_parser-enum_serialization_runtime +) +target_sources(cpp-threading-thread_local PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/thread_local/thread_local.cpp +) +generate_enum_serilization(cpp-threading-thread_local + ${CMAKE_SOURCE_DIR}/library/cpp/threading/thread_local/thread_local.h + INCLUDE_HEADERS + library/cpp/threading/thread_local/thread_local.h +) diff --git a/library/cpp/threading/thread_local/CMakeLists.linux-x86_64.txt b/library/cpp/threading/thread_local/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..69ae5b9e2e --- /dev/null +++ b/library/cpp/threading/thread_local/CMakeLists.linux-x86_64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(cpp-threading-thread_local) +target_link_libraries(cpp-threading-thread_local PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-threading-hot_swap + cpp-threading-skip_list + tools-enum_parser-enum_serialization_runtime +) +target_sources(cpp-threading-thread_local PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/thread_local/thread_local.cpp +) +generate_enum_serilization(cpp-threading-thread_local + ${CMAKE_SOURCE_DIR}/library/cpp/threading/thread_local/thread_local.h + INCLUDE_HEADERS + library/cpp/threading/thread_local/thread_local.h +) diff --git a/library/cpp/threading/thread_local/CMakeLists.txt b/library/cpp/threading/thread_local/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/threading/thread_local/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/threading/thread_local/CMakeLists.windows-x86_64.txt b/library/cpp/threading/thread_local/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..2a282845d1 --- /dev/null +++ b/library/cpp/threading/thread_local/CMakeLists.windows-x86_64.txt @@ -0,0 +1,31 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(cpp-threading-thread_local) +target_link_libraries(cpp-threading-thread_local PUBLIC + contrib-libs-cxxsupp + yutil + cpp-threading-hot_swap + cpp-threading-skip_list + tools-enum_parser-enum_serialization_runtime +) +target_sources(cpp-threading-thread_local PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/threading/thread_local/thread_local.cpp +) +generate_enum_serilization(cpp-threading-thread_local + ${CMAKE_SOURCE_DIR}/library/cpp/threading/thread_local/thread_local.h + INCLUDE_HEADERS + library/cpp/threading/thread_local/thread_local.h +) diff --git a/library/cpp/type_info/CMakeLists.darwin-x86_64.txt b/library/cpp/type_info/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..a1fe5a9709 --- /dev/null +++ b/library/cpp/type_info/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,38 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(library-cpp-type_info) +target_link_libraries(library-cpp-type_info PUBLIC + contrib-libs-cxxsupp + yutil + tools-enum_parser-enum_serialization_runtime + yson_pull +) +target_sources(library-cpp-type_info PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_info.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/builder.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/error.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_complexity.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_equivalence.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_factory.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_io.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_list.cpp +) +generate_enum_serilization(library-cpp-type_info + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_list.h + INCLUDE_HEADERS + library/cpp/type_info/type_list.h +) diff --git a/library/cpp/type_info/CMakeLists.linux-aarch64.txt b/library/cpp/type_info/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..388946eff8 --- /dev/null +++ b/library/cpp/type_info/CMakeLists.linux-aarch64.txt @@ -0,0 +1,39 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(library-cpp-type_info) +target_link_libraries(library-cpp-type_info PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + tools-enum_parser-enum_serialization_runtime + yson_pull +) +target_sources(library-cpp-type_info PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_info.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/builder.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/error.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_complexity.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_equivalence.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_factory.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_io.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_list.cpp +) +generate_enum_serilization(library-cpp-type_info + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_list.h + INCLUDE_HEADERS + library/cpp/type_info/type_list.h +) diff --git a/library/cpp/type_info/CMakeLists.linux-x86_64.txt b/library/cpp/type_info/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..388946eff8 --- /dev/null +++ b/library/cpp/type_info/CMakeLists.linux-x86_64.txt @@ -0,0 +1,39 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(library-cpp-type_info) +target_link_libraries(library-cpp-type_info PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + tools-enum_parser-enum_serialization_runtime + yson_pull +) +target_sources(library-cpp-type_info PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_info.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/builder.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/error.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_complexity.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_equivalence.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_factory.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_io.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_list.cpp +) +generate_enum_serilization(library-cpp-type_info + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_list.h + INCLUDE_HEADERS + library/cpp/type_info/type_list.h +) diff --git a/library/cpp/type_info/CMakeLists.txt b/library/cpp/type_info/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/type_info/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/type_info/CMakeLists.windows-x86_64.txt b/library/cpp/type_info/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..a1fe5a9709 --- /dev/null +++ b/library/cpp/type_info/CMakeLists.windows-x86_64.txt @@ -0,0 +1,38 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) + +add_library(library-cpp-type_info) +target_link_libraries(library-cpp-type_info PUBLIC + contrib-libs-cxxsupp + yutil + tools-enum_parser-enum_serialization_runtime + yson_pull +) +target_sources(library-cpp-type_info PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_info.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/builder.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/error.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_complexity.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_equivalence.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_factory.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_io.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_list.cpp +) +generate_enum_serilization(library-cpp-type_info + ${CMAKE_SOURCE_DIR}/library/cpp/type_info/type_list.h + INCLUDE_HEADERS + library/cpp/type_info/type_list.h +) diff --git a/library/cpp/type_info/Readme.md b/library/cpp/type_info/Readme.md new file mode 100644 index 0000000000..e4501c8c11 --- /dev/null +++ b/library/cpp/type_info/Readme.md @@ -0,0 +1,9 @@ +# TI — unified in-memory representation for Common Yandex Typesystem + +Common Yandex Typesystem is a type system used across data storage and processing technologies developed by Yandex. A list of systems that support it includes YT, YDB, RTMR, YQL and others. + +The Type Info library provides classes that represent types from Common Yandex Typesystem. It allows constructing in-memory representations of types, inspecting said representations, combining them to derive new types, as well as serializing and deserializing them. + +For those familiar with Protobuf, Type Info implements the concept of message descriptors for Yandex type system. + +Here you can find open issues: https://st.yandex-team.ru/YT/order:updated:false/filter?tags=type_info&resolution=empty()
\ No newline at end of file diff --git a/library/cpp/type_info/builder.cpp b/library/cpp/type_info/builder.cpp new file mode 100644 index 0000000000..008e0af754 --- /dev/null +++ b/library/cpp/type_info/builder.cpp @@ -0,0 +1,458 @@ +#include "builder.h" + +#include "type_factory.h" + +namespace NTi { + TStructBuilderRaw::TStructBuilderRaw(IPoolTypeFactory& factory) noexcept + : Factory_(&factory) + { + } + + TStructBuilderRaw::TStructBuilderRaw(IPoolTypeFactory& factory, TStructTypePtr prototype) noexcept + : TStructBuilderRaw(factory, prototype.Get()) + { + } + + TStructBuilderRaw::TStructBuilderRaw(IPoolTypeFactory& factory, const TStructType* prototype) noexcept + : TStructBuilderRaw(factory) + { + Members_.reserve(prototype->GetMembers().size()); + + if (prototype->GetFactory() == Factory_) { + // members are in the same factory -- can reuse them. + for (auto member : prototype->GetMembers()) { + Members_.push_back(member); + } + } else { + // members are in a different factory -- should copy them. + for (auto member : prototype->GetMembers()) { + AddMember(member.GetName(), member.GetTypeRaw()); + } + } + } + + TStructBuilderRaw& TStructBuilderRaw::SetName(TMaybe<TStringBuf> name) & noexcept { + Name_ = Factory_->AllocateStringMaybe(name); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::SetName(TMaybe<TStringBuf> name) && noexcept { + return std::move(SetName(name)); + } + + bool TStructBuilderRaw::HasName() const noexcept { + return Name_.Defined(); + } + + TMaybe<TStringBuf> TStructBuilderRaw::GetName() const noexcept { + return Name_; + } + + TStructBuilderRaw& TStructBuilderRaw::Reserve(size_t size) & noexcept { + Members_.reserve(size); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::Reserve(size_t size) && noexcept { + return std::move(Reserve(size)); + } + + TStructBuilderRaw& TStructBuilderRaw::AddMember(TStringBuf name, TTypePtr type) & noexcept { + return AddMember(name, type.Get()); + } + + TStructBuilderRaw TStructBuilderRaw::AddMember(TStringBuf name, TTypePtr type) && noexcept { + return std::move(AddMember(name, type)); + } + + TStructBuilderRaw& TStructBuilderRaw::AddMember(TStringBuf name, const TType* type) & noexcept { + Members_.emplace_back(Factory_->AllocateString(name), Factory_->Own(type)); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::AddMember(TStringBuf name, const TType* type) && noexcept { + return std::move(AddMember(name, type)); + } + + TStructBuilderRaw& TStructBuilderRaw::AddMemberName(TStringBuf name) & noexcept { + PendingMemberName_ = Factory_->AllocateString(name); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::AddMemberName(TStringBuf name) && noexcept { + return std::move(AddMemberName(name)); + } + + TStructBuilderRaw& TStructBuilderRaw::DiscardMemberName() & noexcept { + PendingMemberName_.Clear(); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::DiscardMemberName() && noexcept { + return std::move(DiscardMemberName()); + } + + bool TStructBuilderRaw::HasMemberName() const noexcept { + return PendingMemberName_.Defined(); + } + + TMaybe<TStringBuf> TStructBuilderRaw::GetMemberName() const noexcept { + return PendingMemberName_; + } + + TStructBuilderRaw& TStructBuilderRaw::AddMemberType(TTypePtr type) & noexcept { + return AddMemberType(type.Get()); + } + + TStructBuilderRaw TStructBuilderRaw::AddMemberType(TTypePtr type) && noexcept { + return std::move(AddMemberType(type)); + } + + TStructBuilderRaw& TStructBuilderRaw::AddMemberType(const TType* type) & noexcept { + PendingMemberType_ = Factory_->Own(type); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::AddMemberType(const TType* type) && noexcept { + return std::move(AddMemberType(type)); + } + + TStructBuilderRaw& TStructBuilderRaw::DiscardMemberType() & noexcept { + PendingMemberType_.Clear(); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::DiscardMemberType() && noexcept { + return std::move(DiscardMemberType()); + } + + bool TStructBuilderRaw::HasMemberType() const noexcept { + return PendingMemberType_.Defined(); + } + + TMaybe<const TType*> TStructBuilderRaw::GetMemberType() const noexcept { + return PendingMemberType_; + } + + bool TStructBuilderRaw::CanAddMember() const noexcept { + return HasMemberName() && HasMemberType(); + } + + TStructBuilderRaw& TStructBuilderRaw::AddMember() & noexcept { + Y_VERIFY(CanAddMember()); + Members_.emplace_back(*PendingMemberName_, *PendingMemberType_); + DiscardMember(); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::AddMember() && noexcept { + return std::move(AddMember()); + } + + TStructBuilderRaw& TStructBuilderRaw::DiscardMember() & noexcept { + DiscardMemberName(); + DiscardMemberType(); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::DiscardMember() && noexcept { + return std::move(DiscardMember()); + } + + TStructType::TMembers TStructBuilderRaw::GetMembers() const noexcept { + return Members_; + } + + TStructBuilderRaw& TStructBuilderRaw::Reset() & noexcept { + Name_ = {}; + Members_.clear(); + return *this; + } + + TStructBuilderRaw TStructBuilderRaw::Reset() && noexcept { + return std::move(Reset()); + } + + TStructTypePtr TStructBuilderRaw::Build() { + return BuildRaw()->AsPtr(); + } + + const TStructType* TStructBuilderRaw::BuildRaw() { + return DoBuildRaw(Name_); + } + + TVariantTypePtr TStructBuilderRaw::BuildVariant() { + return BuildVariantRaw()->AsPtr(); + } + + const TVariantType* TStructBuilderRaw::BuildVariantRaw() { + return Factory_->New<TVariantType>(Nothing(), Name_, DoBuildRaw(Nothing())); + } + + const TStructType* TStructBuilderRaw::DoBuildRaw(TMaybe<TStringBuf> name) { + auto members = Factory_->NewArray<TStructType::TMember>(Members_.size(), [this](TStructType::TMember* member, size_t i) { + new (member) TStructType::TMember(Members_[i]); + }); + + auto sortedMembersArray = Factory_->AllocateArrayFor<size_t>(Members_.size()); + auto sortedMembers = TArrayRef(sortedMembersArray, members.size()); + TStructType::MakeSortedMembers(members, sortedMembers); + + return Factory_->New<TStructType>(Nothing(), name, members, sortedMembers); + } + + TTupleBuilderRaw::TTupleBuilderRaw(IPoolTypeFactory& factory) noexcept + : Factory_(&factory) + { + } + + TTupleBuilderRaw::TTupleBuilderRaw(IPoolTypeFactory& factory, TTupleTypePtr prototype) noexcept + : TTupleBuilderRaw(factory, prototype.Get()) + { + } + + TTupleBuilderRaw::TTupleBuilderRaw(IPoolTypeFactory& factory, const TTupleType* prototype) noexcept + : TTupleBuilderRaw(factory) + { + Elements_.reserve(prototype->GetElements().size()); + + if (prototype->GetFactory() == Factory_) { + // elements are in the same factory -- can reuse them + for (auto element : prototype->GetElements()) { + Elements_.push_back(element); + } + } else { + // items are in a different factory -- should copy them + for (auto element : prototype->GetElements()) { + AddElement(element.GetTypeRaw()); + } + } + } + + TTupleBuilderRaw& TTupleBuilderRaw::SetName(TMaybe<TStringBuf> name) & noexcept { + Name_ = Factory_->AllocateStringMaybe(name); + return *this; + } + + TTupleBuilderRaw TTupleBuilderRaw::SetName(TMaybe<TStringBuf> name) && noexcept { + return std::move(SetName(name)); + } + + bool TTupleBuilderRaw::HasName() const noexcept { + return Name_.Defined(); + } + + TMaybe<TStringBuf> TTupleBuilderRaw::GetName() const noexcept { + return Name_; + } + + TTupleBuilderRaw& TTupleBuilderRaw::Reserve(size_t size) & noexcept { + Elements_.reserve(size); + return *this; + } + + TTupleBuilderRaw TTupleBuilderRaw::Reserve(size_t size) && noexcept { + return std::move(Reserve(size)); + } + + TTupleBuilderRaw& TTupleBuilderRaw::AddElement(TTypePtr type) & noexcept { + return AddElement(type.Get()); + } + + TTupleBuilderRaw TTupleBuilderRaw::AddElement(TTypePtr type) && noexcept { + return std::move(AddElement(type)); + } + + TTupleBuilderRaw& TTupleBuilderRaw::AddElement(const TType* type) & noexcept { + Elements_.emplace_back(Factory_->Own(type)); + return *this; + } + + TTupleBuilderRaw TTupleBuilderRaw::AddElement(const TType* type) && noexcept { + return std::move(AddElement(type)); + } + + TTupleBuilderRaw& TTupleBuilderRaw::AddElementType(TTypePtr type) & noexcept { + return AddElementType(type.Get()); + } + + TTupleBuilderRaw TTupleBuilderRaw::AddElementType(TTypePtr type) && noexcept { + return std::move(AddElementType(type)); + } + + TTupleBuilderRaw& TTupleBuilderRaw::AddElementType(const TType* type) & noexcept { + PendingElementType_ = Factory_->Own(type); + return *this; + } + + TTupleBuilderRaw TTupleBuilderRaw::AddElementType(const TType* type) && noexcept { + return std::move(AddElementType(type)); + } + + TTupleBuilderRaw& TTupleBuilderRaw::DiscardElementType() & noexcept { + PendingElementType_.Clear(); + return *this; + } + + TTupleBuilderRaw TTupleBuilderRaw::DiscardElementType() && noexcept { + return std::move(DiscardElementType()); + } + + bool TTupleBuilderRaw::HasElementType() const noexcept { + return PendingElementType_.Defined(); + } + + TMaybe<const TType*> TTupleBuilderRaw::GetElementType() const noexcept { + return PendingElementType_; + } + + bool TTupleBuilderRaw::CanAddElement() const noexcept { + return HasElementType(); + } + + TTupleBuilderRaw& TTupleBuilderRaw::AddElement() & noexcept { + Y_VERIFY(CanAddElement()); + Elements_.emplace_back(*PendingElementType_); + DiscardElement(); + return *this; + } + + TTupleBuilderRaw TTupleBuilderRaw::AddElement() && noexcept { + return std::move(AddElement()); + } + + TTupleBuilderRaw& TTupleBuilderRaw::DiscardElement() & noexcept { + DiscardElementType(); + return *this; + } + + TTupleBuilderRaw TTupleBuilderRaw::DiscardElement() && noexcept { + return std::move(DiscardElement()); + } + + TTupleBuilderRaw& TTupleBuilderRaw::Reset() & noexcept { + Name_ = {}; + Elements_.clear(); + return *this; + } + + TTupleType::TElements TTupleBuilderRaw::GetElements() const noexcept { + return Elements_; + } + + TTupleBuilderRaw TTupleBuilderRaw::Reset() && noexcept { + return std::move(Reset()); + } + + TTupleTypePtr TTupleBuilderRaw::Build() { + return BuildRaw()->AsPtr(); + } + + const TTupleType* TTupleBuilderRaw::BuildRaw() { + return DoBuildRaw(Name_); + } + + TVariantTypePtr TTupleBuilderRaw::BuildVariant() { + return BuildVariantRaw()->AsPtr(); + } + + const TVariantType* TTupleBuilderRaw::BuildVariantRaw() { + return Factory_->New<TVariantType>(Nothing(), Name_, DoBuildRaw(Nothing())); + } + + const TTupleType* TTupleBuilderRaw::DoBuildRaw(TMaybe<TStringBuf> name) { + auto items = Factory_->NewArray<TTupleType::TElement>(Elements_.size(), [this](TTupleType::TElement* element, size_t i) { + new (element) TTupleType::TElement(Elements_[i]); + }); + + return Factory_->New<TTupleType>(Nothing(), name, items); + } + + TTaggedBuilderRaw::TTaggedBuilderRaw(IPoolTypeFactory& factory) noexcept + : Factory_(&factory) + { + } + + TTaggedBuilderRaw& TTaggedBuilderRaw::SetTag(TStringBuf tag) & noexcept { + Tag_ = Factory_->AllocateString(tag); + return *this; + } + + TTaggedBuilderRaw TTaggedBuilderRaw::SetTag(TStringBuf tag) && noexcept { + return std::move(SetTag(tag)); + } + + TTaggedBuilderRaw& TTaggedBuilderRaw::DiscardTag() & noexcept { + Tag_.Clear(); + return *this; + } + + TTaggedBuilderRaw TTaggedBuilderRaw::DiscardTag() && noexcept { + return std::move(DiscardTag()); + } + + bool TTaggedBuilderRaw::HasTag() const noexcept { + return Tag_.Defined(); + } + + TMaybe<TStringBuf> TTaggedBuilderRaw::GetTag() const noexcept { + return Tag_; + } + + TTaggedBuilderRaw& TTaggedBuilderRaw::SetItem(TTypePtr type) & noexcept { + return SetItem(type.Get()); + } + + TTaggedBuilderRaw TTaggedBuilderRaw::SetItem(TTypePtr type) && noexcept { + return std::move(SetItem(std::move(type))); + } + + TTaggedBuilderRaw& TTaggedBuilderRaw::SetItem(const TType* type) & noexcept { + Item_ = Factory_->Own(type); + return *this; + } + + TTaggedBuilderRaw TTaggedBuilderRaw::SetItem(const TType* type) && noexcept { + return std::move(SetItem(type)); + } + + TTaggedBuilderRaw& TTaggedBuilderRaw::DiscardItem() & noexcept { + Item_.Clear(); + return *this; + } + + TTaggedBuilderRaw TTaggedBuilderRaw::DiscardItem() && noexcept { + return std::move(DiscardItem()); + } + + bool TTaggedBuilderRaw::HasItem() const noexcept { + return Item_.Defined(); + } + + TMaybe<const TType*> TTaggedBuilderRaw::GetItem() const noexcept { + return Item_; + } + + bool TTaggedBuilderRaw::CanBuild() const noexcept { + return HasTag() && HasItem(); + } + + TTaggedBuilderRaw& TTaggedBuilderRaw::Reset() & noexcept { + DiscardTag(); + DiscardItem(); + return *this; + } + + TTaggedBuilderRaw TTaggedBuilderRaw::Reset() && noexcept { + return std::move(Reset()); + } + + TTaggedTypePtr TTaggedBuilderRaw::Build() { + return BuildRaw()->AsPtr(); + } + + const TTaggedType* TTaggedBuilderRaw::BuildRaw() { + Y_VERIFY(CanBuild()); + return Factory_->New<TTaggedType>(Nothing(), *Item_, *Tag_); + } +} diff --git a/library/cpp/type_info/builder.h b/library/cpp/type_info/builder.h new file mode 100644 index 0000000000..faae45ab51 --- /dev/null +++ b/library/cpp/type_info/builder.h @@ -0,0 +1,346 @@ +#pragma once + +//! @file builder.h +//! +//! Builders help with creating complex types piece-by-piece. + +#include <util/generic/vector.h> + +#include "type.h" + +#include "fwd.h" + +namespace NTi { + /// An interface for building structs using the raw interface (via memory-pool-based factory). + /// + /// This interface allows allocating structs piece-by-piece. You can feed it struct items, one-by-one, and they'll + /// be copied into the memory pool before the struct is created. This way, you don't have to allocate heap memory + /// to temporarily store item names or types. + /// + /// Note: this builder doesn't own the underlying factory. + class TStructBuilderRaw { + public: + /// Create a new builder with the given factory. + TStructBuilderRaw(IPoolTypeFactory& factory) noexcept; + + /// Create a new builder with the given factory and add fields from the given struct. + /// Note that struct name is not copied to the builder. + TStructBuilderRaw(IPoolTypeFactory& factory, TStructTypePtr prototype) noexcept; + TStructBuilderRaw(IPoolTypeFactory& factory, const TStructType* prototype) noexcept; + + /// Set a new struct name. + /// + /// Note that the name is copied to the factory right away. If you call this function twice, second call will + /// overwrite name from the first call, but will not remove it from the memory pool. + TStructBuilderRaw& SetName(TMaybe<TStringBuf> name) & noexcept; + TStructBuilderRaw SetName(TMaybe<TStringBuf> name) && noexcept; + + /// Check if there's a struct name set. + bool HasName() const noexcept; + + /// Get a struct name. + /// + /// Name was copied to the factory's memory pool and will live as long as the factory lives. + TMaybe<TStringBuf> GetName() const noexcept; + + /// Reserve some place in the underlying vector that collects struct items. + TStructBuilderRaw& Reserve(size_t size) & noexcept; + TStructBuilderRaw Reserve(size_t size) && noexcept; + + /// Append a new struct member. + TStructBuilderRaw& AddMember(TStringBuf name, TTypePtr type) & noexcept; + TStructBuilderRaw AddMember(TStringBuf name, TTypePtr type) && noexcept; + TStructBuilderRaw& AddMember(TStringBuf name, const TType* type) & noexcept; + TStructBuilderRaw AddMember(TStringBuf name, const TType* type) && noexcept; + + /// Partial member creation interface. + /// + /// This interface allows building individual struct items piece-by-piece. You can pass member's name + /// and copy it to the pool without passing its type. For example: + /// + /// ``` + /// auto builder = TStructBuilderRaw(factory); + /// builder.AddMemberName("name"); // add name for a new member. + /// builder.AddMemberType(type); // add type for a new member. + /// builder.AddMember(); // use added name and type to construct and append an member. + /// ``` + /// + /// This interface is useful when you can't store member name for a long time. For example, if you building + /// a parser, at some point have an member name, but you don't have an member type yet. Then you can add member name + /// when you have it, and add member type later. + /// + /// @{ + //- + /// Set name for pending member. + /// + /// Note that the name is copied to the factory right away. If you call this function twice, second call will + /// overwrite name from the first call, but will not remove it from the memory pool. + TStructBuilderRaw& AddMemberName(TStringBuf name) & noexcept; + TStructBuilderRaw AddMemberName(TStringBuf name) && noexcept; + + /// Unset name for pending member. + /// + /// Note that member name is copied to the factory right away. If you discard it via this method, + /// it will not be removed from the memory pool. + TStructBuilderRaw& DiscardMemberName() & noexcept; + TStructBuilderRaw DiscardMemberName() && noexcept; + + /// Check if there's a name set for pending member. + bool HasMemberName() const noexcept; + + /// Get name for pending member. + /// + /// Name was copied to the factory's memory pool and will live as long as the factory lives. + TMaybe<TStringBuf> GetMemberName() const noexcept; + + /// Set type for pending member. + /// + /// Note that the type is copied to the factory right away. If you call this function twice, second call will + /// overwrite type from the first call, but will not remove it from the memory pool. + TStructBuilderRaw& AddMemberType(TTypePtr type) & noexcept; + TStructBuilderRaw AddMemberType(TTypePtr type) && noexcept; + TStructBuilderRaw& AddMemberType(const TType* type) & noexcept; + TStructBuilderRaw AddMemberType(const TType* type) && noexcept; + + /// Unset type for pending member. + /// + /// Note that member type is copied to the factory right away. If you discard it via this method, + /// it will not be removed from the memory pool. + TStructBuilderRaw& DiscardMemberType() & noexcept; + TStructBuilderRaw DiscardMemberType() && noexcept; + + /// Check if there's a type set for pending member. + bool HasMemberType() const noexcept; + + /// Get type for pending member. + TMaybe<const TType*> GetMemberType() const noexcept; + + /// Check if both name and type are set for pending member. + bool CanAddMember() const noexcept; + + /// Use data added via `AddMemberName` and `AddMemberType` to construct a new struct member + /// and append it to this builder. This function panics if there's no name or no type set for pending member. + TStructBuilderRaw& AddMember() & noexcept; + TStructBuilderRaw AddMember() && noexcept; + + /// Discard all data added via `AddMemberName` and `AddMemberType` functions. + /// + /// Note that member name and type are copied to the factory right away. If you discard them via this method, + /// they will not be removed from the memory pool. + TStructBuilderRaw& DiscardMember() & noexcept; + TStructBuilderRaw DiscardMember() && noexcept; + + /// @} + + /// Get immutable list of items that've been added to this builder. + /// The returned object is invalidated when this builder dies or when `AddMember` is called. + TStructType::TMembers GetMembers() const noexcept; + + /// Reset struct name and items. + TStructBuilderRaw& Reset() & noexcept; + TStructBuilderRaw Reset() && noexcept; + + /// Create a new struct type using name and items from this builder. + TStructTypePtr Build(); + + /// Like `Build`, but returns a raw pointer. + const TStructType* BuildRaw(); + + /// Create a new variant over struct using items from this builder. + TVariantTypePtr BuildVariant(); + + /// Like `BuildVariant`, but returns a raw pointer. + const TVariantType* BuildVariantRaw(); + + private: + const TStructType* DoBuildRaw(TMaybe<TStringBuf> name); + + private: + IPoolTypeFactory* Factory_; + TMaybe<TStringBuf> Name_; + TVector<TStructType::TMember> Members_; + TMaybe<TStringBuf> PendingMemberName_; + TMaybe<const TType*> PendingMemberType_; + }; + + /// An interface for building tuples using the raw interface (via memory-pool-based factory). + /// + /// Note: this builder doesn't own the underlying factory. + class TTupleBuilderRaw { + public: + /// Create a new builder with the given factory. + TTupleBuilderRaw(IPoolTypeFactory& factory) noexcept; + + /// Create a new builder with the given factory and add fields from the given tuple. + /// Note that tuple name is not copied to the builder. + TTupleBuilderRaw(IPoolTypeFactory& factory, TTupleTypePtr prototype) noexcept; + TTupleBuilderRaw(IPoolTypeFactory& factory, const TTupleType* prototype) noexcept; + + /// Set a new tuple name. + /// Note that the name is copied to the factory right away. If you call this function twice, second call will + /// overwrite name from the first call, but will not remove it from the memory pool. + TTupleBuilderRaw& SetName(TMaybe<TStringBuf> name) & noexcept; + TTupleBuilderRaw SetName(TMaybe<TStringBuf> name) && noexcept; + + /// Check if there's a tuple name set. + bool HasName() const noexcept; + + /// Get a tuple name. + /// + /// Name was copied to the factory's memory pool and will live as long as the factory lives. + TMaybe<TStringBuf> GetName() const noexcept; + + /// Reserve some place in the underlying vector that collects tuple items. + TTupleBuilderRaw& Reserve(size_t size) & noexcept; + TTupleBuilderRaw Reserve(size_t size) && noexcept; + + /// Append a new tuple item. + TTupleBuilderRaw& AddElement(TTypePtr type) & noexcept; + TTupleBuilderRaw AddElement(TTypePtr type) && noexcept; + TTupleBuilderRaw& AddElement(const TType* type) & noexcept; + TTupleBuilderRaw AddElement(const TType* type) && noexcept; + + /// Partial item creation interface. + /// + /// This interface allows building individual tuple items piece-by-piece. It mirrors the same type of interface + /// in the struct builder. + /// + /// @{ + //- + /// Set type for pending item. + /// + /// Note that the type is copied to the factory right away. If you call this function twice, second call will + /// overwrite type from the first call, but will not remove it from the memory pool. + TTupleBuilderRaw& AddElementType(TTypePtr type) & noexcept; + TTupleBuilderRaw AddElementType(TTypePtr type) && noexcept; + TTupleBuilderRaw& AddElementType(const TType* type) & noexcept; + TTupleBuilderRaw AddElementType(const TType* type) && noexcept; + + /// Unset type for pending item. + /// + /// Note that item type is copied to the factory right away. If you discard it via this method, + /// it will not be removed from the memory pool. + TTupleBuilderRaw& DiscardElementType() & noexcept; + TTupleBuilderRaw DiscardElementType() && noexcept; + + /// Check if there's a type set for pending item. + bool HasElementType() const noexcept; + + /// Get type for pending item. + TMaybe<const TType*> GetElementType() const noexcept; + + /// Check if type is set for pending item. + bool CanAddElement() const noexcept; + + /// Use data added via `AddElementType` to construct a new tuple item and append it to this builder. + /// This function panics if there's no type set for pending item. + TTupleBuilderRaw& AddElement() & noexcept; + TTupleBuilderRaw AddElement() && noexcept; + + /// Discard all data added via `AddElementType` function. + /// + /// Note that item type is copied to the factory right away. If you discard it via this method, + /// it will not be removed from the memory pool. + TTupleBuilderRaw& DiscardElement() & noexcept; + TTupleBuilderRaw DiscardElement() && noexcept; + + /// @} + + /// Get immutable list of items that've been added to this builder. + /// The returned object is invalidated when this builder dies or when `AddElement` is called. + TTupleType::TElements GetElements() const noexcept; + + /// Reset tuple name and items. + TTupleBuilderRaw& Reset() & noexcept; + TTupleBuilderRaw Reset() && noexcept; + + /// Create a new tuple type using name and items from this builder. + TTupleTypePtr Build(); + + /// Like `Build`, but returns a raw pointer. + const TTupleType* BuildRaw(); + + /// Create a new variant over tuple using items from this builder. + TVariantTypePtr BuildVariant(); + + /// Like `BuildVariant`, but returns a raw pointer. + const TVariantType* BuildVariantRaw(); + + private: + const TTupleType* DoBuildRaw(TMaybe<TStringBuf> name); + + private: + IPoolTypeFactory* Factory_; + TMaybe<TStringBuf> Name_; + TVector<TTupleType::TElement> Elements_; + TMaybe<const TType*> PendingElementType_; + }; + + /// An interface for building tagged types using the raw interface (via memory-pool-based factory). + /// + /// Note: this builder doesn't own the underlying factory. + class TTaggedBuilderRaw { + public: + /// Create a new builder with the given factory. + TTaggedBuilderRaw(IPoolTypeFactory& factory) noexcept; + + /// Set a new tag. + /// + /// Note that the tag is copied to the factory right away. If you call this function twice, second call will + /// overwrite tag from the first call, but will not remove it from the memory pool. + TTaggedBuilderRaw& SetTag(TStringBuf tag) & noexcept; + TTaggedBuilderRaw SetTag(TStringBuf tag) && noexcept; + + /// Unset tag. + /// + /// Note that tag is copied to the factory right away. If you discard it via this method, + /// it will not be removed from the memory pool. + TTaggedBuilderRaw& DiscardTag() & noexcept; + TTaggedBuilderRaw DiscardTag() && noexcept; + + /// Check if a tag is set. + bool HasTag() const noexcept; + + /// Get a tag. + /// + /// The tag was copied to the factory's memory pool and will live as long as the factory lives. + TMaybe<TStringBuf> GetTag() const noexcept; + + /// Set type that's being tagged. + TTaggedBuilderRaw& SetItem(TTypePtr type) & noexcept; + TTaggedBuilderRaw SetItem(TTypePtr type) && noexcept; + TTaggedBuilderRaw& SetItem(const TType* type) & noexcept; + TTaggedBuilderRaw SetItem(const TType* type) && noexcept; + + /// Unset item type. + /// + /// Note that item type is copied to the factory right away. If you discard it via this method, + /// it will not be removed from the memory pool. + TTaggedBuilderRaw& DiscardItem() & noexcept; + TTaggedBuilderRaw DiscardItem() && noexcept; + + /// Check if there's an item type set. + bool HasItem() const noexcept; + + /// Get item type. + TMaybe<const TType*> GetItem() const noexcept; + + /// Check if there's both name and item type set. + bool CanBuild() const noexcept; + + /// Discard both tag and item. + TTaggedBuilderRaw& Reset() & noexcept; + TTaggedBuilderRaw Reset() && noexcept; + + /// Create a new tagged type using name and item from this builder. + TTaggedTypePtr Build(); + + /// Like `Build`, but returns a raw pointer. + const TTaggedType* BuildRaw(); + + private: + IPoolTypeFactory* Factory_; + TMaybe<TStringBuf> Tag_; + TMaybe<const TType*> Item_; + }; +} diff --git a/library/cpp/type_info/error.cpp b/library/cpp/type_info/error.cpp new file mode 100644 index 0000000000..fe9ecf1d9f --- /dev/null +++ b/library/cpp/type_info/error.cpp @@ -0,0 +1 @@ +#include "error.h" diff --git a/library/cpp/type_info/error.h b/library/cpp/type_info/error.h new file mode 100644 index 0000000000..26fd8853f9 --- /dev/null +++ b/library/cpp/type_info/error.h @@ -0,0 +1,33 @@ +#pragma once + +//! @file error.h +//! +//! All error classes that one can encounter when working with type info library. + +#include <util/generic/yexception.h> + +namespace NTi { + /// Base class for all exceptions that arise when working with Type Info library. + class TException: public yexception { + }; + + /// Type Info API used in an unintended way. + class TApiException: public TException { + }; + + /// Attempting to create an illegal type. + /// + /// For example, this exception is raised when attempting to create a struct with non-unique item names. + class TIllegalTypeException: public TException { + }; + + /// Type deserializer got an invalid input. + /// + /// See `TType::Serialize()` and `TType::Deserialize()` for more info on type serialization/deserialization. + class TDeserializationException: public TException { + }; + + /// No such item in type. + class TItemNotFound: public TException { + }; +} diff --git a/library/cpp/type_info/fwd.h b/library/cpp/type_info/fwd.h new file mode 100644 index 0000000000..644932f677 --- /dev/null +++ b/library/cpp/type_info/fwd.h @@ -0,0 +1,123 @@ +#pragma once + +#include <util/generic/fwd.h> + +namespace NTi { + class ITypeFactoryInternal; + + class TNamedTypeBuilderRaw; + class TStructBuilderRaw; + class TTupleBuilderRaw; + class TTaggedBuilderRaw; + + class ITypeFactory; + using ITypeFactoryPtr = TIntrusivePtr<ITypeFactory>; + + class IPoolTypeFactory; + using IPoolTypeFactoryPtr = TIntrusivePtr<IPoolTypeFactory>; + + class TType; + using TTypePtr = TIntrusiveConstPtr<TType>; + + class TVoidType; + using TVoidTypePtr = TIntrusiveConstPtr<TVoidType>; + + class TNullType; + using TNullTypePtr = TIntrusiveConstPtr<TNullType>; + + class TPrimitiveType; + using TPrimitiveTypePtr = TIntrusiveConstPtr<TPrimitiveType>; + + class TBoolType; + using TBoolTypePtr = TIntrusiveConstPtr<TBoolType>; + + class TInt8Type; + using TInt8TypePtr = TIntrusiveConstPtr<TInt8Type>; + + class TInt16Type; + using TInt16TypePtr = TIntrusiveConstPtr<TInt16Type>; + + class TInt32Type; + using TInt32TypePtr = TIntrusiveConstPtr<TInt32Type>; + + class TInt64Type; + using TInt64TypePtr = TIntrusiveConstPtr<TInt64Type>; + + class TUint8Type; + using TUint8TypePtr = TIntrusiveConstPtr<TUint8Type>; + + class TUint16Type; + using TUint16TypePtr = TIntrusiveConstPtr<TUint16Type>; + + class TUint32Type; + using TUint32TypePtr = TIntrusiveConstPtr<TUint32Type>; + + class TUint64Type; + using TUint64TypePtr = TIntrusiveConstPtr<TUint64Type>; + + class TFloatType; + using TFloatTypePtr = TIntrusiveConstPtr<TFloatType>; + + class TDoubleType; + using TDoubleTypePtr = TIntrusiveConstPtr<TDoubleType>; + + class TStringType; + using TStringTypePtr = TIntrusiveConstPtr<TStringType>; + + class TUtf8Type; + using TUtf8TypePtr = TIntrusiveConstPtr<TUtf8Type>; + + class TDateType; + using TDateTypePtr = TIntrusiveConstPtr<TDateType>; + + class TDatetimeType; + using TDatetimeTypePtr = TIntrusiveConstPtr<TDatetimeType>; + + class TTimestampType; + using TTimestampTypePtr = TIntrusiveConstPtr<TTimestampType>; + + class TTzDateType; + using TTzDateTypePtr = TIntrusiveConstPtr<TTzDateType>; + + class TTzDatetimeType; + using TTzDatetimeTypePtr = TIntrusiveConstPtr<TTzDatetimeType>; + + class TTzTimestampType; + using TTzTimestampTypePtr = TIntrusiveConstPtr<TTzTimestampType>; + + class TIntervalType; + using TIntervalTypePtr = TIntrusiveConstPtr<TIntervalType>; + + class TDecimalType; + using TDecimalTypePtr = TIntrusiveConstPtr<TDecimalType>; + + class TJsonType; + using TJsonTypePtr = TIntrusiveConstPtr<TJsonType>; + + class TYsonType; + using TYsonTypePtr = TIntrusiveConstPtr<TYsonType>; + + class TUuidType; + using TUuidTypePtr = TIntrusiveConstPtr<TUuidType>; + + class TOptionalType; + using TOptionalTypePtr = TIntrusiveConstPtr<TOptionalType>; + + class TListType; + using TListTypePtr = TIntrusiveConstPtr<TListType>; + + class TDictType; + using TDictTypePtr = TIntrusiveConstPtr<TDictType>; + + class TStructType; + using TStructTypePtr = TIntrusiveConstPtr<TStructType>; + + class TTupleType; + using TTupleTypePtr = TIntrusiveConstPtr<TTupleType>; + + class TVariantType; + using TVariantTypePtr = TIntrusiveConstPtr<TVariantType>; + + class TTaggedType; + using TTaggedTypePtr = TIntrusiveConstPtr<TTaggedType>; +} diff --git a/library/cpp/type_info/type.cpp b/library/cpp/type_info/type.cpp new file mode 100644 index 0000000000..cee58a0a79 --- /dev/null +++ b/library/cpp/type_info/type.cpp @@ -0,0 +1,1662 @@ +#include "type.h" + +#include "type_factory.h" +#include "type_equivalence.h" + +#include <util/generic/overloaded.h> + +#include <util/digest/murmur.h> +#include <util/generic/hash_set.h> +#include <util/string/escape.h> + +namespace { + inline ui64 Hash(NTi::ETypeName type) { + return IntHash(static_cast<ui64>(type)); + } + + inline ui64 Hash(ui64 value, ui64 seed) { + return MurmurHash(&value, sizeof(value), seed); + } + + inline ui64 Hash(TStringBuf string, ui64 seed) { + seed = ::Hash(string.size(), seed); + return MurmurHash(string.data(), string.size(), seed); + } + + inline ui64 Hash(TMaybe<TStringBuf> string, ui64 seed) { + if (string.Defined()) { + return MurmurHash(string->data(), string->size(), seed); + } else { + return seed; + } + } + + TString Quote(TStringBuf s) { + TString result; + result.push_back('\''); + result += EscapeC(s); + result.push_back('\''); + return result; + } +} + +namespace NTi { + TType::TType(TMaybe<ui64> hash, ETypeName typeName) noexcept + : TypeName_(typeName) + , HasHash_(hash.Defined()) + , Hash_(hash.GetOrElse(0)) + { + } + + ui64 TType::CalculateHash() const noexcept { + return ::Hash(TypeName_); + } + + TMaybe<ui64> TType::GetHashRaw() const noexcept { + if (HasHash_.load(std::memory_order_seq_cst)) { + return Hash_.load(std::memory_order_seq_cst); + } else { + return Nothing(); + } + } + + ui64 TType::GetHash() const { + if (HasHash_.load(std::memory_order_seq_cst)) { + return Hash_.load(std::memory_order_seq_cst); + } else { + ui64 hash = VisitRaw([](const auto* type) { + return type->CalculateHash(); + }); + Hash_.store(hash, std::memory_order_seq_cst); + HasHash_.store(true, std::memory_order_seq_cst); + return hash; + } + } + + TTypePtr TType::StripTags() const noexcept { + return StripTagsRaw()->AsPtr(); + } + + const TType* TType::StripTagsRaw() const noexcept { + auto type = this; + while (type->IsTagged()) { + type = type->AsTaggedRaw()->GetItemTypeRaw(); + } + return type; + } + + TTypePtr TType::StripOptionals() const noexcept { + return StripOptionalsRaw()->AsPtr(); + } + + const TType* TType::StripOptionalsRaw() const noexcept { + auto type = this; + while (type->IsOptional()) { + type = type->AsOptionalRaw()->GetItemTypeRaw(); + } + return type; + } + + TTypePtr TType::StripTagsAndOptionals() const noexcept { + return StripTagsAndOptionalsRaw()->AsPtr(); + } + + const TType* TType::StripTagsAndOptionalsRaw() const noexcept { + auto type = this; + while (type->IsTagged() || type->IsOptional()) { + if (type->IsTagged()) { + type = type->AsTaggedRaw()->GetItemTypeRaw(); + } else { + type = type->AsOptionalRaw()->GetItemTypeRaw(); + } + } + return type; + } + + const TType* TType::Clone(ITypeFactoryInternal& factory) const noexcept { + return VisitRaw([&factory](const auto* type) -> const NTi::TType* { + return type->Clone(factory); + }); + } + + void TType::Drop(ITypeFactoryInternal& factory) noexcept { + VisitRaw([&factory](const auto* type) { + using T = std::remove_const_t<std::remove_pointer_t<decltype(type)>>; + return const_cast<T*>(type)->Drop(factory); + }); + } + + ITypeFactoryInternal* TType::GetFactory() const noexcept { + size_t manager_or_rc = FactoryOrRc_.load(std::memory_order_relaxed); + if (IsRc(manager_or_rc)) { + return NPrivate::GetDefaultHeapFactory(); + } else if (IsFactory(manager_or_rc)) { + return CastToFactory(manager_or_rc); + } else { + return nullptr; + } + } + + void TType::SetFactory(ITypeFactoryInternal* factory) noexcept { + if (factory == NPrivate::GetDefaultHeapFactory()) { + FactoryOrRc_.store(0b1u, std::memory_order_release); + } else { + FactoryOrRc_.store(CastFromFactory(factory), std::memory_order_release); + } + } + + ITypeFactoryInternal& TType::FactoryInternal(ITypeFactory& factory) noexcept { + return static_cast<ITypeFactoryInternal&>(factory); + } + + template <bool RefFactory> + void TType::RefImpl() noexcept { + size_t factoryOrRc = FactoryOrRc_.load(std::memory_order_relaxed); + if (Y_LIKELY(IsRc(factoryOrRc))) { + FactoryOrRc_.fetch_add(0b10u, std::memory_order_acq_rel); + } else if (Y_LIKELY(IsFactory(factoryOrRc))) { + auto factory = CastToFactory(factoryOrRc); + if (RefFactory) { + factory->Ref(); + } + factory->RefType(this); + } + } + + template void TType::RefImpl<true>() noexcept; + template void TType::RefImpl<false>() noexcept; + + template <bool UnRefFactory> + void TType::UnRefImpl() noexcept { + size_t factoryOrRc = FactoryOrRc_.load(std::memory_order_relaxed); + if (Y_LIKELY(IsRc(factoryOrRc))) { + size_t rc = FactoryOrRc_.fetch_sub(0b10u, std::memory_order_acq_rel); + if (rc == 0b11u) { + auto factory = NPrivate::GetDefaultHeapFactory(); + Drop(*factory); + factory->Delete(this); + } + } else if (Y_LIKELY(IsFactory(factoryOrRc))) { + auto factory = CastToFactory(factoryOrRc); + factory->UnRefType(this); + if (UnRefFactory) { + factory->UnRef(); + } + } + } + + template void TType::UnRefImpl<true>() noexcept; + template void TType::UnRefImpl<false>() noexcept; + + template <bool DecRefFactory> + void TType::DecRefImpl() noexcept { + size_t factoryOrRc = FactoryOrRc_.load(std::memory_order_relaxed); + if (Y_LIKELY(IsRc(factoryOrRc))) { + size_t rc = FactoryOrRc_.fetch_sub(2, std::memory_order_acq_rel); + if (rc == 2) { + Y_FAIL("DecRef isn't supposed to drop"); + } + } else if (Y_LIKELY(IsFactory(factoryOrRc))) { + auto factory = CastToFactory(factoryOrRc); + factory->DecRefType(this); + if (DecRefFactory) { + factory->DecRef(); + } + } + } + + template void TType::DecRefImpl<true>() noexcept; + template void TType::DecRefImpl<false>() noexcept; + + long TType::RefCountImpl() const noexcept { + size_t factoryOrRc = FactoryOrRc_.load(std::memory_order_relaxed); + if (Y_LIKELY(IsRc(factoryOrRc))) { + return factoryOrRc >> 1u; + } else if (Y_LIKELY(IsFactory(factoryOrRc))) { + return CastToFactory(factoryOrRc)->RefCountType(this); + } else { + return 0; + } + } + + template <typename T, typename TCtor> + const T* TType::Cached(const T* type, ITypeFactoryInternal& factory, TCtor&& ctor) { + const TType* result = factory.LookupCache(type); + + if (result == nullptr) { + result = std::forward<TCtor>(ctor)(); + factory.SaveCache(result); + } + + Y_VERIFY(result->GetTypeName() == type->GetTypeName()); + Y_VERIFY_DEBUG(result->GetHash() == type->GetHash()); + return static_cast<const T*>(result); + } + + bool operator==(const TType& lhs, const TType& rhs) { + Y_VERIFY(&lhs); + Y_VERIFY(&rhs); + return NEq::TStrictlyEqual().IgnoreHash(&lhs, &rhs); + } + + bool operator!=(const TType& lhs, const TType& rhs) + { + return !(lhs == rhs); + } + + TVoidType::TVoidType() + : TType({}, ETypeName::Void) + { + } + + TVoidTypePtr TVoidType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TVoidType* TVoidType::InstanceRaw() { + static auto singleton = TVoidType(); + return &singleton; + } + + const TVoidType* TVoidType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TVoidType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TNullType::TNullType() + : TType({}, ETypeName::Null) + { + } + + TNullTypePtr TNullType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TNullType* TNullType::InstanceRaw() { + static auto singleton = TNullType(); + return &singleton; + } + + const TNullType* TNullType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TNullType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TPrimitiveType::TPrimitiveType(TMaybe<ui64> hash, EPrimitiveTypeName primitiveTypeName) noexcept + : TType(hash, ToTypeName(primitiveTypeName)) + { + } + + TBoolType::TBoolType() + : TPrimitiveType({}, EPrimitiveTypeName::Bool) + { + } + + TBoolTypePtr TBoolType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TBoolType* TBoolType::InstanceRaw() { + static auto singleton = TBoolType(); + return &singleton; + } + + const TBoolType* TBoolType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TBoolType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TInt8Type::TInt8Type() + : TPrimitiveType({}, EPrimitiveTypeName::Int8) + { + } + + TInt8TypePtr TInt8Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TInt8Type* TInt8Type::InstanceRaw() { + static auto singleton = TInt8Type(); + return &singleton; + } + + const TInt8Type* TInt8Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TInt8Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TInt16Type::TInt16Type() + : TPrimitiveType({}, EPrimitiveTypeName::Int16) + { + } + + TInt16TypePtr TInt16Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TInt16Type* TInt16Type::InstanceRaw() { + static auto singleton = TInt16Type(); + return &singleton; + } + + const TInt16Type* TInt16Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TInt16Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TInt32Type::TInt32Type() + : TPrimitiveType({}, EPrimitiveTypeName::Int32) + { + } + + TInt32TypePtr TInt32Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TInt32Type* TInt32Type::InstanceRaw() { + static auto singleton = TInt32Type(); + return &singleton; + } + + const TInt32Type* TInt32Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TInt32Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TInt64Type::TInt64Type() + : TPrimitiveType({}, EPrimitiveTypeName::Int64) + { + } + + TInt64TypePtr TInt64Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TInt64Type* TInt64Type::InstanceRaw() { + static auto singleton = TInt64Type(); + return &singleton; + } + + const TInt64Type* TInt64Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TInt64Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TUint8Type::TUint8Type() + : TPrimitiveType({}, EPrimitiveTypeName::Uint8) + { + } + + TUint8TypePtr TUint8Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TUint8Type* TUint8Type::InstanceRaw() { + static auto singleton = TUint8Type(); + return &singleton; + } + + const TUint8Type* TUint8Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TUint8Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TUint16Type::TUint16Type() + : TPrimitiveType({}, EPrimitiveTypeName::Uint16) + { + } + + TUint16TypePtr TUint16Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TUint16Type* TUint16Type::InstanceRaw() { + static auto singleton = TUint16Type(); + return &singleton; + } + + const TUint16Type* TUint16Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TUint16Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TUint32Type::TUint32Type() + : TPrimitiveType({}, EPrimitiveTypeName::Uint32) + { + } + + TUint32TypePtr TUint32Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TUint32Type* TUint32Type::InstanceRaw() { + static auto singleton = TUint32Type(); + return &singleton; + } + + const TUint32Type* TUint32Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TUint32Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TUint64Type::TUint64Type() + : TPrimitiveType({}, EPrimitiveTypeName::Uint64) + { + } + + TUint64TypePtr TUint64Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TUint64Type* TUint64Type::InstanceRaw() { + static auto singleton = TUint64Type(); + return &singleton; + } + + const TUint64Type* TUint64Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TUint64Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TFloatType::TFloatType() + : TPrimitiveType({}, EPrimitiveTypeName::Float) + { + } + + TFloatTypePtr TFloatType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TFloatType* TFloatType::InstanceRaw() { + static auto singleton = TFloatType(); + return &singleton; + } + + const TFloatType* TFloatType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TFloatType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TDoubleType::TDoubleType() + : TPrimitiveType({}, EPrimitiveTypeName::Double) + { + } + + TDoubleTypePtr TDoubleType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TDoubleType* TDoubleType::InstanceRaw() { + static auto singleton = TDoubleType(); + return &singleton; + } + + const TDoubleType* TDoubleType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TDoubleType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TStringType::TStringType() + : TPrimitiveType({}, EPrimitiveTypeName::String) + { + } + + TStringTypePtr TStringType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TStringType* TStringType::InstanceRaw() { + static auto singleton = TStringType(); + return &singleton; + } + + const TStringType* TStringType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TStringType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TUtf8Type::TUtf8Type() + : TPrimitiveType({}, EPrimitiveTypeName::Utf8) + { + } + + TUtf8TypePtr TUtf8Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TUtf8Type* TUtf8Type::InstanceRaw() { + static auto singleton = TUtf8Type(); + return &singleton; + } + + const TUtf8Type* TUtf8Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TUtf8Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TDateType::TDateType() + : TPrimitiveType({}, EPrimitiveTypeName::Date) + { + } + + TDateTypePtr TDateType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TDateType* TDateType::InstanceRaw() { + static auto singleton = TDateType(); + return &singleton; + } + + const TDateType* TDateType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TDateType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TDatetimeType::TDatetimeType() + : TPrimitiveType({}, EPrimitiveTypeName::Datetime) + { + } + + TDatetimeTypePtr TDatetimeType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TDatetimeType* TDatetimeType::InstanceRaw() { + static auto singleton = TDatetimeType(); + return &singleton; + } + + const TDatetimeType* TDatetimeType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TDatetimeType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TTimestampType::TTimestampType() + : TPrimitiveType({}, EPrimitiveTypeName::Timestamp) + { + } + + TTimestampTypePtr TTimestampType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TTimestampType* TTimestampType::InstanceRaw() { + static auto singleton = TTimestampType(); + return &singleton; + } + + const TTimestampType* TTimestampType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TTimestampType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TTzDateType::TTzDateType() + : TPrimitiveType({}, EPrimitiveTypeName::TzDate) + { + } + + TTzDateTypePtr TTzDateType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TTzDateType* TTzDateType::InstanceRaw() { + static auto singleton = TTzDateType(); + return &singleton; + } + + const TTzDateType* TTzDateType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TTzDateType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TTzDatetimeType::TTzDatetimeType() + : TPrimitiveType({}, EPrimitiveTypeName::TzDatetime) + { + } + + TTzDatetimeTypePtr TTzDatetimeType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TTzDatetimeType* TTzDatetimeType::InstanceRaw() { + static auto singleton = TTzDatetimeType(); + return &singleton; + } + + const TTzDatetimeType* TTzDatetimeType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TTzDatetimeType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TTzTimestampType::TTzTimestampType() + : TPrimitiveType({}, EPrimitiveTypeName::TzTimestamp) + { + } + + TTzTimestampTypePtr TTzTimestampType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TTzTimestampType* TTzTimestampType::InstanceRaw() { + static auto singleton = TTzTimestampType(); + return &singleton; + } + + const TTzTimestampType* TTzTimestampType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TTzTimestampType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TIntervalType::TIntervalType() + : TPrimitiveType({}, EPrimitiveTypeName::Interval) + { + } + + TIntervalTypePtr TIntervalType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TIntervalType* TIntervalType::InstanceRaw() { + static auto singleton = TIntervalType(); + return &singleton; + } + + const TIntervalType* TIntervalType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TIntervalType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TDecimalType::TDecimalType(TMaybe<ui64> hash, ui8 precision, ui8 scale) noexcept + : TPrimitiveType(hash, EPrimitiveTypeName::Decimal) + , Precision_(precision) + , Scale_(scale) + { + } + + TDecimalTypePtr TDecimalType::Create(ITypeFactory& factory, ui8 precision, ui8 scale) { + return CreateRaw(factory, precision, scale)->AsPtr(); + } + + const TDecimalType* TDecimalType::CreateRaw(ITypeFactory& factory, ui8 precision, ui8 scale) { + Y_ENSURE_EX( + scale <= precision, + TIllegalTypeException() << "decimal scale " << (ui32)scale + << " should be no greater than decimal precision " << (ui32)precision); + + return TDecimalType({}, precision, scale).Clone(FactoryInternal(factory)); + } + + ui64 TDecimalType::CalculateHash() const noexcept { + auto hash = TType::CalculateHash(); + hash = ::Hash(Precision_, hash); + hash = ::Hash(Scale_, hash); + return hash; + } + + const TDecimalType* TDecimalType::Clone(ITypeFactoryInternal& factory) const noexcept { + return Cached(this, factory, [this, &factory]() -> const TDecimalType* { + auto hash = GetHashRaw(); + auto precision = Precision_; + auto scale = Scale_; + return factory.New<TDecimalType>(hash, precision, scale); + }); + } + + void TDecimalType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TJsonType::TJsonType() + : TPrimitiveType({}, EPrimitiveTypeName::Json) + { + } + + TJsonTypePtr TJsonType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TJsonType* TJsonType::InstanceRaw() { + static auto singleton = TJsonType(); + return &singleton; + } + + const TJsonType* TJsonType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TJsonType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TYsonType::TYsonType() + : TPrimitiveType({}, EPrimitiveTypeName::Yson) + { + } + + TYsonTypePtr TYsonType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TYsonType* TYsonType::InstanceRaw() { + static auto singleton = TYsonType(); + return &singleton; + } + + const TYsonType* TYsonType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TYsonType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TUuidType::TUuidType() + : TPrimitiveType({}, EPrimitiveTypeName::Uuid) + { + } + + TUuidTypePtr TUuidType::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TUuidType* TUuidType::InstanceRaw() { + static auto singleton = TUuidType(); + return &singleton; + } + + const TUuidType* TUuidType::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TUuidType::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TOptionalType::TOptionalType(TMaybe<ui64> hash, const TType* item) noexcept + : TType(hash, ETypeName::Optional) + , Item_(item) + { + } + + TOptionalTypePtr TOptionalType::Create(ITypeFactory& factory, TTypePtr item) { + return CreateRaw(factory, item.Get())->AsPtr(); + } + + const TOptionalType* TOptionalType::CreateRaw(ITypeFactory& factory, const TType* item) { + return TOptionalType({}, item).Clone(FactoryInternal(factory)); + } + + const TOptionalType* TOptionalType::Clone(ITypeFactoryInternal& factory) const noexcept { + return Cached(this, factory, [this, &factory]() -> const TOptionalType* { + auto hash = GetHashRaw(); + auto item = factory.Own(Item_); + return factory.New<TOptionalType>(hash, item); + }); + } + + void TOptionalType::Drop(ITypeFactoryInternal& factory) noexcept { + factory.Disown(Item_); + } + + ui64 TOptionalType::CalculateHash() const noexcept { + auto hash = TType::CalculateHash(); + hash = ::Hash(Item_->GetHash(), hash); + return hash; + } + + TListType::TListType(TMaybe<ui64> hash, const TType* item) noexcept + : TType(hash, ETypeName::List) + , Item_(item) + { + } + + TListTypePtr TListType::Create(ITypeFactory& factory, TTypePtr item) { + return CreateRaw(factory, item.Get())->AsPtr(); + } + + const TListType* TListType::CreateRaw(ITypeFactory& factory, const TType* item) { + return TListType({}, item).Clone(FactoryInternal(factory)); + } + + const TListType* TListType::Clone(ITypeFactoryInternal& factory) const noexcept { + return Cached(this, factory, [this, &factory]() -> const TListType* { + auto hash = GetHashRaw(); + auto item = factory.Own(Item_); + return factory.New<TListType>(hash, item); + }); + } + + void TListType::Drop(ITypeFactoryInternal& factory) noexcept { + factory.Disown(Item_); + } + + ui64 TListType::CalculateHash() const noexcept { + auto hash = TType::CalculateHash(); + hash = ::Hash(Item_->GetHash(), hash); + return hash; + } + + TDictType::TDictType(TMaybe<ui64> hash, const TType* key, const TType* value) noexcept + : TType(hash, ETypeName::Dict) + , Key_(key) + , Value_(value) + { + } + + TDictTypePtr TDictType::Create(ITypeFactory& factory, TTypePtr key, TTypePtr value) { + return CreateRaw(factory, key.Get(), value.Get())->AsPtr(); + } + + const TDictType* TDictType::CreateRaw(ITypeFactory& factory, const TType* key, const TType* value) { + return TDictType({}, key, value).Clone(FactoryInternal(factory)); + } + + const TDictType* TDictType::Clone(ITypeFactoryInternal& factory) const noexcept { + return Cached(this, factory, [this, &factory]() -> const TDictType* { + auto hash = GetHashRaw(); + auto key = factory.Own(Key_); + auto value = factory.Own(Value_); + return factory.New<TDictType>(hash, key, value); + }); + } + + void TDictType::Drop(ITypeFactoryInternal& factory) noexcept { + factory.Disown(Key_); + factory.Disown(Value_); + } + + ui64 TDictType::CalculateHash() const noexcept { + auto hash = TType::CalculateHash(); + hash = ::Hash(Key_->GetHash(), hash); + hash = ::Hash(Value_->GetHash(), hash); + return hash; + } + + TStructType::TMember::TMember(TStringBuf name, const TType* type) + : Name_(name) + , Type_(type) + { + } + + ui64 TStructType::TMember::Hash() const { + auto hash = 0x10000; + hash = ::Hash(Name_, hash); + hash = ::Hash(Type_->GetHash(), hash); + return hash; + } + + TStructType::TOwnedMember::TOwnedMember(TString name, TTypePtr type) + : Name_(std::move(name)) + , Type_(std::move(type)) + { + } + + TStructType::TOwnedMember::operator TStructType::TMember() const& { + return TStructType::TMember(Name_, Type_.Get()); + } + + TStructType::TStructType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, TMembers members, TConstArrayRef<size_t> sortedItems) noexcept + : TType(hash, ETypeName::Struct) + , Name_(name) + , Members_(members) + , SortedMembers_(sortedItems) + { + } + + TStructTypePtr TStructType::Create(ITypeFactory& factory, TStructType::TOwnedMembers members) { + return Create(factory, Nothing(), members); + } + + TStructTypePtr TStructType::Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TStructType::TOwnedMembers members) { + auto rawItems = TTempArray<TMember>(members.size()); + for (size_t i = 0; i < members.size(); ++i) { + new (rawItems.Data() + i) TMember(members[i]); + } + return CreateRaw(factory, name, TArrayRef(rawItems.Data(), members.size()))->AsPtr(); + } + + const TStructType* TStructType::CreateRaw(ITypeFactory& factory, TStructType::TMembers members) { + return CreateRaw(factory, Nothing(), members); + } + + const TStructType* TStructType::CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, TStructType::TMembers members) { + auto sortedMembersArray = TTempArray<size_t>(members.size()); + auto sortedMembers = TArrayRef(sortedMembersArray.Data(), members.size()); + MakeSortedMembers(members, sortedMembers); + return TStructType({}, name, members, sortedMembers).Clone(FactoryInternal(factory)); + } + + void TStructType::MakeSortedMembers(TStructType::TMembers members, TArrayRef<size_t> sortedItems) { + Y_VERIFY(members.size() == sortedItems.size()); + + for (size_t i = 0; i < members.size(); ++i) { + sortedItems[i] = i; + } + + Sort(sortedItems.begin(), sortedItems.end(), [members](size_t lhs, size_t rhs) { + return members[lhs].GetName() < members[rhs].GetName(); + }); + + for (size_t i = 1; i < members.size(); ++i) { + if (members[sortedItems[i - 1]].GetName() == members[sortedItems[i]].GetName()) { + ythrow TIllegalTypeException() << "duplicate struct item " << Quote(members[sortedItems[i]].GetName()); + } + } + } + + const TStructType* TStructType::Clone(ITypeFactoryInternal& factory) const noexcept { + return Cached(this, factory, [this, &factory]() -> const TStructType* { + auto hash = GetHashRaw(); + auto name = factory.AllocateStringMaybe(Name_); + auto members = factory.NewArray<TMember>(Members_.size(), [this, &factory](TMember* item, size_t i) { + auto name = factory.AllocateString(Members_[i].GetName()); + auto type = factory.Own(Members_[i].GetTypeRaw()); + new (item) TMember(name, type); + }); + auto sortedItems = factory.AllocateArrayFor<size_t>(SortedMembers_.size()); + Copy(SortedMembers_.begin(), SortedMembers_.end(), sortedItems); + return factory.New<TStructType>(hash, name, members, TArrayRef{sortedItems, SortedMembers_.size()}); + }); + } + + void TStructType::Drop(ITypeFactoryInternal& factory) noexcept { + factory.FreeStringMaybe(Name_); + factory.DeleteArray(Members_, [&factory](const TMember* item, size_t) { + factory.FreeString(item->GetName()); + factory.Disown(item->GetTypeRaw()); + }); + factory.Free(const_cast<void*>(static_cast<const void*>(SortedMembers_.data()))); + } + + ui64 TStructType::CalculateHash() const noexcept { + auto hash = TType::CalculateHash(); + hash = ::Hash(Name_, hash); + hash = ::Hash(Members_.size(), hash); + for (auto& item : Members_) { + hash = ::Hash(item.Hash(), hash); + } + return hash; + } + + bool TStructType::HasMember(TStringBuf name) const noexcept { + return GetMemberIndex(name) != -1; + } + + const TStructType::TMember& TStructType::GetMember(TStringBuf name) const { + auto idx = GetMemberIndex(name); + if (idx == -1) { + ythrow TItemNotFound() << "no item named " << Quote(name); + } else { + return Members_[idx]; + } + } + + ssize_t TStructType::GetMemberIndex(TStringBuf name) const noexcept { + auto it = LowerBound(SortedMembers_.begin(), SortedMembers_.end(), name, [this](size_t i, TStringBuf name) { + return Members_[i].GetName() < name; + }); + + if (it == SortedMembers_.end() || Members_[*it].GetName() != name) { + return -1; + } else { + return *it; + } + } + + TTupleType::TElement::TElement(const TType* type) + : Type_(type) + { + } + + ui64 TTupleType::TElement::Hash() const { + auto hash = 0x10001; + hash = ::Hash(Type_->GetHash(), hash); + return hash; + } + + TTupleType::TOwnedElement::TOwnedElement(TTypePtr type) + : Type_(std::move(type)) + { + } + + TTupleType::TOwnedElement::operator TTupleType::TElement() const& { + return TTupleType::TElement(Type_.Get()); + } + + TTupleType::TTupleType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, TElements elements) noexcept + : TType(hash, ETypeName::Tuple) + , Name_(name) + , Elements_(elements) + { + } + + TTupleTypePtr TTupleType::Create(ITypeFactory& factory, TTupleType::TOwnedElements elements) { + return Create(factory, Nothing(), elements); + } + + TTupleTypePtr TTupleType::Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TTupleType::TOwnedElements elements) { + auto rawItems = TTempArray<TElement>(elements.size()); + for (size_t i = 0; i < elements.size(); ++i) { + new (rawItems.Data() + i) TElement(elements[i]); + } + return CreateRaw(factory, name, TArrayRef(rawItems.Data(), elements.size()))->AsPtr(); + } + + const TTupleType* TTupleType::CreateRaw(ITypeFactory& factory, TTupleType::TElements elements) { + return CreateRaw(factory, Nothing(), elements); + } + + const TTupleType* TTupleType::CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, TTupleType::TElements elements) { + return TTupleType({}, name, elements).Clone(FactoryInternal(factory)); + } + + const TTupleType* TTupleType::Clone(ITypeFactoryInternal& factory) const noexcept { + return Cached(this, factory, [this, &factory]() -> const TTupleType* { + auto hash = GetHashRaw(); + auto name = factory.AllocateStringMaybe(Name_); + auto elements = factory.NewArray<TElement>(Elements_.size(), [this, &factory](TElement* item, size_t i) { + auto type = factory.Own(Elements_[i].GetTypeRaw()); + new (item) TElement(type); + }); + return factory.New<TTupleType>(hash, name, elements); + }); + } + + void TTupleType::Drop(ITypeFactoryInternal& factory) noexcept { + factory.FreeStringMaybe(Name_); + factory.DeleteArray(Elements_, [&factory](const TElement* item, size_t) { + factory.Disown(item->GetTypeRaw()); + }); + } + + ui64 TTupleType::CalculateHash() const noexcept { + auto hash = TType::CalculateHash(); + hash = ::Hash(Name_, hash); + hash = ::Hash(Elements_.size(), hash); + for (auto& item : Elements_) { + hash = ::Hash(item.Hash(), hash); + } + return hash; + } + + TVariantType::TVariantType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, const TType* inner) noexcept + : TType(hash, ETypeName::Variant) + , Name_(name) + , Underlying_(inner) + { + } + + TVariantTypePtr TVariantType::Create(ITypeFactory& factory, TTypePtr inner) { + return Create(factory, Nothing(), std::move(inner)); + } + + TVariantTypePtr TVariantType::Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TTypePtr inner) { + return CreateRaw(factory, name, inner.Get())->AsPtr(); + } + + const TVariantType* TVariantType::CreateRaw(ITypeFactory& factory, const TType* inner) { + return CreateRaw(factory, Nothing(), inner); + } + + const TVariantType* TVariantType::CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, const TType* inner) { + inner->VisitRaw(TOverloaded{ + [&](const TStructType* s) { + Y_ENSURE_EX( + !s->GetMembers().empty(), + TIllegalTypeException() << "variant should contain at least one alternative"); + }, + [&](const TTupleType* t) { + Y_ENSURE_EX( + !t->GetElements().empty(), + TIllegalTypeException() << "variant should contain at least one alternative"); + }, + [](const TType* t) { + ythrow TIllegalTypeException() << "variants can only contain structs and tuples, got " + << t->GetTypeName() << " instead"; + }}); + + return TVariantType({}, name, inner).Clone(FactoryInternal(factory)); + } + + const TVariantType* TVariantType::Clone(ITypeFactoryInternal& factory) const noexcept { + return Cached(this, factory, [this, &factory]() -> const TVariantType* { + auto hash = GetHashRaw(); + auto name = factory.AllocateStringMaybe(Name_); + auto inner = factory.Own(Underlying_); + return factory.New<TVariantType>(hash, name, inner); + }); + } + + void TVariantType::Drop(ITypeFactoryInternal& factory) noexcept { + factory.FreeStringMaybe(Name_); + factory.Disown(Underlying_); + } + + ui64 TVariantType::CalculateHash() const noexcept { + auto hash = TType::CalculateHash(); + hash = ::Hash(Name_, hash); + hash = ::Hash(Underlying_->GetHash(), hash); + return hash; + } + + TTaggedType::TTaggedType(TMaybe<ui64> hash, const TType* item, TStringBuf tag) noexcept + : TType(hash, ETypeName::Tagged) + , Item_(item) + , Tag_(tag) + { + } + + TTaggedTypePtr TTaggedType::Create(ITypeFactory& factory, TTypePtr type, TStringBuf tag) { + return CreateRaw(factory, type.Get(), tag)->AsPtr(); + } + + const TTaggedType* TTaggedType::CreateRaw(ITypeFactory& factory, const TType* type, TStringBuf tag) { + return TTaggedType({}, type, tag).Clone(FactoryInternal(factory)); + } + + const TTaggedType* TTaggedType::Clone(ITypeFactoryInternal& factory) const noexcept { + return Cached(this, factory, [this, &factory]() -> const TTaggedType* { + auto hash = GetHashRaw(); + auto item = factory.Own(Item_); + auto tag = factory.AllocateString(Tag_); + return factory.New<TTaggedType>(hash, item, tag); + }); + } + + void TTaggedType::Drop(ITypeFactoryInternal& factory) noexcept { + factory.FreeString(Tag_); + factory.Disown(Item_); + } + + ui64 TTaggedType::CalculateHash() const noexcept { + auto hash = TType::CalculateHash(); + hash = ::Hash(Tag_, hash); + hash = ::Hash(Item_->GetHash(), hash); + return hash; + } + + TVoidTypePtr Void() { + return NPrivate::GetDefaultHeapFactory()->Void(); + } + + TNullTypePtr Null() { + return NPrivate::GetDefaultHeapFactory()->Null(); + } + + TBoolTypePtr Bool() { + return NPrivate::GetDefaultHeapFactory()->Bool(); + } + + TInt8TypePtr Int8() { + return NPrivate::GetDefaultHeapFactory()->Int8(); + } + + TInt16TypePtr Int16() { + return NPrivate::GetDefaultHeapFactory()->Int16(); + } + + TInt32TypePtr Int32() { + return NPrivate::GetDefaultHeapFactory()->Int32(); + } + + TInt64TypePtr Int64() { + return NPrivate::GetDefaultHeapFactory()->Int64(); + } + + TUint8TypePtr Uint8() { + return NPrivate::GetDefaultHeapFactory()->Uint8(); + } + + TUint16TypePtr Uint16() { + return NPrivate::GetDefaultHeapFactory()->Uint16(); + } + + TUint32TypePtr Uint32() { + return NPrivate::GetDefaultHeapFactory()->Uint32(); + } + + TUint64TypePtr Uint64() { + return NPrivate::GetDefaultHeapFactory()->Uint64(); + } + + TFloatTypePtr Float() { + return NPrivate::GetDefaultHeapFactory()->Float(); + } + + TDoubleTypePtr Double() { + return NPrivate::GetDefaultHeapFactory()->Double(); + } + + TStringTypePtr String() { + return NPrivate::GetDefaultHeapFactory()->String(); + } + + TUtf8TypePtr Utf8() { + return NPrivate::GetDefaultHeapFactory()->Utf8(); + } + + TDateTypePtr Date() { + return NPrivate::GetDefaultHeapFactory()->Date(); + } + + TDatetimeTypePtr Datetime() { + return NPrivate::GetDefaultHeapFactory()->Datetime(); + } + + TTimestampTypePtr Timestamp() { + return NPrivate::GetDefaultHeapFactory()->Timestamp(); + } + + TTzDateTypePtr TzDate() { + return NPrivate::GetDefaultHeapFactory()->TzDate(); + } + + TTzDatetimeTypePtr TzDatetime() { + return NPrivate::GetDefaultHeapFactory()->TzDatetime(); + } + + TTzTimestampTypePtr TzTimestamp() { + return NPrivate::GetDefaultHeapFactory()->TzTimestamp(); + } + + TIntervalTypePtr Interval() { + return NPrivate::GetDefaultHeapFactory()->Interval(); + } + + TDecimalTypePtr Decimal(ui8 precision, ui8 scale) { + return NPrivate::GetDefaultHeapFactory()->Decimal(precision, scale); + } + + TJsonTypePtr Json() { + return NPrivate::GetDefaultHeapFactory()->Json(); + } + + TYsonTypePtr Yson() { + return NPrivate::GetDefaultHeapFactory()->Yson(); + } + + TUuidTypePtr Uuid() { + return NPrivate::GetDefaultHeapFactory()->Uuid(); + } + + TOptionalTypePtr Optional(TTypePtr item) { + return NPrivate::GetDefaultHeapFactory()->Optional(std::move(item)); + } + + TListTypePtr List(TTypePtr item) { + return NPrivate::GetDefaultHeapFactory()->List(std::move(item)); + } + + TDictTypePtr Dict(TTypePtr key, TTypePtr value) { + return NPrivate::GetDefaultHeapFactory()->Dict(std::move(key), std::move(value)); + } + + TStructTypePtr Struct(TStructType::TOwnedMembers members) { + return NPrivate::GetDefaultHeapFactory()->Struct(members); + } + + TStructTypePtr Struct(TMaybe<TStringBuf> name, TStructType::TOwnedMembers members) { + return NPrivate::GetDefaultHeapFactory()->Struct(name, members); + } + + TTupleTypePtr Tuple(TTupleType::TOwnedElements elements) { + return NPrivate::GetDefaultHeapFactory()->Tuple(elements); + } + + TTupleTypePtr Tuple(TMaybe<TStringBuf> name, TTupleType::TOwnedElements elements) { + return NPrivate::GetDefaultHeapFactory()->Tuple(name, elements); + } + + TVariantTypePtr Variant(TTypePtr underlying) { + return NPrivate::GetDefaultHeapFactory()->Variant(std::move(underlying)); + } + + TVariantTypePtr Variant(TMaybe<TStringBuf> name, TTypePtr underlying) { + return NPrivate::GetDefaultHeapFactory()->Variant(name, std::move(underlying)); + } + + TTaggedTypePtr Tagged(TTypePtr type, TStringBuf tag) { + return NPrivate::GetDefaultHeapFactory()->Tagged(std::move(type), tag); + } +} + +Y_DECLARE_OUT_SPEC(, NTi::TType, o, v) { + v.VisitRaw([&o](const auto* v) { o << *v; }); +} + +Y_DECLARE_OUT_SPEC(, NTi::TPrimitiveType, o, v) { + v.VisitPrimitiveRaw([&o](const auto* v) { o << *v; }); +} + +Y_DECLARE_OUT_SPEC(, NTi::TVoidType, o, v) { + Y_UNUSED(v); + o << "Void"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TNullType, o, v) { + Y_UNUSED(v); + o << "Null"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TBoolType, o, v) { + Y_UNUSED(v); + o << "Bool"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TInt8Type, o, v) { + Y_UNUSED(v); + o << "Int8"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TInt16Type, o, v) { + Y_UNUSED(v); + o << "Int16"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TInt32Type, o, v) { + Y_UNUSED(v); + o << "Int32"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TInt64Type, o, v) { + Y_UNUSED(v); + o << "Int64"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TUint8Type, o, v) { + Y_UNUSED(v); + o << "Uint8"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TUint16Type, o, v) { + Y_UNUSED(v); + o << "Uint16"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TUint32Type, o, v) { + Y_UNUSED(v); + o << "Uint32"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TUint64Type, o, v) { + Y_UNUSED(v); + o << "Uint64"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TFloatType, o, v) { + Y_UNUSED(v); + o << "Float"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TDoubleType, o, v) { + Y_UNUSED(v); + o << "Double"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TStringType, o, v) { + Y_UNUSED(v); + o << "String"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TUtf8Type, o, v) { + Y_UNUSED(v); + o << "Utf8"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TDateType, o, v) { + Y_UNUSED(v); + o << "Date"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TDatetimeType, o, v) { + Y_UNUSED(v); + o << "Datetime"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TTimestampType, o, v) { + Y_UNUSED(v); + o << "Timestamp"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TTzDateType, o, v) { + Y_UNUSED(v); + o << "TzDate"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TTzDatetimeType, o, v) { + Y_UNUSED(v); + o << "TzDatetime"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TTzTimestampType, o, v) { + Y_UNUSED(v); + o << "TzTimestamp"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TIntervalType, o, v) { + Y_UNUSED(v); + o << "Interval"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TDecimalType, o, v) { + o << "Decimal(" << (i32)v.GetPrecision() << ", " << (i32)v.GetScale() << ')'; +} + +Y_DECLARE_OUT_SPEC(, NTi::TJsonType, o, v) { + Y_UNUSED(v); + o << "Json"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TYsonType, o, v) { + Y_UNUSED(v); + o << "Yson"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TUuidType, o, v) { + Y_UNUSED(v); + o << "Uuid"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TOptionalType, o, v) { + o << "Optional<" << *v.GetItemTypeRaw() << ">"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TListType, o, v) { + o << "List<" << *v.GetItemTypeRaw() << ">"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TDictType, o, v) { + o << "Dict<" << *v.GetKeyTypeRaw() << ", " << *v.GetValueTypeRaw() << ">"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TStructType, o, v) { + o << "Struct"; + + if (v.GetName().Defined()) { + o << "[" << Quote(*v.GetName()) << "]"; + } + + o << "<"; + const char* sep = ""; + for (auto& item : v.GetMembers()) { + o << sep << Quote(item.GetName()) << ": " << *item.GetTypeRaw(); + sep = ", "; + } + o << ">"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TTupleType, o, v) { + o << "Tuple"; + + if (v.GetName().Defined()) { + o << "[" << Quote(*v.GetName()) << "]"; + } + + o << "<"; + const char* sep = ""; + for (auto& item : v.GetElements()) { + o << sep << *item.GetTypeRaw(); + sep = ", "; + } + o << ">"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TVariantType, o, v) { + o << "Variant"; + + if (v.GetName().Defined()) { + o << "[" << Quote(*v.GetName()) << "]"; + } + + o << "<"; + v.VisitUnderlyingRaw( + TOverloaded{ + [&o](const NTi::TStructType* s) { + const char* sep = ""; + for (auto& item : s->GetMembers()) { + o << sep << Quote(item.GetName()) << ": " << *item.GetTypeRaw(); + sep = ", "; + } + }, + [&o](const NTi::TTupleType* t) { + const char* sep = ""; + for (auto& item : t->GetElements()) { + o << sep << *item.GetTypeRaw(); + sep = ", "; + } + }}); + o << ">"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TTaggedType, o, v) { + o << "Tagged<" << *v.GetItemTypeRaw() << ", " << Quote(v.GetTag()) << ">"; +} + +static_assert(std::is_trivially_destructible_v<NTi::TVoidType>); +static_assert(std::is_trivially_destructible_v<NTi::TNullType>); +static_assert(std::is_trivially_destructible_v<NTi::TPrimitiveType>); +static_assert(std::is_trivially_destructible_v<NTi::TBoolType>); +static_assert(std::is_trivially_destructible_v<NTi::TInt8Type>); +static_assert(std::is_trivially_destructible_v<NTi::TInt16Type>); +static_assert(std::is_trivially_destructible_v<NTi::TInt32Type>); +static_assert(std::is_trivially_destructible_v<NTi::TInt64Type>); +static_assert(std::is_trivially_destructible_v<NTi::TUint8Type>); +static_assert(std::is_trivially_destructible_v<NTi::TUint16Type>); +static_assert(std::is_trivially_destructible_v<NTi::TUint32Type>); +static_assert(std::is_trivially_destructible_v<NTi::TUint64Type>); +static_assert(std::is_trivially_destructible_v<NTi::TFloatType>); +static_assert(std::is_trivially_destructible_v<NTi::TDoubleType>); +static_assert(std::is_trivially_destructible_v<NTi::TStringType>); +static_assert(std::is_trivially_destructible_v<NTi::TUtf8Type>); +static_assert(std::is_trivially_destructible_v<NTi::TDateType>); +static_assert(std::is_trivially_destructible_v<NTi::TDatetimeType>); +static_assert(std::is_trivially_destructible_v<NTi::TTimestampType>); +static_assert(std::is_trivially_destructible_v<NTi::TTzDateType>); +static_assert(std::is_trivially_destructible_v<NTi::TTzDatetimeType>); +static_assert(std::is_trivially_destructible_v<NTi::TTzTimestampType>); +static_assert(std::is_trivially_destructible_v<NTi::TIntervalType>); +static_assert(std::is_trivially_destructible_v<NTi::TDecimalType>); +static_assert(std::is_trivially_destructible_v<NTi::TJsonType>); +static_assert(std::is_trivially_destructible_v<NTi::TYsonType>); +static_assert(std::is_trivially_destructible_v<NTi::TUuidType>); +static_assert(std::is_trivially_destructible_v<NTi::TOptionalType>); +static_assert(std::is_trivially_destructible_v<NTi::TListType>); +static_assert(std::is_trivially_destructible_v<NTi::TDictType>); +static_assert(std::is_trivially_destructible_v<NTi::TStructType>); +static_assert(std::is_trivially_destructible_v<NTi::TStructType::TMember>); +static_assert(std::is_trivially_destructible_v<NTi::TTupleType>); +static_assert(std::is_trivially_destructible_v<NTi::TTupleType::TElement>); +static_assert(std::is_trivially_destructible_v<NTi::TVariantType>); +static_assert(std::is_trivially_destructible_v<NTi::TTaggedType>); diff --git a/library/cpp/type_info/type.h b/library/cpp/type_info/type.h new file mode 100644 index 0000000000..1577f1cee3 --- /dev/null +++ b/library/cpp/type_info/type.h @@ -0,0 +1,2421 @@ +#pragma once + +//! @file type.h +//! +//! Hierarchy of classes that represent types. +#include "fwd.h" + +#include "error.h" +#include "type_list.h" + +#include <atomic> +#include <util/generic/array_ref.h> +#include <util/generic/maybe.h> +#include <util/generic/strbuf.h> +#include <util/generic/string.h> + +namespace NTi { + /// Represents a single type. + /// + /// Create instances of types using type factory (see `NTi::ITypeFactory`). + /// + /// Introspect them using associated methods and functions that work with `ETypeName`. + /// + /// Pattern-match them using the `Visit` method. + /// + /// Serialize and deserialize them using functions from `NTi::NIo`. + class TType { + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + template <typename T> + friend class ::TDefaultIntrusivePtrOps; + + public: + TTypePtr AsPtr() const noexcept { + return const_cast<TType*>(this); + } + + protected: + explicit TType(TMaybe<ui64> hash, ETypeName typeName) noexcept; + + protected: + /// Calculate hash for this type. This function is lazily called by `GetHash`. + /// + /// Note: this function is not marked as `virtual` because we use our own dispatch via `Visit`. + ui64 CalculateHash() const noexcept; + + public: + /// Get hash of this type. Hashes follow the 'strict equivalence' relation (see `type_equivalence.h`). + ui64 GetHash() const; + + /// Get hash of this type. If hash is not calculated, returns nothing. + TMaybe<ui64> GetHashRaw() const noexcept; + + /// Get name of this type as a `NTi::ETypeName` enumerator. + ETypeName GetTypeName() const noexcept { + return TypeName_; + } + + /// @name Simple type downcast functions + /// + /// Check if type is of the given subclass and convert between subclasses. + /// Conversions panic if downcasting into an incompatible type. + /// + /// @{ + inline bool IsVoid() const noexcept; + inline TVoidTypePtr AsVoid() const noexcept; + inline const TVoidType* AsVoidRaw() const noexcept; + + inline bool IsNull() const noexcept; + inline TNullTypePtr AsNull() const noexcept; + inline const TNullType* AsNullRaw() const noexcept; + + inline bool IsPrimitive() const noexcept; + inline TPrimitiveTypePtr AsPrimitive() const noexcept; + inline const TPrimitiveType* AsPrimitiveRaw() const noexcept; + + inline bool IsBool() const noexcept; + inline TBoolTypePtr AsBool() const noexcept; + inline const TBoolType* AsBoolRaw() const noexcept; + + inline bool IsInt8() const noexcept; + inline TInt8TypePtr AsInt8() const noexcept; + inline const TInt8Type* AsInt8Raw() const noexcept; + + inline bool IsInt16() const noexcept; + inline TInt16TypePtr AsInt16() const noexcept; + inline const TInt16Type* AsInt16Raw() const noexcept; + + inline bool IsInt32() const noexcept; + inline TInt32TypePtr AsInt32() const noexcept; + inline const TInt32Type* AsInt32Raw() const noexcept; + + inline bool IsInt64() const noexcept; + inline TInt64TypePtr AsInt64() const noexcept; + inline const TInt64Type* AsInt64Raw() const noexcept; + + inline bool IsUint8() const noexcept; + inline TUint8TypePtr AsUint8() const noexcept; + inline const TUint8Type* AsUint8Raw() const noexcept; + + inline bool IsUint16() const noexcept; + inline TUint16TypePtr AsUint16() const noexcept; + inline const TUint16Type* AsUint16Raw() const noexcept; + + inline bool IsUint32() const noexcept; + inline TUint32TypePtr AsUint32() const noexcept; + inline const TUint32Type* AsUint32Raw() const noexcept; + + inline bool IsUint64() const noexcept; + inline TUint64TypePtr AsUint64() const noexcept; + inline const TUint64Type* AsUint64Raw() const noexcept; + + inline bool IsFloat() const noexcept; + inline TFloatTypePtr AsFloat() const noexcept; + inline const TFloatType* AsFloatRaw() const noexcept; + + inline bool IsDouble() const noexcept; + inline TDoubleTypePtr AsDouble() const noexcept; + inline const TDoubleType* AsDoubleRaw() const noexcept; + + inline bool IsString() const noexcept; + inline TStringTypePtr AsString() const noexcept; + inline const TStringType* AsStringRaw() const noexcept; + + inline bool IsUtf8() const noexcept; + inline TUtf8TypePtr AsUtf8() const noexcept; + inline const TUtf8Type* AsUtf8Raw() const noexcept; + + inline bool IsDate() const noexcept; + inline TDateTypePtr AsDate() const noexcept; + inline const TDateType* AsDateRaw() const noexcept; + + inline bool IsDatetime() const noexcept; + inline TDatetimeTypePtr AsDatetime() const noexcept; + inline const TDatetimeType* AsDatetimeRaw() const noexcept; + + inline bool IsTimestamp() const noexcept; + inline TTimestampTypePtr AsTimestamp() const noexcept; + inline const TTimestampType* AsTimestampRaw() const noexcept; + + inline bool IsTzDate() const noexcept; + inline TTzDateTypePtr AsTzDate() const noexcept; + inline const TTzDateType* AsTzDateRaw() const noexcept; + + inline bool IsTzDatetime() const noexcept; + inline TTzDatetimeTypePtr AsTzDatetime() const noexcept; + inline const TTzDatetimeType* AsTzDatetimeRaw() const noexcept; + + inline bool IsTzTimestamp() const noexcept; + inline TTzTimestampTypePtr AsTzTimestamp() const noexcept; + inline const TTzTimestampType* AsTzTimestampRaw() const noexcept; + + inline bool IsInterval() const noexcept; + inline TIntervalTypePtr AsInterval() const noexcept; + inline const TIntervalType* AsIntervalRaw() const noexcept; + + inline bool IsDecimal() const noexcept; + inline TDecimalTypePtr AsDecimal() const noexcept; + inline const TDecimalType* AsDecimalRaw() const noexcept; + + inline bool IsJson() const noexcept; + inline TJsonTypePtr AsJson() const noexcept; + inline const TJsonType* AsJsonRaw() const noexcept; + + inline bool IsYson() const noexcept; + inline TYsonTypePtr AsYson() const noexcept; + inline const TYsonType* AsYsonRaw() const noexcept; + + inline bool IsUuid() const noexcept; + inline TUuidTypePtr AsUuid() const noexcept; + inline const TUuidType* AsUuidRaw() const noexcept; + + inline bool IsOptional() const noexcept; + inline TOptionalTypePtr AsOptional() const noexcept; + inline const TOptionalType* AsOptionalRaw() const noexcept; + + inline bool IsList() const noexcept; + inline TListTypePtr AsList() const noexcept; + inline const TListType* AsListRaw() const noexcept; + + inline bool IsDict() const noexcept; + inline TDictTypePtr AsDict() const noexcept; + inline const TDictType* AsDictRaw() const noexcept; + + inline bool IsStruct() const noexcept; + inline TStructTypePtr AsStruct() const noexcept; + inline const TStructType* AsStructRaw() const noexcept; + + inline bool IsTuple() const noexcept; + inline TTupleTypePtr AsTuple() const noexcept; + inline const TTupleType* AsTupleRaw() const noexcept; + + inline bool IsVariant() const noexcept; + inline TVariantTypePtr AsVariant() const noexcept; + inline const TVariantType* AsVariantRaw() const noexcept; + + inline bool IsTagged() const noexcept; + inline TTaggedTypePtr AsTagged() const noexcept; + inline const TTaggedType* AsTaggedRaw() const noexcept; + + /// @} + + /// Recursively descends to tagged types and returns first non-tagged type. + TTypePtr StripTags() const noexcept; + + /// Like `StripTags`, but returns a raw pointer. + const TType* StripTagsRaw() const noexcept; + + /// Recursively descends to optional types and returns first non-optional type. + TTypePtr StripOptionals() const noexcept; + + /// Like `StripOptionals`, but returns a raw pointer. + const TType* StripOptionalsRaw() const noexcept; + + /// Recursively descends to tagged and optional types and returns first non-tagged non-optional type. + TTypePtr StripTagsAndOptionals() const noexcept; + + /// Like `StripTagsAndOptionals`, but returns a raw pointer. + const TType* StripTagsAndOptionalsRaw() const noexcept; + + /// Cast this base class down to the most-derived class and pass it to the `visitor`. + /// + /// This function is used as a safer alternative to manually downcasting types via `IsType`/`AsType` calls. + /// It works like `std::visit` for types, except that it doesn't produce as much code bloat as `std::visit` + /// does, and can be optimized better. It casts an instance of `NTi::TType` down to the most-derived class, + /// and passes an intrusive pointer to that concrete type to the `visitor` functor. That is, `visitor` should + /// be a callable which can handle `NTi::TVoidTypePtr`, `NTi::TOptionalTypePtr`, etc. + /// + /// This function returns whatever the `visitor` returns. + /// + /// + /// # Example: visitor + /// + /// A simple visitor that returns name for a type would look like this: + /// + /// ``` + /// struct TGetNameVisitor { + /// TString operator()(TVoidTypePtr) { + /// return "Void"; + /// } + /// + /// TString operator()(TStringTypePtr) { + /// return "String"; + /// } + /// + /// // ... + /// + /// TString operator()(TStructTypePtr type) { + /// return TString(type->GetName().GetOrElse("Struct"));; + /// } + /// + /// // ... + /// } + /// ``` + /// + /// Now, we can use this visitor as following: + /// + /// ``` + /// TString typeName = type->Visit(TGetNameVisitor()); + /// ``` + /// + /// + /// # Example: overloaded struct + /// + /// Writing a separate struct each time one needs a visitor is tedious. Thanks to C++17 magic, we may avoid it. + /// Using lambdas and `TOverloaded` from `library/cpp/overloaded` allows replacing separate struct with + /// a bunch of lambdas: + /// + /// ``` + /// TString typeName = type->Visit(TOverloaded{ + /// [](TVoidTypePtr) -> TString { + /// return "Void"; + /// }, + /// [](TStringTypePtr) -> TString { + /// return "String"; + /// }, + /// + /// // ... + /// + /// }); + /// ``` + /// + /// + /// # Example: handling all primitives at once + /// + /// Since all primitives derive from `TPrimitiveType`, they can be handled all at once, + /// by accepting `TPrimitiveTypePtr`: + /// + /// ``` + /// TString typeName = type->Visit(TOverloaded{ + /// // All primitive types are handled by this lambda. + /// [](TPrimitiveTypePtr) -> TString { + /// return "Primitive"; + /// }, + /// + /// // Special handler for string type. Strings are handled by this lambda + /// // because of how C++ prioritizes overloads. + /// [](TStringTypePtr) -> TString { + /// return "String"; + /// }, + /// + /// // ... + /// + /// }); + /// ``` + template <typename V> + inline decltype(auto) Visit(V&& visitor) const; + + /// Like `Visit`, but passes const raw pointers to the visitor. + template <typename V> + inline decltype(auto) VisitRaw(V&& visitor) const; + + /// @} + + protected: + /// @name Internal interface for adoption semantics support + /// + /// Do not call these functions manually! + /// + /// See `type_factory.h`'s section on implementation details for more info. + /// + /// @{ + //- + /// Create a new instance of the class using this instance as a prototype. + /// + /// This is a [virtual copy constructor]. Typical implementation does the following: + /// + /// 1. for nested types, if any, it calls the factory's `Own` function. The `Own` acquires internal + /// ownership over the nested types, thus guaranteeing that they'll outlive the object that've + /// owned them. Depending on the particular factory implementation, `Own` may either recursively deepcopy + /// the whole nested type, increment some reference counter, or do nothing; + /// 2. for other resources owned by this type (i.e. arrays, strings, etc.), it copies them into the given + /// factory by calling factory's `New` and `Allocate` functions; + /// 3. finally, it creates a new instance of the type by invoking its constructor via the factory's + /// `New` function. + /// + /// Note: there are no guarantees on the value stored in `FactoryOrRc_` during invocation of this function. + /// Specifically, creating types on the stack (`FactoryOrRc_` is `0` in this case) and moving them + /// into a factory is a valid technique used extensively throughout this library. + /// + /// See `type_factory.h`'s section on implementation details for more info. + /// + /// Note: this function is not marked as `virtual` because we use our own dispatch via `Visit`. + /// + /// [virtual move constructor]: https://isocpp.org/wiki/faq/virtual-functions#virtual-ctors + const TType* Clone(ITypeFactoryInternal& factory) const noexcept; + + /// Release internal resources that were allocated in `Clone`. + /// + /// This function is the opposite of `Clone`. It releases all memory that was allocated within `Clone`, + /// and disowns nested types. + /// + /// This function is called by factories that perform active memory management, such as the default + /// heap factory. Typical implementation does the following: + /// + /// 1. for each `Clone`'s call to `Own` it calls `Disown`. The `Disown` releases internal ownership + /// over the nested types, thus allowing factory to free their memory. Depending + /// on the particular factory implementation, `Disown` may either decrement some reference counter, + /// call `Drop` and free the underlying memory, or do nothing; + /// 2. for each `Clone`'s call to `New` and `Allocate`, it calls `Delete` and `Free`; + /// 3. it should *not* call `Delete(this)` to mirror `Clone`'s final call to `New` (see the third bullet + /// in the `Clone`'s documentation). It is the factory's job to release memory under the type that's + /// being dropped. + /// + /// Note: there are no guarantees on whether this method will be called or not. For example, the default + /// heap factory will call it when some type's reference counter reaches zero. The default memory pool + /// factory will not call it. + /// + /// Note: this function is not marked as `virtual` because we use our own dispatch via `Visit`. + /// + /// See `type_factory.h`'s section on implementation details for more info. + void Drop(ITypeFactoryInternal& factory) noexcept; + + /// Get factory that manages this instance. + /// + /// If this instance is refcounted, returns the default heap factory. If it is unmanaged, returns `nullptr`. + /// Otherwise, returns pointer to the instance's factory. + /// + /// Remember that factories are not thread safe, thus using factory from this method may not be safe. + ITypeFactoryInternal* GetFactory() const noexcept; + + /// Mark this instance as managed by the given factory. + void SetFactory(ITypeFactoryInternal* factory) noexcept; + + /// Get factory's internal interface. + static ITypeFactoryInternal& FactoryInternal(ITypeFactory& factory) noexcept; + + /// @} + + protected: + /// @name Internal interface for reference counting + /// + /// See `type_factory.h`'s section on implementation details for more info. + /// + /// @{ + //- + /// Increase reference count of this type. + void RefSelf() noexcept { + RefImpl</* RefFactory = */ false>(); + } + + /// Increase reference count of this type and its factory. + void Ref() noexcept { + RefImpl</* RefFactory = */ true>(); + } + + /// Decrease reference count of this type. + void UnRefSelf() noexcept { + UnRefImpl</* UnRefFactory = */ false>(); + } + + /// Decrease reference count of this type and its factory. + void UnRef() noexcept { + UnRefImpl</* UnRefFactory = */ true>(); + } + + /// Decrease reference count of this type. Panic if any of it reaches zero. + void DecRefSelf() noexcept { + DecRefImpl</* DecRefFactory = */ false>(); + } + + /// Decrease reference count of type and its factory. Panic if any of it reaches zero. + void DecRef() noexcept { + DecRefImpl</* DecRefFactory = */ true>(); + } + + /// Get current reference count for this type. + long RefCount() const noexcept { + return RefCountImpl(); + } + + /// @} + + private: + template <bool RefFactory> + void RefImpl() noexcept; + + template <bool UnRefFactory> + void UnRefImpl() noexcept; + + template <bool DecRefFactory> + void DecRefImpl() noexcept; + + long RefCountImpl() const noexcept; + + protected: + /// Helper for implementing `Clone` with caching. + template <typename T, typename TCtor> + static const T* Cached(const T* type, ITypeFactoryInternal& factory, TCtor&& ctor); + + private: + /// Pointer to the type factory that've created this instance. + /// If this instance is static, this variable contains zero. + /// If this instance was created by the default heap factory, this variable is used as a reference counter. + std::atomic<size_t> FactoryOrRc_ = 0; + + /// Name of this type. Can be used to check before downcast. + ETypeName TypeName_; + + /// Hash is calculated lazily. + mutable std::atomic<bool> HasHash_; + mutable std::atomic<ui64> Hash_; + + private: + static bool IsRc(size_t factoryOrRc) noexcept { + return factoryOrRc & 1u; + } + static bool IsFactory(size_t factoryOrRc) noexcept { + return factoryOrRc != 0 && !IsRc(factoryOrRc); + } + static size_t CastFromFactory(ITypeFactoryInternal* factory) noexcept { + return reinterpret_cast<size_t>(factory); + } + static ITypeFactoryInternal* CastToFactory(size_t factoryOrRc) noexcept { + return reinterpret_cast<ITypeFactoryInternal*>(factoryOrRc); + } + }; + + static_assert(sizeof(TType) == 24); + + bool operator==(const TTypePtr& lhs, const TTypePtr& rhs) = delete; + bool operator!=(const TTypePtr& lhs, const TTypePtr& rhs) = delete; + + /// @brief Check for strict equivalence of the types. + /// + /// @see NTi::NEq::TStrictlyEqual + /// + /// @{ + bool operator==(const TType& lhs, const TType& rhs); + bool operator!=(const TType& lhs, const TType& rhs); + /// @} + + /// A singular type. This type has only one value. When serialized, it takes no space because it carries no data. + /// + /// Historically, YQL's `Void` is what's known as unit type (see https://en.wikipedia.org/wiki/Unit_type), + /// i.e. a type with only one possible value. This is similar to Python's `NoneType` or Rust's `()`. + class TVoidType final: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TVoidTypePtr AsPtr() const noexcept { + return const_cast<TVoidType*>(this); + } + + private: + explicit TVoidType(); + + public: + static TVoidTypePtr Instance(); + static const TVoidType* InstanceRaw(); + + protected: + const TVoidType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// Create new `Void` type using the default heap factory. + TVoidTypePtr Void(); + + /// A singular type, value of an empty optional. + /// + /// This type is used by YQL for `NULL` literal. Use `TVoidType` unless you have reasons to use this one. + class TNullType final: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TNullTypePtr AsPtr() const noexcept { + return const_cast<TNullType*>(this); + } + + private: + explicit TNullType(); + + public: + static TNullTypePtr Instance(); + static const TNullType* InstanceRaw(); + + protected: + const TNullType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// Base class for all primitive types. + class TPrimitiveType: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TPrimitiveTypePtr AsPtr() const noexcept { + return const_cast<TPrimitiveType*>(this); + } + + protected: + explicit TPrimitiveType(TMaybe<ui64> hash, EPrimitiveTypeName primitiveTypeName) noexcept; + + public: + /// Get name of this primitive type as a `NTi::EPrimitiveTypeName` enumerator. + EPrimitiveTypeName GetPrimitiveTypeName() const noexcept { + return ToPrimitiveTypeName(GetTypeName()); + } + + /// Cast this scalar class down to the most-derived type and pass it to the `visitor`. + /// + /// This function works like `NTi::TType::Visit`, but only handles primitive types. + /// + /// + /// # Example: + /// + /// ``` + /// auto name = scalar->VisitPrimitive(TOverloaded{ + /// [](TBoolTypePtr t) { return "Bool" }, + /// [](TStringTypePtr t) { return "String" }, + /// // ... + /// }); + /// ``` + template <typename V> + inline decltype(auto) VisitPrimitive(V&& visitor) const; + + /// Like `VisitPrimitive`, but passes raw pointers to the visitor. + template <typename V> + inline decltype(auto) VisitPrimitiveRaw(V&& visitor) const; + }; + + /// A logical type capable of holding one of the two values: true or false. + class TBoolType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TBoolTypePtr AsPtr() const noexcept { + return const_cast<TBoolType*>(this); + } + + private: + explicit TBoolType(); + + public: + static TBoolTypePtr Instance(); + static const TBoolType* InstanceRaw(); + + protected: + const TBoolType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A signed integer, one byte. + class TInt8Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TInt8TypePtr AsPtr() const noexcept { + return const_cast<TInt8Type*>(this); + } + + private: + explicit TInt8Type(); + + public: + static TInt8TypePtr Instance(); + static const TInt8Type* InstanceRaw(); + + protected: + const TInt8Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A signed integer, two bytes. + class TInt16Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TInt16TypePtr AsPtr() const noexcept { + return const_cast<TInt16Type*>(this); + } + + private: + explicit TInt16Type(); + + public: + static TInt16TypePtr Instance(); + static const TInt16Type* InstanceRaw(); + + protected: + const TInt16Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A signed integer, four bytes. + class TInt32Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TInt32TypePtr AsPtr() const noexcept { + return const_cast<TInt32Type*>(this); + } + + private: + explicit TInt32Type(); + + public: + static TInt32TypePtr Instance(); + static const TInt32Type* InstanceRaw(); + + protected: + const TInt32Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A signed integer, eight bytes. + class TInt64Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TInt64TypePtr AsPtr() const noexcept { + return const_cast<TInt64Type*>(this); + } + + private: + explicit TInt64Type(); + + public: + static TInt64TypePtr Instance(); + static const TInt64Type* InstanceRaw(); + + protected: + const TInt64Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// An unsigned integer, one byte. + class TUint8Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TUint8TypePtr AsPtr() const noexcept { + return const_cast<TUint8Type*>(this); + } + + private: + explicit TUint8Type(); + + public: + static TUint8TypePtr Instance(); + static const TUint8Type* InstanceRaw(); + + protected: + const TUint8Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// An unsigned integer, two bytes. + class TUint16Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TUint16TypePtr AsPtr() const noexcept { + return const_cast<TUint16Type*>(this); + } + + private: + explicit TUint16Type(); + + public: + static TUint16TypePtr Instance(); + static const TUint16Type* InstanceRaw(); + + protected: + const TUint16Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// An unsigned integer, four bytes. + class TUint32Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TUint32TypePtr AsPtr() const noexcept { + return const_cast<TUint32Type*>(this); + } + + private: + explicit TUint32Type(); + + public: + static TUint32TypePtr Instance(); + static const TUint32Type* InstanceRaw(); + + protected: + const TUint32Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// An unsigned integer, eight bytes. + class TUint64Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TUint64TypePtr AsPtr() const noexcept { + return const_cast<TUint64Type*>(this); + } + + private: + explicit TUint64Type(); + + public: + static TUint64TypePtr Instance(); + static const TUint64Type* InstanceRaw(); + + protected: + const TUint64Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A floating point number, four bytes. + class TFloatType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TFloatTypePtr AsPtr() const noexcept { + return const_cast<TFloatType*>(this); + } + + private: + explicit TFloatType(); + + public: + static TFloatTypePtr Instance(); + static const TFloatType* InstanceRaw(); + + protected: + const TFloatType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A floating point number, eight bytes. + class TDoubleType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TDoubleTypePtr AsPtr() const noexcept { + return const_cast<TDoubleType*>(this); + } + + private: + explicit TDoubleType(); + + public: + static TDoubleTypePtr Instance(); + static const TDoubleType* InstanceRaw(); + + protected: + const TDoubleType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A binary blob. + /// + /// This type can be used for any binary data. For text, consider using type `Utf8`. + class TStringType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TStringTypePtr AsPtr() const noexcept { + return const_cast<TStringType*>(this); + } + + private: + explicit TStringType(); + + public: + static TStringTypePtr Instance(); + static const TStringType* InstanceRaw(); + + protected: + const TStringType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A utf-8 encoded text. + class TUtf8Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TUtf8TypePtr AsPtr() const noexcept { + return const_cast<TUtf8Type*>(this); + } + + private: + explicit TUtf8Type(); + + public: + static TUtf8TypePtr Instance(); + static const TUtf8Type* InstanceRaw(); + + protected: + const TUtf8Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// An absolute point in time in range `[1970-01-01, 2106-01-01)`, precision up to days. + class TDateType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TDateTypePtr AsPtr() const noexcept { + return const_cast<TDateType*>(this); + } + + private: + explicit TDateType(); + + public: + static TDateTypePtr Instance(); + static const TDateType* InstanceRaw(); + + protected: + const TDateType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// An absolute point in time in range `[1970-01-01, 2106-01-01)`, precision up to seconds. + class TDatetimeType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TDatetimeTypePtr AsPtr() const noexcept { + return const_cast<TDatetimeType*>(this); + } + + private: + explicit TDatetimeType(); + + public: + static TDatetimeTypePtr Instance(); + static const TDatetimeType* InstanceRaw(); + + protected: + const TDatetimeType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// An absolute point in time in range `[1970-01-01, 2106-01-01)`, precision up to microseconds. + class TTimestampType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TTimestampTypePtr AsPtr() const noexcept { + return const_cast<TTimestampType*>(this); + } + + private: + explicit TTimestampType(); + + public: + static TTimestampTypePtr Instance(); + static const TTimestampType* InstanceRaw(); + + protected: + const TTimestampType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// `TDateType` with additional timezone mark. + class TTzDateType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TTzDateTypePtr AsPtr() const noexcept { + return const_cast<TTzDateType*>(this); + } + + private: + explicit TTzDateType(); + + public: + static TTzDateTypePtr Instance(); + static const TTzDateType* InstanceRaw(); + + protected: + const TTzDateType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// `TDatetimeType` with additional timezone mark. + class TTzDatetimeType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TTzDatetimeTypePtr AsPtr() const noexcept { + return const_cast<TTzDatetimeType*>(this); + } + + private: + explicit TTzDatetimeType(); + + public: + static TTzDatetimeTypePtr Instance(); + static const TTzDatetimeType* InstanceRaw(); + + protected: + const TTzDatetimeType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// `TTimestampType` with additional timezone mark. + class TTzTimestampType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TTzTimestampTypePtr AsPtr() const noexcept { + return const_cast<TTzTimestampType*>(this); + } + + private: + explicit TTzTimestampType(); + + public: + static TTzTimestampTypePtr Instance(); + static const TTzTimestampType* InstanceRaw(); + + protected: + const TTzTimestampType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// Signed delta between two timestamps. + class TIntervalType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TIntervalTypePtr AsPtr() const noexcept { + return const_cast<TIntervalType*>(this); + } + + private: + explicit TIntervalType(); + + public: + static TIntervalTypePtr Instance(); + static const TIntervalType* InstanceRaw(); + + protected: + const TIntervalType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A 128-bit number with controlled exponent/significand size. + /// + /// Decimal is a type for extra precise calculations. Internally, it is represented by a float-like 128-bit number. + /// + /// Two parameters control number of decimal digits in the decimal value. `Precision` is the total number + /// of decimal digits. `Scale` is the number of digits after the decimal point. + class TDecimalType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TDecimalTypePtr AsPtr() const noexcept { + return const_cast<TDecimalType*>(this); + } + + private: + explicit TDecimalType(TMaybe<ui64> hash, ui8 precision, ui8 scale) noexcept; + static TDecimalTypePtr Create(ITypeFactory& factory, ui8 precision, ui8 scale); + static const TDecimalType* CreateRaw(ITypeFactory& factory, ui8 precision, ui8 scale); + + protected: + ui64 CalculateHash() const noexcept; + + public: + /// Get total number of decimal digits. + ui8 GetPrecision() const noexcept { + return Precision_; + } + + /// Get number of decimal digits after the decimal point. + ui8 GetScale() const noexcept { + return Scale_; + } + + protected: + const TDecimalType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + + private: + ui8 Precision_; + ui8 Scale_; + }; + + /// A string with valid JSON. + class TJsonType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TJsonTypePtr AsPtr() const noexcept { + return const_cast<TJsonType*>(this); + } + + private: + explicit TJsonType(); + + public: + static TJsonTypePtr Instance(); + static const TJsonType* InstanceRaw(); + + protected: + const TJsonType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A string with valid YSON. + class TYsonType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TYsonTypePtr AsPtr() const noexcept { + return const_cast<TYsonType*>(this); + } + + private: + explicit TYsonType(); + + public: + static TYsonTypePtr Instance(); + static const TYsonType* InstanceRaw(); + + protected: + const TYsonType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// A string with valid UUID. + class TUuidType final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TUuidTypePtr AsPtr() const noexcept { + return const_cast<TUuidType*>(this); + } + + private: + explicit TUuidType(); + + public: + static TUuidTypePtr Instance(); + static const TUuidType* InstanceRaw(); + + protected: + const TUuidType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// Object which can store a value or a singular `NULL` value. + /// + /// This type is used to encode a value or its absence. + class TOptionalType final: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TOptionalTypePtr AsPtr() const noexcept { + return const_cast<TOptionalType*>(this); + } + + private: + explicit TOptionalType(TMaybe<ui64> hash, const TType* item) noexcept; + static TOptionalTypePtr Create(ITypeFactory& factory, TTypePtr item); + static const TOptionalType* CreateRaw(ITypeFactory& factory, const TType* item); + + protected: + ui64 CalculateHash() const noexcept; + + public: + /// Get underlying type. + TTypePtr GetItemType() const noexcept { + return GetItemTypeRaw()->AsPtr(); + } + + /// Like `GetMemberType`, but returns a raw pointer. + const TType* GetItemTypeRaw() const noexcept { + return Item_; + } + + protected: + const TOptionalType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + + private: + const TType* Item_; + }; + + /// A variable-size collection of homogeneous values. + class TListType final: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TListTypePtr AsPtr() const noexcept { + return const_cast<TListType*>(this); + } + + private: + explicit TListType(TMaybe<ui64> hash, const TType* item) noexcept; + static TListTypePtr Create(ITypeFactory& factory, TTypePtr item); + static const TListType* CreateRaw(ITypeFactory& factory, const TType* item); + + protected: + ui64 CalculateHash() const noexcept; + + public: + /// Get underlying type. + TTypePtr GetItemType() const noexcept { + return GetItemTypeRaw()->AsPtr(); + } + + /// Like `GetMemberType`, but returns a raw pointer. + const TType* GetItemTypeRaw() const noexcept { + return Item_; + } + + protected: + const TListType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + + private: + const TType* Item_; + }; + + /// An associative key-value container. + /// + /// Values of this type are usually represented as hashmaps. + class TDictType final: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TDictTypePtr AsPtr() const noexcept { + return const_cast<TDictType*>(this); + } + + private: + explicit TDictType(TMaybe<ui64> hash, const TType* key, const TType* value) noexcept; + static TDictTypePtr Create(ITypeFactory& factory, TTypePtr key, TTypePtr value); + static const TDictType* CreateRaw(ITypeFactory& factory, const TType* key, const TType* value); + + protected: + ui64 CalculateHash() const noexcept; + + public: + /// Get the key type. + TTypePtr GetKeyType() const noexcept { + return GetKeyTypeRaw()->AsPtr(); + } + + /// Like `GetKeyType`, but returns a raw pointer. + const TType* GetKeyTypeRaw() const noexcept { + return Key_; + } + + /// Get the value type. + TTypePtr GetValueType() const noexcept { + return GetValueTypeRaw()->AsPtr(); + } + + /// Like `GetValueType`, but returns a raw pointer. + const TType* GetValueTypeRaw() const noexcept { + return Value_; + } + + protected: + const TDictType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + + private: + const TType* Key_; + const TType* Value_; + }; + + /// A fixed-size collection of named heterogeneous values. + /// + /// Structs represent multiple values grouped into a single entity. Values stored in a struct are called items. + /// Each item have an associated type and a name. + /// + /// Struct type is represented by an array of item types and their names. + /// + /// Even though struct elements are accessed by their names, we use vector to represent struct type because order + /// of struct items matters. If affects memory layout of a struct, how struct is serialized and deserialized. + /// Items order is vital for struct versioning. New fields should always be added to the end of the struct. + /// This way older parsers can read values serialized by newer writers: they'll simply read known head of a struct + /// and skip tail that contains unknown fields. + /// + /// + /// # Struct names + /// + /// Each struct defined by YDL must have an associated name. This name is used to refer struct in code, to generate + /// code representing this struct in other programming languages, and to report errors. The struct's name is saved + /// in this field. + /// + /// Note that, even though YDL requires struct names, name field is optional because other systems might + /// use anonymous structs (or maybe they dont't use struct names at all). + /// + /// Note also that type aliases (especially `newtype`) use tags to name types, so primitives don't have name field. + class TStructType final: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + friend class TStructBuilderRaw; + + public: + TStructTypePtr AsPtr() const noexcept { + return const_cast<TStructType*>(this); + } + + public: + /// A single struct element. + class TMember { + public: + TMember(TStringBuf name, const TType* type); + + public: + /// Get name of this item. + TStringBuf GetName() const { + return Name_; + } + + /// Get type of this item. + TTypePtr GetType() const { + return GetTypeRaw()->AsPtr(); + } + + /// Like `GetType`, but returns a raw pointer. + const TType* GetTypeRaw() const { + return Type_; + } + + /// Calculate this item's hash. Hashes follow the 'strict equivalence' (see type_equivalence.h). + ui64 Hash() const; + + private: + TStringBuf Name_; + const TType* Type_; + }; + + using TMembers = TConstArrayRef<TMember>; + + /// Like `TMember`, but owns its contents. Used in non-raw constructors to guarantee data validity. + class TOwnedMember { + public: + TOwnedMember(TString name, TTypePtr type); + + public: + operator TMember() const&; + + private: + TString Name_; + TTypePtr Type_; + }; + + using TOwnedMembers = TConstArrayRef<TOwnedMember>; + + private: + explicit TStructType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, TMembers members, TConstArrayRef<size_t> sortedMembers) noexcept; + static TStructTypePtr Create(ITypeFactory& factory, TOwnedMembers members); + static TStructTypePtr Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TOwnedMembers members); + static const TStructType* CreateRaw(ITypeFactory& factory, TMembers members); + static const TStructType* CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, TMembers members); + static void MakeSortedMembers(TMembers members, TArrayRef<size_t> sortedItems); + + protected: + ui64 CalculateHash() const noexcept; + + public: + /// Get name of this struct. + TMaybe<TStringBuf> GetName() const noexcept { + return Name_; + } + + /// Get description of structure members. + TMembers GetMembers() const noexcept { + return Members_; + } + + /// Check if there is an item with the given name in this struct. + bool HasMember(TStringBuf name) const noexcept; + + /// Lookup struct item by name. Throw an error if there is no such item. + const TMember& GetMember(TStringBuf name) const; + + /// Lookup struct item by name, return its index or `-1`, if item was not found. + /// This function works in `O(log(n))` time, where `n` is number of struct members. + ssize_t GetMemberIndex(TStringBuf name) const noexcept; + + protected: + const TStructType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + + private: + TMaybe<TStringBuf> Name_; + TMembers Members_; + TConstArrayRef<size_t> SortedMembers_; + }; + + /// A fixed-size collection of named heterogeneous values. + /// + /// Tuples, like structs, represent multiple values, also called items, grouped into a single entity. Unlike + /// structs, though, tuple items are unnamed. Instead of names, they are accessed by their indexes. + /// + /// For a particular tuple type, number of items, their order and types are fixed, they should be known before + /// creating instances of tuples and can't change over time. + /// + /// + /// # Tuple names + /// + /// YDL requires each tuple definition to have a name (see `TStructType`). The name might not be mandatory + /// in other systems, so name field is optional. + class TTupleType: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + friend class TTupleBuilderRaw; + + public: + TTupleTypePtr AsPtr() const noexcept { + return const_cast<TTupleType*>(this); + } + + public: + /// A single tuple element. + class TElement { + public: + TElement(const TType* type); + + public: + /// Get type of this item. + TTypePtr GetType() const { + return GetTypeRaw()->AsPtr(); + } + + /// Like `GetType`, but returns a raw pointer. + const TType* GetTypeRaw() const { + return Type_; + } + + /// Calculate this item's hash. Hashes follow the 'strict equivalence' (see type_equivalence.h). + ui64 Hash() const; + + private: + const TType* Type_; + }; + + using TElements = TConstArrayRef<TElement>; + + /// Like `TElement`, but owns its contents. Used in non-raw constructors to guarantee data validity. + class TOwnedElement { + public: + TOwnedElement(TTypePtr type); + + public: + operator TElement() const&; + + private: + TTypePtr Type_; + }; + + using TOwnedElements = TConstArrayRef<TOwnedElement>; + + private: + explicit TTupleType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, TElements items) noexcept; + static TTupleTypePtr Create(ITypeFactory& factory, TOwnedElements items); + static TTupleTypePtr Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TOwnedElements items); + static const TTupleType* CreateRaw(ITypeFactory& factory, TElements items); + static const TTupleType* CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, TElements items); + + protected: + ui64 CalculateHash() const noexcept; + + public: + /// Get name of this type. + TMaybe<TStringBuf> GetName() const noexcept { + return Name_; + } + + /// Get description of tuple items. + TElements GetElements() const noexcept { + return Elements_; + } + + protected: + const TTupleType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + + private: + TMaybe<TStringBuf> Name_; + TElements Elements_; + }; + + /// A tagged union with named or unnamed alternatives (a.k.a. variant over struct or variant over tuple). + /// + /// Variants are used to store values of different types. + /// + /// For example, a variant over struct which holds an ip address could look like + /// `Ip = Variant<v4: IpV4, v6: IpV6>`, where `IpV4` and `IpV6` are some other types. Now, a value of type `Ip` + /// could store either a value of type `IpV4` or a value of type `IpV6`. + /// + /// Even though item types can be the same, each item represents a distinct state of a variant. For example, + /// a variant for a user identifier can look like `Uid = Variant<yuid: String, ip: String>`. This variant can + /// contain either user's yandexuid of user's ip. Despite both items are of the same type `String`, `Uid` which + /// contains a `yuid` and `Uid` which contains an `ip` are never equal. + /// That is, `Uid.yuid("000000") != Uid.ip("000000")`. + /// + /// Exactly like with structs or tuples, order of variant items matter. Indexes of variant items are used + /// instead of names when variant is serialized. + /// + /// + /// # Variant names + /// + /// YDL requires each variant definition to have a name (see `TStructType`). The name might not be mandatory + /// in other systems, so name field is optional. + class TVariantType final: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + friend class TStructBuilderRaw; + friend class TTupleBuilderRaw; + + public: + TVariantTypePtr AsPtr() const noexcept { + return const_cast<TVariantType*>(this); + } + + private: + explicit TVariantType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, const TType* inner) noexcept; + static TVariantTypePtr Create(ITypeFactory& factory, TTypePtr inner); + static TVariantTypePtr Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TTypePtr inner); + static const TVariantType* CreateRaw(ITypeFactory& factory, const TType* inner); + static const TVariantType* CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, const TType* inner); + + protected: + ui64 CalculateHash() const noexcept; + + public: + /// Get name of this variant. + TMaybe<TStringBuf> GetName() const noexcept { + return Name_; + } + + /// Get vector of variant items. + TTypePtr GetUnderlyingType() const noexcept { + return Underlying_->AsPtr(); + } + + /// Like `GetUnderlyingType`, but returns a raw pointer. + const TType* GetUnderlyingTypeRaw() const noexcept { + return Underlying_; + } + + /// Check if this variant's inner type is a struct. + bool IsVariantOverStruct() const noexcept { + return GetUnderlyingTypeRaw()->GetTypeName() == ETypeName::Struct; + } + + /// Check if this variant's inner type is a tuple. + bool IsVariantOverTuple() const noexcept { + return GetUnderlyingTypeRaw()->GetTypeName() == ETypeName::Tuple; + } + + /// Visit inner type of this variant. This function works like `Visit`, but only casts inner type + /// to struct or tuple, so you don't need to handle other types in a visitor. + template <typename V> + inline decltype(auto) VisitUnderlying(V&& visitor) const; + + /// Like `VisitUnderlying`, but passes const raw pointers to the visitor. + template <typename V> + inline decltype(auto) VisitUnderlyingRaw(V&& visitor) const; + + protected: + const TVariantType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + + private: + TMaybe<TStringBuf> Name_; + const TType* Underlying_; + }; + + /// Named or tagged or user-defined type. + /// + /// Tags are used to create new types from existing ones by assigning them a tag, i.e. a name. They wrap other types + /// adding them additional semantics. + /// + /// On physical level, tags do not change types. Both `Tagged<Int32, 'GeoId'>` and `Int32` have exactly the same + /// representation when serialized. + /// + /// On logical level, tags change semantics of a type. This can affect how types are displayed, how they + /// are checked and converted, etc. + class TTaggedType final: public TType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TTaggedTypePtr AsPtr() const noexcept { + return const_cast<TTaggedType*>(this); + } + + private: + explicit TTaggedType(TMaybe<ui64> hash, const TType* type, TStringBuf tag) noexcept; + static TTaggedTypePtr Create(ITypeFactory& factory, TTypePtr type, TStringBuf tag); + static const TTaggedType* CreateRaw(ITypeFactory& factory, const TType* type, TStringBuf tag); + + protected: + ui64 CalculateHash() const noexcept; + + public: + /// Get tag content, i.e. name of the new type. + TStringBuf GetTag() const noexcept { + return Tag_; + } + + /// Get the wrapped type. + TTypePtr GetItemType() const noexcept { + return GetItemTypeRaw()->AsPtr(); + } + + /// Like `GetMemberType`, but returns a raw pointer. + const TType* GetItemTypeRaw() const noexcept { + return Item_; + } + + protected: + const TTaggedType* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + + private: + const TType* Item_; + TStringBuf Tag_; + }; + +#ifdef __JETBRAINS_IDE__ +#pragma clang diagnostic push +#pragma ide diagnostic ignored "cppcoreguidelines-pro-type-static-cast-downcast" +#endif + + bool TType::IsPrimitive() const noexcept { + return ::NTi::IsPrimitive(TypeName_); + } + + TPrimitiveTypePtr TType::AsPrimitive() const noexcept { + return AsPrimitiveRaw()->AsPtr(); + } + + const TPrimitiveType* TType::AsPrimitiveRaw() const noexcept { + Y_VERIFY(IsPrimitive()); + return static_cast<const TPrimitiveType*>(this); + } + + bool TType::IsVoid() const noexcept { + return TypeName_ == ETypeName::Void; + } + + TVoidTypePtr TType::AsVoid() const noexcept { + return AsVoidRaw()->AsPtr(); + } + + const TVoidType* TType::AsVoidRaw() const noexcept { + Y_VERIFY(IsVoid()); + return static_cast<const TVoidType*>(this); + } + + bool TType::IsNull() const noexcept { + return TypeName_ == ETypeName::Null; + } + + TNullTypePtr TType::AsNull() const noexcept { + return AsNullRaw()->AsPtr(); + } + + const TNullType* TType::AsNullRaw() const noexcept { + Y_VERIFY(IsNull()); + return static_cast<const TNullType*>(this); + } + + bool TType::IsBool() const noexcept { + return TypeName_ == ETypeName::Bool; + } + + TBoolTypePtr TType::AsBool() const noexcept { + return AsBoolRaw()->AsPtr(); + } + + const TBoolType* TType::AsBoolRaw() const noexcept { + Y_VERIFY(IsBool()); + return static_cast<const TBoolType*>(this); + } + + bool TType::IsInt8() const noexcept { + return TypeName_ == ETypeName::Int8; + } + + TInt8TypePtr TType::AsInt8() const noexcept { + return AsInt8Raw()->AsPtr(); + } + + const TInt8Type* TType::AsInt8Raw() const noexcept { + Y_VERIFY(IsInt8()); + return static_cast<const TInt8Type*>(this); + } + + bool TType::IsInt16() const noexcept { + return TypeName_ == ETypeName::Int16; + } + + TInt16TypePtr TType::AsInt16() const noexcept { + return AsInt16Raw()->AsPtr(); + } + + const TInt16Type* TType::AsInt16Raw() const noexcept { + Y_VERIFY(IsInt16()); + return static_cast<const TInt16Type*>(this); + } + + bool TType::IsInt32() const noexcept { + return TypeName_ == ETypeName::Int32; + } + + TInt32TypePtr TType::AsInt32() const noexcept { + return AsInt32Raw()->AsPtr(); + } + + const TInt32Type* TType::AsInt32Raw() const noexcept { + Y_VERIFY(IsInt32()); + return static_cast<const TInt32Type*>(this); + } + + bool TType::IsInt64() const noexcept { + return TypeName_ == ETypeName::Int64; + } + + TInt64TypePtr TType::AsInt64() const noexcept { + return AsInt64Raw()->AsPtr(); + } + + const TInt64Type* TType::AsInt64Raw() const noexcept { + Y_VERIFY(IsInt64()); + return static_cast<const TInt64Type*>(this); + } + + bool TType::IsUint8() const noexcept { + return TypeName_ == ETypeName::Uint8; + } + + TUint8TypePtr TType::AsUint8() const noexcept { + return AsUint8Raw()->AsPtr(); + } + + const TUint8Type* TType::AsUint8Raw() const noexcept { + Y_VERIFY(IsUint8()); + return static_cast<const TUint8Type*>(this); + } + + bool TType::IsUint16() const noexcept { + return TypeName_ == ETypeName::Uint16; + } + + TUint16TypePtr TType::AsUint16() const noexcept { + return AsUint16Raw()->AsPtr(); + } + + const TUint16Type* TType::AsUint16Raw() const noexcept { + Y_VERIFY(IsUint16()); + return static_cast<const TUint16Type*>(this); + } + + bool TType::IsUint32() const noexcept { + return TypeName_ == ETypeName::Uint32; + } + + TUint32TypePtr TType::AsUint32() const noexcept { + return AsUint32Raw()->AsPtr(); + } + + const TUint32Type* TType::AsUint32Raw() const noexcept { + Y_VERIFY(IsUint32()); + return static_cast<const TUint32Type*>(this); + } + + bool TType::IsUint64() const noexcept { + return TypeName_ == ETypeName::Uint64; + } + + TUint64TypePtr TType::AsUint64() const noexcept { + return AsUint64Raw()->AsPtr(); + } + + const TUint64Type* TType::AsUint64Raw() const noexcept { + Y_VERIFY(IsUint64()); + return static_cast<const TUint64Type*>(this); + } + + bool TType::IsFloat() const noexcept { + return TypeName_ == ETypeName::Float; + } + + TFloatTypePtr TType::AsFloat() const noexcept { + return AsFloatRaw()->AsPtr(); + } + + const TFloatType* TType::AsFloatRaw() const noexcept { + Y_VERIFY(IsFloat()); + return static_cast<const TFloatType*>(this); + } + + bool TType::IsDouble() const noexcept { + return TypeName_ == ETypeName::Double; + } + + TDoubleTypePtr TType::AsDouble() const noexcept { + return AsDoubleRaw()->AsPtr(); + } + + const TDoubleType* TType::AsDoubleRaw() const noexcept { + Y_VERIFY(IsDouble()); + return static_cast<const TDoubleType*>(this); + } + + bool TType::IsString() const noexcept { + return TypeName_ == ETypeName::String; + } + + TStringTypePtr TType::AsString() const noexcept { + return AsStringRaw()->AsPtr(); + } + + const TStringType* TType::AsStringRaw() const noexcept { + Y_VERIFY(IsString()); + return static_cast<const TStringType*>(this); + } + + bool TType::IsUtf8() const noexcept { + return TypeName_ == ETypeName::Utf8; + } + + TUtf8TypePtr TType::AsUtf8() const noexcept { + return AsUtf8Raw()->AsPtr(); + } + + const TUtf8Type* TType::AsUtf8Raw() const noexcept { + Y_VERIFY(IsUtf8()); + return static_cast<const TUtf8Type*>(this); + } + + bool TType::IsDate() const noexcept { + return TypeName_ == ETypeName::Date; + } + + TDateTypePtr TType::AsDate() const noexcept { + return AsDateRaw()->AsPtr(); + } + + const TDateType* TType::AsDateRaw() const noexcept { + Y_VERIFY(IsDate()); + return static_cast<const TDateType*>(this); + } + + bool TType::IsDatetime() const noexcept { + return TypeName_ == ETypeName::Datetime; + } + + TDatetimeTypePtr TType::AsDatetime() const noexcept { + return AsDatetimeRaw()->AsPtr(); + } + + const TDatetimeType* TType::AsDatetimeRaw() const noexcept { + Y_VERIFY(IsDatetime()); + return static_cast<const TDatetimeType*>(this); + } + + bool TType::IsTimestamp() const noexcept { + return TypeName_ == ETypeName::Timestamp; + } + + TTimestampTypePtr TType::AsTimestamp() const noexcept { + return AsTimestampRaw()->AsPtr(); + } + + const TTimestampType* TType::AsTimestampRaw() const noexcept { + Y_VERIFY(IsTimestamp()); + return static_cast<const TTimestampType*>(this); + } + + bool TType::IsTzDate() const noexcept { + return TypeName_ == ETypeName::TzDate; + } + + TTzDateTypePtr TType::AsTzDate() const noexcept { + return AsTzDateRaw()->AsPtr(); + } + + const TTzDateType* TType::AsTzDateRaw() const noexcept { + Y_VERIFY(IsTzDate()); + return static_cast<const TTzDateType*>(this); + } + + bool TType::IsTzDatetime() const noexcept { + return TypeName_ == ETypeName::TzDatetime; + } + + TTzDatetimeTypePtr TType::AsTzDatetime() const noexcept { + return AsTzDatetimeRaw()->AsPtr(); + } + + const TTzDatetimeType* TType::AsTzDatetimeRaw() const noexcept { + Y_VERIFY(IsTzDatetime()); + return static_cast<const TTzDatetimeType*>(this); + } + + bool TType::IsTzTimestamp() const noexcept { + return TypeName_ == ETypeName::TzTimestamp; + } + + TTzTimestampTypePtr TType::AsTzTimestamp() const noexcept { + return AsTzTimestampRaw()->AsPtr(); + } + + const TTzTimestampType* TType::AsTzTimestampRaw() const noexcept { + Y_VERIFY(IsTzTimestamp()); + return static_cast<const TTzTimestampType*>(this); + } + + bool TType::IsInterval() const noexcept { + return TypeName_ == ETypeName::Interval; + } + + TIntervalTypePtr TType::AsInterval() const noexcept { + return AsIntervalRaw()->AsPtr(); + } + + const TIntervalType* TType::AsIntervalRaw() const noexcept { + Y_VERIFY(IsInterval()); + return static_cast<const TIntervalType*>(this); + } + + bool TType::IsDecimal() const noexcept { + return TypeName_ == ETypeName::Decimal; + } + + TDecimalTypePtr TType::AsDecimal() const noexcept { + return AsDecimalRaw()->AsPtr(); + } + + const TDecimalType* TType::AsDecimalRaw() const noexcept { + Y_VERIFY(IsDecimal()); + return static_cast<const TDecimalType*>(this); + } + + bool TType::IsJson() const noexcept { + return TypeName_ == ETypeName::Json; + } + + TJsonTypePtr TType::AsJson() const noexcept { + return AsJsonRaw()->AsPtr(); + } + + const TJsonType* TType::AsJsonRaw() const noexcept { + Y_VERIFY(IsJson()); + return static_cast<const TJsonType*>(this); + } + + bool TType::IsYson() const noexcept { + return TypeName_ == ETypeName::Yson; + } + + TYsonTypePtr TType::AsYson() const noexcept { + return AsYsonRaw()->AsPtr(); + } + + const TYsonType* TType::AsYsonRaw() const noexcept { + Y_VERIFY(IsYson()); + return static_cast<const TYsonType*>(this); + } + + bool TType::IsUuid() const noexcept { + return TypeName_ == ETypeName::Uuid; + } + + TUuidTypePtr TType::AsUuid() const noexcept { + return AsUuidRaw()->AsPtr(); + } + + const TUuidType* TType::AsUuidRaw() const noexcept { + Y_VERIFY(IsUuid()); + return static_cast<const TUuidType*>(this); + } + + bool TType::IsOptional() const noexcept { + return TypeName_ == ETypeName::Optional; + } + + TOptionalTypePtr TType::AsOptional() const noexcept { + return AsOptionalRaw()->AsPtr(); + } + + const TOptionalType* TType::AsOptionalRaw() const noexcept { + Y_VERIFY(IsOptional()); + return static_cast<const TOptionalType*>(this); + } + + bool TType::IsList() const noexcept { + return TypeName_ == ETypeName::List; + } + + TListTypePtr TType::AsList() const noexcept { + return AsListRaw()->AsPtr(); + } + + const TListType* TType::AsListRaw() const noexcept { + Y_VERIFY(IsList()); + return static_cast<const TListType*>(this); + } + + bool TType::IsDict() const noexcept { + return TypeName_ == ETypeName::Dict; + } + + TDictTypePtr TType::AsDict() const noexcept { + return AsDictRaw()->AsPtr(); + } + + const TDictType* TType::AsDictRaw() const noexcept { + Y_VERIFY(IsDict()); + return static_cast<const TDictType*>(this); + } + + bool TType::IsStruct() const noexcept { + return TypeName_ == ETypeName::Struct; + } + + TStructTypePtr TType::AsStruct() const noexcept { + return AsStructRaw()->AsPtr(); + } + + const TStructType* TType::AsStructRaw() const noexcept { + Y_VERIFY(IsStruct()); + return static_cast<const TStructType*>(this); + } + + bool TType::IsTuple() const noexcept { + return TypeName_ == ETypeName::Tuple; + } + + TTupleTypePtr TType::AsTuple() const noexcept { + return AsTupleRaw()->AsPtr(); + } + + const TTupleType* TType::AsTupleRaw() const noexcept { + Y_VERIFY(IsTuple()); + return static_cast<const TTupleType*>(this); + } + + bool TType::IsVariant() const noexcept { + return TypeName_ == ETypeName::Variant; + } + + TVariantTypePtr TType::AsVariant() const noexcept { + return AsVariantRaw()->AsPtr(); + } + + const TVariantType* TType::AsVariantRaw() const noexcept { + Y_VERIFY(IsVariant()); + return static_cast<const TVariantType*>(this); + } + + bool TType::IsTagged() const noexcept { + return TypeName_ == ETypeName::Tagged; + } + + TTaggedTypePtr TType::AsTagged() const noexcept { + return AsTaggedRaw()->AsPtr(); + } + + const TTaggedType* TType::AsTaggedRaw() const noexcept { + Y_VERIFY(IsTagged()); + return static_cast<const TTaggedType*>(this); + } + +#ifdef __JETBRAINS_IDE__ +#pragma clang diagnostic pop +#endif + + template <typename V> + decltype(auto) TType::Visit(V&& visitor) const { + switch (TypeName_) { + case ETypeName::Bool: + return std::forward<V>(visitor)(this->AsBool()); + case ETypeName::Int8: + return std::forward<V>(visitor)(this->AsInt8()); + case ETypeName::Int16: + return std::forward<V>(visitor)(this->AsInt16()); + case ETypeName::Int32: + return std::forward<V>(visitor)(this->AsInt32()); + case ETypeName::Int64: + return std::forward<V>(visitor)(this->AsInt64()); + case ETypeName::Uint8: + return std::forward<V>(visitor)(this->AsUint8()); + case ETypeName::Uint16: + return std::forward<V>(visitor)(this->AsUint16()); + case ETypeName::Uint32: + return std::forward<V>(visitor)(this->AsUint32()); + case ETypeName::Uint64: + return std::forward<V>(visitor)(this->AsUint64()); + case ETypeName::Float: + return std::forward<V>(visitor)(this->AsFloat()); + case ETypeName::Double: + return std::forward<V>(visitor)(this->AsDouble()); + case ETypeName::String: + return std::forward<V>(visitor)(this->AsString()); + case ETypeName::Utf8: + return std::forward<V>(visitor)(this->AsUtf8()); + case ETypeName::Date: + return std::forward<V>(visitor)(this->AsDate()); + case ETypeName::Datetime: + return std::forward<V>(visitor)(this->AsDatetime()); + case ETypeName::Timestamp: + return std::forward<V>(visitor)(this->AsTimestamp()); + case ETypeName::TzDate: + return std::forward<V>(visitor)(this->AsTzDate()); + case ETypeName::TzDatetime: + return std::forward<V>(visitor)(this->AsTzDatetime()); + case ETypeName::TzTimestamp: + return std::forward<V>(visitor)(this->AsTzTimestamp()); + case ETypeName::Interval: + return std::forward<V>(visitor)(this->AsInterval()); + case ETypeName::Decimal: + return std::forward<V>(visitor)(this->AsDecimal()); + case ETypeName::Json: + return std::forward<V>(visitor)(this->AsJson()); + case ETypeName::Yson: + return std::forward<V>(visitor)(this->AsYson()); + case ETypeName::Uuid: + return std::forward<V>(visitor)(this->AsUuid()); + case ETypeName::Void: + return std::forward<V>(visitor)(this->AsVoid()); + case ETypeName::Null: + return std::forward<V>(visitor)(this->AsNull()); + case ETypeName::Optional: + return std::forward<V>(visitor)(this->AsOptional()); + case ETypeName::List: + return std::forward<V>(visitor)(this->AsList()); + case ETypeName::Dict: + return std::forward<V>(visitor)(this->AsDict()); + case ETypeName::Struct: + return std::forward<V>(visitor)(this->AsStruct()); + case ETypeName::Tuple: + return std::forward<V>(visitor)(this->AsTuple()); + case ETypeName::Variant: + return std::forward<V>(visitor)(this->AsVariant()); + case ETypeName::Tagged: + return std::forward<V>(visitor)(this->AsTagged()); + } + + Y_UNREACHABLE(); + } + + template <typename V> + decltype(auto) TType::VisitRaw(V&& visitor) const { + switch (TypeName_) { + case ETypeName::Bool: + return std::forward<V>(visitor)(this->AsBoolRaw()); + case ETypeName::Int8: + return std::forward<V>(visitor)(this->AsInt8Raw()); + case ETypeName::Int16: + return std::forward<V>(visitor)(this->AsInt16Raw()); + case ETypeName::Int32: + return std::forward<V>(visitor)(this->AsInt32Raw()); + case ETypeName::Int64: + return std::forward<V>(visitor)(this->AsInt64Raw()); + case ETypeName::Uint8: + return std::forward<V>(visitor)(this->AsUint8Raw()); + case ETypeName::Uint16: + return std::forward<V>(visitor)(this->AsUint16Raw()); + case ETypeName::Uint32: + return std::forward<V>(visitor)(this->AsUint32Raw()); + case ETypeName::Uint64: + return std::forward<V>(visitor)(this->AsUint64Raw()); + case ETypeName::Float: + return std::forward<V>(visitor)(this->AsFloatRaw()); + case ETypeName::Double: + return std::forward<V>(visitor)(this->AsDoubleRaw()); + case ETypeName::String: + return std::forward<V>(visitor)(this->AsStringRaw()); + case ETypeName::Utf8: + return std::forward<V>(visitor)(this->AsUtf8Raw()); + case ETypeName::Date: + return std::forward<V>(visitor)(this->AsDateRaw()); + case ETypeName::Datetime: + return std::forward<V>(visitor)(this->AsDatetimeRaw()); + case ETypeName::Timestamp: + return std::forward<V>(visitor)(this->AsTimestampRaw()); + case ETypeName::TzDate: + return std::forward<V>(visitor)(this->AsTzDateRaw()); + case ETypeName::TzDatetime: + return std::forward<V>(visitor)(this->AsTzDatetimeRaw()); + case ETypeName::TzTimestamp: + return std::forward<V>(visitor)(this->AsTzTimestampRaw()); + case ETypeName::Interval: + return std::forward<V>(visitor)(this->AsIntervalRaw()); + case ETypeName::Decimal: + return std::forward<V>(visitor)(this->AsDecimalRaw()); + case ETypeName::Json: + return std::forward<V>(visitor)(this->AsJsonRaw()); + case ETypeName::Yson: + return std::forward<V>(visitor)(this->AsYsonRaw()); + case ETypeName::Uuid: + return std::forward<V>(visitor)(this->AsUuidRaw()); + case ETypeName::Void: + return std::forward<V>(visitor)(this->AsVoidRaw()); + case ETypeName::Null: + return std::forward<V>(visitor)(this->AsNullRaw()); + case ETypeName::Optional: + return std::forward<V>(visitor)(this->AsOptionalRaw()); + case ETypeName::List: + return std::forward<V>(visitor)(this->AsListRaw()); + case ETypeName::Dict: + return std::forward<V>(visitor)(this->AsDictRaw()); + case ETypeName::Struct: + return std::forward<V>(visitor)(this->AsStructRaw()); + case ETypeName::Tuple: + return std::forward<V>(visitor)(this->AsTupleRaw()); + case ETypeName::Variant: + return std::forward<V>(visitor)(this->AsVariantRaw()); + case ETypeName::Tagged: + return std::forward<V>(visitor)(this->AsTaggedRaw()); + } + + Y_UNREACHABLE(); + } + + template <typename V> + decltype(auto) TPrimitiveType::VisitPrimitive(V&& visitor) const { + switch (GetPrimitiveTypeName()) { + case EPrimitiveTypeName::Bool: + return std::forward<V>(visitor)(this->AsBool()); + case EPrimitiveTypeName::Int8: + return std::forward<V>(visitor)(this->AsInt8()); + case EPrimitiveTypeName::Int16: + return std::forward<V>(visitor)(this->AsInt16()); + case EPrimitiveTypeName::Int32: + return std::forward<V>(visitor)(this->AsInt32()); + case EPrimitiveTypeName::Int64: + return std::forward<V>(visitor)(this->AsInt64()); + case EPrimitiveTypeName::Uint8: + return std::forward<V>(visitor)(this->AsUint8()); + case EPrimitiveTypeName::Uint16: + return std::forward<V>(visitor)(this->AsUint16()); + case EPrimitiveTypeName::Uint32: + return std::forward<V>(visitor)(this->AsUint32()); + case EPrimitiveTypeName::Uint64: + return std::forward<V>(visitor)(this->AsUint64()); + case EPrimitiveTypeName::Float: + return std::forward<V>(visitor)(this->AsFloat()); + case EPrimitiveTypeName::Double: + return std::forward<V>(visitor)(this->AsDouble()); + case EPrimitiveTypeName::String: + return std::forward<V>(visitor)(this->AsString()); + case EPrimitiveTypeName::Utf8: + return std::forward<V>(visitor)(this->AsUtf8()); + case EPrimitiveTypeName::Date: + return std::forward<V>(visitor)(this->AsDate()); + case EPrimitiveTypeName::Datetime: + return std::forward<V>(visitor)(this->AsDatetime()); + case EPrimitiveTypeName::Timestamp: + return std::forward<V>(visitor)(this->AsTimestamp()); + case EPrimitiveTypeName::TzDate: + return std::forward<V>(visitor)(this->AsTzDate()); + case EPrimitiveTypeName::TzDatetime: + return std::forward<V>(visitor)(this->AsTzDatetime()); + case EPrimitiveTypeName::TzTimestamp: + return std::forward<V>(visitor)(this->AsTzTimestamp()); + case EPrimitiveTypeName::Interval: + return std::forward<V>(visitor)(this->AsInterval()); + case EPrimitiveTypeName::Decimal: + return std::forward<V>(visitor)(this->AsDecimal()); + case EPrimitiveTypeName::Json: + return std::forward<V>(visitor)(this->AsJson()); + case EPrimitiveTypeName::Yson: + return std::forward<V>(visitor)(this->AsYson()); + case EPrimitiveTypeName::Uuid: + return std::forward<V>(visitor)(this->AsUuid()); + } + + Y_UNREACHABLE(); + } + + template <typename V> + decltype(auto) TPrimitiveType::VisitPrimitiveRaw(V&& visitor) const { + switch (GetPrimitiveTypeName()) { + case EPrimitiveTypeName::Bool: + return std::forward<V>(visitor)(this->AsBoolRaw()); + case EPrimitiveTypeName::Int8: + return std::forward<V>(visitor)(this->AsInt8Raw()); + case EPrimitiveTypeName::Int16: + return std::forward<V>(visitor)(this->AsInt16Raw()); + case EPrimitiveTypeName::Int32: + return std::forward<V>(visitor)(this->AsInt32Raw()); + case EPrimitiveTypeName::Int64: + return std::forward<V>(visitor)(this->AsInt64Raw()); + case EPrimitiveTypeName::Uint8: + return std::forward<V>(visitor)(this->AsUint8Raw()); + case EPrimitiveTypeName::Uint16: + return std::forward<V>(visitor)(this->AsUint16Raw()); + case EPrimitiveTypeName::Uint32: + return std::forward<V>(visitor)(this->AsUint32Raw()); + case EPrimitiveTypeName::Uint64: + return std::forward<V>(visitor)(this->AsUint64Raw()); + case EPrimitiveTypeName::Float: + return std::forward<V>(visitor)(this->AsFloatRaw()); + case EPrimitiveTypeName::Double: + return std::forward<V>(visitor)(this->AsDoubleRaw()); + case EPrimitiveTypeName::String: + return std::forward<V>(visitor)(this->AsStringRaw()); + case EPrimitiveTypeName::Utf8: + return std::forward<V>(visitor)(this->AsUtf8Raw()); + case EPrimitiveTypeName::Date: + return std::forward<V>(visitor)(this->AsDateRaw()); + case EPrimitiveTypeName::Datetime: + return std::forward<V>(visitor)(this->AsDatetimeRaw()); + case EPrimitiveTypeName::Timestamp: + return std::forward<V>(visitor)(this->AsTimestampRaw()); + case EPrimitiveTypeName::TzDate: + return std::forward<V>(visitor)(this->AsTzDateRaw()); + case EPrimitiveTypeName::TzDatetime: + return std::forward<V>(visitor)(this->AsTzDatetimeRaw()); + case EPrimitiveTypeName::TzTimestamp: + return std::forward<V>(visitor)(this->AsTzTimestampRaw()); + case EPrimitiveTypeName::Interval: + return std::forward<V>(visitor)(this->AsIntervalRaw()); + case EPrimitiveTypeName::Decimal: + return std::forward<V>(visitor)(this->AsDecimalRaw()); + case EPrimitiveTypeName::Json: + return std::forward<V>(visitor)(this->AsJsonRaw()); + case EPrimitiveTypeName::Yson: + return std::forward<V>(visitor)(this->AsYsonRaw()); + case EPrimitiveTypeName::Uuid: + return std::forward<V>(visitor)(this->AsUuidRaw()); + } + + Y_UNREACHABLE(); + } + + template <typename V> + decltype(auto) TVariantType::VisitUnderlying(V&& visitor) const { + switch (GetUnderlyingTypeRaw()->GetTypeName()) { + case ETypeName::Struct: + return std::forward<V>(visitor)(this->GetUnderlyingTypeRaw()->AsStruct()); + case ETypeName::Tuple: + return std::forward<V>(visitor)(this->GetUnderlyingTypeRaw()->AsTuple()); + default: + Y_UNREACHABLE(); + } + } + + template <typename V> + decltype(auto) TVariantType::VisitUnderlyingRaw(V&& visitor) const { + switch (GetUnderlyingTypeRaw()->GetTypeName()) { + case ETypeName::Struct: + return std::forward<V>(visitor)(this->GetUnderlyingTypeRaw()->AsStructRaw()); + case ETypeName::Tuple: + return std::forward<V>(visitor)(this->GetUnderlyingTypeRaw()->AsTupleRaw()); + default: + Y_UNREACHABLE(); + } + } +} // namespace NTi diff --git a/library/cpp/type_info/type_complexity.cpp b/library/cpp/type_info/type_complexity.cpp new file mode 100644 index 0000000000..073c24b909 --- /dev/null +++ b/library/cpp/type_info/type_complexity.cpp @@ -0,0 +1,78 @@ +#include "type_complexity.h" + +#include "type.h" + + +namespace NTi { + +int ComputeTypeComplexity(const TTypePtr& type) +{ + return ComputeTypeComplexity(type.Get()); +} + +int ComputeTypeComplexity(const TType* type) +{ + switch (type->GetTypeName()) { + case ETypeName::Bool: + case ETypeName::Int8: + case ETypeName::Int16: + case ETypeName::Int32: + case ETypeName::Int64: + case ETypeName::Uint8: + case ETypeName::Uint16: + case ETypeName::Uint32: + case ETypeName::Uint64: + case ETypeName::Float: + case ETypeName::Double: + case ETypeName::String: + case ETypeName::Utf8: + case ETypeName::Date: + case ETypeName::Datetime: + case ETypeName::Timestamp: + case ETypeName::TzDate: + case ETypeName::TzDatetime: + case ETypeName::TzTimestamp: + case ETypeName::Interval: + case ETypeName::Decimal: + case ETypeName::Json: + case ETypeName::Yson: + case ETypeName::Uuid: + case ETypeName::Void: + case ETypeName::Null: + return 1; + + case ETypeName::Optional: + return 1 + ComputeTypeComplexity(type->AsOptionalRaw()->GetItemTypeRaw()); + + case ETypeName::List: + return 1 + ComputeTypeComplexity(type->AsListRaw()->GetItemTypeRaw()); + + case ETypeName::Dict: + return 1 + ComputeTypeComplexity(type->AsDictRaw()->GetKeyTypeRaw()) + + ComputeTypeComplexity(type->AsDictRaw()->GetValueTypeRaw()); + case ETypeName::Struct: { + int result = 1; + for (const auto& member : type->AsStructRaw()->GetMembers()) { + result += ComputeTypeComplexity(member.GetTypeRaw()); + } + return result; + } + case ETypeName::Tuple: { + int result = 1; + for (const auto& element : type->AsTupleRaw()->GetElements()) { + result += ComputeTypeComplexity(element.GetTypeRaw()); + } + return result; + } + case ETypeName::Variant: { + return ComputeTypeComplexity(type->AsVariantRaw()->GetUnderlyingTypeRaw()); + } + case ETypeName::Tagged: { + return 1 + ComputeTypeComplexity(type->AsTaggedRaw()->GetItemType()); + } + } + Y_FAIL("internal error: unreachable code"); +} + +} // namespace NTi + diff --git a/library/cpp/type_info/type_complexity.h b/library/cpp/type_info/type_complexity.h new file mode 100644 index 0000000000..fcf367eb81 --- /dev/null +++ b/library/cpp/type_info/type_complexity.h @@ -0,0 +1,18 @@ +#pragma once +#include "fwd.h" + +namespace NTi { + + /// + /// Compute type complexity. + /// Roughly speaking type complexity is a number of nodes in the schema tree. + /// Examples: + /// - Type complexity of simple or singular type (i.e. Int64, String, Null, Decimal) is 1. + /// - Type complexity of `Optional<Int64>` is 2 + /// - Type complexity of `Struct<a:Int64,b:Optional<String>` is 4 (1 for Int64, 2 for Optional<String> and 1 for struct) + /// + /// Systems might impose restrictions on the type complexity. + int ComputeTypeComplexity(const TTypePtr& type); + int ComputeTypeComplexity(const TType* type); + +} // namespace NTi diff --git a/library/cpp/type_info/type_constructors.h b/library/cpp/type_info/type_constructors.h new file mode 100644 index 0000000000..439d5a1887 --- /dev/null +++ b/library/cpp/type_info/type_constructors.h @@ -0,0 +1,110 @@ +#pragma once + +//! @file type_constructors.h + +#include "type.h" + +namespace NTi { + /// Create new `Null` type using the default heap factory. + TNullTypePtr Null(); + + /// Create new `Bool` type using the default heap factory. + TBoolTypePtr Bool(); + + /// Create new `Int8` type using the default heap factory. + TInt8TypePtr Int8(); + + /// Create new `Int16` type using the default heap factory. + TInt16TypePtr Int16(); + + /// Create new `Int32` type using the default heap factory. + TInt32TypePtr Int32(); + + /// Create new `Int64` type using the default heap factory. + TInt64TypePtr Int64(); + + /// Create new `Uint8` type using the default heap factory. + TUint8TypePtr Uint8(); + + /// Create new `Uint16` type using the default heap factory. + TUint16TypePtr Uint16(); + + /// Create new `Uint32` type using the default heap factory. + TUint32TypePtr Uint32(); + + /// Create new `Uint64` type using the default heap factory. + TUint64TypePtr Uint64(); + + /// Create new `Float` type using the default heap factory. + TFloatTypePtr Float(); + + /// Create new `Double` type using the default heap factory. + TDoubleTypePtr Double(); + + /// Create new `String` type using the default heap factory. + TStringTypePtr String(); + + /// Create new `Utf8` type using the default heap factory. + TUtf8TypePtr Utf8(); + + /// Create new `Date` type using the default heap factory. + TDateTypePtr Date(); + + /// Create new `Datetime` type using the default heap factory. + TDatetimeTypePtr Datetime(); + + /// Create new `Timestamp` type using the default heap factory. + TTimestampTypePtr Timestamp(); + + /// Create new `TzDate` type using the default heap factory. + TTzDateTypePtr TzDate(); + + /// Create new `TzDatetime` type using the default heap factory. + TTzDatetimeTypePtr TzDatetime(); + + /// Create new `TzTimestamp` type using the default heap factory. + TTzTimestampTypePtr TzTimestamp(); + + /// Create new `Interval` type using the default heap factory. + TIntervalTypePtr Interval(); + + /// Create new `Decimal` type using the default heap factory. + TDecimalTypePtr Decimal(ui8 precision, ui8 scale); + + /// Create new `Json` type using the default heap factory. + TJsonTypePtr Json(); + + /// Create new `Yson` type using the default heap factory. + TYsonTypePtr Yson(); + + /// Create new `Uuid` type using the default heap factory. + TUuidTypePtr Uuid(); + + /// Create new `Optional` type using the default heap factory. + TOptionalTypePtr Optional(TTypePtr item); + + /// Create new `List` type using the default heap factory. + TListTypePtr List(TTypePtr item); + + /// Create new `Dict` type using the default heap factory. + TDictTypePtr Dict(TTypePtr key, TTypePtr value); + + /// Create new `Struct` type using the default heap factory. + TStructTypePtr Struct(TStructType::TOwnedMembers items); + /// Create new `Struct` type using the default heap factory. + TStructTypePtr Struct(TMaybe<TStringBuf> name, TStructType::TOwnedMembers items); + + /// Create new `Tuple` type using the default heap factory. + TTupleTypePtr Tuple(TTupleType::TOwnedElements items); + /// Create new `Tuple` type using the default heap factory. + TTupleTypePtr Tuple(TMaybe<TStringBuf> name, TTupleType::TOwnedElements items); + + /// Create new `Variant` type using the default heap factory. + TVariantTypePtr Variant(TTypePtr underlying); + /// Create new `Variant` type using the default heap factory. + TVariantTypePtr Variant(TMaybe<TStringBuf> name, TTypePtr underlying); + + /// Create new `Tagged` type using the default heap factory. + TTaggedTypePtr Tagged(TTypePtr type, TStringBuf tag); + +} // namespace NTi diff --git a/library/cpp/type_info/type_equivalence.cpp b/library/cpp/type_info/type_equivalence.cpp new file mode 100644 index 0000000000..9f22f0308c --- /dev/null +++ b/library/cpp/type_info/type_equivalence.cpp @@ -0,0 +1,286 @@ +#include "type_equivalence.h" + +#include <util/generic/overloaded.h> + +#include "type.h" + +namespace NTi::NEq { + namespace { + template <bool IgnoreHash> + bool StrictlyEqual(const TType* lhs, const TType* rhs); + + // template <bool IgnoreHash> + // bool StrictlyEqual(const TDocumentation& lhs, const TDocumentation& rhs) { + // ... + // } + + // template <bool IgnoreHash> + // bool StrictlyEqual(const TAnnotations& lhs, const TAnnotations& rhs) { + // ... + // } + + template <bool IgnoreHash> + bool StrictlyEqual(const TVoidType&, const TVoidType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TNullType&, const TNullType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TBoolType&, const TBoolType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TInt8Type&, const TInt8Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TInt16Type&, const TInt16Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TInt32Type&, const TInt32Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TInt64Type&, const TInt64Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TUint8Type&, const TUint8Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TUint16Type&, const TUint16Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TUint32Type&, const TUint32Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TUint64Type&, const TUint64Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TFloatType&, const TFloatType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TDoubleType&, const TDoubleType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TStringType&, const TStringType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TUtf8Type&, const TUtf8Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TDateType&, const TDateType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TDatetimeType&, const TDatetimeType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TTimestampType&, const TTimestampType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TTzDateType&, const TTzDateType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TTzDatetimeType&, const TTzDatetimeType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TTzTimestampType&, const TTzTimestampType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TIntervalType&, const TIntervalType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TDecimalType& lhs, const TDecimalType& rhs) { + return lhs.GetPrecision() == rhs.GetPrecision() && lhs.GetScale() == rhs.GetScale(); + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TJsonType&, const TJsonType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TYsonType&, const TYsonType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TUuidType&, const TUuidType&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TOptionalType& lhs, const TOptionalType& rhs) { + return StrictlyEqual<IgnoreHash>(lhs.GetItemTypeRaw(), rhs.GetItemTypeRaw()); + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TListType& lhs, const TListType& rhs) { + return StrictlyEqual<IgnoreHash>(lhs.GetItemTypeRaw(), rhs.GetItemTypeRaw()); + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TDictType& lhs, const TDictType& rhs) { + return StrictlyEqual<IgnoreHash>(lhs.GetKeyTypeRaw(), rhs.GetKeyTypeRaw()) && + StrictlyEqual<IgnoreHash>(lhs.GetValueTypeRaw(), rhs.GetValueTypeRaw()); + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TStructType& lhs, const TStructType& rhs) { + if (lhs.GetName() != rhs.GetName()) { + return false; + } + + return std::equal( + lhs.GetMembers().begin(), lhs.GetMembers().end(), + rhs.GetMembers().begin(), rhs.GetMembers().end(), + [](const TStructType::TMember& lhs, const TStructType::TMember& rhs) -> bool { + return lhs.GetName() == rhs.GetName() && + StrictlyEqual<IgnoreHash>(lhs.GetTypeRaw(), rhs.GetTypeRaw()); + // && StrictlyEqual(lhs.GetAnnotations(), rhs.GetAnnotations()) + // && StrictlyEqual(lhs.GetDocumentation(), rhs.GetDocumentation()); + }); + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TTupleType& lhs, const TTupleType& rhs) { + if (lhs.GetName() != rhs.GetName()) { + return false; + } + + return std::equal( + lhs.GetElements().begin(), lhs.GetElements().end(), + rhs.GetElements().begin(), rhs.GetElements().end(), + [](const TTupleType::TElement& lhs, const TTupleType::TElement& rhs) -> bool { + return StrictlyEqual<IgnoreHash>(lhs.GetTypeRaw(), rhs.GetTypeRaw()); + // && StrictlyEqual(lhs.GetAnnotations(), rhs.GetAnnotations()) + // && StrictlyEqual(lhs.GetDocumentation(), rhs.GetDocumentation()); + }); + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TVariantType& lhs, const TVariantType& rhs) { + if (lhs.GetName() != rhs.GetName()) { + return false; + } + + return StrictlyEqual<IgnoreHash>(lhs.GetUnderlyingTypeRaw(), rhs.GetUnderlyingTypeRaw()); + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TTaggedType& lhs, const TTaggedType& rhs) { + return lhs.GetTag() == rhs.GetTag() && + StrictlyEqual<IgnoreHash>(lhs.GetItemTypeRaw(), rhs.GetItemTypeRaw()); + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TType* lhs, const TType* rhs) { + if (lhs == rhs) { + return true; + } + + if (lhs == nullptr || rhs == nullptr) { + return false; + } + + if (!IgnoreHash && lhs->GetHash() != rhs->GetHash()) { + return false; + } + + if (lhs->GetTypeName() != rhs->GetTypeName()) { + return false; + } + + // FIXME: update `TStrictlyEqual`'s docs to explicitly state that documentation and annotations + // must be the same for types to compare strictly equal. + + // if (!StrictlyEqual(lhs->GetAnnotations(), rhs->GetAnnotations())) { + // return false; + // } + + // if (!StrictlyEqual(lhs->GetDocumentation(), rhs->GetDocumentation())) { + // return false; + // } + + return lhs->VisitRaw([&rhs](const auto* lhs) { + using TSameType = decltype(lhs); + return rhs->VisitRaw(TOverloaded{ + [lhs](TSameType rhs) { + return StrictlyEqual<IgnoreHash>(*lhs, *rhs); + }, + [](const auto* rhs) { + static_assert(!std::is_same_v<decltype(lhs), decltype(rhs)>); + return false; + }}); + }); + } + } + + bool TStrictlyEqual::operator()(const TType* lhs, const TType* rhs) const { + return StrictlyEqual<false>(lhs, rhs); + } + + bool TStrictlyEqual::operator()(TTypePtr lhs, TTypePtr rhs) const { + return operator()(lhs.Get(), rhs.Get()); + } + + bool TStrictlyEqual::IgnoreHash(const TType* lhs, const TType* rhs) const { + return StrictlyEqual<true>(lhs, rhs); + } + + bool TStrictlyEqual::IgnoreHash(TTypePtr lhs, TTypePtr rhs) const { + return IgnoreHash(lhs.Get(), rhs.Get()); + } + + ui64 TStrictlyEqualHash::operator()(const TType* type) const { + if (type == nullptr) { + return 0; + } + + return type->GetHash(); + } + + ui64 TStrictlyEqualHash::operator()(TTypePtr type) const { + return operator()(type.Get()); + } +} diff --git a/library/cpp/type_info/type_equivalence.h b/library/cpp/type_info/type_equivalence.h new file mode 100644 index 0000000000..70351dc689 --- /dev/null +++ b/library/cpp/type_info/type_equivalence.h @@ -0,0 +1,36 @@ +#pragma once + +//! @file type_equivalence.h +//! +//! Relations between types. +//! +//! Type info declares multiple ways to compare types. There's strict, nominal, structural and other equivalences, +//! as well as subtyping. See corresponding functors for more info. +//! +//! At the moment, only strict equivalence is implemented because others are not yet standartized. + +#include <type_traits> + +#include "fwd.h" + +#include <util/system/types.h> + +namespace NTi::NEq { + /// Strict equivalence is the strongest form of type equivalence. If two types are strictly equal, + /// they're literally the same type for all intents and purposes. This includes struct, tuple, variant and enum + /// names, order of their items, names of their items, names of tagged types, and so on. + struct TStrictlyEqual { + bool operator()(const TType* lhs, const TType* rhs) const; + bool operator()(TTypePtr lhs, TTypePtr rhs) const; + + /// Compare types without calculating and comparing their hashes first. + bool IgnoreHash(const TType* lhs, const TType* rhs) const; + bool IgnoreHash(TTypePtr lhs, TTypePtr rhs) const; + }; + + /// Hash that follows the strict equality rules (see `TStrictlyEqual`). + struct TStrictlyEqualHash { + ui64 operator()(const TType* type) const; + ui64 operator()(TTypePtr type) const; + }; +} diff --git a/library/cpp/type_info/type_factory.cpp b/library/cpp/type_info/type_factory.cpp new file mode 100644 index 0000000000..9d60307938 --- /dev/null +++ b/library/cpp/type_info/type_factory.cpp @@ -0,0 +1,495 @@ +#include "type_factory.h" + +#include "type.h" +#include "type_equivalence.h" + +#include <util/memory/pool.h> +#include <util/generic/hash_set.h> + +#include <cstdlib> + +namespace NTi { + TVoidTypePtr ITypeFactory::Void() { + return TVoidType::Instance(); + } + + const TVoidType* IPoolTypeFactory::VoidRaw() { + return TVoidType::InstanceRaw(); + } + + TNullTypePtr ITypeFactory::Null() { + return TNullType::Instance(); + } + + const TNullType* IPoolTypeFactory::NullRaw() { + return TNullType::InstanceRaw(); + } + + TBoolTypePtr ITypeFactory::Bool() { + return TBoolType::Instance(); + } + + const TBoolType* IPoolTypeFactory::BoolRaw() { + return TBoolType::InstanceRaw(); + } + + TInt8TypePtr ITypeFactory::Int8() { + return TInt8Type::Instance(); + } + + const TInt8Type* IPoolTypeFactory::Int8Raw() { + return TInt8Type::InstanceRaw(); + } + + TInt16TypePtr ITypeFactory::Int16() { + return TInt16Type::Instance(); + } + + const TInt16Type* IPoolTypeFactory::Int16Raw() { + return TInt16Type::InstanceRaw(); + } + + TInt32TypePtr ITypeFactory::Int32() { + return TInt32Type::Instance(); + } + + const TInt32Type* IPoolTypeFactory::Int32Raw() { + return TInt32Type::InstanceRaw(); + } + + TInt64TypePtr ITypeFactory::Int64() { + return TInt64Type::Instance(); + } + + const TInt64Type* IPoolTypeFactory::Int64Raw() { + return TInt64Type::InstanceRaw(); + } + + TUint8TypePtr ITypeFactory::Uint8() { + return TUint8Type::Instance(); + } + + const TUint8Type* IPoolTypeFactory::Uint8Raw() { + return TUint8Type::InstanceRaw(); + } + + TUint16TypePtr ITypeFactory::Uint16() { + return TUint16Type::Instance(); + } + + const TUint16Type* IPoolTypeFactory::Uint16Raw() { + return TUint16Type::InstanceRaw(); + } + + TUint32TypePtr ITypeFactory::Uint32() { + return TUint32Type::Instance(); + } + + const TUint32Type* IPoolTypeFactory::Uint32Raw() { + return TUint32Type::InstanceRaw(); + } + + TUint64TypePtr ITypeFactory::Uint64() { + return TUint64Type::Instance(); + } + + const TUint64Type* IPoolTypeFactory::Uint64Raw() { + return TUint64Type::InstanceRaw(); + } + + TFloatTypePtr ITypeFactory::Float() { + return TFloatType::Instance(); + } + + const TFloatType* IPoolTypeFactory::FloatRaw() { + return TFloatType::InstanceRaw(); + } + + TDoubleTypePtr ITypeFactory::Double() { + return TDoubleType::Instance(); + } + + const TDoubleType* IPoolTypeFactory::DoubleRaw() { + return TDoubleType::InstanceRaw(); + } + + TStringTypePtr ITypeFactory::String() { + return TStringType::Instance(); + } + + const TStringType* IPoolTypeFactory::StringRaw() { + return TStringType::InstanceRaw(); + } + + TUtf8TypePtr ITypeFactory::Utf8() { + return TUtf8Type::Instance(); + } + + const TUtf8Type* IPoolTypeFactory::Utf8Raw() { + return TUtf8Type::InstanceRaw(); + } + + TDateTypePtr ITypeFactory::Date() { + return TDateType::Instance(); + } + + const TDateType* IPoolTypeFactory::DateRaw() { + return TDateType::InstanceRaw(); + } + + TDatetimeTypePtr ITypeFactory::Datetime() { + return TDatetimeType::Instance(); + } + + const TDatetimeType* IPoolTypeFactory::DatetimeRaw() { + return TDatetimeType::InstanceRaw(); + } + + TTimestampTypePtr ITypeFactory::Timestamp() { + return TTimestampType::Instance(); + } + + const TTimestampType* IPoolTypeFactory::TimestampRaw() { + return TTimestampType::InstanceRaw(); + } + + TTzDateTypePtr ITypeFactory::TzDate() { + return TTzDateType::Instance(); + } + + const TTzDateType* IPoolTypeFactory::TzDateRaw() { + return TTzDateType::InstanceRaw(); + } + + TTzDatetimeTypePtr ITypeFactory::TzDatetime() { + return TTzDatetimeType::Instance(); + } + + const TTzDatetimeType* IPoolTypeFactory::TzDatetimeRaw() { + return TTzDatetimeType::InstanceRaw(); + } + + TTzTimestampTypePtr ITypeFactory::TzTimestamp() { + return TTzTimestampType::Instance(); + } + + const TTzTimestampType* IPoolTypeFactory::TzTimestampRaw() { + return TTzTimestampType::InstanceRaw(); + } + + TIntervalTypePtr ITypeFactory::Interval() { + return TIntervalType::Instance(); + } + + const TIntervalType* IPoolTypeFactory::IntervalRaw() { + return TIntervalType::InstanceRaw(); + } + + TDecimalTypePtr ITypeFactory::Decimal(ui8 precision, ui8 scale) { + return TDecimalType::Create(*this, precision, scale); + } + + const TDecimalType* IPoolTypeFactory::DecimalRaw(ui8 precision, ui8 scale) { + return TDecimalType::CreateRaw(*this, precision, scale); + } + + TJsonTypePtr ITypeFactory::Json() { + return TJsonType::Instance(); + } + + const TJsonType* IPoolTypeFactory::JsonRaw() { + return TJsonType::InstanceRaw(); + } + + TYsonTypePtr ITypeFactory::Yson() { + return TYsonType::Instance(); + } + + const TYsonType* IPoolTypeFactory::YsonRaw() { + return TYsonType::InstanceRaw(); + } + + TUuidTypePtr ITypeFactory::Uuid() { + return TUuidType::Instance(); + } + + const TUuidType* IPoolTypeFactory::UuidRaw() { + return TUuidType::InstanceRaw(); + } + + TOptionalTypePtr ITypeFactory::Optional(TTypePtr item) { + return TOptionalType::Create(*this, std::move(item)); + } + + const TOptionalType* IPoolTypeFactory::OptionalRaw(const TType* item) { + return TOptionalType::CreateRaw(*this, item); + } + + TListTypePtr ITypeFactory::List(TTypePtr item) { + return TListType::Create(*this, std::move(item)); + } + + const TListType* IPoolTypeFactory::ListRaw(const TType* item) { + return TListType::CreateRaw(*this, item); + } + + TDictTypePtr ITypeFactory::Dict(TTypePtr key, TTypePtr value) { + return TDictType::Create(*this, std::move(key), std::move(value)); + } + + const TDictType* IPoolTypeFactory::DictRaw(const TType* key, const TType* value) { + return TDictType::CreateRaw(*this, key, value); + } + + TStructTypePtr ITypeFactory::Struct(TStructType::TOwnedMembers items) { + return TStructType::Create(*this, items); + } + + TStructTypePtr ITypeFactory::Struct(TMaybe<TStringBuf> name, TStructType::TOwnedMembers items) { + return TStructType::Create(*this, name, items); + } + + const TStructType* IPoolTypeFactory::StructRaw(TStructType::TMembers items) { + return TStructType::CreateRaw(*this, items); + } + + const TStructType* IPoolTypeFactory::StructRaw(TMaybe<TStringBuf> name, TStructType::TMembers items) { + return TStructType::CreateRaw(*this, name, items); + } + + TTupleTypePtr ITypeFactory::Tuple(TTupleType::TOwnedElements items) { + return TTupleType::Create(*this, items); + } + + TTupleTypePtr ITypeFactory::Tuple(TMaybe<TStringBuf> name, TTupleType::TOwnedElements items) { + return TTupleType::Create(*this, name, items); + } + + const TTupleType* IPoolTypeFactory::TupleRaw(TTupleType::TElements items) { + return TTupleType::CreateRaw(*this, items); + } + + const TTupleType* IPoolTypeFactory::TupleRaw(TMaybe<TStringBuf> name, TTupleType::TElements items) { + return TTupleType::CreateRaw(*this, name, items); + } + + TVariantTypePtr ITypeFactory::Variant(TTypePtr inner) { + return TVariantType::Create(*this, std::move(inner)); + } + + TVariantTypePtr ITypeFactory::Variant(TMaybe<TStringBuf> name, TTypePtr inner) { + return TVariantType::Create(*this, name, std::move(inner)); + } + + const TVariantType* IPoolTypeFactory::VariantRaw(const TType* inner) { + return TVariantType::CreateRaw(*this, inner); + } + + const TVariantType* IPoolTypeFactory::VariantRaw(TMaybe<TStringBuf> name, const TType* inner) { + return TVariantType::CreateRaw(*this, name, inner); + } + + TTaggedTypePtr ITypeFactory::Tagged(TTypePtr type, TStringBuf tag) { + return TTaggedType::Create(*this, std::move(type), tag); + } + + const TTaggedType* IPoolTypeFactory::TaggedRaw(const TType* type, TStringBuf tag) { + return TTaggedType::CreateRaw(*this, type, tag); + } + + namespace { + class TPoolFactory: public NTi::IPoolTypeFactory { + public: + TPoolFactory(size_t initial, TMemoryPool::IGrowPolicy* grow, IAllocator* alloc, TMemoryPool::TOptions options) + : Pool_(initial, grow, alloc, options) + { + } + + public: + void* Allocate(size_t size, size_t align) noexcept override { + return Pool_.Allocate(size, align); + } + + void Free(void* data) noexcept override { + Y_UNUSED(data); + } + + protected: + const NTi::TType* LookupCache(const NTi::TType* type) noexcept override { + Y_UNUSED(type); + return nullptr; + } + + void SaveCache(const NTi::TType* type) noexcept override { + Y_UNUSED(type); + } + + void Ref() noexcept override { + Counter_.Inc(); + } + + void UnRef() noexcept override { + if (Counter_.Dec() == 0) { + delete this; + } + } + + void DecRef() noexcept override { + if (Counter_.Dec() == 0) { + Y_FAIL("DecRef is not supposed to drop"); + } + } + + long RefCount() const noexcept override { + return Counter_.Val(); + } + + void RefType(NTi::TType* type) noexcept override { + Y_UNUSED(type); + } + + void UnRefType(NTi::TType* type) noexcept override { + Y_UNUSED(type); + } + + void DecRefType(NTi::TType* type) noexcept override { + Y_UNUSED(type); + } + + long RefCountType(const NTi::TType* type) const noexcept override { + Y_UNUSED(type); + return RefCount(); + } + + public: + size_t Available() const noexcept override { + return Pool_.Available(); + } + + size_t MemoryAllocated() const noexcept override { + return Pool_.MemoryAllocated(); + } + + size_t MemoryWaste() const noexcept override { + return Pool_.MemoryWaste(); + } + + private: + TAtomicCounter Counter_; + TMemoryPool Pool_; + }; + + class TPoolFactoryDedup: public TPoolFactory { + public: + using TPoolFactory::TPoolFactory; + + public: + const NTi::TType* LookupCache(const NTi::TType* type) noexcept override { + if (auto it = Cache_.find(type); it != Cache_.end()) { + return *it; + } else { + return nullptr; + } + } + + void SaveCache(const NTi::TType* type) noexcept override { + Cache_.insert(type); + } + + protected: + TStringBuf AllocateString(TStringBuf str) noexcept override { + if (str.empty()) { + return TStringBuf(); // `str` could still point somewhere whereas empty strbuf points to NULL + } + + if (auto it = StringCache_.find(str); it != StringCache_.end()) { + return *it; + } else { + return *StringCache_.insert(it, ITypeFactoryInternal::AllocateString(str)); + } + } + + private: + THashSet<const NTi::TType*, NTi::NEq::TStrictlyEqualHash, NTi::NEq::TStrictlyEqual> Cache_; + THashSet<TStringBuf> StringCache_; + }; + + class THeapFactory: public NTi::ITypeFactory { + public: + void* Allocate(size_t size, size_t align) noexcept override { + Y_UNUSED(align); + return malloc(size); + } + + void Free(void* data) noexcept override { + free(data); + } + + protected: + const NTi::TType* LookupCache(const NTi::TType* type) noexcept override { + Y_UNUSED(type); + return nullptr; + } + + void SaveCache(const NTi::TType* type) noexcept override { + Y_UNUSED(type); + } + + void Ref() noexcept override { + // nothing + } + + void UnRef() noexcept override { + // nothing + } + + void DecRef() noexcept override { + // nothing + } + + long RefCount() const noexcept override { + return 0; + } + + void RefType(NTi::TType* type) noexcept override { + Y_UNUSED(type); + Y_FAIL("not supposed to be called"); + } + + void UnRefType(NTi::TType* type) noexcept override { + Y_UNUSED(type); + Y_FAIL("not supposed to be called"); + } + + void DecRefType(NTi::TType* type) noexcept override { + Y_UNUSED(type); + Y_FAIL("not supposed to be called"); + } + + long RefCountType(const NTi::TType* type) const noexcept override { + Y_UNUSED(type); + Y_FAIL("not supposed to be called"); + } + }; + + THeapFactory HEAP_FACTORY; + } + + IPoolTypeFactoryPtr PoolFactory(bool deduplicate, size_t initial, TMemoryPool::IGrowPolicy* grow, IAllocator* alloc, TMemoryPool::TOptions options) { + if (deduplicate) { + return new TPoolFactoryDedup(initial, grow, alloc, options); + } else { + return new TPoolFactory(initial, grow, alloc, options); + } + } + + namespace NPrivate { + ITypeFactory* GetDefaultHeapFactory() { + return &HEAP_FACTORY; + } + } + + ITypeFactoryPtr HeapFactory() { + return NPrivate::GetDefaultHeapFactory(); + } +} diff --git a/library/cpp/type_info/type_factory.h b/library/cpp/type_info/type_factory.h new file mode 100644 index 0000000000..df47c7082b --- /dev/null +++ b/library/cpp/type_info/type_factory.h @@ -0,0 +1,906 @@ +#pragma once + +//! @file type_factory.h +//! +//! Type factory creates type instances and manages their lifetimes and destruction. +//! +//! Type info supports multiple ways of allocating type instances and managing their lifetimes: +//! +//! - [heap-based allocation] is the standard memory allocation method for C++. It offers great flexibility +//! as it can allocate memory regions of almost arbitrary size and alignment, deallocate them, re-use them +//! for new allocations or return the to the operating system. This flexibility comes with a price, though. +//! Heap allocators usually require synchronisation (i.e. a mutex or a spinlock), and some memory overhead to track +//! allocated regions. Also, they provide no guarantees on data locality and CPU cache friendliness whatsoever. +//! +//! When using a heap-based factory, each type instance have a separate reference counter. `TTypePtr`s will +//! increment and decrement this counter. Whenever it reaches zero, they will destroy the type instance and free +//! its memory. This is the standard reference counting technique, just like in `TRefCounted`. +//! +//! - [memory pool] is a data structure for faster memory allocation. It pre-allocates large chunks of memory and uses +//! them to allocate smaller objects. This way, we don't have to `malloc` memory for each new object. +//! Also, we can drop all objects at once by discarding all memory chunks. +//! +//! When using memory-pool-based factory, types don't have individual reference counters. Instead, they store +//! a pointer to the factory that've created them. Whenever such type is referenced or unreferenced, the factory's +//! own reference counter is incremented or decremented. Factory dies and releases all pool's memory when the last +//! reference to a type in its pool dies. +//! +//! [heap-based allocation]: https://en.wikipedia.org/wiki/Memory_management#Dynamic_memory_allocation +//! [Reference counting]: https://en.wikipedia.org/wiki/Reference_counting +//! [memory pool]: https://en.wikipedia.org/wiki/Region-based_memory_management +//! +//! The rule of thumb is: if you have an intrusive pointer to a type, you can be sure it points to an alive object; +//! if you have a raw pointer to a type, it's your responsibility to ensure its validity. +//! +//! Whenever you have a valid raw pointer to a type, you can promote it to an intrusive pointer by calling `AsPtr`. +//! This is always safe because there's no way to create a type instance on the stack or in the static memory +//! of a program. +//! +//! +//! # Implementation details +//! +//! We're building a hierarchy of classes that work equally well with both heap-based allocation model +//! and memory-pool-based allocation model. In order to understand how we do it, we should first understand how +//! object ownership works in both models. +//! +//! +//! ## Ownership schema +//! +//! Depending on the chosen factory implementation, ownership schema may vary. +//! +//! When using the heap-based factory, each type has its own reference counter. User own types, and types own +//! their nested types. For `Optional<String>`, user owns the `Optional` type, and `Optional` type +//! owns the `String` type: +//! +//! ```text +//! Legend: A ───> B -- A owns B | A ─ ─> B -- A points to B +//! +//! User code: Objects: +//! ┌─────────────┐ ┌────────────────┐ +//! │ IFactoryPtr │───────>│ Factory │ +//! └─────────────┘ └────────────────┘ +//! ┌─────────────┐ ┌────────────────┐ +//! │ TTypePtr │───────>│ Type: Optional │──┐ +//! └─────────────┘ └────────────────┘ │ +//! ┌──────────────────────┘ +//! │ ┌────────────────┐ +//! └─>│ Type: String │ +//! └────────────────┘ +//! ``` +//! +//! When using a pool-based factory, all allocated types are owned by pool. So, in the `Optional<String>` example, +//! user owns factory, factory owns the `Optional` type and the `String` type. All references between types +//! are just borrows: +//! +//! ```text +//! Legend: A ───> B -- A owns B | A ─ ─> B -- A points to B +//! +//! User code: Objects: +//! ┌─────────────┐ ┌────────────────┐ +//! │ IFactoryPtr │─┌─────>│ Factory │──┐ +//! └─────────────┘ │ └────────────────┘ │ +//! ┌─────────────┐ │ ┌────────────────┐ │ +//! │ TTypePtr │─┘─ ─ ─>│ Type: Optional │<─┤ +//! └─────────────┘ └───────┬────────┘ │ +//! ┌ ─ ─ ─ ─ ─ ─┘ │ +//! ╎ ┌────────────────┐ │ +//! └─ ─>│ Type: String │<─┘ +//! └────────────────┘ +//! ``` +//! +//! Notice how `TTypePtr` points to `Optional`, but doesn't own it. Instead, it owns the factory which, +//! transitively, owns `Optional`. This way we can be sure that the factory will not be destroyed before +//! all `IFactoryPtr`s and `TTypePtr`s die, thus guaranteeing that all `TTypePtr` always stay valid. +//! +//! With that in mind, we can derive two types of ownership here. +//! +//! Whenever `TTypePtr` owns some type, we call it 'external ownership' (because it's code that is external +//! to this library owns something). In this situation, ref and unref procedures must increment type's own reference +//! counter, if there is one, and also increment factory's reference counter. `NTi::TType::Ref` and `NTi::TType::Unref` +//! do this by calling `NTi::ITypeFactoryInternal::Ref`, `NTi::ITypeFactoryInternal::RefType`, +//! `NTi::ITypeFactoryInternal::UnRef`, `NTi::ITypeFactoryInternal::UnRefType`. +//! +//! Whenever one type owns another type, we call it 'internal ownership'. In this situation, ref and unref procedures +//! must only increment and decrement type's own reference counter. They should *not* increment or decrement +//! factory's reference counter. `NTi::TType::RefSelf` and `NTi::TType::UnrefSelf` do this by calling +//! `NTi::ITypeFactoryInternal::RefType` and `NTi::ITypeFactoryInternal::UnRefType`, and not calling +//! `NTi::ITypeFactoryInternal::Ref` and `NTi::ITypeFactoryInternal::UnRef`. +//! +//! +//! ## Adoption semantics +//! +//! Let's see now what should happen when we create some complex type using more that one factory. +//! +//! Suppose we're creating a container type, such as `Optional`, using factory `a`. If its nested type is +//! managed by the same factory, we have no issues at all: +//! +//! ``` +//! auto a = NTi::PoolFactory(); +//! auto string = a->String(); +//! auto optional = a->Optional(string); +//! ``` +//! +//! But what if the nested type is managed by some other factory `b`? Consider: +//! +//! ``` +//! auto b = NTi::PoolFactory(); +//! auto a = NTi::PoolFactory(); +//! auto string = b->String(); +//! auto optional = a->Optional(string); +//! ``` +//! +//! Well, we're in trouble. It's not enough for `optional` to just acquire internal ownership over `string`. +//! Actually, `optional` has to own both `string` and its factory, `b`. Otherwise, if `b` dies, `string` will +//! die with it, leaving `optional` with a dangling pointer. +//! +//! However, we can't have a type owning its factory. It will create a loop. The only safe solution is to copy +//! the nested type from one factory to another. If we have some type managed by factory `a`, all nested types +//! must also be managed by the same factory `a`. +//! +//! So, whenever we create a type, we must ensure that all its nested types are managed by the same factory. +//! If they're not, we must deep-copy them to the current factory. This is called the 'adoption semantics'. That is, +//! we adopt nested types into the current factory. +//! +//! In fact, adoption semantics is a bit more complicated that this. Some types are not managed by any factory. +//! They have static storage duration and therefore need no management. So, if we see that there's no factory +//! associated with some type, we don't perform deep-copy. Instead, we just return it as is. +//! +//! +//! ## Overview +//! +//! So, now we're ready to put this puzzle together. +//! +//! When we create a new type instance, we do the following: +//! +//! 1: we adopt all nested types and acquire internal ownership over them. This is done by the `Own` function; +//! 2: we allocate some memory for the type. This is done by `New`, `Allocate` and other functions; +//! 3: we initialize all this memory; +//! 4: finally, we acquire external ownership over the newly created type and return it to the user. +//! +//! Steps 1 through 3 are performed by the `NTi::TType::Clone` function. Step 4 is performed by the factory. +//! +//! When we adopt some type, we check if it's managed by any factory other that the current one. If needed, +//! we deep-copy it by the `NTi::TType::Clone` function. That is, `Clone` copies type to a new factory and adopts +//! nested types, adoption calls nested type's `Clone`, causing recursive deep-copying. +//! +//! When we release external or internal ownership over some type, factory might decide to destroy it (if, for +//! example, this type's reference count reached zero). In this case, we need to release internal ownership +//! over the nested types, and release all memory that was allocated in step 3. +//! This is done by `NTi::TType::Drop` function. + +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> +#include <util/memory/pool.h> + +#include "type.h" + +#include "fwd.h" + +namespace NTi { + namespace NPrivate { + /// If `T` is derived from `NTi::TType` (ignoring cv-qualification), provides the member constant `value` + /// equal to `true`. Otherwise `value` is `false`. + template <typename T> + struct TIsType: public std::is_base_of<TType, T> { + }; + + /// Helper template variable for `TIsType`. + template <typename T> + inline constexpr const bool IsTypeV = TIsType<T>::value; + + /// Statically assert that `T` is a managed object. + template <typename T> + void AssertIsType() { + static_assert(IsTypeV<T>, "object should be derived from 'NTi::TType'"); + } + } + + /// Internal interface for type factory. + /// + /// This interface is accessible from `TType` implementation, but not from outside. + class ITypeFactoryInternal { + friend class TType; + template <typename T> + friend class ::TDefaultIntrusivePtrOps; + + public: + virtual ~ITypeFactoryInternal() = default; + + public: + /// @name Object creation and lifetime management interface + /// + /// @{ + //- + /// Create a new object of type `T` by allocating memory for it and constructing it with arguments `args`. + /// + /// If `T` is derived from `TType`, it will be linked to this factory via the `NTi::TType::SetFactory` method. + /// + /// This method does not acquire any ownership over the created object, not internal nor external. It just + /// allocates some memory and calls a constructor and that's it. + template <typename T, typename... Args> + inline T* New(Args&&... args) { + // Note: it's important to understand difference between `New` and `Create` - read the docs! + + static_assert(std::is_trivially_destructible_v<T>, "can only create trivially destructible types"); + + auto value = new (AllocateFor<T>()) T(std::forward<Args>(args)...); + + if constexpr (NPrivate::IsTypeV<T>) { + value->SetFactory(this); + } + + return value; + } + + /// Delete an object that was created via the `New` function. + /// + /// This function essentially just calls `Free`. Note that `New` can only create trivially destructible types, + /// thus `Delete` does not neet to call object's destructor. + template <typename T> + inline void Delete(T* obj) noexcept { + Free(obj); + } + + /// Create a new array of `count` objects of type `T`. + /// + /// This function will allocate an array of `count` objects of type `T`. Then, for each object in the array, + /// it will call the `ctor(T* memory, size_t index)`, passing a pointer to the uninitialized object + /// and its index in the array. `ctor` should call placement new on the passed pointer. + /// + /// If `T` is derived from `TType`, it will be linked to this factory via the `NTi::TType::SetFactory` method. + /// + /// This method does not acquire any ownership over the created array elements, not internal nor external. + /// It just allocates some memory and calls a constructor and that's it. + /// + /// + /// # Examples + /// + /// Copy vector to factory: + /// + /// ``` + /// TVector<T> source = ...; + /// TArrayRef<T> copied = NMem::NewArray<T>(factory, source.size(), [&source](T* ptr, size_t i) { + /// new (ptr) T(source[i]); + /// }); + /// ``` + template <typename T, typename TCtor> + inline TArrayRef<T> NewArray(size_t count, TCtor&& ctor) { + // Note: it's important to understand difference between `New` and `Create` - read the docs! + + static_assert(std::is_trivially_destructible_v<T>, "can only create trivially destructible types"); + + auto items = AllocateArrayFor<T>(count); + + for (size_t i = 0; i < count; ++i) { + auto value = items + i; + ctor(value, i); + if constexpr (NPrivate::IsTypeV<T>) { + value->SetFactory(this); + } + } + + return TArrayRef(items, count); + } + + /// Delete an object that was created via the `NewArray` function. + /// + /// This function runs `dtor` on each element of the array and then just calls `Free`. Note that `NewArray` + /// can only create arrays for trivially destructible types, thus `DeleteArray` + /// does not call object destructors. + template <typename T, typename TDtor> + inline void DeleteArray(TArrayRef<T> obj, TDtor&& dtor) noexcept { + for (size_t i = 0; i < obj.size(); ++i) { + auto value = obj.data() + i; + dtor(value, i); + } + + Free(const_cast<void*>(static_cast<const void*>(obj.data()))); + } + + /// Adopt `type` into this factory. + /// + /// This function works like `AdoptRaw`, but it wraps its result to an intrusive pointer, thus acquiring + /// external ownership over it. + /// + /// It is used by external code to copy types between factories. + template <typename T> + inline TIntrusiveConstPtr<T> Adopt(TIntrusiveConstPtr<T> type) noexcept { + NPrivate::AssertIsType<T>(); + + return TIntrusiveConstPtr<T>(const_cast<T*>(AdoptRaw<T>(type.Get()))); + } + + /// Adopt `type` into this factory. + /// + /// This function is used to transfer types between factories. It has the following semantics: + /// + /// - if `type` is managed by this factory, this function returns its argument unchanged; + /// - if `type` is not managed by any factory, this function also returns its argument unchanged; + /// - if `type` is managed by another factory, this function deep-copies `type` into this factory and returns + /// a pointer to the new deep-copied value. + /// + /// This function uses `NTi::TType::Clone` to deep-copy types. It does not acquire any ownership over + /// the returned object, not internal nor external. If you need to adopt and acquire external ownership + /// (i.e. you're returning type to a user), use `Adopt`. If you need to adopt and acquire internal ownership + /// (i.e. you're writing `NTi::TType::Clone` implementation), use `Own`. + template <typename T> + inline const T* AdoptRaw(const T* type) noexcept { + NPrivate::AssertIsType<T>(); + + ITypeFactoryInternal* typeFactory = type->GetFactory(); + if (typeFactory == nullptr || typeFactory == this) { + return type; + } else { + return type->Clone(*this); + } + } + + /// Adopt some type and acquire internal ownership over it. + /// + /// This function works like `AdoptRaw`, but acquires internal ownership over the returned type. + /// + /// It is used by `NTi::TType::Clone` implementations to acquire ownership over the nested types. + template <typename T> + inline const T* Own(const T* type) noexcept { + NPrivate::AssertIsType<T>(); + + type = AdoptRaw(type); + const_cast<T*>(type)->RefSelf(); + return type; + } + + /// Release internal ownership over some type. + /// + /// This function is used by `NTi::TType::Drop` to release internal ownership over nested types. + template <typename T> + inline void Disown(const T* type) noexcept { + NPrivate::AssertIsType<T>(); + + const_cast<T*>(type)->UnRefSelf(); + } + + /// @} + + public: + /// @name Memory management interface + /// + /// Functions for dealing with raw unmanaged memory. + /// + /// @{ + //- + /// Allocate a new chunk of memory of size `size` aligned on `align`. + /// + /// The behaviour is undefined if `size` is zero, `align` is zero, is not a power of two, + /// or greater than `PLATFORM_DATA_ALIGN` (i.e., over-aligned). + /// + /// This function panics if memory can't be allocated. + /// + /// Returned memory may or may not be initialized. + virtual void* Allocate(size_t size, size_t align) noexcept = 0; + + /// Allocate a chunk of memory suitable for storing an object of type `T`. + /// + /// This function panics if memory can't be allocated. + /// + /// Returned memory may or may not be initialized. + template <typename T> + T* AllocateFor() noexcept { + static_assert(alignof(T) <= PLATFORM_DATA_ALIGN, "over-aligned types are not supported"); + return static_cast<T*>(Allocate(sizeof(T), alignof(T))); + } + + /// Allocate a new chunk of memory suitable for storing `count` of objects of size `size` aligned on `align`. + /// + /// The behaviour is undefined if `count` is zero, `size` is zero, `align` is zero, is not a power of two, + /// or greater than `PLATFORM_DATA_ALIGN` (i.e., over-aligned). + /// + /// This function panics if memory can't be allocated. + /// + /// Returned memory may or may not be initialized. + void* AllocateArray(size_t count, size_t size, size_t align) noexcept { + return Allocate(size * count, align); + } + + /// Allocate a chunk of memory suitable for storing an array of `count` objects of type `T`. + /// + /// This function panics if memory can't be allocated. + /// + /// Returned memory may or may not be initialized. + template <typename T> + T* AllocateArrayFor(size_t count) noexcept { + static_assert(alignof(T) <= PLATFORM_DATA_ALIGN, "over-aligned types are not supported"); + return static_cast<T*>(AllocateArray(count, sizeof(T), alignof(T))); + } + + /// Reclaim a chunk of memory memory that was allocated via one of the above `Allocate*` functions. + /// + /// This function is not suitable for freeing memory allocated via `AllocateString` or `AllocateStringMaybe`. + /// + /// Behaviour of this function varies between implementations. Specifically, in the memory-pool-based factory, + /// this function does nothing, while in heap-based factory, it calls `free`. + /// + /// Passing a null pointer here is ok and does nothing. + /// + /// The behaviour is undefined if the given pointer is not null and it wasn't obtained from a call + /// to the `Allocate` function or it was obtained from a call to the `Allocate` function + /// of a different factory. + virtual void Free(void* data) noexcept = 0; + + /// Allocate memory for string `str` and copy `str` to the allocated memory. + /// + /// Note: the returned string is not guaranteed to be null-terminated. + /// + /// Note: some factories may cache results of this function to avoid repeated allocations. Read documentation + /// for specific factory for more info. + virtual TStringBuf AllocateString(TStringBuf str) noexcept { + return TStringBuf(MemCopy(AllocateArrayFor<char>(str.size()), str.data(), str.size()), str.size()); + } + + /// Reclaim a chunk of memory memory that was allocated via the `AllocateString` function. + virtual void FreeString(TStringBuf str) noexcept { + Free(const_cast<char*>(str.Data())); + } + + /// Like `AllocateString`, but works with `TMaybe<TStringBuf>`. + /// + /// If the given `str` contains a value, pass it to `AllocateString`. + /// If it contains no value, return an empty `TMaybe`. + TMaybe<TStringBuf> AllocateStringMaybe(TMaybe<TStringBuf> str) noexcept { + if (str.Defined()) { + return AllocateString(str.GetRef()); + } else { + return Nothing(); + } + } + + /// Reclaim a chunk of memory memory that was allocated via the `AllocateStringMaybe` function. + void FreeStringMaybe(TMaybe<TStringBuf> str) noexcept { + if (str.Defined()) { + FreeString(str.GetRef()); + } + } + + /// @} + + public: + /// @name Type caching and deduplication interface + /// + /// @{ + //- + /// Lookup the given type in the factory cache. Return cached version of the type or `nullptr`. + virtual const TType* LookupCache(const TType* type) noexcept = 0; + + /// Save the given type to the factory cache. Type must be allocated via this factory. + virtual void SaveCache(const TType* type) noexcept = 0; + + /// @} + + public: + /// @name Factory reference counting interface + /// + /// @{ + //- + /// Increase reference count of this factory. + virtual void Ref() noexcept = 0; + + /// Decrease reference count of this factory. + /// If reference count reaches zero, drop this factory. + virtual void UnRef() noexcept = 0; + + /// Decrease reference count of this factory. + /// If reference count reaches zero, panic. + virtual void DecRef() noexcept = 0; + + /// Get current reference count of this factory. + virtual long RefCount() const noexcept = 0; + + /// @} + + public: + /// @name Type instance reference counting interface + /// + /// @{ + //- + /// Increase reference count of some object managed by this factory. + virtual void RefType(TType*) noexcept = 0; + + /// Decrease reference count of some object managed by this factory. + /// If reference count reaches zero, call `TType::Drop` and free object's memory, if needed. + virtual void UnRefType(TType*) noexcept = 0; + + /// Decrease reference count of some object managed by this factory. + /// If reference count reaches zero, panic. + virtual void DecRefType(TType*) noexcept = 0; + + /// Get current reference count of some object managed by this factory. + virtual long RefCountType(const TType*) const noexcept = 0; + + /// @} + }; + + /// Base interface for type factory. There are functions to create type instances, and also a function to + /// adopt a type from one factory to another. See the file-level documentation for more info. + class ITypeFactory: protected ITypeFactoryInternal { + friend class TType; + template <typename T> + friend class ::TDefaultIntrusivePtrOps; + + public: + /// Adopt type into this factory. + /// + /// This function is used to transfer types between factories. It has the following semantics: + /// + /// - if `type` is managed by this factory, this function returns its argument unchanged; + /// - if `type` is not managed by any factory, this function also returns its argument unchanged; + /// - if `type` is managed by another factory, this function deep-copies `type` into this factory and returns + /// a pointer to the new deep-copied value. + /// + /// This function should be used on API borders, whenever you receive ownership over some type that wasn't + /// created by you. For example, you may ensure that the type you got is allocated in heap, or you may want to + /// copy some type into a memory pool that you have control over. + template <typename T> + inline TIntrusiveConstPtr<T> Adopt(TIntrusiveConstPtr<T> value) noexcept { + return ITypeFactoryInternal::Adopt<T>(std::move(value)); + } + + public: + /// Create a new instance of a type using this factory. + /// @{ + //- + /// Create a new `Void` type. See `NTi::TVoidType` for more info. + TVoidTypePtr Void(); + + /// Create a new `Null` type. See `NTi::TNullType` for more info. + TNullTypePtr Null(); + + /// Create a new `Bool` type. See `NTi::TBoolType` for more info. + TBoolTypePtr Bool(); + + /// Create a new `Int8` type. See `NTi::TInt8Type` for more info. + TInt8TypePtr Int8(); + + /// Create a new `Int16` type. See `NTi::TInt16Type` for more info. + TInt16TypePtr Int16(); + + /// Create a new `Int32` type. See `NTi::TInt32Type` for more info. + TInt32TypePtr Int32(); + + /// Create a new `Int64` type. See `NTi::TInt64Type` for more info. + TInt64TypePtr Int64(); + + /// Create a new `Uint8` type. See `NTi::TUint8Type` for more info. + TUint8TypePtr Uint8(); + + /// Create a new `Uint16` type. See `NTi::TUint16Type` for more info. + TUint16TypePtr Uint16(); + + /// Create a new `Uint32` type. See `NTi::TUint32Type` for more info. + TUint32TypePtr Uint32(); + + /// Create a new `Uint64` type. See `NTi::TUint64Type` for more info. + TUint64TypePtr Uint64(); + + /// Create a new `Float` type. See `NTi::TFloatType` for more info. + TFloatTypePtr Float(); + + /// Create a new `Double` type. See `NTi::TDoubleType` for more info. + TDoubleTypePtr Double(); + + /// Create a new `String` type. See `NTi::TStringType` for more info. + TStringTypePtr String(); + + /// Create a new `Utf8` type. See `NTi::TUtf8Type` for more info. + TUtf8TypePtr Utf8(); + + /// Create a new `Date` type. See `NTi::TDateType` for more info. + TDateTypePtr Date(); + + /// Create a new `Datetime` type. See `NTi::TDatetimeType` for more info. + TDatetimeTypePtr Datetime(); + + /// Create a new `Timestamp` type. See `NTi::TTimestampType` for more info. + TTimestampTypePtr Timestamp(); + + /// Create a new `TzDate` type. See `NTi::TTzDateType` for more info. + TTzDateTypePtr TzDate(); + + /// Create a new `TzDatetime` type. See `NTi::TTzDatetimeType` for more info. + TTzDatetimeTypePtr TzDatetime(); + + /// Create a new `TzTimestamp` type. See `NTi::TTzTimestampType` for more info. + TTzTimestampTypePtr TzTimestamp(); + + /// Create a new `Interval` type. See `NTi::TIntervalType` for more info. + TIntervalTypePtr Interval(); + + /// Create a new `Decimal` type. See `NTi::TDecimalType` for more info. + TDecimalTypePtr Decimal(ui8 precision, ui8 scale); + + /// Create a new `Json` type. See `NTi::TJsonType` for more info. + TJsonTypePtr Json(); + + /// Create a new `Yson` type. See `NTi::TYsonType` for more info. + TYsonTypePtr Yson(); + + /// Create a new `Uuid` type. See `NTi::TUuidType` for more info. + TUuidTypePtr Uuid(); + + /// Create a new `Optional` type. See `NTi::TOptionalType` for more info. + /// If `item` is managed by some other factory, it will be deep-copied into this factory. + TOptionalTypePtr Optional(TTypePtr item); + + /// Create a new `List` type. See `NTi::TListType` for more info. + /// If `item` is managed by some other factory, it will be deep-copied into this factory. + TListTypePtr List(TTypePtr item); + + /// Create a new `Dict` type. See `NTi::TDictType` for more info. + /// If `key` or `value` are managed by some other factory, they will be deep-copied into this factory. + TDictTypePtr Dict(TTypePtr key, TTypePtr value); + + /// Create a new `Struct` type. See `NTi::TStructType` for more info. + /// If item types are managed by some other factory, they will be deep-copied into this factory. + TStructTypePtr Struct(TStructType::TOwnedMembers items); + TStructTypePtr Struct(TMaybe<TStringBuf> name, TStructType::TOwnedMembers items); + + /// Create a new `Tuple` type. See `NTi::TTupleType` for more info. + /// If item types are managed by some other factory, they will be deep-copied into this factory. + TTupleTypePtr Tuple(TTupleType::TOwnedElements items); + TTupleTypePtr Tuple(TMaybe<TStringBuf> name, TTupleType::TOwnedElements items); + + /// Create a new `Variant` type. See `NTi::TVariantType` for more info. + /// If `inner` is managed by some other factory, it will be deep-copied into this factory. + TVariantTypePtr Variant(TTypePtr inner); + TVariantTypePtr Variant(TMaybe<TStringBuf> name, TTypePtr inner); + + /// Create a new `Tagged` type. See `NTi::TTaggedType` for more info. + /// If `type` is managed by some other factory, it will be deep-copied into this factory. + TTaggedTypePtr Tagged(TTypePtr type, TStringBuf tag); + + /// @} + }; + + /// Type factory over a memory pool. + /// + /// This interface provides some additional info related to the underlying memory pool state. + /// + /// Also, since all types that're created via this factory will live exactly as long as this factory lives, + /// one doesn't have to actually refcount them. Thus, it is safe to work with raw pointers instead of intrusive + /// pointers when using this kind of factory. So we provide the 'raw' interface that creates type instances + /// as usual, but doesn't increment any reference counters, thus saving some atomic operations. It still + /// adopts nested types into this factory, though. + class IPoolTypeFactory: public ITypeFactory { + friend class TNamedTypeBuilderRaw; + friend class TStructBuilderRaw; + friend class TTupleBuilderRaw; + friend class TTaggedBuilderRaw; + + public: + /// Memory available in the current chunk. + /// + /// Allocating object of size greater than this number will cause allocation of a new chunk. When this happens, + /// available memory in the current chunk is lost, i.e. it'll never be used again. + virtual size_t Available() const noexcept = 0; + + /// Number of bytes that're currently used by some object (i.e. they were allocated). + /// + /// Total memory consumed by arena is `MemoryAllocated() + MemoryWaste()`. + virtual size_t MemoryAllocated() const noexcept = 0; + + /// Number of bytes that're not used by anyone. + /// + /// Total memory consumed by arena is `MemoryAllocated() + MemoryWaste()`. + /// + /// Memory that's lost and will not be reused is `MemoryWaste() - Available()`. + virtual size_t MemoryWaste() const noexcept = 0; + + public: + /// Adopt type into this factory. + /// + /// This works like `ITypeFactory::Adopt`, but returns a raw pointer. + template <typename T> + inline const T* AdoptRaw(const T* value) noexcept { + return ITypeFactoryInternal::AdoptRaw<T>(value); + } + + public: + /// Raw versions of type constructors. See the class-level documentation for more info. + /// + /// @{ + //- + /// Create a new `Void` type. See `NTi::TVoidType` for more info. + /// The returned object will live for as long as this factory lives. + const TVoidType* VoidRaw(); + + /// Create a new `Null` type. See `NTi::TNullType` for more info. + /// The returned object will live for as long as this factory lives. + const TNullType* NullRaw(); + + /// Create a new `Bool` type. See `NTi::TBoolType` for more info. + /// The returned object will live for as long as this factory lives. + const TBoolType* BoolRaw(); + + /// Create a new `Int8` type. See `NTi::TInt8Type` for more info. + /// The returned object will live for as long as this factory lives. + const TInt8Type* Int8Raw(); + + /// Create a new `Int16` type. See `NTi::TInt16Type` for more info. + /// The returned object will live for as long as this factory lives. + const TInt16Type* Int16Raw(); + + /// Create a new `Int32` type. See `NTi::TInt32Type` for more info. + /// The returned object will live for as long as this factory lives. + const TInt32Type* Int32Raw(); + + /// Create a new `Int64` type. See `NTi::TInt64Type` for more info. + /// The returned object will live for as long as this factory lives. + const TInt64Type* Int64Raw(); + + /// Create a new `Uint8` type. See `NTi::TUint8Type` for more info. + /// The returned object will live for as long as this factory lives. + const TUint8Type* Uint8Raw(); + + /// Create a new `Uint16` type. See `NTi::TUint16Type` for more info. + /// The returned object will live for as long as this factory lives. + const TUint16Type* Uint16Raw(); + + /// Create a new `Uint32` type. See `NTi::TUint32Type` for more info. + /// The returned object will live for as long as this factory lives. + const TUint32Type* Uint32Raw(); + + /// Create a new `Uint64` type. See `NTi::TUint64Type` for more info. + /// The returned object will live for as long as this factory lives. + const TUint64Type* Uint64Raw(); + + /// Create a new `Float` type. See `NTi::TFloatType` for more info. + /// The returned object will live for as long as this factory lives. + const TFloatType* FloatRaw(); + + /// Create a new `Double` type. See `NTi::TDoubleType` for more info. + /// The returned object will live for as long as this factory lives. + const TDoubleType* DoubleRaw(); + + /// Create a new `String` type. See `NTi::TStringType` for more info. + /// The returned object will live for as long as this factory lives. + const TStringType* StringRaw(); + + /// Create a new `Utf8` type. See `NTi::TUtf8Type` for more info. + /// The returned object will live for as long as this factory lives. + const TUtf8Type* Utf8Raw(); + + /// Create a new `Date` type. See `NTi::TDateType` for more info. + /// The returned object will live for as long as this factory lives. + const TDateType* DateRaw(); + + /// Create a new `Datetime` type. See `NTi::TDatetimeType` for more info. + /// The returned object will live for as long as this factory lives. + const TDatetimeType* DatetimeRaw(); + + /// Create a new `Timestamp` type. See `NTi::TTimestampType` for more info. + /// The returned object will live for as long as this factory lives. + const TTimestampType* TimestampRaw(); + + /// Create a new `TzDate` type. See `NTi::TTzDateType` for more info. + /// The returned object will live for as long as this factory lives. + const TTzDateType* TzDateRaw(); + + /// Create a new `TzDatetime` type. See `NTi::TTzDatetimeType` for more info. + /// The returned object will live for as long as this factory lives. + const TTzDatetimeType* TzDatetimeRaw(); + + /// Create a new `TzTimestamp` type. See `NTi::TTzTimestampType` for more info. + /// The returned object will live for as long as this factory lives. + const TTzTimestampType* TzTimestampRaw(); + + /// Create a new `Interval` type. See `NTi::TIntervalType` for more info. + /// The returned object will live for as long as this factory lives. + const TIntervalType* IntervalRaw(); + + /// Create a new `Decimal` type. See `NTi::TDecimalType` for more info. + /// The returned object will live for as long as this factory lives. + const TDecimalType* DecimalRaw(ui8 precision, ui8 scale); + + /// Create a new `Json` type. See `NTi::TJsonType` for more info. + /// The returned object will live for as long as this factory lives. + const TJsonType* JsonRaw(); + + /// Create a new `Yson` type. See `NTi::TYsonType` for more info. + /// The returned object will live for as long as this factory lives. + const TYsonType* YsonRaw(); + + /// Create a new `Uuid` type. See `NTi::TUuidType` for more info. + /// The returned object will live for as long as this factory lives. + const TUuidType* UuidRaw(); + + /// Create a new `Optional` type. See `NTi::TOptionalType` for more info. + /// The returned object will live for as long as this factory lives. + const TOptionalType* OptionalRaw(const TType* item); + + /// Create a new `List` type. See `NTi::TListType` for more info. + /// The returned object will live for as long as this factory lives. + const TListType* ListRaw(const TType* item); + + /// Create a new `Dict` type. See `NTi::TDictType` for more info. + /// The returned object will live for as long as this factory lives. + const TDictType* DictRaw(const TType* key, const TType* value); + + /// Create a new `Struct` type. See `NTi::TStructType` for more info. + /// The returned object will live for as long as this factory lives. + const TStructType* StructRaw(TStructType::TMembers items); + const TStructType* StructRaw(TMaybe<TStringBuf> name, TStructType::TMembers items); + + /// Create a new `Tuple` type. See `NTi::TTupleType` for more info. + /// The returned object will live for as long as this factory lives. + const TTupleType* TupleRaw(TTupleType::TElements items); + const TTupleType* TupleRaw(TMaybe<TStringBuf> name, TTupleType::TElements items); + + /// Create a new `Variant` type. See `NTi::TVariantType` for more info. + /// The returned object will live for as long as this factory lives. + const TVariantType* VariantRaw(const TType* inner); + const TVariantType* VariantRaw(TMaybe<TStringBuf> name, const TType* inner); + + /// Create a new `Tagged` type. See `NTi::TTaggedType` for more info. + /// The returned object will live for as long as this factory lives. + const TTaggedType* TaggedRaw(const TType* type, TStringBuf tag); + + /// @} + }; + + namespace NPrivate { + ITypeFactory* GetDefaultHeapFactory(); + } + + /// Create a heap-based factory. + /// + /// This factory uses heap to allocate type instances, and reference counting to track individual type lifetimes. + /// + /// Choose this factory if you need safety and flexibility. + ITypeFactoryPtr HeapFactory(); + + /// Create a memory-pool-based factory. + /// + /// This factory uses a memory pool to allocate type instances. All allocated memory will be released when + /// the factory dies. + /// + /// Choose this factory if you need speed and memory efficiency. You have to understand how memory pool works. + /// We advise testing your code with msan when using this factory. + /// + /// If in doubt, use a heap-based factory instead. + /// + /// @param deduplicate + /// if enabled, factory will cache all types and avoid allocations when creating a new type that is + /// strictly-equal (see `type_equivalence.h`) to some previously created type. + /// + /// For example: + /// + /// ``` + /// auto a = factory.Optional(factory.String()); + /// auto b = factory.Optional(factory.String()); + /// Y_ASSERT(a == b); + /// ``` + /// + /// If `deduplicate` is disabled, this assert will fail. If, however, `deduplicate` is enabled, + /// the assert will not fail because `a` and `b` will point to the same type instance. + /// + /// This behaviour slows down creation process, but can save some memory when working with complex types. + /// + /// @param initial + /// number of bytes in the first page of the memory pool. + /// By default, we pre-allocate `64` bytes. + /// + /// @param grow + /// policy for calculating size for a next pool page. + /// By default, next page is twice as big as the previous one. + /// + /// @param alloc + /// underlying allocator that'll be used to allocate pool pages. + /// + /// @param options + /// other memory pool options. + IPoolTypeFactoryPtr PoolFactory( + bool deduplicate = true, + size_t initial = 64, + TMemoryPool::IGrowPolicy* grow = TMemoryPool::TExpGrow::Instance(), + IAllocator* alloc = TDefaultAllocator::Instance(), + TMemoryPool::TOptions options = {}); +} diff --git a/library/cpp/type_info/type_info.cpp b/library/cpp/type_info/type_info.cpp new file mode 100644 index 0000000000..8c14b158d0 --- /dev/null +++ b/library/cpp/type_info/type_info.cpp @@ -0,0 +1 @@ +#include "type_info.h" diff --git a/library/cpp/type_info/type_info.h b/library/cpp/type_info/type_info.h new file mode 100644 index 0000000000..5c6e249c11 --- /dev/null +++ b/library/cpp/type_info/type_info.h @@ -0,0 +1,10 @@ +#pragma once + +#include "builder.h" +#include "error.h" +#include "type.h" +#include "type_constructors.h" +#include "type_equivalence.h" +#include "type_factory.h" +#include "type_io.h" +#include "type_list.h" diff --git a/library/cpp/type_info/type_io.cpp b/library/cpp/type_info/type_io.cpp new file mode 100644 index 0000000000..90b98c8baa --- /dev/null +++ b/library/cpp/type_info/type_io.cpp @@ -0,0 +1,1186 @@ +#include "type_io.h" + +#include "builder.h" +#include "type_constructors.h" +#include "type_factory.h" + +#include <util/generic/overloaded.h> + +#include <library/cpp/yson_pull/read_ops.h> + +#include <util/stream/mem.h> +#include <util/string/cast.h> +#include <util/generic/vector.h> +#include <util/generic/scope.h> + +namespace NTi::NIo { + namespace { + class TYsonDeserializer: private TNonCopyable { + public: + TYsonDeserializer(IPoolTypeFactory* factory, NYsonPull::TReader* reader) + : Factory_(factory) + , Reader_(reader) + { + } + + public: + const TType* ReadType() { + if (++Depth_ > 100) { + ythrow TDeserializationException() << "types are nested too deep"; + } + + Y_DEFER { + Depth_--; + }; + + auto event = Reader_->NextEvent(); + + if (event.Type() == NYsonPull::EEventType::BeginStream) { + event = Reader_->NextEvent(); + } + + if (event.Type() == NYsonPull::EEventType::EndStream) { + if (Depth_ == 1) { + return nullptr; + } else { + ythrow TDeserializationException() << "unexpected end of stream"; + } + } + + if (event.Type() == NYsonPull::EEventType::Scalar && event.AsScalar().Type() == NYsonPull::EScalarType::String) { + return ReadTypeFromData(TypeNameStringToEnum(event.AsString()), std::monostate{}); + } else if (event.Type() == NYsonPull::EEventType::BeginMap) { + return ReadTypeFromMap(); + } else { + ythrow TDeserializationException() << "type must be either a string or a map"; + } + } + + private: + struct TDictData { + const TType *Key, *Value; + }; + struct TDecimalData { + ui8 Precision, Scale; + }; + using TTypeData = std::variant< + std::monostate, + TDictData, + TDecimalData, + TStructBuilderRaw, + TTupleBuilderRaw, + TTaggedBuilderRaw>; + + const TType* ReadTypeFromMap() { + TMaybe<ETypeName> typeName; + TTypeData data; + + while (true) { + auto event = Reader_->NextEvent(); + if (event.Type() == NYsonPull::EEventType::Key) { + auto mapKey = event.AsString(); + + if (mapKey == "type_name") { + if (typeName.Defined()) { + ythrow TDeserializationException() << R"(duplicate key R"(type_name"))"; + } else { + typeName = TypeNameStringToEnum(ReadString(R"("type_name")")); + } + } else if (mapKey == "item") { + if (std::holds_alternative<std::monostate>(data)) { + data = TTaggedBuilderRaw(*Factory_); + } + if (std::holds_alternative<TTaggedBuilderRaw>(data)) { + auto& builder = std::get<TTaggedBuilderRaw>(data); + if (!builder.HasItem()) { + builder.SetItem(ReadType()); + } else { + ythrow TDeserializationException() << R"(duplicate key "item")"; + } + } else { + ythrow TDeserializationException() << R"(unexpected key "item")"; + } + } else if (mapKey == "key") { + if (std::holds_alternative<std::monostate>(data)) { + data = TDictData{nullptr, nullptr}; + } + if (std::holds_alternative<TDictData>(data)) { + auto& dictData = std::get<TDictData>(data); + if (dictData.Key == nullptr) { + dictData.Key = ReadType(); + } else { + ythrow TDeserializationException() << R"(duplicate key "key")"; + } + } else { + ythrow TDeserializationException() << R"(unexpected key "key")"; + } + } else if (mapKey == "value") { + if (std::holds_alternative<std::monostate>(data)) { + data = TDictData{nullptr, nullptr}; + } + if (std::holds_alternative<TDictData>(data)) { + auto& dictData = std::get<TDictData>(data); + if (dictData.Value == nullptr) { + dictData.Value = ReadType(); + } else { + ythrow TDeserializationException() << R"(duplicate key "value")"; + } + } else { + ythrow TDeserializationException() << R"(unexpected key "value")"; + } + } else if (mapKey == "members") { + if (std::holds_alternative<std::monostate>(data)) { + data = TStructBuilderRaw(*Factory_); + } + if (std::holds_alternative<TStructBuilderRaw>(data)) { + ReadMembers(std::get<TStructBuilderRaw>(data)); + } else { + ythrow TDeserializationException() << R"(unexpected key "members")"; + } + } else if (mapKey == "elements") { + if (std::holds_alternative<std::monostate>(data)) { + data = TTupleBuilderRaw(*Factory_); + } + if (std::holds_alternative<TTupleBuilderRaw>(data)) { + ReadElements(std::get<TTupleBuilderRaw>(data)); + } else { + ythrow TDeserializationException() << R"(unexpected key "elements")"; + } + } else if (mapKey == "tag") { + if (std::holds_alternative<std::monostate>(data)) { + data = TTaggedBuilderRaw(*Factory_); + } + if (std::holds_alternative<TTaggedBuilderRaw>(data)) { + auto& builder = std::get<TTaggedBuilderRaw>(data); + if (!builder.HasTag()) { + builder.SetTag(ReadString(R"("tag")")); + } else { + ythrow TDeserializationException() << R"(duplicate key "tag")"; + } + } else { + ythrow TDeserializationException() << R"(unexpected key "tag")"; + } + } else if (mapKey == "precision") { + if (std::holds_alternative<std::monostate>(data)) { + data = TDecimalData{ReadSmallInt(R"("precision")"), 0}; + } else if (std::holds_alternative<TDecimalData>(data)) { + auto& decimalData = std::get<TDecimalData>(data); + if (decimalData.Precision == 0) { + decimalData.Precision = ReadSmallInt(R"("precision")"); + } else { + ythrow TDeserializationException() << R"(duplicate key "precision")"; + } + } else { + ythrow TDeserializationException() << R"(unexpected key "precision")"; + } + } else if (mapKey == "scale") { + if (std::holds_alternative<std::monostate>(data)) { + data = TDecimalData{0, ReadSmallInt(R"("scale")")}; + } else if (std::holds_alternative<TDecimalData>(data)) { + auto& decimalData = std::get<TDecimalData>(data); + if (decimalData.Scale == 0) { + decimalData.Scale = ReadSmallInt(R"("scale")"); + } else { + ythrow TDeserializationException() << R"(duplicate key "scale")"; + } + } else { + ythrow TDeserializationException() << R"(unexpected key "scale")"; + } + } else { + NYsonPull::NReadOps::SkipValue(*Reader_); + } + } else if (event.Type() == NYsonPull::EEventType::EndMap) { + if (!typeName.Defined()) { + ythrow TDeserializationException() << R"(missing required key "type_name")"; + } + + return ReadTypeFromData(*typeName, std::move(data)); + } else { + ythrow TDeserializationException() << "unexpected event " << event.Type(); + } + } + } + + const TType* ReadTypeFromData(ETypeName typeName, TTypeData data) { + const TType* type; + + switch (typeName) { + case ETypeName::Bool: + type = TBoolType::InstanceRaw(); + break; + case ETypeName::Int8: + type = TInt8Type::InstanceRaw(); + break; + case ETypeName::Int16: + type = TInt16Type::InstanceRaw(); + break; + case ETypeName::Int32: + type = TInt32Type::InstanceRaw(); + break; + case ETypeName::Int64: + type = TInt64Type::InstanceRaw(); + break; + case ETypeName::Uint8: + type = TUint8Type::InstanceRaw(); + break; + case ETypeName::Uint16: + type = TUint16Type::InstanceRaw(); + break; + case ETypeName::Uint32: + type = TUint32Type::InstanceRaw(); + break; + case ETypeName::Uint64: + type = TUint64Type::InstanceRaw(); + break; + case ETypeName::Float: + type = TFloatType::InstanceRaw(); + break; + case ETypeName::Double: + type = TDoubleType::InstanceRaw(); + break; + case ETypeName::String: + type = TStringType::InstanceRaw(); + break; + case ETypeName::Utf8: + type = TUtf8Type::InstanceRaw(); + break; + case ETypeName::Date: + type = TDateType::InstanceRaw(); + break; + case ETypeName::Datetime: + type = TDatetimeType::InstanceRaw(); + break; + case ETypeName::Timestamp: + type = TTimestampType::InstanceRaw(); + break; + case ETypeName::TzDate: + type = TTzDateType::InstanceRaw(); + break; + case ETypeName::TzDatetime: + type = TTzDatetimeType::InstanceRaw(); + break; + case ETypeName::TzTimestamp: + type = TTzTimestampType::InstanceRaw(); + break; + case ETypeName::Interval: + type = TIntervalType::InstanceRaw(); + break; + case ETypeName::Decimal: { + if (!std::holds_alternative<TDecimalData>(data)) { + ythrow TDeserializationException() << R"(missing required keys "precision" and "scale" for type Decimal)"; + } + + auto& decimalData = std::get<TDecimalData>(data); + + if (decimalData.Precision == 0) { + ythrow TDeserializationException() << R"(missing required key "precision" for type Decimal)"; + } + + if (decimalData.Scale == 0) { + ythrow TDeserializationException() << R"(missing required key "scale" for type Decimal)"; + } + + return Factory_->DecimalRaw(decimalData.Precision, decimalData.Scale); + } + case ETypeName::Json: + type = TJsonType::InstanceRaw(); + break; + case ETypeName::Yson: + type = TYsonType::InstanceRaw(); + break; + case ETypeName::Uuid: + type = TUuidType::InstanceRaw(); + break; + case ETypeName::Void: + type = TVoidType::InstanceRaw(); + break; + case ETypeName::Null: + type = TNullType::InstanceRaw(); + break; + case ETypeName::Optional: { + if (!std::holds_alternative<TTaggedBuilderRaw>(data)) { + ythrow TDeserializationException() << R"(missing required key "item" for type Optional)"; + } + + auto& builder = std::get<TTaggedBuilderRaw>(data); + + if (!builder.HasItem()) { + ythrow TDeserializationException() << R"(missing required key "item" for type Optional)"; + } + + return Factory_->OptionalRaw(*builder.GetItem()); + } + case ETypeName::List: { + if (!std::holds_alternative<TTaggedBuilderRaw>(data)) { + ythrow TDeserializationException() << R"(missing required key "item" for type List)"; + } + + auto& builder = std::get<TTaggedBuilderRaw>(data); + + if (!builder.HasItem()) { + ythrow TDeserializationException() << R"(missing required key "item" for type List)"; + } + + return Factory_->ListRaw(*builder.GetItem()); + } + case ETypeName::Dict: { + if (!std::holds_alternative<TDictData>(data)) { + ythrow TDeserializationException() << R"(missing required keys "key" and "value" for type Dict)"; + } + + auto& dictData = std::get<TDictData>(data); + + if (dictData.Key == nullptr) { + ythrow TDeserializationException() << R"(missing required key "key" for type Dict)"; + } + + if (dictData.Value == nullptr) { + ythrow TDeserializationException() << R"(missing required key "value" for type Dict)"; + } + + return Factory_->DictRaw(dictData.Key, dictData.Value); + } + case ETypeName::Struct: { + if (!std::holds_alternative<TStructBuilderRaw>(data)) { + ythrow TDeserializationException() << R"(missing required key "members" for type Struct)"; + } + + return std::get<TStructBuilderRaw>(data).BuildRaw(); + } + case ETypeName::Tuple: { + if (!std::holds_alternative<TTupleBuilderRaw>(data)) { + ythrow TDeserializationException() << R"(missing required key "elements" for type Tuple)"; + } + + return std::get<TTupleBuilderRaw>(data).BuildRaw(); + } + case ETypeName::Variant: { + if (std::holds_alternative<TStructBuilderRaw>(data)) { + return std::get<TStructBuilderRaw>(data).BuildVariantRaw(); + } else if (std::holds_alternative<TTupleBuilderRaw>(data)) { + return std::get<TTupleBuilderRaw>(data).BuildVariantRaw(); + } else { + ythrow TDeserializationException() << R"(missing both keys "members" and "elements" for type Variant)"; + } + } + case ETypeName::Tagged: { + if (!std::holds_alternative<TTaggedBuilderRaw>(data)) { + ythrow TDeserializationException() << R"(missing required keys "tag" and "item" for type Tagged)"; + } + + auto& builder = std::get<TTaggedBuilderRaw>(data); + + if (!builder.HasItem()) { + ythrow TDeserializationException() << R"(missing required key "item" for type Tagged)"; + } + + if (!builder.HasTag()) { + ythrow TDeserializationException() << R"(missing required key "tag" for type Tagged)"; + } + + return builder.BuildRaw(); + } + } + + if (!std::holds_alternative<std::monostate>(data)) { + ythrow TDeserializationException() << "unexpected key for type " << typeName; + } + + return type; + } + + TStringBuf ReadString(TStringBuf what) { + auto event = Reader_->NextEvent(); + + if (event.Type() != NYsonPull::EEventType::Scalar || event.AsScalar().Type() != NYsonPull::EScalarType::String) { + ythrow TDeserializationException() << what << " must contain a string"; + } + + return event.AsString(); + } + + ui8 ReadSmallInt(TStringBuf what) { + auto event = Reader_->NextEvent(); + + if (event.Type() != NYsonPull::EEventType::Scalar || event.AsScalar().Type() != NYsonPull::EScalarType::Int64) { + ythrow TDeserializationException() << what << " must contain a signed integer"; + } + + auto result = event.AsScalar().AsInt64(); + + if (result <= 0) { + ythrow TDeserializationException() << what << " must be greater than zero"; + } + + if (result > Max<ui8>()) { + ythrow TDeserializationException() << what << " is too big"; + } + + return static_cast<ui8>(result); + } + + void ReadMembers(TStructBuilderRaw& builder) { + if (Reader_->NextEvent().Type() != NYsonPull::EEventType::BeginList) { + ythrow TDeserializationException() << R"("members" must contain a list)"; + } + + while (true) { + auto event = Reader_->NextEvent(); + + if (event.Type() == NYsonPull::EEventType::BeginMap) { + while (true) { + auto event = Reader_->NextEvent(); + + if (event.Type() == NYsonPull::EEventType::Key) { + auto mapKey = event.AsString(); + if (mapKey == "name") { + if (builder.HasMemberName()) { + ythrow TDeserializationException() << R"(duplicate key "name")"; + } + + builder.AddMemberName(ReadString(R"("name")")); + } else if (mapKey == "type") { + if (builder.HasMemberType()) { + ythrow TDeserializationException() << R"(duplicate key "type")"; + } + + builder.AddMemberType(ReadType()); + } else { + NYsonPull::NReadOps::SkipValue(*Reader_); + } + } else if (event.Type() == NYsonPull::EEventType::EndMap) { + if (!builder.HasMemberName()) { + ythrow TDeserializationException() << R"(missing required key "name")"; + } + if (!builder.HasMemberType()) { + ythrow TDeserializationException() << R"(missing required key "type")"; + } + + builder.AddMember(); + break; + } else { + ythrow TDeserializationException() << "unexpected event " << event.Type(); + } + } + } else if (event.Type() == NYsonPull::EEventType::EndList) { + break; + } else { + ythrow TDeserializationException() << R"("members" must contain a list of maps)"; + } + } + } + + void ReadElements(TTupleBuilderRaw& builder) { + if (Reader_->NextEvent().Type() != NYsonPull::EEventType::BeginList) { + ythrow TDeserializationException() << R"("elements" must contain a list)"; + } + + while (true) { + auto event = Reader_->NextEvent(); + + if (event.Type() == NYsonPull::EEventType::BeginMap) { + while (true) { + auto event = Reader_->NextEvent(); + + if (event.Type() == NYsonPull::EEventType::Key) { + auto mapKey = event.AsString(); + if (mapKey == "type") { + if (builder.HasElementType()) { + ythrow TDeserializationException() << R"(duplicate key "type")"; + } + + builder.AddElementType(ReadType()); + } else { + NYsonPull::NReadOps::SkipValue(*Reader_); + } + } else if (event.Type() == NYsonPull::EEventType::EndMap) { + if (!builder.HasElementType()) { + ythrow TDeserializationException() << R"(missing required key "type")"; + } + + builder.AddElement(); + break; + } else { + ythrow TDeserializationException() << "unexpected event " << event.Type(); + } + } + } else if (event.Type() == NYsonPull::EEventType::EndList) { + break; + } else { + ythrow TDeserializationException() << R"("elements" must contain a list of maps)"; + } + } + } + + static ETypeName TypeNameStringToEnum(TStringBuf name) { + static const THashMap<TStringBuf, ETypeName> dispatch = { + {"void", ETypeName::Void}, + {"null", ETypeName::Null}, + {"bool", ETypeName::Bool}, + {"int8", ETypeName::Int8}, + {"int16", ETypeName::Int16}, + {"int32", ETypeName::Int32}, + {"int64", ETypeName::Int64}, + {"uint8", ETypeName::Uint8}, + {"uint16", ETypeName::Uint16}, + {"uint32", ETypeName::Uint32}, + {"uint64", ETypeName::Uint64}, + {"float", ETypeName::Float}, + {"double", ETypeName::Double}, + {"string", ETypeName::String}, + {"utf8", ETypeName::Utf8}, + {"date", ETypeName::Date}, + {"datetime", ETypeName::Datetime}, + {"timestamp", ETypeName::Timestamp}, + {"tz_date", ETypeName::TzDate}, + {"tz_datetime", ETypeName::TzDatetime}, + {"tz_timestamp", ETypeName::TzTimestamp}, + {"interval", ETypeName::Interval}, + {"json", ETypeName::Json}, + {"yson", ETypeName::Yson}, + {"uuid", ETypeName::Uuid}, + {"decimal", ETypeName::Decimal}, + {"optional", ETypeName::Optional}, + {"list", ETypeName::List}, + {"dict", ETypeName::Dict}, + {"struct", ETypeName::Struct}, + {"tuple", ETypeName::Tuple}, + {"variant", ETypeName::Variant}, + {"tagged", ETypeName::Tagged}, + }; + + if (auto it = dispatch.find(name); it != dispatch.end()) { + return it->second; + } else { + ythrow TDeserializationException() << "unknown type " << TString{name}.Quote(); + } + } + + private: + IPoolTypeFactory* Factory_; + NYsonPull::TReader* Reader_; + size_t Depth_ = 0; + }; + } + + TTypePtr DeserializeYson(ITypeFactory& factory, NYsonPull::TReader& reader, bool deduplicate) { + auto pool = PoolFactory(deduplicate, 2048); + auto type = DeserializeYsonRaw(*pool, reader); + return factory.Adopt(type->AsPtr()); + } + + TTypePtr DeserializeYson(ITypeFactory& factory, TStringBuf data, bool deduplicate) { + auto reader = NYsonPull::TReader(NYsonPull::NInput::FromMemory(data), NYsonPull::EStreamType::Node); + return DeserializeYson(factory, reader, deduplicate); + } + + TTypePtr DeserializeYson(ITypeFactory& factory, IInputStream& input, bool deduplicate) { + auto reader = NYsonPull::TReader(NYsonPull::NInput::FromInputStream(&input), NYsonPull::EStreamType::Node); + return DeserializeYson(factory, reader, deduplicate); + } + + const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, NYsonPull::TReader& reader) { + if (reader.LastEvent().Type() != NYsonPull::EEventType::BeginStream) { + ythrow TDeserializationException() << "stream contains extraneous data"; + } + + auto type = DeserializeYsonMultipleRaw(factory, reader); + + if (type == nullptr) { + ythrow TDeserializationException() << "unexpected end of stream"; + } + + if (reader.NextEvent().Type() != NYsonPull::EEventType::EndStream) { + ythrow TDeserializationException() << "stream contains extraneous data"; + } + + return type; + } + + const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, TStringBuf data) { + auto reader = NYsonPull::TReader(NYsonPull::NInput::FromMemory(data), NYsonPull::EStreamType::Node); + return DeserializeYsonRaw(factory, reader); + } + + const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, IInputStream& input) { + auto reader = NYsonPull::TReader(NYsonPull::NInput::FromInputStream(&input), NYsonPull::EStreamType::Node); + return DeserializeYsonRaw(factory, reader); + } + + const TType* DeserializeYsonMultipleRaw(IPoolTypeFactory& factory, NYsonPull::TReader& reader) { + return TYsonDeserializer(&factory, &reader).ReadType(); + } + + TString SerializeYson(const TType* type, bool humanReadable, bool includeTags) { + auto result = TString(); + auto writer = humanReadable + ? NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node) + : NYsonPull::MakeBinaryWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node); + SerializeYson(type, writer.GetConsumer(), includeTags); + return result; + } + + void SerializeYson(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags) { + consumer.OnBeginStream(); + SerializeYsonMultiple(type, consumer, includeTags); + consumer.OnEndStream(); + } + + void SerializeYson(const TType* type, IOutputStream& stream, bool humanReadable, bool includeTags) { + auto writer = humanReadable + ? NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromOutputStream(&stream), NYsonPull::EStreamType::Node) + : NYsonPull::MakeBinaryWriter(NYsonPull::NOutput::FromOutputStream(&stream), NYsonPull::EStreamType::Node); + SerializeYson(type, writer.GetConsumer(), includeTags); + } + + void SerializeYsonMultiple(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags) { + type->VisitRaw(TOverloaded{ + [&consumer](const TVoidType*) { + consumer.OnScalarString("void"); + }, + [&consumer](const TNullType*) { + consumer.OnScalarString("null"); + }, + [&consumer](const TBoolType*) { + consumer.OnScalarString("bool"); + }, + [&consumer](const TInt8Type*) { + consumer.OnScalarString("int8"); + }, + [&consumer](const TInt16Type*) { + consumer.OnScalarString("int16"); + }, + [&consumer](const TInt32Type*) { + consumer.OnScalarString("int32"); + }, + [&consumer](const TInt64Type*) { + consumer.OnScalarString("int64"); + }, + [&consumer](const TUint8Type*) { + consumer.OnScalarString("uint8"); + }, + [&consumer](const TUint16Type*) { + consumer.OnScalarString("uint16"); + }, + [&consumer](const TUint32Type*) { + consumer.OnScalarString("uint32"); + }, + [&consumer](const TUint64Type*) { + consumer.OnScalarString("uint64"); + }, + [&consumer](const TFloatType*) { + consumer.OnScalarString("float"); + }, + [&consumer](const TDoubleType*) { + consumer.OnScalarString("double"); + }, + [&consumer](const TStringType*) { + consumer.OnScalarString("string"); + }, + [&consumer](const TUtf8Type*) { + consumer.OnScalarString("utf8"); + }, + [&consumer](const TDateType*) { + consumer.OnScalarString("date"); + }, + [&consumer](const TDatetimeType*) { + consumer.OnScalarString("datetime"); + }, + [&consumer](const TTimestampType*) { + consumer.OnScalarString("timestamp"); + }, + [&consumer](const TTzDateType*) { + consumer.OnScalarString("tz_date"); + }, + [&consumer](const TTzDatetimeType*) { + consumer.OnScalarString("tz_datetime"); + }, + [&consumer](const TTzTimestampType*) { + consumer.OnScalarString("tz_timestamp"); + }, + [&consumer](const TIntervalType*) { + consumer.OnScalarString("interval"); + }, + [&consumer](const TJsonType*) { + consumer.OnScalarString("json"); + }, + [&consumer](const TYsonType*) { + consumer.OnScalarString("yson"); + }, + [&consumer](const TUuidType*) { + consumer.OnScalarString("uuid"); + }, + [&consumer](const TDecimalType* t) { + consumer.OnBeginMap(); + + consumer.OnKey("type_name"); + consumer.OnScalarString("decimal"); + + consumer.OnKey("precision"); + consumer.OnScalarInt64(t->GetPrecision()); + + consumer.OnKey("scale"); + consumer.OnScalarInt64(t->GetScale()); + + consumer.OnEndMap(); + }, + [&consumer, includeTags](const TOptionalType* t) { + consumer.OnBeginMap(); + + consumer.OnKey("type_name"); + consumer.OnScalarString("optional"); + + consumer.OnKey("item"); + SerializeYsonMultiple(t->GetItemTypeRaw(), consumer, includeTags); + + consumer.OnEndMap(); + }, + [&consumer, includeTags](const TListType* t) { + consumer.OnBeginMap(); + + consumer.OnKey("type_name"); + consumer.OnScalarString("list"); + + consumer.OnKey("item"); + SerializeYsonMultiple(t->GetItemTypeRaw(), consumer, includeTags); + + consumer.OnEndMap(); + }, + [&consumer, includeTags](const TDictType* t) { + consumer.OnBeginMap(); + + consumer.OnKey("type_name"); + consumer.OnScalarString("dict"); + + consumer.OnKey("key"); + SerializeYsonMultiple(t->GetKeyTypeRaw(), consumer, includeTags); + + consumer.OnKey("value"); + SerializeYsonMultiple(t->GetValueTypeRaw(), consumer, includeTags); + + consumer.OnEndMap(); + }, + [&consumer, includeTags](const TStructType* t) { + consumer.OnBeginMap(); + + consumer.OnKey("type_name"); + consumer.OnScalarString("struct"); + + consumer.OnKey("members"); + consumer.OnBeginList(); + for (auto& item : t->GetMembers()) { + consumer.OnBeginMap(); + + consumer.OnKey("name"); + consumer.OnScalarString(item.GetName()); + + consumer.OnKey("type"); + SerializeYsonMultiple(item.GetTypeRaw(), consumer, includeTags); + + consumer.OnEndMap(); + } + consumer.OnEndList(); + + consumer.OnEndMap(); + }, + [&consumer, includeTags](const TTupleType* t) { + consumer.OnBeginMap(); + + consumer.OnKey("type_name"); + consumer.OnScalarString("tuple"); + + consumer.OnKey("elements"); + consumer.OnBeginList(); + for (auto& item : t->GetElements()) { + consumer.OnBeginMap(); + + consumer.OnKey("type"); + SerializeYsonMultiple(item.GetTypeRaw(), consumer, includeTags); + + consumer.OnEndMap(); + } + consumer.OnEndList(); + + consumer.OnEndMap(); + }, + [&consumer, includeTags](const TVariantType* t) { + consumer.OnBeginMap(); + + consumer.OnKey("type_name"); + consumer.OnScalarString("variant"); + + t->VisitUnderlyingRaw( + TOverloaded{ + [&consumer, includeTags](const TStructType* t) { + // Warning: we loose struct's name here. + // See https://ml.yandex-team.ru/thread/data-com-dev/171136785840079161/ + + consumer.OnKey("members"); + consumer.OnBeginList(); + for (auto& item : t->GetMembers()) { + consumer.OnBeginMap(); + + consumer.OnKey("name"); + consumer.OnScalarString(item.GetName()); + + consumer.OnKey("type"); + SerializeYsonMultiple(item.GetTypeRaw(), consumer, includeTags); + + consumer.OnEndMap(); + } + consumer.OnEndList(); + }, + [&consumer, includeTags](const TTupleType* t) { + // Warning: we loose tuple's name here. + // See https://ml.yandex-team.ru/thread/data-com-dev/171136785840079161/ + + consumer.OnKey("elements"); + consumer.OnBeginList(); + for (auto& item : t->GetElements()) { + consumer.OnBeginMap(); + + consumer.OnKey("type"); + SerializeYsonMultiple(item.GetTypeRaw(), consumer, includeTags); + + consumer.OnEndMap(); + } + consumer.OnEndList(); + }}); + + consumer.OnEndMap(); + }, + [&consumer, includeTags](const TTaggedType* t) { + if (includeTags) { + consumer.OnBeginMap(); + + consumer.OnKey("type_name"); + consumer.OnScalarString("tagged"); + + consumer.OnKey("tag"); + consumer.OnScalarString(t->GetTag()); + + consumer.OnKey("item"); + SerializeYsonMultiple(t->GetItemTypeRaw(), consumer, includeTags); + + consumer.OnEndMap(); + } else { + SerializeYsonMultiple(t->GetItemTypeRaw(), consumer, includeTags); + } + }, + }); + } + + namespace { + void WriteVoidType(NYsonPull::IConsumer& consumer) { + consumer.OnBeginList(); + consumer.OnScalarString("VoidType"); + consumer.OnEndList(); + } + + void WriteNullType(NYsonPull::IConsumer& consumer) { + consumer.OnBeginList(); + consumer.OnScalarString("NullType"); + consumer.OnEndList(); + } + + void WriteDataType(NYsonPull::IConsumer& consumer, EPrimitiveTypeName name) { + consumer.OnBeginList(); + consumer.OnScalarString("DataType"); + consumer.OnScalarString(ToString(name)); + consumer.OnEndList(); + } + } + + void AsYqlType(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags) { + type->VisitRaw(TOverloaded{ + [&consumer](const TVoidType*) { + WriteVoidType(consumer); + }, + [&consumer](const TNullType*) { + WriteNullType(consumer); + }, + [&consumer](const TBoolType*) { + WriteDataType(consumer, EPrimitiveTypeName::Bool); + }, + [&consumer](const TInt8Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Int8); + }, + [&consumer](const TInt16Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Int16); + }, + [&consumer](const TInt32Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Int32); + }, + [&consumer](const TInt64Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Int64); + }, + [&consumer](const TUint8Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Uint8); + }, + [&consumer](const TUint16Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Uint16); + }, + [&consumer](const TUint32Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Uint32); + }, + [&consumer](const TUint64Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Uint64); + }, + [&consumer](const TFloatType*) { + WriteDataType(consumer, EPrimitiveTypeName::Float); + }, + [&consumer](const TDoubleType*) { + WriteDataType(consumer, EPrimitiveTypeName::Double); + }, + [&consumer](const TStringType*) { + WriteDataType(consumer, EPrimitiveTypeName::String); + }, + [&consumer](const TUtf8Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Utf8); + }, + [&consumer](const TDateType*) { + WriteDataType(consumer, EPrimitiveTypeName::Date); + }, + [&consumer](const TDatetimeType*) { + WriteDataType(consumer, EPrimitiveTypeName::Datetime); + }, + [&consumer](const TTimestampType*) { + WriteDataType(consumer, EPrimitiveTypeName::Timestamp); + }, + [&consumer](const TTzDateType*) { + WriteDataType(consumer, EPrimitiveTypeName::TzDate); + }, + [&consumer](const TTzDatetimeType*) { + WriteDataType(consumer, EPrimitiveTypeName::TzDatetime); + }, + [&consumer](const TTzTimestampType*) { + WriteDataType(consumer, EPrimitiveTypeName::TzTimestamp); + }, + [&consumer](const TIntervalType*) { + WriteDataType(consumer, EPrimitiveTypeName::Interval); + }, + [&consumer](const TJsonType*) { + WriteDataType(consumer, EPrimitiveTypeName::Json); + }, + [&consumer](const TYsonType*) { + WriteDataType(consumer, EPrimitiveTypeName::Yson); + }, + [&consumer](const TUuidType*) { + WriteDataType(consumer, EPrimitiveTypeName::Uuid); + }, + [&consumer](const TDecimalType* t) { + consumer.OnBeginList(); + consumer.OnScalarString("DataType"); + consumer.OnScalarString("Decimal"); + consumer.OnScalarString(ToString(t->GetPrecision())); + consumer.OnScalarString(ToString(t->GetScale())); + consumer.OnEndList(); + }, + [&consumer, includeTags](const TOptionalType* t) { + consumer.OnBeginList(); + consumer.OnScalarString("OptionalType"); + AsYqlType(t->GetItemTypeRaw(), consumer, includeTags); + consumer.OnEndList(); + }, + [&consumer, includeTags](const TListType* t) { + consumer.OnBeginList(); + consumer.OnScalarString("ListType"); + AsYqlType(t->GetItemTypeRaw(), consumer, includeTags); + consumer.OnEndList(); + }, + [&consumer, includeTags](const TDictType* t) { + consumer.OnBeginList(); + consumer.OnScalarString("DictType"); + AsYqlType(t->GetKeyTypeRaw(), consumer, includeTags); + AsYqlType(t->GetValueTypeRaw(), consumer, includeTags); + consumer.OnEndList(); + }, + [&consumer, includeTags](const TStructType* t) { + consumer.OnBeginList(); + consumer.OnScalarString("StructType"); + { + consumer.OnBeginList(); + for (auto& item : t->GetMembers()) { + consumer.OnBeginList(); + consumer.OnScalarString(item.GetName()); + AsYqlType(item.GetTypeRaw(), consumer, includeTags); + consumer.OnEndList(); + } + consumer.OnEndList(); + } + consumer.OnEndList(); + }, + [&consumer, includeTags](const TTupleType* t) { + consumer.OnBeginList(); + consumer.OnScalarString("TupleType"); + { + consumer.OnBeginList(); + for (auto& item : t->GetElements()) { + AsYqlType(item.GetTypeRaw(), consumer, includeTags); + } + consumer.OnEndList(); + } + + consumer.OnEndList(); + }, + [&consumer, includeTags](const TVariantType* t) { + consumer.OnBeginList(); + consumer.OnScalarString("VariantType"); + AsYqlType(t->GetUnderlyingTypeRaw(), consumer, includeTags); + consumer.OnEndList(); + }, + [&consumer, includeTags](const TTaggedType* t) { + if (includeTags) { + consumer.OnBeginList(); + consumer.OnScalarString("TaggedType"); + consumer.OnScalarString(t->GetTag()); + AsYqlType(t->GetItemTypeRaw(), consumer, includeTags); + consumer.OnEndList(); + } else { + AsYqlType(t->GetItemTypeRaw(), consumer, includeTags); + } + }, + }); + } + + TString AsYqlType(const NTi::TType* type, bool includeTags) { + auto result = TString(); + auto writer = NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node); + writer.BeginStream(); + AsYqlType(type, writer.GetConsumer(), includeTags); + writer.EndStream(); + return result; + } + + void AsYqlRowSpec(const TType* maybeTagged, NYsonPull::IConsumer& consumer, bool includeTags) { + auto* type = maybeTagged->StripTagsRaw(); + + if (!type->IsStruct()) { + ythrow TApiException() << "AsYqlRowSpec expected a struct type but got " << type->GetTypeName(); + } + + consumer.OnBeginMap(); + consumer.OnKey("StrictSchema"); + consumer.OnScalarBoolean(true); + consumer.OnKey("Type"); + AsYqlType(type, consumer, includeTags); + consumer.OnEndMap(); + } + + TString AsYqlRowSpec(const NTi::TType* type, bool includeTags) { + auto result = TString(); + auto writer = NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node); + writer.BeginStream(); + AsYqlRowSpec(type, writer.GetConsumer(), includeTags); + writer.EndStream(); + return result; + } + + void AsYtSchema(const TType* maybeTagged, NYsonPull::IConsumer& consumer, bool failOnEmptyStruct) { + auto* type = maybeTagged->StripTagsRaw(); + + if (!type->IsStruct()) { + ythrow TApiException() << "AsYtSchema expected a struct type but got " << type->GetTypeName(); + } + + auto* structType = type->AsStructRaw(); + + if (structType->GetMembers().empty()) { + if (failOnEmptyStruct) { + ythrow TApiException() << "AsYtSchema expected a non-empty struct"; + } + + AsYtSchema(Struct({{"_yql_fake_column", Optional(Bool())}}).Get(), consumer); + return; + } + + consumer.OnBeginAttributes(); + + consumer.OnKey("strict"); + consumer.OnScalarBoolean(true); + + consumer.OnKey("unique_keys"); + consumer.OnScalarBoolean(false); + + consumer.OnEndAttributes(); + + consumer.OnBeginList(); + for (auto& item : structType->GetMembers()) { + auto* itemType = item.GetTypeRaw()->StripTagsRaw(); + + bool required = true; + + if (itemType->IsOptional()) { + // toplevel optionals make non-required columns + itemType = itemType->AsOptionalRaw()->GetItemTypeRaw(); + required = false; + } + + TStringBuf typeString = itemType->VisitRaw(TOverloaded{ + [](const TVoidType*) -> TStringBuf { return "any"; }, + [](const TNullType*) -> TStringBuf { return "any"; }, + [](const TBoolType*) -> TStringBuf { return "boolean"; }, + [](const TInt8Type*) -> TStringBuf { return "int8"; }, + [](const TInt16Type*) -> TStringBuf { return "int16"; }, + [](const TInt32Type*) -> TStringBuf { return "int32"; }, + [](const TInt64Type*) -> TStringBuf { return "int64"; }, + [](const TUint8Type*) -> TStringBuf { return "uint8"; }, + [](const TUint16Type*) -> TStringBuf { return "uint16"; }, + [](const TUint32Type*) -> TStringBuf { return "uint32"; }, + [](const TUint64Type*) -> TStringBuf { return "uint64"; }, + [](const TFloatType*) -> TStringBuf { return "double"; }, + [](const TDoubleType*) -> TStringBuf { return "double"; }, + [](const TStringType*) -> TStringBuf { return "string"; }, + [](const TUtf8Type*) -> TStringBuf { return "utf8"; }, + [](const TDateType*) -> TStringBuf { return "uint16"; }, + [](const TDatetimeType*) -> TStringBuf { return "uint32"; }, + [](const TTimestampType*) -> TStringBuf { return "uint64"; }, + [](const TTzDateType*) -> TStringBuf { return "string"; }, + [](const TTzDatetimeType*) -> TStringBuf { return "string"; }, + [](const TTzTimestampType*) -> TStringBuf { return "string"; }, + [](const TIntervalType*) -> TStringBuf { return "int64"; }, + [](const TJsonType*) -> TStringBuf { return "string"; }, + [](const TYsonType*) -> TStringBuf { return "any"; }, + [](const TUuidType*) -> TStringBuf { return "string"; }, + [](const TDecimalType*) -> TStringBuf { return "string"; }, + [](const TOptionalType*) -> TStringBuf { return "any"; }, + [](const TListType*) -> TStringBuf { return "any"; }, + [](const TDictType*) -> TStringBuf { return "any"; }, + [](const TStructType*) -> TStringBuf { return "any"; }, + [](const TTupleType*) -> TStringBuf { return "any"; }, + [](const TVariantType*) -> TStringBuf { return "any"; }, + [](const TTaggedType*) -> TStringBuf { return "any"; }, + }); + + if (typeString == "any") { + // columns of type `any` cannot be required + required = false; + } + + { + consumer.OnBeginMap(); + + consumer.OnKey("name"); + consumer.OnScalarString(item.GetName()); + + consumer.OnKey("required"); + consumer.OnScalarBoolean(required); + + consumer.OnKey("type"); + consumer.OnScalarString(typeString); + + consumer.OnEndMap(); + } + } + consumer.OnEndList(); + } + + TString AsYtSchema(const NTi::TType* type, bool failOnEmptyStruct) { + auto result = TString(); + auto writer = NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node); + writer.BeginStream(); + AsYtSchema(type, writer.GetConsumer(), failOnEmptyStruct); + writer.EndStream(); + return result; + } +} diff --git a/library/cpp/type_info/type_io.h b/library/cpp/type_info/type_io.h new file mode 100644 index 0000000000..e400414cff --- /dev/null +++ b/library/cpp/type_info/type_io.h @@ -0,0 +1,115 @@ +#pragma once + +//! @file type_io.h +//! +//! Utilities for serializing and deserializing type instances. + +#include "type.h" + +#include <library/cpp/yson_pull/yson.h> + +namespace NTi::NIo { + /// Load type from a serialized representation. + /// + /// Serialization uses YSON (either binary or text). Contents are described in the [docs page]. + /// + /// Throws `TTypeDeserializationException` if input is not valid. + /// + /// [docs page]: https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/type_info/docs/types_serialization.md + /// + /// @param factory factory that will be used to allocate new type. Technically, type will be deserialized into + /// a temporary pool factory and then adopted into a given one. + /// @param reader yson pull reader that'll be used to read types. + /// @param deduplicate use deduplication while creating new types. See `NTi::PoolFactory` function for more info. + /// @{ + TTypePtr DeserializeYson(ITypeFactory& factory, NYsonPull::TReader& reader, bool deduplicate = true); + TTypePtr DeserializeYson(ITypeFactory& factory, TStringBuf data, bool deduplicate = true); + TTypePtr DeserializeYson(ITypeFactory& factory, IInputStream& input, bool deduplicate = true); + /// @} + + /// Like `Deserialize`, but returns a raw pointer. + /// @{ + const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, NYsonPull::TReader& reader); + const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, TStringBuf data); + const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, IInputStream& input); + /// @} + + /// Like `Deserialize`, but allows deserializing multiple types from the same reader. + /// + /// This function takes a reader created with `NYsonPull::EStreamType::ListFragment` mode. It reads a type, + /// but doesn't fails if there is no `BeginStream` event. If the reader is empty, it returns nullptr. + /// + /// This function mirrors `SerializeMultiple`. Call it multiple times on the same reader to read multiple types. + const TType* DeserializeYsonMultipleRaw(IPoolTypeFactory& factory, NYsonPull::TReader& reader); + + /// Serialize this type info. + /// + /// Serialization uses YSON (either binary or text). Contents are described in the [RFC]. + /// + /// [RFC]: https://a.yandex-team.ru/arc/trunk/arcadia/logfeller/mvp/docs/types_serialization.md + /// + /// @param humanReadable use pretty textual format instead of a binary one. + /// @param includeTags when disabled, tagged types will be removed from the result, only tagged type contents + /// will be dumped. For example, `Tagged<'Url', String>` will be rendered as just `String`. + /// This is useful if you only care about physical layout of a type and don't want to export + /// any semantical meaning. + /// @{ + void SerializeYson(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags = true); + void SerializeYson(const TType* type, IOutputStream& stream, bool humanReadable = false, bool includeTags = true); + TString SerializeYson(const TType* type, bool humanReadable = false, bool includeTags = true); + /// @} + + /// Like `Serialize`, but allows serializing multiple types into the same consumer. + /// + /// This function takes a consumer created with `NYsonPull::EStreamType::ListFragment` mode. It writes type, + /// but doesn't emit the `BeginStream` and `EndStream` commands. + /// + /// Call this function multiple times on the same consumer to write multiple types. Note that you must emit + /// the `BeginStream` and `EndStream` commands to the consumer yourself. + void SerializeYsonMultiple(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags = true); + + /// Convert type to the lisp-like representation used in YQL row specs. + /// + /// TODO: move this code to yql/ + /// + /// @param includeTags when disabled, tagged types will be removed from the result, only tagged type contents + /// will be dumped. For example, `Tagged<'Url', String>` will be rendered as just `String`. + /// This is useful if you only care about physical layout of a type and don't want to export + /// any semantic meaning. + /// @{ + void AsYqlType(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags = true); + TString AsYqlType(const TType* type, bool includeTags = true); + /// @} + + /// Generate a strict YQL row spec. Toplevel tags will be ignored. + /// + /// Throws `TApiException` if the type is not a (possibly tagged) struct. + /// + /// TODO: move this code to yql/ + /// + /// @param type type that'll be converted to YQL row spec. + /// @param consumer yson pull consumer. Attention: `OnBeginStream` and `OnEndStream` should be emitted manually + /// before and after calling this function. + /// @param includeTags same as in `TType::AsYqlType`. + /// @{ + void AsYqlRowSpec(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags = true); + TString AsYqlRowSpec(const TType* type, bool includeTags = true); + /// @} + + /// Generate a strict YT schema (only types are exported, no index/sorting information). + /// + /// Throws `TApiException` if the type is not a (possibly tagged) struct. + /// + /// The schema is generated according to the translation rules of YQL types, i.e. container types translate + /// to `Any`. + /// + /// TODO: move this code to mapreduce/yt/ + /// + /// @param failOnEmptyStruct if true, will throw `TApiException` if called on a struct with no fields; + /// if false, will emit a strict YT schema with a single column `'_yql_fake_column'` + /// of type `Optional<Bool>` (this is how YQL handles empty tables). + /// @{ + void AsYtSchema(const TType* type, NYsonPull::IConsumer& consumer, bool failOnEmptyStruct = true); + TString AsYtSchema(const TType* type, bool failOnEmptyStruct = true); + /// @} +} diff --git a/library/cpp/type_info/type_list.cpp b/library/cpp/type_info/type_list.cpp new file mode 100644 index 0000000000..b741b2858b --- /dev/null +++ b/library/cpp/type_info/type_list.cpp @@ -0,0 +1 @@ +#include "type_list.h" diff --git a/library/cpp/type_info/type_list.h b/library/cpp/type_info/type_list.h new file mode 100644 index 0000000000..c87c51918d --- /dev/null +++ b/library/cpp/type_info/type_list.h @@ -0,0 +1,183 @@ +#pragma once + +//! @file type_list.h +//! +//! Enum with all type names that are included in the Common Type System. +//! +//! +//! # Primitive and non-primitive types +//! +//! Some systems only work with primitive types, so we've split the enum in two: the first contains primitive types, +//! and the second contains all types, including the primitive ones. This way systems that are only interested +//! in primitives can stop handling containers and make all their switch-cases exhaustive. +//! +//! Consequently, the class hierarchy follows the same division: there is `NTi::TPrimitiveType`, +//! from which all primitives are derived. +//! +//! +//! # Enumerator values +//! +//! Enumerator values are implementation detail and should not be relied upon. In particular, use `NTi::ToTypeName` +//! and `NTi::ToPrimitiveTypeName` to safely cast between `NTi::EPrimitiveTypeName` and `NTi::ETypeName`. Also, don't +//! use enumerator values for serialization and deserialization — convert enumerator values to strings +//! it you need persistence. + +#include <util/system/types.h> +#include <util/generic/variant.h> + +namespace NTi { + /// Enum with names of all primitive types. + /// + /// See the file-level documentation. + enum class EPrimitiveTypeName : i32 { + Bool, + + Int8, + Int16, + Int32, + Int64, + Uint8, + Uint16, + Uint32, + Uint64, + + Float, + Double, + + String, + Utf8, + + Date, + Datetime, + Timestamp, + TzDate, + TzDatetime, + TzTimestamp, + Interval, + + Decimal, + Json, + Yson, + Uuid, + }; + + /// Enum with names of all types, including primitives. + /// + /// See the file-level documentation. + enum class ETypeName : i32 { + // + // # Primitive types + + Bool, + + Int8, + Int16, + Int32, + Int64, + Uint8, + Uint16, + Uint32, + Uint64, + + Float, + Double, + + String, + Utf8, + + Date, + Datetime, + Timestamp, + TzDate, + TzDatetime, + TzTimestamp, + Interval, + + Decimal, + Json, + Yson, + Uuid, + + FIRST_PRIMITIVE = Bool, + LAST_PRIMITIVE = Uuid, + + // + // # Singular types + + Void, + Null, + + FIRST_SINGULAR = Void, + LAST_SINGULAR = Null, + + // + // # Containers + + Optional, + List, + Dict, + Struct, + Tuple, + Variant, + Tagged, + + FIRST_CONTAINER = Optional, + LAST_CONTAINER = Tagged, + }; + + /// Return true if the given type is a primitive one. + /// + /// Primitive type is a type that have no type parameters and is not a singular one. + inline constexpr bool IsPrimitive(ETypeName typeName) { + return ETypeName::FIRST_PRIMITIVE <= typeName && typeName <= ETypeName::LAST_PRIMITIVE; + } + + /// Return true if the given type is one of singular types. + /// + /// Singular type is a type that has only one instance and therefore carries no information, + /// i.e. occupy zero-length memory buffer. + inline constexpr bool IsSingular(ETypeName typeName) { + return ETypeName::FIRST_SINGULAR <= typeName && typeName <= ETypeName::LAST_SINGULAR; + } + + /// Return true if the given type is one of containers. + /// + /// Container type is a type that has type parameters. + inline constexpr bool IsContainer(ETypeName typeName) { + return ETypeName::FIRST_CONTAINER <= typeName && typeName <= ETypeName::LAST_CONTAINER; + } + + /// Return true if the given type has any type parameters. + inline constexpr bool HasTypeParameters(ETypeName typeName) { + return IsContainer(typeName); + } + + /// Return true if the given type has any non-type parameters. + inline constexpr bool HasNonTypeParameters(ETypeName typeName) { + return typeName == ETypeName::Decimal; + } + + /// Return true if the given type has any type or non-type parameters. + inline constexpr bool HasParameters(ETypeName typeName) { + return HasTypeParameters(typeName) || HasNonTypeParameters(typeName); + } + + /// Safely cast `NTi::EPrimitiveTypeName` to `NTi::ETypeName`. + /// + /// Enumerator values should not relied upon, therefore users should not cast `NTi::EPrimitiveTypeName` + /// to `NTi::ETypeName` using `static_cast`. + inline constexpr ETypeName ToTypeName(EPrimitiveTypeName primitiveTypeName) { + // Note: there's a test in ut/type_list.cpp that checks this is a safe conversion + return static_cast<ETypeName>(primitiveTypeName); + } + + /// Cast `NTi::ETypeName` to `NTi::EPrimitiveTypeName`, panic if the given type is not a primitive one. + /// + /// Enumerator values should not relied upon, therefore users should not cast `NTi::ETypeName` + /// to `NTi::EPrimitiveTypeName` using `static_cast`. + inline constexpr EPrimitiveTypeName ToPrimitiveTypeName(ETypeName typeName) { + Y_VERIFY(IsPrimitive(typeName)); + // Note: there's a test in ut/type_list.cpp that checks this is a safe conversion + return static_cast<EPrimitiveTypeName>(typeName); + } +} diff --git a/library/cpp/type_info/ut/builder.cpp b/library/cpp/type_info/ut/builder.cpp new file mode 100644 index 0000000000..43b7bb8c9b --- /dev/null +++ b/library/cpp/type_info/ut/builder.cpp @@ -0,0 +1,125 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include "utils.h" + +class Builder: public NTesting::TTest { +public: + void SetUp() override { + F = NTi::PoolFactory(false); + } + + void TearDown() override { + F.Reset(); + } + + NTi::IPoolTypeFactoryPtr F; +}; + +TEST_F(Builder, TaggedBuilder) { + auto builder = NTi::TTaggedBuilderRaw(*F); + + ASSERT_FALSE(builder.CanBuild()); + ASSERT_FALSE(builder.HasTag()); + ASSERT_FALSE(builder.HasItem()); + + UNIT_ASSERT_EQUAL(builder.GetTag(), Nothing()); + UNIT_ASSERT_EQUAL(builder.GetItem(), Nothing()); + + { + auto tag = TString("Url"); + builder.SetTag(tag); + } + + ASSERT_FALSE(builder.CanBuild()); + ASSERT_TRUE(builder.HasTag()); + ASSERT_FALSE(builder.HasItem()); + + UNIT_ASSERT_EQUAL(builder.GetTag(), MakeMaybe<TStringBuf>("Url")); + UNIT_ASSERT_EQUAL(builder.GetItem(), Nothing()); + + { + auto ty = NTi::String(); + builder.SetItem(ty); + } + + ASSERT_TRUE(builder.CanBuild()); + ASSERT_TRUE(builder.HasTag()); + ASSERT_TRUE(builder.HasItem()); + + UNIT_ASSERT_EQUAL(builder.GetTag(), MakeMaybe<TStringBuf>("Url")); + UNIT_ASSERT_EQUAL(builder.GetItem(), MakeMaybe<const NTi::TType*>(NTi::TStringType::InstanceRaw())); + + { + auto tagged = builder.Build(); + ASSERT_STRICT_EQ(tagged, NTi::Tagged(NTi::String(), "Url")); + } + + ASSERT_TRUE(builder.CanBuild()); + + builder.Reset(); + + ASSERT_FALSE(builder.CanBuild()); + ASSERT_FALSE(builder.HasTag()); + ASSERT_FALSE(builder.HasItem()); + + UNIT_ASSERT_EQUAL(builder.GetTag(), Nothing()); + UNIT_ASSERT_EQUAL(builder.GetItem(), Nothing()); + + builder.SetTag("T"); + builder.SetItem(NTi::String()); + + ASSERT_TRUE(builder.CanBuild()); + ASSERT_TRUE(builder.HasTag()); + ASSERT_TRUE(builder.HasItem()); + + UNIT_ASSERT_EQUAL(builder.GetTag(), MakeMaybe<TStringBuf>("T")); + UNIT_ASSERT_EQUAL(builder.GetItem(), MakeMaybe<const NTi::TType*>(NTi::TStringType::InstanceRaw())); + + builder.DiscardItem(); + + ASSERT_FALSE(builder.CanBuild()); + ASSERT_TRUE(builder.HasTag()); + ASSERT_FALSE(builder.HasItem()); + + UNIT_ASSERT_EQUAL(builder.GetTag(), MakeMaybe<TStringBuf>("T")); + UNIT_ASSERT_EQUAL(builder.GetItem(), Nothing()); + + builder.DiscardTag(); + + builder.SetTag("T"); + builder.SetItem(NTi::String()); + + ASSERT_TRUE(builder.CanBuild()); + ASSERT_TRUE(builder.HasTag()); + ASSERT_TRUE(builder.HasItem()); +} + +TEST_F(Builder, TaggedBuilderChaining) { + auto builder = NTi::TTaggedBuilderRaw(*F) + .SetTag("Uuu") + .SetItem(NTi::Optional(NTi::String())); + + ASSERT_STRICT_EQ(builder.Build(), NTi::Tagged(NTi::Optional(NTi::String()), "Uuu")); + + builder = std::move(builder) + .DiscardTag() + .SetTag("Urls"); + + ASSERT_STRICT_EQ(builder.Build(), NTi::Tagged(NTi::Optional(NTi::String()), "Urls")); + + builder = std::move(builder) + .DiscardItem() + .SetItem(F->ListRaw(F->StringRaw())); + + ASSERT_STRICT_EQ(builder.Build(), NTi::Tagged(NTi::List(NTi::String()), "Urls")); + + auto type = std::move(builder) + .Reset() + .SetTag("Url") + .SetItem(F->StringRaw()) + .Build(); + + ASSERT_STRICT_EQ(type, NTi::Tagged(NTi::String(), "Url")); +} diff --git a/library/cpp/type_info/ut/test-data/README.txt b/library/cpp/type_info/ut/test-data/README.txt new file mode 100644 index 0000000000..e4ce965357 --- /dev/null +++ b/library/cpp/type_info/ut/test-data/README.txt @@ -0,0 +1,5 @@ +The format of the text files is as follows: + 1. Each line starting with '#' is comment and must be ignored. + 2. Test case are separated with ';;' + 3. Each test case has predefined number of fields. Fields are separated with '::'. + 4. Meaning of the fields are described in test case. diff --git a/library/cpp/type_info/ut/test-data/bad-types.txt b/library/cpp/type_info/ut/test-data/bad-types.txt new file mode 100644 index 0000000000..c8f3120f63 --- /dev/null +++ b/library/cpp/type_info/ut/test-data/bad-types.txt @@ -0,0 +1,229 @@ +# +# The format of this file is described in README.txt +# +# Each test case contains 3 fields: +# - Yson representation (that cannot be parsed to type). +# - Expected error description. +# - Path in yson that we expect to see in error. +# +# Suggested approach to writing test: +# 1. Try to deserialize type from yson (1st field). +# 2. Ensure that error is raised. +# 3. (Optionally) ensure that error description matches (contains) 2nd field. +# 4. (Optionally) ensure that error description contains path from 3rd field + +5 :: type must be either a string or a map :: ;; + +"" :: unknown type "" :: ;; + +"int" :: unknown type "int" :: ;; + +# Type names must be in lowercase. +"Int32" :: unknown type "Int32" :: ;; + +{typename=int32} :: missing required key "type_name" :: ;; + +{type_name=5} :: "type_name" must contain a string :: ;; + +# +# Decimal +# + +{ + type_name=decimal; + precision=3; +} :: missing required key "scale" :: ;; + +{ + type_name=decimal; + scale=3; +} :: missing required key "precision" :: ;; + +# +# Optional +# + +{ + type_name=optional; +} :: missing required key "item" :: ;; + +{ + item=int32; +} :: missing required key "type_name" :: ;; + +# +# List +# + +{ + type_name=list; +} :: missing required key "item" :: ;; + +# +# Struct +# + +{ + type_name=struct; +} :: missing required key "members" :: ;; + +{ + type_name=struct; + members=5; +} :: "members" must contain a list :: ;; + +{ + type_name=struct; + members=[ + {name=foo; type=int32;}; + 5; + ]; +} :: "members" must contain a list of maps :: ;; + +{ + type_name=struct; + members=[ + {type=int32;}; + ]; +} :: missing required key "name" :: ;; + +{ + type_name=struct; + members=[ + {name=foo;}; + ]; +} :: missing required key "type" :: ;; + +{ + type_name=struct; + members=[ + {name=foo; type=int32}; + {name=bar; type=5}; + ]; +} :: type must be either a string or a map :: /members/1/type ;; + +{ + type_name=struct; + members=[ + {name=4; type=int32}; + ]; +} :: "name" must contain a string :: ;; + +{ + type_name=struct; + elements=[ + {name=4; type=int32}; + ]; +} :: missing required key "members" :: ;; + +# +# Tuple +# + +{ + type_name=tuple; +} :: missing required key "elements" :: ;; + +{ + type_name=tuple; + elements=5; +} :: "elements" must contain a list :: ;; + +{ + type_name=tuple; + elements=[ + {type=int32;}; + 5; + ]; +} :: "elements" must contain a list of maps :: ;; + +{ + type_name=tuple; + elements=[ + {}; + ]; +} :: missing required key "type" :: /elements/0 ;; + +{ + type_name=tuple; + elements=[ + {type=5}; + ]; +} :: type must be either a string or a map :: /elements/1/type ;; + +{ + type_name=tuple; + members=[ + {name=foo; type=int32}; + ]; +} :: missing required key "elements" :: ;; + +# +# Variant +# + +# We don't specify exception message here because in C++ library that message is not good and we don't want to canonize it +# Though fixing it is not that easy too. +{ + type_name=variant; + members=[ + {name=foo; type=int32}; + ]; + elements=[ + {type=int8}; + ] +} :: :: ;; + +{ + type_name=variant; +} :: missing both keys "members" and "elements" :: ;; + +# +# Dict +# +{ + type_name=dict; + key=string; +} :: missing required key "value" :: ;; + +{ + type_name=dict; + value=string; +} :: missing required key "key" :: ;; + +{ + type_name=dict; + value=string; +} :: missing required key "key" :: ;; + +{ + type_name=dict; + key=5; + value=string; +} :: type must be either a string or a map :: /key ;; + +{ + type_name=dict; + key=string; + value=5; +} :: type must be either a string or a map :: /value ;; + +# +# Tagged +# + +{ + type_name=tagged; + item=string; +} :: missing required key "tag" :: ;; + +{ + type_name=tagged; + tag=string; +} :: missing required key "item" :: ;; + +{ + type_name=tagged; + tag=5; + item=string; +} :: "tag" must contain a string :: /tag ;; diff --git a/library/cpp/type_info/ut/test-data/good-types.txt b/library/cpp/type_info/ut/test-data/good-types.txt new file mode 100644 index 0000000000..cb082707b6 --- /dev/null +++ b/library/cpp/type_info/ut/test-data/good-types.txt @@ -0,0 +1,478 @@ +# +# The format of this file is described in README.txt +# +# Each test case contains 2 fields: +# - Yson reporesentation of the type. +# - String representation of the type. +# +# Suggested approach to writing test: +# 1. Try to deserialize type from yson (1st field). +# 2. Check that text representation of parsed type matches 2nd field. +# 3. Serialize type to yson and deserialize it from it. +# 4. Check that typef from 1. and 3. are equal. + +# Integer +int8 :: Int8 ;; +int16 :: Int16 ;; +int32 :: Int32 ;; +int64 :: Int64 ;; +{type_name=int8} :: Int8 ;; +{type_name=int16} :: Int16 ;; +{type_name=int32} :: Int32 ;; +{type_name=int64} :: Int64 ;; + +# Unsigned integer +uint8 :: Uint8 ;; +uint16 :: Uint16 ;; +uint32 :: Uint32 ;; +uint64 :: Uint64 ;; +{type_name=uint8} :: Uint8 ;; +{type_name=uint16} :: Uint16 ;; +{type_name=uint32} :: Uint32 ;; +{type_name=uint64} :: Uint64 ;; + +# Floating +float :: Float ;; +double :: Double ;; +{type_name=float} :: Float ;; +{type_name=double} :: Double ;; + +# Strings +string :: String ;; +utf8 :: Utf8 ;; +{type_name=string} :: String ;; +{type_name=utf8} :: Utf8 ;; + +# Time +date :: Date ;; +datetime :: Datetime ;; +timestamp :: Timestamp ;; +tz_date :: TzDate ;; +tz_datetime :: TzDatetime ;; +tz_timestamp :: TzTimestamp ;; +interval :: Interval ;; +{type_name=date} :: Date ;; +{type_name=datetime} :: Datetime ;; +{type_name=timestamp} :: Timestamp ;; +{type_name=tz_date} :: TzDate ;; +{type_name=tz_datetime} :: TzDatetime ;; +{type_name=tz_timestamp} :: TzTimestamp ;; +{type_name=interval} :: Interval ;; + +# Singular +void :: Void ;; +null :: Null ;; +{type_name=void} :: Void ;; +{type_name=null} :: Null ;; + +# UUID +uuid :: Uuid ;; +{type_name=uuid} :: Uuid ;; + +# Json / Yson +yson :: Yson ;; +json :: Json ;; +{type_name=yson} :: Yson ;; +{type_name=json} :: Json ;; + +{ + type_name=string; + unknown_key=bar; +} :: String ;; + +# Decimal +{ + type_name=decimal; + precision=3; + scale=2; +} :: Decimal(3, 2) ;; + +{ + type_name=decimal; + precision=3; + scale=2; + unknown_column=ha; +} :: Decimal(3, 2) ;; + +# +# Optional +# +{ + type_name=optional; + item=string; +} :: Optional<String> ;; + +{ + type_name=optional; + item={ + type_name=string; + } +} :: Optional<String> ;; + +{ + type_name=optional; + item={ + type_name=optional; + item={ + type_name=list; + item={ + type_name=decimal; + precision=10; + scale=5; + } + + } + } +} :: Optional<Optional<List<Decimal(10, 5)>>> ;; + +{ + type_name=optional; + item={ + type_name=tagged; + item=int32; + tag="foo"; + }; +} :: Optional<Tagged<Int32, 'foo'>> ;; + +{ + type_name=optional; + item={ + type_name=string; + }; + unknown_column=ha; +} :: Optional<String> ;; + +# +# List +# +{ + type_name=list; + item=string; +} :: List<String> ;; + +{ + type_name=list; + item={ + type_name=string; + } +} :: List<String> ;; + +{ + type_name=list; + item={ + type_name=list; + item={ + type_name=optional; + item=string; + } + }; +} :: List<List<Optional<String>>> ;; + +{ + type_name=list; + item={ + type_name=tagged; + item=int32; + tag="foo"; + }; +} :: List<Tagged<Int32, 'foo'>> ;; + +{ + type_name=list; + item={ + type_name=string; + }; + unknown_column=ha; +} :: List<String> ;; + +# +# Dict +# + +{ + type_name=dict; + key=int32; + value=string; +} :: Dict<Int32, String> ;; + +{ + type_name=dict; + key={ + type_name=optional; + item=string; + }; + value={ + type_name=list; + item={ + type_name=int32; + } + }; +} :: Dict<Optional<String>, List<Int32>> ;; + +{ + type_name=dict; + key={ + type_name=tagged; + item=int32; + tag="foo"; + }; + value={ + type_name=tagged; + item=string; + tag="bar"; + }; +} :: Dict<Tagged<Int32, 'foo'>, Tagged<String, 'bar'>> ;; + +{ + type_name=dict; + key=int32; + value=string; + unknown_column=ha; +} :: Dict<Int32, String> ;; + +# +# Struct +# + +{ + type_name=struct; + members=[]; +} :: Struct<> ;; + +{ + type_name=struct; + members=[ + { + name=foo; + type=int32; + }; + { + name=bar; + type={ + type_name=optional; + item=string; + }; + }; + ]; +} :: Struct<'foo': Int32, 'bar': Optional<String>> ;; + +{ + type_name=struct; + members=[ + { + name=foo; + type={ + type_name=tagged; + item=string; + tag=foo; + }; + }; + ]; +} :: Struct<'foo': Tagged<String, 'foo'>> ;; + +{ + type_name=struct; + unknown_column=ha; + members=[ + { + unknown_column=ha; + name=foo; + type=int32; + }; + { + name=bar; + type={ + type_name=optional; + item=string; + }; + }; + ]; +} :: Struct<'foo': Int32, 'bar': Optional<String>> ;; + +# +# Tuple +# + +{ + type_name=tuple; + elements=[]; +} :: Tuple<> ;; + +{ + type_name=tuple; + elements=[ + { + type=int32; + }; + { + type={ + type_name=optional; + item=string; + }; + }; + ]; +} :: Tuple<Int32, Optional<String>> ;; + +{ + type_name=tuple; + elements=[ + { + type={ + type_name=tagged; + item=string; + tag=foo; + }; + }; + ]; +} :: Tuple<Tagged<String, 'foo'>> ;; + +{ + type_name=tuple; + unknown_column=ha; + elements=[ + { + type=int32; + unknown_column=ha; + }; + { + type={ + type_name=optional; + item=string; + }; + }; + ]; +} :: Tuple<Int32, Optional<String>> ;; + +# +# Variant +# + +{ + type_name=variant; + elements=[ + { + type=int32; + }; + { + type={type_name=string}; + }; + ]; +} :: Variant<Int32, String> ;; + +{ + type_name=variant; + members=[ + { + name=foo; + type=int32; + }; + { + name=bar; + type={ + type_name=optional; + item=string; + }; + }; + ]; +} :: Variant<'foo': Int32, 'bar': Optional<String>> ;; + +{ + type_name=variant; + elements=[ + { + type={ + type_name=tagged; + item=string; + tag=foo; + } + }; + ]; +} :: Variant<Tagged<String, 'foo'>> ;; + +{ + type_name=variant; + members=[ + { + name=bar; + type={ + type_name=tagged; + item=string; + tag=foo; + }; + }; + ]; +} :: Variant<'bar': Tagged<String, 'foo'>> ;; + +{ + type_name=variant; + unknown_column=ha; + elements=[ + { + type=int32; + unknown_column=ha; + }; + { + type={ + type_name=string; + unknown_column=ha; + }; + unknown_column=ha; + }; + ]; +} :: Variant<Int32, String> ;; + +{ + type_name=variant; + unknown_column=ha; + members=[ + { + unknown_column=ha; + name=foo; + type=int32; + }; + { + name=bar; + type={ + type_name=optional; + unknown_column=ha; + item=string; + }; + }; + ]; +} :: Variant<'foo': Int32, 'bar': Optional<String>> ;; + +# +# Tagged +# + +{ + type_name=tagged; + item=string; + tag="image/png" +} :: Tagged<String, 'image/png'> ;; + +{ + type_name=tagged; + item={ + type_name=optional; + item=string; + }; + tag="image/png" +} :: Tagged<Optional<String>, 'image/png'> ;; + +{ + type_name=tagged; + item={ + type_name=tagged; + item=string; + tag=foo; + }; + tag=bar; +} :: Tagged<Tagged<String, 'foo'>, 'bar'> ;; + +{ + type_name=tagged; + item=string; + unknown_column=ha; + tag="image/png" +} :: Tagged<String, 'image/png'> ;; diff --git a/library/cpp/type_info/ut/test_data.cpp b/library/cpp/type_info/ut/test_data.cpp new file mode 100644 index 0000000000..36944e7bc4 --- /dev/null +++ b/library/cpp/type_info/ut/test_data.cpp @@ -0,0 +1,91 @@ +#include <library/cpp/testing/unittest/gtest.h> +#include <library/cpp/resource/resource.h> + +#include <library/cpp/type_info/type_info.h> +#include <library/cpp/type_info/type_io.h> + +#include <util/string/strip.h> +#include <util/string/split.h> + +using namespace NTi; + +std::vector<std::vector<TString>> ParseData(TStringBuf data, int expectedFieldsCount) { + TString noComments; + { + TMemoryInput in(data); + TString line; + while (in.ReadLine(line)) { + if (StripString(line).StartsWith('#')) { + continue; + } + noComments += line; + } + } + + std::vector<std::vector<TString>> result; + for (TStringBuf record : StringSplitter(noComments).SplitByString(";;")) { + record = StripString(record); + if (record.Empty()) { + continue; + } + std::vector<TString> fields; + for (TStringBuf field : StringSplitter(record).SplitByString("::")) { + fields.emplace_back(StripString(field)); + } + if (static_cast<int>(fields.size()) != expectedFieldsCount) { + ythrow yexception() << "Unexpected field count expected: " << expectedFieldsCount << " actual: " << fields.size(); + } + result.push_back(fields); + } + return result; +} + +TEST(TestData, GoodTypes) { + auto records = ParseData(NResource::Find("/good"), 2); + + for (const auto& record : records) { + const auto& typeYson = record.at(0); + const auto& typeText = record.at(1); + TString context = TStringBuilder() + << "text: " << typeText << Endl + << "yson: " << typeYson << Endl; + auto wrapError = [&] (const std::exception& ex) { + return yexception() << "Unexpected error: " << ex.what() << '\n' << context; + }; + + TTypePtr type; + try { + type = NIo::DeserializeYson(*HeapFactory(), typeYson); + } catch (const std::exception& ex) { + ythrow wrapError(ex); + } + UNIT_ASSERT_VALUES_EQUAL_C(ToString(*type), typeText, context); + + TTypePtr type2; + try { + auto yson2 = NIo::SerializeYson(type.Get(), true); + type2 = NIo::DeserializeYson(*HeapFactory(), yson2); + } catch (const std::exception& ex) { + ythrow wrapError(ex); + } + UNIT_ASSERT_VALUES_EQUAL_C(*type, *type2, context); + } +} + +TEST(TestData, BadTypes) { + auto records = ParseData(NResource::Find("/bad"), 3); + + for (const auto& record : records) { + const auto& typeYson = record.at(0); + const auto& exceptionMessage = record.at(1); + + TString context = TStringBuilder() + << "exception: " << exceptionMessage << Endl + << "yson: " << typeYson << Endl; + UNIT_ASSERT_EXCEPTION_CONTAINS_C( + NIo::DeserializeYson(*HeapFactory(), typeYson), + yexception, + exceptionMessage, + context); + } +}
\ No newline at end of file diff --git a/library/cpp/type_info/ut/type_basics.cpp b/library/cpp/type_info/ut/type_basics.cpp new file mode 100644 index 0000000000..83996d63d3 --- /dev/null +++ b/library/cpp/type_info/ut/type_basics.cpp @@ -0,0 +1,381 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include "utils.h" + +TEST_TF(TypeBasics, Void) { + auto t = f.Void(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Void); + ASSERT_TRUE(t->IsVoid()); +} + +TEST_TF(TypeBasics, Bool) { + auto t = f.Bool(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Bool); + ASSERT_TRUE(t->IsBool()); +} + +TEST_TF(TypeBasics, Int8) { + auto t = f.Int8(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int8); + ASSERT_TRUE(t->IsInt8()); +} + +TEST_TF(TypeBasics, Int16) { + auto t = f.Int16(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int16); + ASSERT_TRUE(t->IsInt16()); +} + +TEST_TF(TypeBasics, Int32) { + auto t = f.Int32(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int32); + ASSERT_TRUE(t->IsInt32()); +} + +TEST_TF(TypeBasics, Int64) { + auto t = f.Int64(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int64); + ASSERT_TRUE(t->IsInt64()); +} + +TEST_TF(TypeBasics, Uint8) { + auto t = f.Uint8(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint8); + ASSERT_TRUE(t->IsUint8()); +} + +TEST_TF(TypeBasics, Uint16) { + auto t = f.Uint16(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint16); + ASSERT_TRUE(t->IsUint16()); +} + +TEST_TF(TypeBasics, Uint32) { + auto t = f.Uint32(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint32); + ASSERT_TRUE(t->IsUint32()); +} + +TEST_TF(TypeBasics, Uint64) { + auto t = f.Uint64(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint64); + ASSERT_TRUE(t->IsUint64()); +} + +TEST_TF(TypeBasics, Float) { + auto t = f.Float(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Float); + ASSERT_TRUE(t->IsFloat()); +} + +TEST_TF(TypeBasics, Double) { + auto t = f.Double(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Double); + ASSERT_TRUE(t->IsDouble()); +} + +TEST_TF(TypeBasics, String) { + auto t = f.String(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::String); + ASSERT_TRUE(t->IsString()); +} + +TEST_TF(TypeBasics, Utf8) { + auto t = f.Utf8(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Utf8); + ASSERT_TRUE(t->IsUtf8()); +} + +TEST_TF(TypeBasics, Date) { + auto t = f.Date(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Date); + ASSERT_TRUE(t->IsDate()); +} + +TEST_TF(TypeBasics, Datetime) { + auto t = f.Datetime(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Datetime); + ASSERT_TRUE(t->IsDatetime()); +} + +TEST_TF(TypeBasics, Timestamp) { + auto t = f.Timestamp(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Timestamp); + ASSERT_TRUE(t->IsTimestamp()); +} + +TEST_TF(TypeBasics, TzDate) { + auto t = f.TzDate(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzDate); + ASSERT_TRUE(t->IsTzDate()); +} + +TEST_TF(TypeBasics, TzDatetime) { + auto t = f.TzDatetime(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzDatetime); + ASSERT_TRUE(t->IsTzDatetime()); +} + +TEST_TF(TypeBasics, TzTimestamp) { + auto t = f.TzTimestamp(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzTimestamp); + ASSERT_TRUE(t->IsTzTimestamp()); +} + +TEST_TF(TypeBasics, Interval) { + auto t = f.Interval(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Interval); + ASSERT_TRUE(t->IsInterval()); +} + +TEST_TF(TypeBasics, Decimal) { + auto t = f.Decimal(20, 10); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Decimal); + ASSERT_TRUE(t->IsDecimal()); + ASSERT_EQ(t->GetPrecision(), 20); + ASSERT_EQ(t->GetScale(), 10); +} + +TEST_TF(TypeBasics, Json) { + auto t = f.Json(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Json); + ASSERT_TRUE(t->IsJson()); +} + +TEST_TF(TypeBasics, Yson) { + auto t = f.Yson(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Yson); + ASSERT_TRUE(t->IsYson()); +} + +TEST_TF(TypeBasics, Uuid) { + auto t = f.Uuid(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uuid); + ASSERT_TRUE(t->IsUuid()); +} + +TEST_TF(TypeBasics, Optional) { + auto t = f.Optional(f.Void()); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Optional); + ASSERT_TRUE(t->IsOptional()); + ASSERT_TRUE(t->GetItemType()->IsVoid()); + ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw()); +} + +TEST_TF(TypeBasics, List) { + auto t = f.List(f.Void()); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::List); + ASSERT_TRUE(t->IsList()); + ASSERT_TRUE(t->GetItemType()->IsVoid()); + ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw()); +} + +TEST_TF(TypeBasics, Dict) { + auto t = f.Dict(f.Void(), f.String()); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Dict); + ASSERT_TRUE(t->IsDict()); + ASSERT_TRUE(t->GetKeyType()->IsVoid()); + ASSERT_EQ(t->GetKeyType().Get(), t->GetKeyTypeRaw()); + ASSERT_TRUE(t->GetValueType()->IsString()); + ASSERT_EQ(t->GetValueType().Get(), t->GetValueTypeRaw()); +} + +TEST_TF(TypeBasics, EmptyStruct) { + auto t = f.Struct({}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct); + ASSERT_TRUE(t->IsStruct()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_EQ(t->GetMembers().size(), 0); +} + +TEST_TF(TypeBasics, NamedEmptyStruct) { + auto t = f.Struct("S", {}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct); + ASSERT_TRUE(t->IsStruct()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "S"); + ASSERT_EQ(t->GetMembers().size(), 0); +} + +TEST_TF(TypeBasics, Struct) { + auto t = f.Struct({{"a", f.Int64()}, {"b", f.Int8()}}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct); + ASSERT_TRUE(t->IsStruct()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_EQ(t->GetMembers().size(), 2); + ASSERT_EQ(t->GetMembers()[0].GetName(), "a"); + ASSERT_TRUE(t->GetMembers()[0].GetTypeRaw()->IsInt64()); + ASSERT_EQ(t->GetMembers()[0].GetType().Get(), t->GetMembers()[0].GetTypeRaw()); + ASSERT_EQ(t->GetMembers()[1].GetName(), "b"); + ASSERT_TRUE(t->GetMembers()[1].GetTypeRaw()->IsInt8()); + ASSERT_EQ(t->GetMembers()[1].GetType().Get(), t->GetMembers()[1].GetTypeRaw()); +} + +TEST_TF(TypeBasics, NamedStruct) { + auto t = f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct); + ASSERT_TRUE(t->IsStruct()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "S"); + ASSERT_EQ(t->GetMembers().size(), 2); + ASSERT_EQ(t->GetMembers()[0].GetName(), "a"); + ASSERT_TRUE(t->GetMembers()[0].GetTypeRaw()->IsInt64()); + ASSERT_EQ(t->GetMembers()[0].GetType().Get(), t->GetMembers()[0].GetTypeRaw()); + ASSERT_EQ(t->GetMembers()[1].GetName(), "b"); + ASSERT_TRUE(t->GetMembers()[1].GetTypeRaw()->IsInt8()); + ASSERT_EQ(t->GetMembers()[1].GetType().Get(), t->GetMembers()[1].GetTypeRaw()); +} + +TEST_TF(TypeBasics, EmptyTuple) { + auto t = f.Tuple({}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple); + ASSERT_TRUE(t->IsTuple()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_EQ(t->GetElements().size(), 0); +} + +TEST_TF(TypeBasics, NamedEmptyTuple) { + auto t = f.Tuple("T", {}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple); + ASSERT_TRUE(t->IsTuple()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "T"); + ASSERT_EQ(t->GetElements().size(), 0); +} + +TEST_TF(TypeBasics, Tuple) { + auto t = f.Tuple({{f.Int64()}, {f.Int8()}}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple); + ASSERT_TRUE(t->IsTuple()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_EQ(t->GetElements().size(), 2); + ASSERT_TRUE(t->GetElements()[0].GetTypeRaw()->IsInt64()); + ASSERT_EQ(t->GetElements()[0].GetType().Get(), t->GetElements()[0].GetTypeRaw()); + ASSERT_TRUE(t->GetElements()[1].GetTypeRaw()->IsInt8()); + ASSERT_EQ(t->GetElements()[1].GetType().Get(), t->GetElements()[1].GetTypeRaw()); +} + +TEST_TF(TypeBasics, NamedTuple) { + auto t = f.Tuple("T", {{f.Int64()}, {f.Int8()}}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple); + ASSERT_TRUE(t->IsTuple()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "T"); + ASSERT_EQ(t->GetElements().size(), 2); + ASSERT_TRUE(t->GetElements()[0].GetTypeRaw()->IsInt64()); + ASSERT_EQ(t->GetElements()[0].GetType().Get(), t->GetElements()[0].GetTypeRaw()); + ASSERT_TRUE(t->GetElements()[1].GetTypeRaw()->IsInt8()); + ASSERT_EQ(t->GetElements()[1].GetType().Get(), t->GetElements()[1].GetTypeRaw()); +} + +TEST_TF(TypeBasics, VariantOverStruct) { + auto t = f.Variant(f.Struct("Inner", {{"x", f.Void()}})); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant); + ASSERT_TRUE(t->IsVariant()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_TRUE(t->IsVariantOverStruct()); + ASSERT_FALSE(t->IsVariantOverTuple()); + ASSERT_EQ(t->GetUnderlyingType()->AsStruct()->GetName().GetRef(), "Inner"); + ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw()); +} + +TEST_TF(TypeBasics, NamedVariantOverStruct) { + auto t = f.Variant("V", f.Struct("Inner", {{"x", f.Void()}})); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant); + ASSERT_TRUE(t->IsVariant()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "V"); + ASSERT_TRUE(t->IsVariantOverStruct()); + ASSERT_FALSE(t->IsVariantOverTuple()); + ASSERT_EQ(t->GetUnderlyingType()->AsStruct()->GetName().GetRef(), "Inner"); + ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw()); +} + +TEST_TF(TypeBasics, VariantOverTuple) { + auto t = f.Variant(f.Tuple("Inner", {{f.Void()}})); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant); + ASSERT_TRUE(t->IsVariant()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_FALSE(t->IsVariantOverStruct()); + ASSERT_TRUE(t->IsVariantOverTuple()); + ASSERT_EQ(t->GetUnderlyingType()->AsTuple()->GetName().GetRef(), "Inner"); + ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw()); +} + +TEST_TF(TypeBasics, NamedVariantOverTuple) { + auto t = f.Variant("V", f.Tuple("Inner", {{f.Void()}})); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant); + ASSERT_TRUE(t->IsVariant()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "V"); + ASSERT_FALSE(t->IsVariantOverStruct()); + ASSERT_TRUE(t->IsVariantOverTuple()); + ASSERT_EQ(t->GetUnderlyingType()->AsTuple()->GetName().GetRef(), "Inner"); + ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw()); +} + +TEST_TF(TypeBasics, Tagged) { + auto t = f.Tagged(f.Void(), "T"); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tagged); + ASSERT_TRUE(t->IsTagged()); + ASSERT_EQ(t->GetTag(), "T"); + ASSERT_TRUE(t->GetItemType()->IsVoid()); + ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw()); +} + +TEST_TF(TypeBasics, EmptyStructLookupByName) { + auto s = f.Struct({}); + + ASSERT_FALSE(s->HasMember("item")); + ASSERT_EQ(s->GetMemberIndex("item"), -1); + + UNIT_ASSERT_EXCEPTION_CONTAINS( + [=]() { + s->GetMember("item"); + }(), + NTi::TItemNotFound, "no item named 'item'"); +} + +TEST_TF(TypeBasics, StructLookupByName) { + auto s = f.Struct({ + {"a", f.Void()}, + {"field2", f.String()}, + {"field1", f.Utf8()}, + {"", f.Null()}, + }); + + ASSERT_TRUE(s->HasMember("a")); + ASSERT_EQ(s->GetMemberIndex("a"), 0); + + ASSERT_TRUE(s->HasMember("field2")); + ASSERT_EQ(s->GetMemberIndex("field2"), 1); + + ASSERT_TRUE(s->HasMember("field1")); + ASSERT_EQ(s->GetMemberIndex("field1"), 2); + + ASSERT_TRUE(s->HasMember("")); + ASSERT_EQ(s->GetMemberIndex(""), 3); + + ASSERT_FALSE(s->HasMember("b")); + ASSERT_EQ(s->GetMemberIndex("b"), -1); + + ASSERT_EQ(s->GetMember("a").GetName(), "a"); + ASSERT_TRUE(s->GetMember("a").GetTypeRaw()->IsVoid()); + + ASSERT_EQ(s->GetMember("field2").GetName(), "field2"); + ASSERT_TRUE(s->GetMember("field2").GetTypeRaw()->IsString()); + + ASSERT_EQ(s->GetMember("field1").GetName(), "field1"); + ASSERT_TRUE(s->GetMember("field1").GetTypeRaw()->IsUtf8()); + + ASSERT_EQ(s->GetMember("").GetName(), ""); + ASSERT_TRUE(s->GetMember("").GetTypeRaw()->IsNull()); + + UNIT_ASSERT_EXCEPTION_CONTAINS( + [=]() { + s->GetMember("b"); + }(), + NTi::TItemNotFound, "no item named 'b'"); +} diff --git a/library/cpp/type_info/ut/type_complexity_ut.cpp b/library/cpp/type_info/ut/type_complexity_ut.cpp new file mode 100644 index 0000000000..3b4f6e5372 --- /dev/null +++ b/library/cpp/type_info/ut/type_complexity_ut.cpp @@ -0,0 +1,33 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_complexity.h> +#include <library/cpp/type_info/type_constructors.h> + +using namespace NTi; + +TEST(TypeComplexity, Test) +{ + EXPECT_EQ(ComputeTypeComplexity(Int64()), 1); + EXPECT_EQ(ComputeTypeComplexity(String()), 1); + EXPECT_EQ(ComputeTypeComplexity(Null()), 1); + EXPECT_EQ(ComputeTypeComplexity(Decimal(4, 2)), 1); + + EXPECT_EQ(ComputeTypeComplexity(Optional(Utf8())), 2); + EXPECT_EQ(ComputeTypeComplexity(List(Json())), 2); + EXPECT_EQ(ComputeTypeComplexity(Tagged(String(), "jpeg")), 2); + EXPECT_EQ(ComputeTypeComplexity(Dict(String(), Optional(Int64()))), 4); + EXPECT_EQ(ComputeTypeComplexity(Struct({ + {"a", String()}, + {"b", List(Optional(Int64()))}, + })), 5); + EXPECT_EQ(ComputeTypeComplexity(Tuple({{Float()}, {Float()}})), 3); + + EXPECT_EQ(ComputeTypeComplexity(Variant(Struct({ + {"a", String()}, + {"b", Int64()}, + }))), 3); + EXPECT_EQ(ComputeTypeComplexity(Tuple({ + {String()}, + {Int64()}, + })), 3); +}
\ No newline at end of file diff --git a/library/cpp/type_info/ut/type_constraints.cpp b/library/cpp/type_info/ut/type_constraints.cpp new file mode 100644 index 0000000000..0f56bd89b0 --- /dev/null +++ b/library/cpp/type_info/ut/type_constraints.cpp @@ -0,0 +1,53 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include "utils.h" + +TEST_TF(TypeConstraints, DecimalScale) { + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + f.Decimal(20, 21); + }(), + NTi::TIllegalTypeException, "decimal scale 21 should be no greater than decimal precision 20"); +} + +TEST_TF(TypeConstraints, StructDuplicateItem) { + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + f.Struct({{"a", f.Void()}, {"a", f.String()}}); + }(), + NTi::TIllegalTypeException, "duplicate struct item 'a'"); + + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + f.Struct({{"a", f.Void()}, {"b", f.Bool()}, {"a", f.String()}}); + }(), + NTi::TIllegalTypeException, "duplicate struct item 'a'"); +} + +TEST_TF(TypeConstraints, StructEmpty) { + f.Struct({}); // empty structs are ok, this should not fail +} + +TEST_TF(TypeConstraints, TupleEmpty) { + f.Tuple({}); // empty tuples are ok, this should not fail +} + +TEST_TF(TypeConstraints, VariantStructEmpty) { + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + f.Variant(f.Struct({})); + }(), + NTi::TIllegalTypeException, "variant should contain at least one alternative"); +} + +TEST_TF(TypeConstraints, VariantTupleEmpty) { + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + f.Variant(f.Tuple({})); + }(), + NTi::TIllegalTypeException, "variant should contain at least one alternative"); +} + +TEST_TF(TypeConstraints, VariantWrongInnerType) { + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + f.Variant(f.String()); + }(), + NTi::TIllegalTypeException, "variants can only contain structs and tuples, got String instead"); +} diff --git a/library/cpp/type_info/ut/type_deserialize.cpp b/library/cpp/type_info/ut/type_deserialize.cpp new file mode 100644 index 0000000000..9e93a26bee --- /dev/null +++ b/library/cpp/type_info/ut/type_deserialize.cpp @@ -0,0 +1,528 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include <library/cpp/yson_pull/yson.h> + +#include "utils.h" + +TEST(TypeDeserialize, Void) { + ASSERT_DESERIALIZED_EQ(NTi::Void(), R"(void)"); + ASSERT_DESERIALIZED_EQ(NTi::Void(), R"({type_name=void})"); +} + +TEST(TypeDeserialize, Null) { + ASSERT_DESERIALIZED_EQ(NTi::Null(), R"(null)"); + ASSERT_DESERIALIZED_EQ(NTi::Null(), R"({type_name=null})"); +} + +TEST(TypeDeserialize, Bool) { + ASSERT_DESERIALIZED_EQ(NTi::Bool(), R"(bool)"); + ASSERT_DESERIALIZED_EQ(NTi::Bool(), R"({type_name=bool})"); +} + +TEST(TypeDeserialize, Int8) { + ASSERT_DESERIALIZED_EQ(NTi::Int8(), R"(int8)"); + ASSERT_DESERIALIZED_EQ(NTi::Int8(), R"({type_name=int8})"); +} + +TEST(TypeDeserialize, Int16) { + ASSERT_DESERIALIZED_EQ(NTi::Int16(), R"(int16)"); + ASSERT_DESERIALIZED_EQ(NTi::Int16(), R"({type_name=int16})"); +} + +TEST(TypeDeserialize, Int32) { + ASSERT_DESERIALIZED_EQ(NTi::Int32(), R"(int32)"); + ASSERT_DESERIALIZED_EQ(NTi::Int32(), R"({type_name=int32})"); +} + +TEST(TypeDeserialize, Int64) { + ASSERT_DESERIALIZED_EQ(NTi::Int64(), R"(int64)"); + ASSERT_DESERIALIZED_EQ(NTi::Int64(), R"({type_name=int64})"); +} + +TEST(TypeDeserialize, Uint8) { + ASSERT_DESERIALIZED_EQ(NTi::Uint8(), R"(uint8)"); + ASSERT_DESERIALIZED_EQ(NTi::Uint8(), R"({type_name=uint8})"); +} + +TEST(TypeDeserialize, Uint16) { + ASSERT_DESERIALIZED_EQ(NTi::Uint16(), R"(uint16)"); + ASSERT_DESERIALIZED_EQ(NTi::Uint16(), R"({type_name=uint16})"); +} + +TEST(TypeDeserialize, Uint32) { + ASSERT_DESERIALIZED_EQ(NTi::Uint32(), R"(uint32)"); + ASSERT_DESERIALIZED_EQ(NTi::Uint32(), R"({type_name=uint32})"); +} + +TEST(TypeDeserialize, Uint64) { + ASSERT_DESERIALIZED_EQ(NTi::Uint64(), R"(uint64)"); + ASSERT_DESERIALIZED_EQ(NTi::Uint64(), R"({type_name=uint64})"); +} + +TEST(TypeDeserialize, Float) { + ASSERT_DESERIALIZED_EQ(NTi::Float(), R"(float)"); + ASSERT_DESERIALIZED_EQ(NTi::Float(), R"({type_name=float})"); +} + +TEST(TypeDeserialize, Double) { + ASSERT_DESERIALIZED_EQ(NTi::Double(), R"(double)"); + ASSERT_DESERIALIZED_EQ(NTi::Double(), R"({type_name=double})"); +} + +TEST(TypeDeserialize, String) { + ASSERT_DESERIALIZED_EQ(NTi::String(), R"(string)"); + ASSERT_DESERIALIZED_EQ(NTi::String(), R"({type_name=string})"); +} + +TEST(TypeDeserialize, Utf8) { + ASSERT_DESERIALIZED_EQ(NTi::Utf8(), R"(utf8)"); + ASSERT_DESERIALIZED_EQ(NTi::Utf8(), R"({type_name=utf8})"); +} + +TEST(TypeDeserialize, Date) { + ASSERT_DESERIALIZED_EQ(NTi::Date(), R"(date)"); + ASSERT_DESERIALIZED_EQ(NTi::Date(), R"({type_name=date})"); +} + +TEST(TypeDeserialize, Datetime) { + ASSERT_DESERIALIZED_EQ(NTi::Datetime(), R"(datetime)"); + ASSERT_DESERIALIZED_EQ(NTi::Datetime(), R"({type_name=datetime})"); +} + +TEST(TypeDeserialize, Timestamp) { + ASSERT_DESERIALIZED_EQ(NTi::Timestamp(), R"(timestamp)"); + ASSERT_DESERIALIZED_EQ(NTi::Timestamp(), R"({type_name=timestamp})"); +} + +TEST(TypeDeserialize, TzDate) { + ASSERT_DESERIALIZED_EQ(NTi::TzDate(), R"(tz_date)"); + ASSERT_DESERIALIZED_EQ(NTi::TzDate(), R"({type_name=tz_date})"); +} + +TEST(TypeDeserialize, TzDatetime) { + ASSERT_DESERIALIZED_EQ(NTi::TzDatetime(), R"(tz_datetime)"); + ASSERT_DESERIALIZED_EQ(NTi::TzDatetime(), R"({type_name=tz_datetime})"); +} + +TEST(TypeDeserialize, TzTimestamp) { + ASSERT_DESERIALIZED_EQ(NTi::TzTimestamp(), R"(tz_timestamp)"); + ASSERT_DESERIALIZED_EQ(NTi::TzTimestamp(), R"({type_name=tz_timestamp})"); +} + +TEST(TypeDeserialize, Interval) { + ASSERT_DESERIALIZED_EQ(NTi::Interval(), R"(interval)"); + ASSERT_DESERIALIZED_EQ(NTi::Interval(), R"({type_name=interval})"); +} + +TEST(TypeDeserialize, Decimal) { + ASSERT_DESERIALIZED_EQ(NTi::Decimal(20, 10), R"({type_name=decimal; precision=20; scale=10})"); + ASSERT_DESERIALIZED_EQ(NTi::Decimal(20, 10), R"({scale=10; type_name=decimal; precision=20})"); + ASSERT_DESERIALIZED_EQ(NTi::Decimal(10, 10), R"({type_name=decimal; precision=10; scale=10})"); +} + +TEST(TypeDeserialize, DecimalMissingTypeParameters) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({precision=20; scale=10})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=decimal; scale=10})"); + }(), + NTi::TDeserializationException, R"(missing required key "precision")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=decimal; precision=20})"); + }(), + NTi::TDeserializationException, R"(missing required key "scale")"); +} + +TEST(TypeDeserialize, Json) { + ASSERT_DESERIALIZED_EQ(NTi::Json(), R"(json)"); + ASSERT_DESERIALIZED_EQ(NTi::Json(), R"({type_name=json})"); +} + +TEST(TypeDeserialize, Yson) { + ASSERT_DESERIALIZED_EQ(NTi::Yson(), R"(yson)"); + ASSERT_DESERIALIZED_EQ(NTi::Yson(), R"({type_name=yson})"); +} + +TEST(TypeDeserialize, Uuid) { + ASSERT_DESERIALIZED_EQ(NTi::Uuid(), R"(uuid)"); + ASSERT_DESERIALIZED_EQ(NTi::Uuid(), R"({type_name=uuid})"); +} + +TEST(TypeDeserialize, Optional) { + ASSERT_DESERIALIZED_EQ(NTi::Optional(NTi::Void()), R"({type_name=optional; item=void})"); + ASSERT_DESERIALIZED_EQ(NTi::Optional(NTi::String()), R"({type_name=optional; item=string})"); + ASSERT_DESERIALIZED_EQ(NTi::Optional(NTi::String()), R"({item=string; type_name=optional})"); +} + +TEST(TypeDeserialize, OptionalMissingTypeParameters) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({item=string})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=optional})"); + }(), + NTi::TDeserializationException, R"(missing required key "item")"); +} + +TEST(TypeDeserialize, List) { + ASSERT_DESERIALIZED_EQ(NTi::List(NTi::Void()), R"({type_name=list; item=void})"); + ASSERT_DESERIALIZED_EQ(NTi::List(NTi::String()), R"({type_name=list; item=string})"); + ASSERT_DESERIALIZED_EQ(NTi::List(NTi::String()), R"({item=string; type_name=list})"); +} + +TEST(TypeDeserialize, ListMissingTypeParameters) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({item=string})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=list})"); + }(), + NTi::TDeserializationException, R"(missing required key "item")"); +} + +TEST(TypeDeserialize, Dict) { + ASSERT_DESERIALIZED_EQ(NTi::Dict(NTi::Void(), NTi::Void()), R"({type_name=dict; key=void; value=void})"); + ASSERT_DESERIALIZED_EQ(NTi::Dict(NTi::Int32(), NTi::String()), R"({type_name=dict; key=int32; value=string})"); + ASSERT_DESERIALIZED_EQ(NTi::Dict(NTi::Int32(), NTi::String()), R"({key=int32; value=string; type_name=dict})"); + ASSERT_DESERIALIZED_EQ(NTi::Dict(NTi::Int32(), NTi::String()), R"({value=string; key=int32; type_name=dict})"); +} + +TEST(TypeDeserialize, DictMissingTypeParameters) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({key=string; value=string})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=dict})"); + }(), + NTi::TDeserializationException, R"(missing required keys "key" and "value")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=dict; value=string})"); + }(), + NTi::TDeserializationException, R"(missing required key "key")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=dict; key=string})"); + }(), + NTi::TDeserializationException, R"(missing required key "value")"); +} + +TEST(TypeDeserialize, StructEmpty) { + ASSERT_DESERIALIZED_EQ( + NTi::Struct({}), + R"({type_name=struct; members=[]})"); + ASSERT_DESERIALIZED_EQ( + NTi::Struct({}), + R"({members=[]; type_name=struct})"); +} + +TEST(TypeDeserialize, Struct) { + auto ty = NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}}); + + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=struct; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]; type_name=struct})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=struct; members=[{type=string; name=ItemB}; {name=ItemA; type={item=int64; type_name=list}}]})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=struct; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]; name=#})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=struct; name=#; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})"); +} + +TEST(TypeDeserialize, StructMissingTypeParameters) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({members=[]})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=struct})"); + }(), + NTi::TDeserializationException, R"(missing required key "members")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=struct; name=S})"); + }(), + NTi::TDeserializationException, R"(missing required key "members")"); +} + +TEST(TypeDeserialize, TupleEmpty) { + ASSERT_DESERIALIZED_EQ( + NTi::Tuple({}), + R"({type_name=tuple; elements=[]})"); + + ASSERT_DESERIALIZED_EQ( + NTi::Tuple({}), + R"({elements=[]; type_name=tuple})"); +} + +TEST(TypeDeserialize, Tuple) { + auto ty = NTi::Tuple({{NTi::String()}, {NTi::List(NTi::Int64())}}); + + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=tuple; elements=[{type=string}; {type={type_name=list; item=int64}}]})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({elements=[{type=string}; {type={item=int64; type_name=list}}]; type_name=tuple})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=tuple; name=#; elements=[{type=string}; {type={type_name=list; item=int64}}]})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=tuple; elements=[{type=string}; {type={type_name=list; item=int64}}]; name=#})"); +} + +TEST(TypeDeserialize, TupleMissingTypeParameters) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({elements=[]})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=tuple})"); + }(), + NTi::TDeserializationException, R"(missing required key "elements")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=tuple; name=T})"); + }(), + NTi::TDeserializationException, R"(missing required key "elements")"); +} + +TEST(TypeDeserialize, VariantStruct) { + auto ty = NTi::Variant(NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}})); + + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=variant; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]; type_name=variant})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=variant; name=#; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=variant; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]; name=#})"); +} + +TEST(TypeDeserialize, VariantTuple) { + auto ty = NTi::Variant(NTi::Tuple({{NTi::String()}, {NTi::List(NTi::Int64())}})); + + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=variant; elements=[{type=string}; {type={type_name=list; item=int64}}]})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({elements=[{type=string}; {type={type_name=list; item=int64}}]; type_name=variant})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=variant; name=#; elements=[{type=string}; {type={type_name=list; item=int64}}]})"); + ASSERT_DESERIALIZED_EQ( + ty, + R"({type_name=variant; elements=[{type=string}; {type={type_name=list; item=int64}}]; name=#})"); +} + +TEST(TypeDeserialize, VariantMissingTypeParameters) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({elements=[]})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({name=X})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=variant})"); + }(), + NTi::TDeserializationException, R"(missing both keys "members" and "elements")"); +} + +TEST(TypeDeserialize, Tagged) { + ASSERT_DESERIALIZED_EQ( + NTi::Tagged(NTi::String(), "Url"), + R"({type_name=tagged; tag=Url; item=string})"); + ASSERT_DESERIALIZED_EQ( + NTi::Tagged(NTi::String(), "Url"), + R"({type_name=tagged; item=string; tag=Url})"); + ASSERT_DESERIALIZED_EQ( + NTi::Tagged(NTi::String(), "Url"), + R"({item=string; tag=Url; type_name=tagged})"); +} + +TEST(TypeDeserialize, TaggedMissingTypeParameters) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({tag=T; item=string})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({tag=T})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=tagged; tag=T})"); + }(), + NTi::TDeserializationException, R"(missing required key "item")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=tagged; item=string})"); + }(), + NTi::TDeserializationException, R"(missing required key "tag")"); +} + +TEST(TypeDeserialize, ComplexTypeAsString) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(decimal)"); + }(), + NTi::TDeserializationException, R"(missing required keys "precision" and "scale")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(optional)"); + }(), + NTi::TDeserializationException, R"(missing required key "item")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(list)"); + }(), + NTi::TDeserializationException, R"(missing required key "item")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(dict)"); + }(), + NTi::TDeserializationException, R"(missing required keys "key" and "value")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(struct)"); + }(), + NTi::TDeserializationException, R"(missing required key "members")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(tuple)"); + }(), + NTi::TDeserializationException, R"(missing required key "elements")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(variant)"); + }(), + NTi::TDeserializationException, R"(missing both keys "members" and "elements")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(tagged)"); + }(), + NTi::TDeserializationException, R"(missing required keys "tag" and "item")"); +} + +TEST(TypeDeserialize, MissingTypeName) { + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({item=string})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({tag=Url})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + []() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({key=string; value=int32})"); + }(), + NTi::TDeserializationException, R"(missing required key "type_name")"); +} + +TEST(TypeDeserialize, UnknownKeys) { + auto tupleType = NTi::Tuple({{NTi::String()}, {NTi::List(NTi::Int64())}}); + + ASSERT_DESERIALIZED_EQ( + tupleType, + R"({type_name=tuple; unknown_key=<>0; elements=[{type=string}; {type={type_name=list; item=int64}}]})"); + + ASSERT_DESERIALIZED_EQ( + tupleType, + R"({type_name=tuple; elements=[{unknown_key={foo=<>0}; type=string}; {type={type_name=list; item=int64}}]})"); + + auto structType = NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}}); + ASSERT_DESERIALIZED_EQ( + structType, + R"({type_name=struct; unknown_key=[]; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})"); + ASSERT_DESERIALIZED_EQ( + structType, + R"({type_name=struct; members=[{name=ItemB; unknown_key=foo; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})"); + + auto utf8Type = NTi::Utf8(); + ASSERT_DESERIALIZED_EQ( + utf8Type, + R"({type_name=utf8; unknown_key=[];})"); +} + + +TEST(TypeDeserialize, DeepType) { + auto ty = TStringBuilder(); + for (size_t i = 0; i < 100; ++i) + ty << "{type_name=optional; item="; + ty << "string"; + for (size_t i = 0; i < 100; ++i) + ty << "}"; + + UNIT_ASSERT_EXCEPTION_CONTAINS([&ty]() { + NTi::NIo::DeserializeYson(*NTi::HeapFactory(), ty); + }(), + NTi::TDeserializationException, R"(too deep)"); +} + +TEST(TypeDeserialize, MultipleTypes) { + auto ty = "{type_name=optional; item=string}; {type_name=list; item=utf8}"; + + auto reader = NYsonPull::TReader(NYsonPull::NInput::FromMemory(ty), NYsonPull::EStreamType::ListFragment); + auto factory = NTi::PoolFactory(); + + { + auto ty1 = NTi::Optional(NTi::String()); + ASSERT_STRICT_EQ(NTi::NIo::DeserializeYsonMultipleRaw(*factory, reader), ty1.Get()); + } + + { + auto ty2 = NTi::List(NTi::Utf8()); + ASSERT_STRICT_EQ(NTi::NIo::DeserializeYsonMultipleRaw(*factory, reader), ty2.Get()); + } + + ASSERT_EQ(NTi::NIo::DeserializeYsonMultipleRaw(*factory, reader), nullptr); +} diff --git a/library/cpp/type_info/ut/type_equivalence.cpp b/library/cpp/type_info/ut/type_equivalence.cpp new file mode 100644 index 0000000000..ca697620ad --- /dev/null +++ b/library/cpp/type_info/ut/type_equivalence.cpp @@ -0,0 +1,394 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include "utils.h" + +TEST_TF(TypeEquivalence, StrictEqSelf) { + ASSERT_STRICT_EQ( + f.Void(), + f.Void()); + ASSERT_STRICT_EQ( + f.Bool(), + f.Bool()); + ASSERT_STRICT_EQ( + f.Int8(), + f.Int8()); + ASSERT_STRICT_EQ( + f.Int16(), + f.Int16()); + ASSERT_STRICT_EQ( + f.Int32(), + f.Int32()); + ASSERT_STRICT_EQ( + f.Int64(), + f.Int64()); + ASSERT_STRICT_EQ( + f.Uint8(), + f.Uint8()); + ASSERT_STRICT_EQ( + f.Uint16(), + f.Uint16()); + ASSERT_STRICT_EQ( + f.Uint32(), + f.Uint32()); + ASSERT_STRICT_EQ( + f.Uint64(), + f.Uint64()); + ASSERT_STRICT_EQ( + f.Float(), + f.Float()); + ASSERT_STRICT_EQ( + f.Double(), + f.Double()); + ASSERT_STRICT_EQ( + f.String(), + f.String()); + ASSERT_STRICT_EQ( + f.Utf8(), + f.Utf8()); + ASSERT_STRICT_EQ( + f.Date(), + f.Date()); + ASSERT_STRICT_EQ( + f.Datetime(), + f.Datetime()); + ASSERT_STRICT_EQ( + f.Timestamp(), + f.Timestamp()); + ASSERT_STRICT_EQ( + f.TzDate(), + f.TzDate()); + ASSERT_STRICT_EQ( + f.TzDatetime(), + f.TzDatetime()); + ASSERT_STRICT_EQ( + f.TzTimestamp(), + f.TzTimestamp()); + ASSERT_STRICT_EQ( + f.Interval(), + f.Interval()); + ASSERT_STRICT_EQ( + f.Decimal(20, 10), + f.Decimal(20, 10)); + ASSERT_STRICT_EQ( + f.Json(), + f.Json()); + ASSERT_STRICT_EQ( + f.Yson(), + f.Yson()); + ASSERT_STRICT_EQ( + f.Uuid(), + f.Uuid()); + ASSERT_STRICT_EQ( + f.Optional(f.Void()), + f.Optional(f.Void())); + ASSERT_STRICT_EQ( + f.List(f.Void()), + f.List(f.Void())); + ASSERT_STRICT_EQ( + f.Dict(f.Void(), f.String()), + f.Dict(f.Void(), f.String())); + ASSERT_STRICT_EQ( + f.Struct({}), + f.Struct({})); + ASSERT_STRICT_EQ( + f.Struct("S", {}), + f.Struct("S", {})); + ASSERT_STRICT_EQ( + f.Struct({{"a", f.Int64()}, {"b", f.Int8()}}), + f.Struct({{"a", f.Int64()}, {"b", f.Int8()}})); + ASSERT_STRICT_EQ( + f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}}), + f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}})); + ASSERT_STRICT_EQ( + f.Tuple({}), + f.Tuple({})); + ASSERT_STRICT_EQ( + f.Tuple("T", {}), + f.Tuple("T", {})); + ASSERT_STRICT_EQ( + f.Tuple({{f.Int64()}, {f.Int8()}}), + f.Tuple({{f.Int64()}, {f.Int8()}})); + ASSERT_STRICT_EQ( + f.Tuple("T", {{f.Int64()}, {f.Int8()}}), + f.Tuple("T", {{f.Int64()}, {f.Int8()}})); + ASSERT_STRICT_EQ( + f.Variant(f.Struct("Inner", {{"x", f.Void()}})), + f.Variant(f.Struct("Inner", {{"x", f.Void()}}))); + ASSERT_STRICT_EQ( + f.Variant("V", f.Struct("Inner", {{"x", f.Void()}})), + f.Variant("V", f.Struct("Inner", {{"x", f.Void()}}))); + ASSERT_STRICT_EQ( + f.Variant(f.Tuple("Inner", {{f.Void()}})), + f.Variant(f.Tuple("Inner", {{f.Void()}}))); + ASSERT_STRICT_EQ( + f.Variant("V", f.Tuple("Inner", {{f.Void()}})), + f.Variant("V", f.Tuple("Inner", {{f.Void()}}))); + ASSERT_STRICT_EQ( + f.Tagged(f.Void(), "T"), + f.Tagged(f.Void(), "T")); +} + +TEST_TF(TypeEquivalence, StrictNeOtherType) { + ASSERT_STRICT_NE( + f.Void(), + f.Bool()); + ASSERT_STRICT_NE( + f.Bool(), + f.Int8()); + ASSERT_STRICT_NE( + f.Int8(), + f.Int16()); + ASSERT_STRICT_NE( + f.Int16(), + f.Int32()); + ASSERT_STRICT_NE( + f.Int32(), + f.Int64()); + ASSERT_STRICT_NE( + f.Int64(), + f.Uint8()); + ASSERT_STRICT_NE( + f.Uint8(), + f.Uint16()); + ASSERT_STRICT_NE( + f.Uint16(), + f.Uint32()); + ASSERT_STRICT_NE( + f.Uint32(), + f.Uint64()); + ASSERT_STRICT_NE( + f.Uint64(), + f.Float()); + ASSERT_STRICT_NE( + f.Float(), + f.Double()); + ASSERT_STRICT_NE( + f.Double(), + f.String()); + ASSERT_STRICT_NE( + f.String(), + f.Utf8()); + ASSERT_STRICT_NE( + f.Utf8(), + f.Date()); + ASSERT_STRICT_NE( + f.Date(), + f.Datetime()); + ASSERT_STRICT_NE( + f.Datetime(), + f.Timestamp()); + ASSERT_STRICT_NE( + f.Timestamp(), + f.TzDate()); + ASSERT_STRICT_NE( + f.TzDate(), + f.TzDatetime()); + ASSERT_STRICT_NE( + f.TzDatetime(), + f.TzTimestamp()); + ASSERT_STRICT_NE( + f.TzTimestamp(), + f.Interval()); + ASSERT_STRICT_NE( + f.Interval(), + f.Decimal(20, 10)); + ASSERT_STRICT_NE( + f.Decimal(20, 10), + f.Json()); + ASSERT_STRICT_NE( + f.Json(), + f.Yson()); + ASSERT_STRICT_NE( + f.Yson(), + f.Uuid()); + ASSERT_STRICT_NE( + f.Uuid(), + f.Optional(f.Void())); + ASSERT_STRICT_NE( + f.Optional(f.Void()), + f.List(f.Void())); + ASSERT_STRICT_NE( + f.List(f.Void()), + f.Dict(f.Void(), f.String())); + ASSERT_STRICT_NE( + f.Dict(f.Void(), f.String()), + f.Struct({})); + ASSERT_STRICT_NE( + f.Struct({}), + f.Struct("S", {})); + ASSERT_STRICT_NE( + f.Struct("S", {}), + f.Struct({{"a", f.Int64()}, {"b", f.Int8()}})); + ASSERT_STRICT_NE( + f.Struct({{"a", f.Int64()}, {"b", f.Int8()}}), + f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}})); + ASSERT_STRICT_NE( + f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}}), + f.Tuple({})); + ASSERT_STRICT_NE( + f.Tuple({}), + f.Tuple("T", {})); + ASSERT_STRICT_NE( + f.Tuple("T", {}), + f.Tuple({{f.Int64()}, {f.Int8()}})); + ASSERT_STRICT_NE( + f.Tuple({{f.Int64()}, {f.Int8()}}), + f.Tuple("T", {{f.Int64()}, {f.Int8()}})); + ASSERT_STRICT_NE( + f.Tuple("T", {{f.Int64()}, {f.Int8()}}), + f.Variant(f.Struct("Inner", {{"x", f.Void()}}))); + ASSERT_STRICT_NE( + f.Variant(f.Struct("Inner", {{"x", f.Void()}})), + f.Variant("V", f.Struct("Inner", {{"x", f.Void()}}))); + ASSERT_STRICT_NE( + f.Variant("V", f.Struct("Inner", {{"x", f.Void()}})), + f.Variant(f.Tuple("Inner", {{f.Void()}}))); + ASSERT_STRICT_NE( + f.Variant(f.Tuple("Inner", {{f.Void()}})), + f.Variant("V", f.Tuple("Inner", {{f.Void()}}))); + ASSERT_STRICT_NE( + f.Variant("V", f.Tuple("Inner", {{f.Void()}})), + f.Tagged(f.Void(), "T")); + ASSERT_STRICT_NE( + f.Tagged(f.Void(), "T"), + f.Void()); +} + +TEST_TF(TypeEquivalence, StrictNeDecimal) { + ASSERT_STRICT_NE( + f.Decimal(20, 10), + f.Decimal(21, 10)); + ASSERT_STRICT_NE( + f.Decimal(20, 10), + f.Decimal(20, 11)); +} + +TEST_TF(TypeEquivalence, StrictNeStruct) { + ASSERT_STRICT_NE( + f.Struct({}), + f.Struct("", {})); + ASSERT_STRICT_NE( + f.Struct("name", {}), + f.Struct("other name", {})); + ASSERT_STRICT_NE( + f.Struct({}), + f.Struct({{"x", f.Void()}})); + ASSERT_STRICT_NE( + f.Struct({{"x", f.Void()}}), + f.Struct({{"y", f.Void()}})); + ASSERT_STRICT_NE( + f.Struct({{"x", f.Void()}}), + f.Struct({{"x", f.String()}})); + ASSERT_STRICT_NE( + f.Struct("name", {}), + f.Struct("name", {{"x", f.Void()}})); + ASSERT_STRICT_NE( + f.Struct("name", {{"x", f.Void()}}), + f.Struct("name", {{"y", f.Void()}})); + ASSERT_STRICT_NE( + f.Struct("name", {{"x", f.Void()}}), + f.Struct("name", {{"x", f.String()}})); + ASSERT_STRICT_NE( + f.Struct({{"x", f.Void()}, {"y", f.String()}}), + f.Struct({{"x", f.String()}, {"y", f.Void()}})); + ASSERT_STRICT_NE( + f.Struct({{"x", f.Void()}, {"y", f.Void()}}), + f.Struct({{"y", f.Void()}, {"x", f.Void()}})); + ASSERT_STRICT_NE( + f.Struct({{"x", f.Void()}, {"y", f.Void()}}), + f.Struct({{"x", f.Void()}, {"y", f.Void()}, {"z", f.Void()}})); +} + +TEST_TF(TypeEquivalence, StrictNeTuple) { + ASSERT_STRICT_NE( + f.Tuple({}), + f.Tuple("", {})); + ASSERT_STRICT_NE( + f.Tuple("name", {}), + f.Tuple("other name", {})); + ASSERT_STRICT_NE( + f.Tuple({}), + f.Tuple({{f.Void()}})); + ASSERT_STRICT_NE( + f.Tuple({{f.Void()}}), + f.Tuple({{f.String()}})); + ASSERT_STRICT_NE( + f.Tuple("name", {}), + f.Tuple("name", {{f.Void()}})); + ASSERT_STRICT_NE( + f.Tuple("name", {{f.Void()}}), + f.Tuple("name", {{f.String()}})); + ASSERT_STRICT_NE( + f.Tuple({{f.String()}, {f.Void()}}), + f.Tuple({{f.Void()}, {f.String()}})); + ASSERT_STRICT_NE( + f.Tuple({{f.Void()}, {f.Void()}}), + f.Tuple({{f.Void()}, {f.Void()}, {f.Void()}})); +} + +TEST_TF(TypeEquivalence, StrictNeVariant) { + ASSERT_STRICT_NE( + f.Variant(f.Tuple({{f.Void()}})), + f.Variant("", f.Tuple({{f.Void()}}))); + ASSERT_STRICT_NE( + f.Variant("", f.Tuple({{f.Void()}})), + f.Variant("X", f.Tuple({{f.Void()}}))); + ASSERT_STRICT_NE( + f.Variant(f.Tuple({{f.Void()}})), + f.Variant(f.Tuple({{f.String()}}))); + ASSERT_STRICT_NE( + f.Variant("X", f.Tuple({{f.Utf8()}})), + f.Variant("X", f.Tuple({{f.String()}}))); + ASSERT_STRICT_NE( + f.Variant(f.Tuple({{f.Utf8()}})), + f.Variant(f.Struct({{"_", f.Utf8()}}))); + ASSERT_STRICT_NE( + f.Variant(f.Struct({{"item1", f.String()}})), + f.Variant(f.Struct({{"item2", f.String()}}))); + ASSERT_STRICT_NE( + f.Variant("X", f.Struct({{"item2", f.String()}})), + f.Variant("X", f.Struct({{"item1", f.String()}}))); +} + +TEST_TF(TypeEquivalence, StrictNeTagged) { + ASSERT_STRICT_NE( + f.Tagged(f.String(), "Tag"), + f.Tagged(f.String(), "Other tag")); + ASSERT_STRICT_NE( + f.Tagged(f.String(), "Tag"), + f.Tagged(f.Utf8(), "Tag")); +} + +TEST_TF(TypeEquivalence, StrictNeDeep) { + auto t1 = f.Struct({ + {"i", f.Optional(f.String())}, + {"don't", f.Utf8()}, + {"have", f.Tuple("GeoCoordinates", {{f.Float()}, {f.Float()}})}, + {"enough", f.List(f.Optional(f.String()))}, + {"fantasy", f.Dict(f.String(), f.List(f.Utf8()))}, // < difference + {"to", f.Yson()}, + {"think", f.Optional(f.Json())}, + {"of", f.Optional(f.String())}, + {"a", f.List(f.Tuple({{f.String()}, {f.Tuple("GeoCoordinates", {{f.Float()}, {f.Float()}})}}))}, + {"meaningful", f.Optional(f.String())}, + {"example", f.Dict(f.Optional(f.Decimal(10, 5)), f.List(f.Dict(f.Int32(), f.Float())))}, + }); + + auto t2 = f.Struct({ + {"i", f.Optional(f.String())}, + {"don't", f.Utf8()}, + {"have", f.Tuple("GeoCoordinates", {{f.Float()}, {f.Float()}})}, + {"enough", f.List(f.Optional(f.String()))}, + {"fantasy", f.Dict(f.String(), f.List(f.String()))}, // < difference + {"to", f.Yson()}, + {"think", f.Optional(f.Json())}, + {"of", f.Optional(f.String())}, + {"a", f.List(f.Tuple({{f.String()}, {f.Tuple("GeoCoordinates", {{f.Float()}, {f.Float()}})}}))}, + {"meaningful", f.Optional(f.String())}, + {"example", f.Dict(f.Optional(f.Decimal(10, 5)), f.List(f.Dict(f.Int32(), f.Float())))}, + }); + + ASSERT_STRICT_NE(t1, t2); +} diff --git a/library/cpp/type_info/ut/type_factory.cpp b/library/cpp/type_info/ut/type_factory.cpp new file mode 100644 index 0000000000..0ab44129ad --- /dev/null +++ b/library/cpp/type_info/ut/type_factory.cpp @@ -0,0 +1,121 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +TEST(TypeFactory, AdoptPoolToPool) { + auto f1 = NTi::PoolFactory(); + auto f2 = NTi::PoolFactory(); + + auto t = f1->Optional(f1->List(f1->String())); + auto ta = f2->Adopt(t); + + ASSERT_NE(t.Get(), ta.Get()); + ASSERT_NE(t->GetItemTypeRaw(), ta->GetItemTypeRaw()); + ASSERT_EQ(t->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw(), ta->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw()); + + f1 = nullptr; + f2 = nullptr; + t = nullptr; + + // `ta` is still alive + + ASSERT_TRUE(ta->IsOptional()); + ASSERT_TRUE(ta->GetItemTypeRaw()->IsList()); +} + +TEST(TypeFactory, AdoptPoolToSamePool) { + auto f = NTi::PoolFactory(); + + auto t = f->Optional(f->List(f->String())); + auto ta = f->Adopt(t); + + ASSERT_EQ(t.Get(), ta.Get()); +} + +TEST(TypeFactory, AdoptHeapToHeap) { + auto f = NTi::HeapFactory(); + + auto t = f->Optional(f->List(f->String())); + auto ta = f->Adopt(t); + + ASSERT_EQ(t.Get(), ta.Get()); +} + +TEST(TypeFactory, AdoptHeapToPool) { + auto f1 = NTi::HeapFactory(); + auto f2 = NTi::PoolFactory(); + + auto t = f1->Optional(f1->List(f1->String())); + auto ta = f2->Adopt(t); + + ASSERT_NE(t.Get(), ta.Get()); + ASSERT_NE(t->GetItemTypeRaw(), ta->GetItemTypeRaw()); + ASSERT_EQ(t->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw(), ta->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw()); + + f1 = nullptr; + f2 = nullptr; + t = nullptr; + + // `ta` is still alive + + ASSERT_TRUE(ta->IsOptional()); + ASSERT_TRUE(ta->GetItemTypeRaw()->IsList()); +} + +TEST(TypeFactory, AdoptPoolToHeap) { + auto f1 = NTi::PoolFactory(); + auto f2 = NTi::HeapFactory(); + + auto t = f1->Optional(f1->List(f1->String())); + auto ta = f2->Adopt(t); + + ASSERT_NE(t.Get(), ta.Get()); + ASSERT_NE(t->GetItemTypeRaw(), ta->GetItemTypeRaw()); + ASSERT_EQ(t->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw(), ta->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw()); + + f1 = nullptr; + f2 = nullptr; + t = nullptr; + + // `ta` is still alive + + ASSERT_TRUE(ta->IsOptional()); + ASSERT_TRUE(ta->GetItemTypeRaw()->IsList()); +} + +TEST(TypeFactory, AdoptStaticToPool) { + auto f = NTi::PoolFactory(); + + auto t = NTi::Void(); + auto ta = f->Adopt(t); + + ASSERT_EQ(t.Get(), ta.Get()); +} + +TEST(TypeFactory, AdoptStaticToHeap) { + auto f = NTi::HeapFactory(); + + auto t = NTi::Void(); + auto ta = f->Adopt(t); + ASSERT_EQ(t.Get(), ta.Get()); +} + +TEST(TypeFactory, Dedup) { + { + auto f = NTi::PoolFactory(/* deduplicate = */ false); + + auto a = f->OptionalRaw(f->StringRaw()); + auto b = f->OptionalRaw(f->StringRaw()); + + ASSERT_NE(a, b); + } + + { + auto f = NTi::PoolFactory(/* deduplicate = */ true); + + auto a = f->OptionalRaw(f->StringRaw()); + auto b = f->OptionalRaw(f->StringRaw()); + + ASSERT_EQ(a, b); + } +} diff --git a/library/cpp/type_info/ut/type_factory_raw.cpp b/library/cpp/type_info/ut/type_factory_raw.cpp new file mode 100644 index 0000000000..37f5d71aa7 --- /dev/null +++ b/library/cpp/type_info/ut/type_factory_raw.cpp @@ -0,0 +1,365 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +TEST(TypeFactoryRaw, Void) { + auto f = NTi::PoolFactory(false); + auto t = f->VoidRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Void); + ASSERT_TRUE(t->IsVoid()); +} + +TEST(TypeFactoryRaw, Bool) { + auto f = NTi::PoolFactory(false); + auto t = f->BoolRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Bool); + ASSERT_TRUE(t->IsBool()); +} + +TEST(TypeFactoryRaw, Int8) { + auto f = NTi::PoolFactory(false); + auto t = f->Int8Raw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int8); + ASSERT_TRUE(t->IsInt8()); +} + +TEST(TypeFactoryRaw, Int16) { + auto f = NTi::PoolFactory(false); + auto t = f->Int16Raw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int16); + ASSERT_TRUE(t->IsInt16()); +} + +TEST(TypeFactoryRaw, Int32) { + auto f = NTi::PoolFactory(false); + auto t = f->Int32Raw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int32); + ASSERT_TRUE(t->IsInt32()); +} + +TEST(TypeFactoryRaw, Int64) { + auto f = NTi::PoolFactory(false); + auto t = f->Int64Raw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int64); + ASSERT_TRUE(t->IsInt64()); +} + +TEST(TypeFactoryRaw, Uint8) { + auto f = NTi::PoolFactory(false); + auto t = f->Uint8Raw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint8); + ASSERT_TRUE(t->IsUint8()); +} + +TEST(TypeFactoryRaw, Uint16) { + auto f = NTi::PoolFactory(false); + auto t = f->Uint16Raw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint16); + ASSERT_TRUE(t->IsUint16()); +} + +TEST(TypeFactoryRaw, Uint32) { + auto f = NTi::PoolFactory(false); + auto t = f->Uint32Raw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint32); + ASSERT_TRUE(t->IsUint32()); +} + +TEST(TypeFactoryRaw, Uint64) { + auto f = NTi::PoolFactory(false); + auto t = f->Uint64Raw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint64); + ASSERT_TRUE(t->IsUint64()); +} + +TEST(TypeFactoryRaw, Float) { + auto f = NTi::PoolFactory(false); + auto t = f->FloatRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Float); + ASSERT_TRUE(t->IsFloat()); +} + +TEST(TypeFactoryRaw, Double) { + auto f = NTi::PoolFactory(false); + auto t = f->DoubleRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Double); + ASSERT_TRUE(t->IsDouble()); +} + +TEST(TypeFactoryRaw, String) { + auto f = NTi::PoolFactory(false); + auto t = f->StringRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::String); + ASSERT_TRUE(t->IsString()); +} + +TEST(TypeFactoryRaw, Utf8) { + auto f = NTi::PoolFactory(false); + auto t = f->Utf8Raw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Utf8); + ASSERT_TRUE(t->IsUtf8()); +} + +TEST(TypeFactoryRaw, Date) { + auto f = NTi::PoolFactory(false); + auto t = f->DateRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Date); + ASSERT_TRUE(t->IsDate()); +} + +TEST(TypeFactoryRaw, Datetime) { + auto f = NTi::PoolFactory(false); + auto t = f->DatetimeRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Datetime); + ASSERT_TRUE(t->IsDatetime()); +} + +TEST(TypeFactoryRaw, Timestamp) { + auto f = NTi::PoolFactory(false); + auto t = f->TimestampRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Timestamp); + ASSERT_TRUE(t->IsTimestamp()); +} + +TEST(TypeFactoryRaw, TzDate) { + auto f = NTi::PoolFactory(false); + auto t = f->TzDateRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzDate); + ASSERT_TRUE(t->IsTzDate()); +} + +TEST(TypeFactoryRaw, TzDatetime) { + auto f = NTi::PoolFactory(false); + auto t = f->TzDatetimeRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzDatetime); + ASSERT_TRUE(t->IsTzDatetime()); +} + +TEST(TypeFactoryRaw, TzTimestamp) { + auto f = NTi::PoolFactory(false); + auto t = f->TzTimestampRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzTimestamp); + ASSERT_TRUE(t->IsTzTimestamp()); +} + +TEST(TypeFactoryRaw, Interval) { + auto f = NTi::PoolFactory(false); + auto t = f->IntervalRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Interval); + ASSERT_TRUE(t->IsInterval()); +} + +TEST(TypeFactoryRaw, Decimal) { + auto f = NTi::PoolFactory(false); + auto t = f->DecimalRaw(20, 10); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Decimal); + ASSERT_TRUE(t->IsDecimal()); + ASSERT_EQ(t->GetPrecision(), 20); + ASSERT_EQ(t->GetScale(), 10); +} + +TEST(TypeFactoryRaw, Json) { + auto f = NTi::PoolFactory(false); + auto t = f->JsonRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Json); + ASSERT_TRUE(t->IsJson()); +} + +TEST(TypeFactoryRaw, Yson) { + auto f = NTi::PoolFactory(false); + auto t = f->YsonRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Yson); + ASSERT_TRUE(t->IsYson()); +} + +TEST(TypeFactoryRaw, Uuid) { + auto f = NTi::PoolFactory(false); + auto t = f->UuidRaw(); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uuid); + ASSERT_TRUE(t->IsUuid()); +} + +TEST(TypeFactoryRaw, Optional) { + auto f = NTi::PoolFactory(false); + auto t = f->OptionalRaw(f->VoidRaw()); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Optional); + ASSERT_TRUE(t->IsOptional()); + ASSERT_TRUE(t->GetItemType()->IsVoid()); + ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw()); +} + +TEST(TypeFactoryRaw, List) { + auto f = NTi::PoolFactory(false); + auto t = f->ListRaw(f->VoidRaw()); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::List); + ASSERT_TRUE(t->IsList()); + ASSERT_TRUE(t->GetItemType()->IsVoid()); + ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw()); +} + +TEST(TypeFactoryRaw, Dict) { + auto f = NTi::PoolFactory(false); + auto t = f->DictRaw(f->VoidRaw(), f->StringRaw()); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Dict); + ASSERT_TRUE(t->IsDict()); + ASSERT_TRUE(t->GetKeyType()->IsVoid()); + ASSERT_EQ(t->GetKeyType().Get(), t->GetKeyTypeRaw()); + ASSERT_TRUE(t->GetValueType()->IsString()); + ASSERT_EQ(t->GetValueType().Get(), t->GetValueTypeRaw()); +} + +TEST(TypeFactoryRaw, EmptyStruct) { + auto f = NTi::PoolFactory(false); + auto t = f->StructRaw({}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct); + ASSERT_TRUE(t->IsStruct()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_EQ(t->GetMembers().size(), 0); +} + +TEST(TypeFactoryRaw, NamedEmptyStruct) { + auto f = NTi::PoolFactory(false); + auto t = f->StructRaw("S", {}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct); + ASSERT_TRUE(t->IsStruct()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "S"); + ASSERT_EQ(t->GetMembers().size(), 0); +} + +TEST(TypeFactoryRaw, Struct) { + auto f = NTi::PoolFactory(false); + auto t = f->StructRaw({{"a", f->Int64Raw()}, {"b", f->Int8Raw()}}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct); + ASSERT_TRUE(t->IsStruct()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_EQ(t->GetMembers().size(), 2); + ASSERT_EQ(t->GetMembers()[0].GetName(), "a"); + ASSERT_TRUE(t->GetMembers()[0].GetTypeRaw()->IsInt64()); + ASSERT_EQ(t->GetMembers()[0].GetType().Get(), t->GetMembers()[0].GetTypeRaw()); + ASSERT_EQ(t->GetMembers()[1].GetName(), "b"); + ASSERT_TRUE(t->GetMembers()[1].GetTypeRaw()->IsInt8()); + ASSERT_EQ(t->GetMembers()[1].GetType().Get(), t->GetMembers()[1].GetTypeRaw()); +} + +TEST(TypeFactoryRaw, NamedStruct) { + auto f = NTi::PoolFactory(false); + auto t = f->StructRaw("S", {{"a", f->Int64Raw()}, {"b", f->Int8Raw()}}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct); + ASSERT_TRUE(t->IsStruct()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "S"); + ASSERT_EQ(t->GetMembers().size(), 2); + ASSERT_EQ(t->GetMembers()[0].GetName(), "a"); + ASSERT_TRUE(t->GetMembers()[0].GetTypeRaw()->IsInt64()); + ASSERT_EQ(t->GetMembers()[0].GetType().Get(), t->GetMembers()[0].GetTypeRaw()); + ASSERT_EQ(t->GetMembers()[1].GetName(), "b"); + ASSERT_TRUE(t->GetMembers()[1].GetTypeRaw()->IsInt8()); + ASSERT_EQ(t->GetMembers()[1].GetType().Get(), t->GetMembers()[1].GetTypeRaw()); +} + +TEST(TypeFactoryRaw, EmptyTuple) { + auto f = NTi::PoolFactory(false); + auto t = f->TupleRaw({}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple); + ASSERT_TRUE(t->IsTuple()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_EQ(t->GetElements().size(), 0); +} + +TEST(TypeFactoryRaw, NamedEmptyTuple) { + auto f = NTi::PoolFactory(false); + auto t = f->TupleRaw("T", {}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple); + ASSERT_TRUE(t->IsTuple()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "T"); + ASSERT_EQ(t->GetElements().size(), 0); +} + +TEST(TypeFactoryRaw, Tuple) { + auto f = NTi::PoolFactory(false); + auto t = f->TupleRaw({{f->Int64Raw()}, {f->Int8Raw()}}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple); + ASSERT_TRUE(t->IsTuple()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_EQ(t->GetElements().size(), 2); + ASSERT_TRUE(t->GetElements()[0].GetTypeRaw()->IsInt64()); + ASSERT_EQ(t->GetElements()[0].GetType().Get(), t->GetElements()[0].GetTypeRaw()); + ASSERT_TRUE(t->GetElements()[1].GetTypeRaw()->IsInt8()); + ASSERT_EQ(t->GetElements()[1].GetType().Get(), t->GetElements()[1].GetTypeRaw()); +} + +TEST(TypeFactoryRaw, NamedTuple) { + auto f = NTi::PoolFactory(false); + auto t = f->TupleRaw("T", {{f->Int64Raw()}, {f->Int8Raw()}}); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple); + ASSERT_TRUE(t->IsTuple()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "T"); + ASSERT_EQ(t->GetElements().size(), 2); + ASSERT_TRUE(t->GetElements()[0].GetTypeRaw()->IsInt64()); + ASSERT_EQ(t->GetElements()[0].GetType().Get(), t->GetElements()[0].GetTypeRaw()); + ASSERT_TRUE(t->GetElements()[1].GetTypeRaw()->IsInt8()); + ASSERT_EQ(t->GetElements()[1].GetType().Get(), t->GetElements()[1].GetTypeRaw()); +} + +TEST(TypeFactoryRaw, VariantOverStruct) { + auto f = NTi::PoolFactory(false); + auto t = f->VariantRaw(f->StructRaw("Inner", {{"x", f->VoidRaw()}})); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant); + ASSERT_TRUE(t->IsVariant()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_TRUE(t->IsVariantOverStruct()); + ASSERT_FALSE(t->IsVariantOverTuple()); + ASSERT_EQ(t->GetUnderlyingType()->AsStruct()->GetName().GetRef(), "Inner"); + ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw()); +} + +TEST(TypeFactoryRaw, NamedVariantOverStruct) { + auto f = NTi::PoolFactory(false); + auto t = f->VariantRaw("V", f->StructRaw("Inner", {{"x", f->VoidRaw()}})); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant); + ASSERT_TRUE(t->IsVariant()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "V"); + ASSERT_TRUE(t->IsVariantOverStruct()); + ASSERT_FALSE(t->IsVariantOverTuple()); + ASSERT_EQ(t->GetUnderlyingType()->AsStruct()->GetName().GetRef(), "Inner"); + ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw()); +} + +TEST(TypeFactoryRaw, VariantOverTuple) { + auto f = NTi::PoolFactory(false); + auto t = f->VariantRaw(f->TupleRaw("Inner", {{f->VoidRaw()}})); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant); + ASSERT_TRUE(t->IsVariant()); + ASSERT_FALSE(t->GetName().Defined()); + ASSERT_FALSE(t->IsVariantOverStruct()); + ASSERT_TRUE(t->IsVariantOverTuple()); + ASSERT_EQ(t->GetUnderlyingType()->AsTuple()->GetName().GetRef(), "Inner"); + ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw()); +} + +TEST(TypeFactoryRaw, NamedVariantOverTuple) { + auto f = NTi::PoolFactory(false); + auto t = f->VariantRaw("V", f->TupleRaw("Inner", {{f->VoidRaw()}})); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant); + ASSERT_TRUE(t->IsVariant()); + ASSERT_TRUE(t->GetName().Defined()); + ASSERT_EQ(t->GetName().GetRef(), "V"); + ASSERT_FALSE(t->IsVariantOverStruct()); + ASSERT_TRUE(t->IsVariantOverTuple()); + ASSERT_EQ(t->GetUnderlyingType()->AsTuple()->GetName().GetRef(), "Inner"); + ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw()); +} + +TEST(TypeFactoryRaw, Tagged) { + auto f = NTi::PoolFactory(false); + auto t = f->TaggedRaw(f->VoidRaw(), "T"); + ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tagged); + ASSERT_TRUE(t->IsTagged()); + ASSERT_EQ(t->GetTag(), "T"); + ASSERT_TRUE(t->GetItemType()->IsVoid()); + ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw()); +} diff --git a/library/cpp/type_info/ut/type_io.cpp b/library/cpp/type_info/ut/type_io.cpp new file mode 100644 index 0000000000..4feb9b7d83 --- /dev/null +++ b/library/cpp/type_info/ut/type_io.cpp @@ -0,0 +1,535 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include "utils.h" + +TEST_TF(TypeIO, AsYqlType) { + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Void().Get()), + "[VoidType]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Null().Get()), + "[NullType]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Bool().Get()), + "[DataType; Bool]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Int8().Get()), + "[DataType; Int8]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Int16().Get()), + "[DataType; Int16]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Int32().Get()), + "[DataType; Int32]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Int64().Get()), + "[DataType; Int64]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Uint8().Get()), + "[DataType; Uint8]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Uint16().Get()), + "[DataType; Uint16]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Uint32().Get()), + "[DataType; Uint32]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Uint64().Get()), + "[DataType; Uint64]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Float().Get()), + "[DataType; Float]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Double().Get()), + "[DataType; Double]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.String().Get()), + "[DataType; String]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Utf8().Get()), + "[DataType; Utf8]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Date().Get()), + "[DataType; Date]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Datetime().Get()), + "[DataType; Datetime]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Timestamp().Get()), + "[DataType; Timestamp]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.TzDate().Get()), + "[DataType; TzDate]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.TzDatetime().Get()), + "[DataType; TzDatetime]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.TzTimestamp().Get()), + "[DataType; TzTimestamp]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Interval().Get()), + "[DataType; Interval]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Json().Get()), + "[DataType; Json]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Yson().Get()), + "[DataType; Yson]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Uuid().Get()), + "[DataType; Uuid]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Decimal(20, 10).Get()), + "[DataType; Decimal; \"20\"; \"10\"]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Decimal(35, 35).Get()), + "[DataType; Decimal; \"35\"; \"35\"]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(Optional(f.Bool()).Get()), + "[OptionalType; [DataType; Bool]]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.List(f.Bool()).Get()), + "[ListType; [DataType; Bool]]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Dict(f.Bool(), f.Int32()).Get()), + "[DictType; [DataType; Bool]; [DataType; Int32]]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Struct({}).Get()), + "[StructType; []]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Struct({{"a", f.Bool()}}).Get()), + "[StructType; [[a; [DataType; Bool]]]]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Struct({{"a", f.Yson()}, {"b", f.Bool()}}).Get()), + "[StructType; [[a; [DataType; Yson]]; [b; [DataType; Bool]]]]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Struct({{"a", f.Int32()}, {"b", f.Int32()}, {"c", f.Int64()}}).Get()), + "[StructType; [[a; [DataType; Int32]]; [b; [DataType; Int32]]; [c; [DataType; Int64]]]]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Tuple({}).Get()), + "[TupleType; []]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Tuple({{f.Bool()}}).Get()), + "[TupleType; [[DataType; Bool]]]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Tuple({{f.Yson()}, {f.Bool()}}).Get()), + "[TupleType; [[DataType; Yson]; [DataType; Bool]]]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Tuple({{f.Int32()}, {f.Int32()}, {f.Int64()}}).Get()), + "[TupleType; [[DataType; Int32]; [DataType; Int32]; [DataType; Int64]]]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Variant(f.Struct({{"a", f.Bool()}})).Get()), + "[VariantType; [StructType; [[a; [DataType; Bool]]]]]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Variant(f.Struct({{"a", f.Yson()}, {"b", f.Bool()}})).Get()), + "[VariantType; [StructType; [[a; [DataType; Yson]]; [b; [DataType; Bool]]]]]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Variant(f.Struct({{"a", f.Int32()}, {"b", f.Int32()}, {"c", f.Int64()}})).Get()), + "[VariantType; [StructType; [[a; [DataType; Int32]]; [b; [DataType; Int32]]; [c; [DataType; Int64]]]]]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Variant(f.Tuple({{f.Bool()}})).Get()), + "[VariantType; [TupleType; [[DataType; Bool]]]]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Variant(f.Tuple({{f.Yson()}, {f.Bool()}})).Get()), + "[VariantType; [TupleType; [[DataType; Yson]; [DataType; Bool]]]]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Variant(f.Tuple({{f.Int32()}, {f.Int32()}, {f.Int32()}})).Get()), + "[VariantType; [TupleType; [[DataType; Int32]; [DataType; Int32]; [DataType; Int32]]]]"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Tagged(f.String(), "Url").Get()), + "[TaggedType; Url; [DataType; String]]"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlType(f.Tagged(f.String(), "Url").Get(), /* includeTags = */ false), + "[DataType; String]"); +} + +TEST_TF(TypeIO, AsYqlRowSpec) { + { + auto type = f.Struct({}); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlRowSpec(type.Get()), + "{StrictSchema=%true; Type=[StructType; []]}"); + } + { + auto type = f.Tagged(f.Struct({}), "Event"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlRowSpec(type.Get()), + "{StrictSchema=%true; Type=[StructType; []]}"); + } + { + auto type = f.Struct({{"x", f.Tagged(f.String(), "Url")}}); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlRowSpec(type.Get()), + "{StrictSchema=%true; Type=[StructType; [[x; [TaggedType; Url; [DataType; String]]]]]}"); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlRowSpec(type.Get(), /* includeTags = */ false), + "{StrictSchema=%true; Type=[StructType; [[x; [DataType; String]]]]}"); + } + { + auto type = f.Struct({{"a", f.Bool()}}); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlRowSpec(type.Get()), + "{StrictSchema=%true; Type=[StructType; [[a; [DataType; Bool]]]]}"); + } + { + auto type = f.Struct({{"a", f.Yson()}, {"b", f.Bool()}}); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlRowSpec(type.Get()), + "{StrictSchema=%true; Type=[StructType; [[a; [DataType; Yson]]; [b; [DataType; Bool]]]]}"); + } + { + auto type = f.Struct({{"a", f.Int32()}, {"b", f.Int32()}, {"c", f.Int32()}}); + ASSERT_YSON_EQ( + NTi::NIo::AsYqlRowSpec(type.Get()), + "{StrictSchema=%true; Type=[StructType; [[a; [DataType; Int32]]; [b; [DataType; Int32]]; [c; [DataType; Int32]]]]}"); + } + + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + NTi::NIo::AsYqlRowSpec(f.Void().Get()); + }(), + NTi::TApiException, "expected a struct type"); + + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + NTi::NIo::AsYqlRowSpec(Optional(f.Struct({})).Get()); + }(), + NTi::TApiException, "expected a struct type"); +} + +TEST_TF(TypeIO, AsYtSchema) { + { + auto type = f.Struct({ + {"void", f.Void()}, + {"null", f.Null()}, + {"bool", f.Bool()}, + {"int8", f.Int8()}, + {"int16", f.Int16()}, + {"int32", f.Int32()}, + {"int64", f.Int64()}, + {"uint8", f.Uint8()}, + {"uint16", f.Uint16()}, + {"uint32", f.Uint32()}, + {"uint64", f.Uint64()}, + {"float", f.Float()}, + {"double", f.Double()}, + {"string", f.String()}, + {"utf8", f.Utf8()}, + {"date", f.Date()}, + {"datetime", f.Datetime()}, + {"timestamp", f.Timestamp()}, + {"tzdate", f.TzDate()}, + {"tzdatetime", f.TzDatetime()}, + {"tztimestamp", f.TzTimestamp()}, + {"interval", f.Interval()}, + {"decimal", f.Decimal(20, 10)}, + {"json", f.Json()}, + {"yson", f.Yson()}, + {"uuid", f.Uuid()}, + {"list", f.List(f.Bool())}, + {"dict", f.Dict(f.Bool(), f.Bool())}, + {"struct", f.Struct({})}, + {"tuple", f.Tuple({})}, + {"variant", f.Variant(f.Struct({{"x", f.Bool()}}))}, + }); + + ASSERT_YSON_EQ( + NTi::NIo::AsYtSchema(type.Get()), + R"( + <strict=%true; unique_keys=%false> + [ + {name=void; required=%false; type=any }; + {name=null; required=%false; type=any }; + {name=bool; required=%true; type=boolean }; + {name=int8; required=%true; type=int8 }; + {name=int16; required=%true; type=int16 }; + {name=int32; required=%true; type=int32 }; + {name=int64; required=%true; type=int64 }; + {name=uint8; required=%true; type=uint8 }; + {name=uint16; required=%true; type=uint16 }; + {name=uint32; required=%true; type=uint32 }; + {name=uint64; required=%true; type=uint64 }; + {name=float; required=%true; type=double }; + {name=double; required=%true; type=double }; + {name=string; required=%true; type=string }; + {name=utf8; required=%true; type=utf8 }; + {name=date; required=%true; type=uint16 }; + {name=datetime; required=%true; type=uint32 }; + {name=timestamp; required=%true; type=uint64 }; + {name=tzdate; required=%true; type=string }; + {name=tzdatetime; required=%true; type=string }; + {name=tztimestamp; required=%true; type=string }; + {name=interval; required=%true; type=int64 }; + {name=decimal; required=%true; type=string }; + {name=json; required=%true; type=string }; + {name=yson; required=%false; type=any }; + {name=uuid; required=%true; type=string }; + {name=list; required=%false; type=any }; + {name=dict; required=%false; type=any }; + {name=struct; required=%false; type=any }; + {name=tuple; required=%false; type=any }; + {name=variant; required=%false; type=any }; + ] + )"); + } + + { + auto type = f.Struct({ + {"void", f.Tagged(f.Void(), "Tag")}, + {"null", f.Tagged(f.Null(), "Tag")}, + {"bool", f.Tagged(f.Bool(), "Tag")}, + {"int8", f.Tagged(f.Int8(), "Tag")}, + {"int16", f.Tagged(f.Int16(), "Tag")}, + {"int32", f.Tagged(f.Int32(), "Tag")}, + {"int64", f.Tagged(f.Int64(), "Tag")}, + {"uint8", f.Tagged(f.Uint8(), "Tag")}, + {"uint16", f.Tagged(f.Uint16(), "Tag")}, + {"uint32", f.Tagged(f.Uint32(), "Tag")}, + {"uint64", f.Tagged(f.Uint64(), "Tag")}, + {"float", f.Tagged(f.Float(), "Tag")}, + {"double", f.Tagged(f.Double(), "Tag")}, + {"string", f.Tagged(f.String(), "Tag")}, + {"utf8", f.Tagged(f.Utf8(), "Tag")}, + {"date", f.Tagged(f.Date(), "Tag")}, + {"datetime", f.Tagged(f.Datetime(), "Tag")}, + {"timestamp", f.Tagged(f.Timestamp(), "Tag")}, + {"tzdate", f.Tagged(f.TzDate(), "Tag")}, + {"tzdatetime", f.Tagged(f.TzDatetime(), "Tag")}, + {"tztimestamp", f.Tagged(f.TzTimestamp(), "Tag")}, + {"interval", f.Tagged(f.Interval(), "Tag")}, + {"decimal", f.Tagged(f.Decimal(20, 10), "Tag")}, + {"json", f.Tagged(f.Json(), "Tag")}, + {"yson", f.Tagged(f.Yson(), "Tag")}, + {"uuid", f.Tagged(f.Uuid(), "Tag")}, + {"list", f.Tagged(f.List(f.Bool()), "Tag")}, + {"dict", f.Tagged(f.Dict(f.Bool(), f.Bool()), "Tag")}, + {"struct", f.Tagged(f.Struct({}), "Tag")}, + {"tuple", f.Tagged(f.Tuple({}), "Tag")}, + {"variant", f.Tagged(f.Variant(f.Struct({{"x", f.Bool()}})), "Tag")}, + }); + + ASSERT_YSON_EQ( + NTi::NIo::AsYtSchema(type.Get()), + R"( + <strict=%true; unique_keys=%false> + [ + {name=void; required=%false; type=any }; + {name=null; required=%false; type=any }; + {name=bool; required=%true; type=boolean }; + {name=int8; required=%true; type=int8 }; + {name=int16; required=%true; type=int16 }; + {name=int32; required=%true; type=int32 }; + {name=int64; required=%true; type=int64 }; + {name=uint8; required=%true; type=uint8 }; + {name=uint16; required=%true; type=uint16 }; + {name=uint32; required=%true; type=uint32 }; + {name=uint64; required=%true; type=uint64 }; + {name=float; required=%true; type=double }; + {name=double; required=%true; type=double }; + {name=string; required=%true; type=string }; + {name=utf8; required=%true; type=utf8 }; + {name=date; required=%true; type=uint16 }; + {name=datetime; required=%true; type=uint32 }; + {name=timestamp; required=%true; type=uint64 }; + {name=tzdate; required=%true; type=string }; + {name=tzdatetime; required=%true; type=string }; + {name=tztimestamp; required=%true; type=string }; + {name=interval; required=%true; type=int64 }; + {name=decimal; required=%true; type=string }; + {name=json; required=%true; type=string }; + {name=yson; required=%false; type=any }; + {name=uuid; required=%true; type=string }; + {name=list; required=%false; type=any }; + {name=dict; required=%false; type=any }; + {name=struct; required=%false; type=any }; + {name=tuple; required=%false; type=any }; + {name=variant; required=%false; type=any }; + ] + )"); + } + + { + auto type = f.Struct({ + {"void", Optional(f.Void())}, + {"null", Optional(f.Null())}, + {"bool", Optional(f.Bool())}, + {"int8", Optional(f.Int8())}, + {"int16", Optional(f.Int16())}, + {"int32", Optional(f.Int32())}, + {"int64", Optional(f.Int64())}, + {"uint8", Optional(f.Uint8())}, + {"uint16", Optional(f.Uint16())}, + {"uint32", Optional(f.Uint32())}, + {"uint64", Optional(f.Uint64())}, + {"float", Optional(f.Float())}, + {"double", Optional(f.Double())}, + {"string", Optional(f.String())}, + {"utf8", Optional(f.Utf8())}, + {"date", Optional(f.Date())}, + {"datetime", Optional(f.Datetime())}, + {"timestamp", Optional(f.Timestamp())}, + {"tzdate", Optional(f.TzDate())}, + {"tzdatetime", Optional(f.TzDatetime())}, + {"tztimestamp", Optional(f.TzTimestamp())}, + {"interval", Optional(f.Interval())}, + {"decimal", Optional(f.Decimal(20, 10))}, + {"json", Optional(f.Json())}, + {"yson", Optional(f.Yson())}, + {"uuid", Optional(f.Uuid())}, + {"list", Optional(f.List(f.Bool()))}, + {"dict", Optional(f.Dict(f.Bool(), f.Bool()))}, + {"struct", Optional(f.Struct({}))}, + {"tuple", Optional(f.Tuple({}))}, + {"variant", Optional(f.Variant(f.Struct({{"x", f.Bool()}})))}, + }); + + ASSERT_YSON_EQ( + NTi::NIo::AsYtSchema(type.Get()), + R"( + <strict=%true; unique_keys=%false> + [ + {name=void; required=%false; type=any }; + {name=null; required=%false; type=any }; + {name=bool; required=%false; type=boolean }; + {name=int8; required=%false; type=int8 }; + {name=int16; required=%false; type=int16 }; + {name=int32; required=%false; type=int32 }; + {name=int64; required=%false; type=int64 }; + {name=uint8; required=%false; type=uint8 }; + {name=uint16; required=%false; type=uint16 }; + {name=uint32; required=%false; type=uint32 }; + {name=uint64; required=%false; type=uint64 }; + {name=float; required=%false; type=double }; + {name=double; required=%false; type=double }; + {name=string; required=%false; type=string }; + {name=utf8; required=%false; type=utf8 }; + {name=date; required=%false; type=uint16 }; + {name=datetime; required=%false; type=uint32 }; + {name=timestamp; required=%false; type=uint64 }; + {name=tzdate; required=%false; type=string }; + {name=tzdatetime; required=%false; type=string }; + {name=tztimestamp; required=%false; type=string }; + {name=interval; required=%false; type=int64 }; + {name=decimal; required=%false; type=string }; + {name=json; required=%false; type=string }; + {name=yson; required=%false; type=any }; + {name=uuid; required=%false; type=string }; + {name=list; required=%false; type=any }; + {name=dict; required=%false; type=any }; + {name=struct; required=%false; type=any }; + {name=tuple; required=%false; type=any }; + {name=variant; required=%false; type=any }; + ] + )"); + } + + { + auto type = f.Struct({ + {"void", Optional(Optional(f.Void()))}, + {"null", Optional(Optional(f.Null()))}, + {"bool", Optional(Optional(f.Bool()))}, + {"int8", Optional(Optional(f.Int8()))}, + {"int16", Optional(Optional(f.Int16()))}, + {"int32", Optional(Optional(f.Int32()))}, + {"int64", Optional(Optional(f.Int64()))}, + {"uint8", Optional(Optional(f.Uint8()))}, + {"uint16", Optional(Optional(f.Uint16()))}, + {"uint32", Optional(Optional(f.Uint32()))}, + {"uint64", Optional(Optional(f.Uint64()))}, + {"float", Optional(Optional(f.Float()))}, + {"double", Optional(Optional(f.Double()))}, + {"string", Optional(Optional(f.String()))}, + {"utf8", Optional(Optional(f.Utf8()))}, + {"date", Optional(Optional(f.Date()))}, + {"datetime", Optional(Optional(f.Datetime()))}, + {"timestamp", Optional(Optional(f.Timestamp()))}, + {"tzdate", Optional(Optional(f.TzDate()))}, + {"tzdatetime", Optional(Optional(f.TzDatetime()))}, + {"tztimestamp", Optional(Optional(f.TzTimestamp()))}, + {"interval", Optional(Optional(f.Interval()))}, + {"decimal", Optional(Optional(f.Decimal(20, 10)))}, + {"json", Optional(Optional(f.Json()))}, + {"yson", Optional(Optional(f.Yson()))}, + {"uuid", Optional(Optional(f.Uuid()))}, + {"list", Optional(Optional(f.List(f.Bool())))}, + {"dict", Optional(Optional(f.Dict(f.Bool(), f.Bool())))}, + {"struct", Optional(Optional(f.Struct({})))}, + {"tuple", Optional(Optional(f.Tuple({})))}, + {"variant", Optional(Optional(f.Variant(f.Struct({{"x", f.Bool()}}))))}, + }); + + ASSERT_YSON_EQ( + NTi::NIo::AsYtSchema(type.Get()), + R"( + <strict=%true; unique_keys=%false> + [ + {name=void; required=%false; type=any }; + {name=null; required=%false; type=any }; + {name=bool; required=%false; type=any }; + {name=int8; required=%false; type=any }; + {name=int16; required=%false; type=any }; + {name=int32; required=%false; type=any }; + {name=int64; required=%false; type=any }; + {name=uint8; required=%false; type=any }; + {name=uint16; required=%false; type=any }; + {name=uint32; required=%false; type=any }; + {name=uint64; required=%false; type=any }; + {name=float; required=%false; type=any }; + {name=double; required=%false; type=any }; + {name=string; required=%false; type=any }; + {name=utf8; required=%false; type=any }; + {name=date; required=%false; type=any }; + {name=datetime; required=%false; type=any }; + {name=timestamp; required=%false; type=any }; + {name=tzdate; required=%false; type=any }; + {name=tzdatetime; required=%false; type=any }; + {name=tztimestamp; required=%false; type=any }; + {name=interval; required=%false; type=any }; + {name=decimal; required=%false; type=any }; + {name=json; required=%false; type=any }; + {name=yson; required=%false; type=any }; + {name=uuid; required=%false; type=any }; + {name=list; required=%false; type=any }; + {name=dict; required=%false; type=any }; + {name=struct; required=%false; type=any }; + {name=tuple; required=%false; type=any }; + {name=variant; required=%false; type=any }; + ] + )"); + } + + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + NTi::NIo::AsYtSchema(f.Void().Get()); + }(), + NTi::TApiException, "expected a struct type"); + + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + NTi::NIo::AsYtSchema(Optional(f.Struct({})).Get()); + }(), + NTi::TApiException, "expected a struct type"); + + UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() { + NTi::NIo::AsYtSchema(f.Struct({}).Get()); + }(), + NTi::TApiException, "expected a non-empty struct"); + + ASSERT_YSON_EQ( + NTi::NIo::AsYtSchema(f.Struct({}).Get(), /* failOnEmptyf.Struct = */ false), + "<strict=%true; unique_keys=%false>[{name=_yql_fake_column; required=%false; type=boolean}]"); +} diff --git a/library/cpp/type_info/ut/type_list.cpp b/library/cpp/type_info/ut/type_list.cpp new file mode 100644 index 0000000000..56d9e16061 --- /dev/null +++ b/library/cpp/type_info/ut/type_list.cpp @@ -0,0 +1,64 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include <util/generic/serialized_enum.h> + +TEST(TypeList, PrimitiveTypeNameSequence) { + auto primitiveNames = GetEnumAllValues<NTi::EPrimitiveTypeName>(); + + ASSERT_GT(primitiveNames.size(), 0); + + ASSERT_EQ(static_cast<i32>(primitiveNames[0]), 0); + + for (size_t i = 0; i < primitiveNames.size() - 1; ++i) { + ASSERT_EQ(static_cast<i32>(primitiveNames[i]) + 1, static_cast<i32>(primitiveNames[i + 1])); + } +} + +TEST(TypeList, PrimitiveTypeGroup) { + auto primitiveNames = GetEnumAllValues<NTi::EPrimitiveTypeName>(); + + for (auto typeName : primitiveNames) { + ASSERT_TRUE(NTi::IsPrimitive(static_cast<NTi::ETypeName>(typeName))); + } +} + +TEST(TypeList, TypeNameInExactlyOneGroup) { + auto allNames = GetEnumAllValues<NTi::ETypeName>(); + + for (auto typeName : allNames) { + int groups = 0; + groups += NTi::IsPrimitive(typeName); + groups += NTi::IsSingular(typeName); + groups += NTi::IsContainer(typeName); + ASSERT_EQ(groups, 1); + } +} + +TEST(TypeList, EnumCoherence) { + auto primitiveNames = GetEnumAllValues<NTi::EPrimitiveTypeName>(); + auto allNames = GetEnumAllValues<NTi::ETypeName>(); + + ASSERT_GT(allNames.size(), primitiveNames.size()); + + size_t i = 0; + + for (; i < primitiveNames.size(); ++i) { + ASSERT_EQ(static_cast<i32>(primitiveNames[i]), static_cast<i32>(allNames[i])); + ASSERT_EQ(ToString(primitiveNames[i]), ToString(allNames[i])); + ASSERT_TRUE(NTi::IsPrimitive(allNames[i])); + } + + for (; i < allNames.size(); ++i) { + ASSERT_FALSE(NTi::IsPrimitive(allNames[i])); + } +} + +TEST(TypeList, Cast) { + auto primitiveNames = GetEnumAllValues<NTi::EPrimitiveTypeName>(); + + for (auto typeName : primitiveNames) { + ASSERT_EQ(typeName, NTi::ToPrimitiveTypeName(NTi::ToTypeName(typeName))); + } +} diff --git a/library/cpp/type_info/ut/type_serialize.cpp b/library/cpp/type_info/ut/type_serialize.cpp new file mode 100644 index 0000000000..0d7e184eca --- /dev/null +++ b/library/cpp/type_info/ut/type_serialize.cpp @@ -0,0 +1,251 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include "utils.h" + +TEST(TypeSerialize, Void) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Void().Get()), R"(void)"); +} + +TEST(TypeSerialize, Null) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Null().Get()), R"(null)"); +} + +TEST(TypeSerialize, Bool) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Bool().Get()), R"(bool)"); +} + +TEST(TypeSerialize, Int8) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Int8().Get()), R"(int8)"); +} + +TEST(TypeSerialize, Int16) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Int16().Get()), R"(int16)"); +} + +TEST(TypeSerialize, Int32) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Int32().Get()), R"(int32)"); +} + +TEST(TypeSerialize, Int64) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Int64().Get()), R"(int64)"); +} + +TEST(TypeSerialize, Uint8) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uint8().Get()), R"(uint8)"); +} + +TEST(TypeSerialize, Uint16) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uint16().Get()), R"(uint16)"); +} + +TEST(TypeSerialize, Uint32) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uint32().Get()), R"(uint32)"); +} + +TEST(TypeSerialize, Uint64) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uint64().Get()), R"(uint64)"); +} + +TEST(TypeSerialize, Float) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Float().Get()), R"(float)"); +} + +TEST(TypeSerialize, Double) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Double().Get()), R"(double)"); +} + +TEST(TypeSerialize, String) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::String().Get()), R"(string)"); +} + +TEST(TypeSerialize, Utf8) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Utf8().Get()), R"(utf8)"); +} + +TEST(TypeSerialize, Date) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Date().Get()), R"(date)"); +} + +TEST(TypeSerialize, Datetime) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Datetime().Get()), R"(datetime)"); +} + +TEST(TypeSerialize, Timestamp) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Timestamp().Get()), R"(timestamp)"); +} + +TEST(TypeSerialize, TzDate) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::TzDate().Get()), R"(tz_date)"); +} + +TEST(TypeSerialize, TzDatetime) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::TzDatetime().Get()), R"(tz_datetime)"); +} + +TEST(TypeSerialize, TzTimestamp) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::TzTimestamp().Get()), R"(tz_timestamp)"); +} + +TEST(TypeSerialize, Interval) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Interval().Get()), R"(interval)"); +} + +TEST(TypeSerialize, Decimal) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Decimal(20, 10).Get()), R"({type_name=decimal; precision=20; scale=10})"); + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Decimal(10, 10).Get()), R"({type_name=decimal; precision=10; scale=10})"); +} + +TEST(TypeSerialize, Json) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Json().Get()), R"(json)"); +} + +TEST(TypeSerialize, Yson) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Yson().Get()), R"(yson)"); +} + +TEST(TypeSerialize, Uuid) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uuid().Get()), R"(uuid)"); +} + +TEST(TypeSerialize, Optional) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Optional(NTi::Void()).Get()), R"({type_name=optional; item=void})"); + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Optional(NTi::String()).Get()), R"({type_name=optional; item=string})"); +} + +TEST(TypeSerialize, List) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::List(NTi::Void()).Get()), R"({type_name=list; item=void})"); + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::List(NTi::String()).Get()), R"({type_name=list; item=string})"); +} + +TEST(TypeSerialize, Dict) { + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Dict(NTi::Void(), NTi::Void()).Get()), R"({type_name=dict; key=void; value=void})"); + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Dict(NTi::Int32(), NTi::String()).Get()), R"({type_name=dict; key=int32; value=string})"); +} + +TEST(TypeSerialize, StructEmpty) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Struct({}).Get()), + R"({type_name=struct; members=[]})"); +} + +TEST(TypeSerialize, Struct) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}}).Get()), + R"({type_name=struct; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})"); +} + +TEST(TypeSerialize, StructNamedEmpty) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Struct("S", {}).Get()), + R"({type_name=struct; members=[]})"); +} + +TEST(TypeSerialize, StructNamedEmptyNoNames) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Struct("S", {}).Get(), + /* binary = */ false, + /* includeTags = */ true), + R"({type_name=struct; members=[]})"); +} + +TEST(TypeSerialize, StructNamed) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Struct("S", {{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}}).Get()), + R"({type_name=struct; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})"); +} + +TEST(TypeSerialize, TupleEmpty) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Tuple({}).Get()), + R"({type_name=tuple; elements=[]})"); +} + +TEST(TypeSerialize, Tuple) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Tuple({{NTi::String()}, {NTi::List(NTi::Int64())}}).Get()), + R"({type_name=tuple; elements=[{type=string}; {type={type_name=list; item=int64}}]})"); +} + +TEST(TypeSerialize, TupleNamedEmpty) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Tuple("S", {}).Get()), + R"({type_name=tuple; elements=[]})"); +} + +TEST(TypeSerialize, TupleNamedEmptyNoNames) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Tuple("S", {}).Get(), + /* binary = */ false), + R"({type_name=tuple; elements=[]})"); +} + +TEST(TypeSerialize, VariantStruct) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Variant( + NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}})) + .Get()), + R"({type_name=variant; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})"); +} + +TEST(TypeSerialize, VariantTuple) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Variant( + NTi::Tuple({ + {NTi::String()}, + {NTi::List(NTi::Int64())}, + })) + .Get()), + R"({type_name=variant; elements=[{type=string}; {type={type_name=list; item=int64}}]})"); +} + +TEST(TypeSerialize, Tagged) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Tagged(NTi::String(), "Url").Get()), + R"({type_name=tagged; tag=Url; item=string})"); +} + +TEST(TypeSerialize, TaggedNoTags) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Tagged(NTi::String(), "Url").Get(), + /* binary = */ false, + /* includeTags = */ false), + R"(string)"); +} + +TEST(TypeSerialize, MultipleType) { + auto result = TString(); + + { + auto writer = NYsonPull::MakeTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::ListFragment); + + writer.BeginStream(); + NTi::NIo::SerializeYsonMultiple(NTi::Optional(NTi::String()).Get(), writer.GetConsumer()); + NTi::NIo::SerializeYsonMultiple(NTi::List(NTi::Utf8()).Get(), writer.GetConsumer()); + writer.EndStream(); + } + + ASSERT_YSON_EQ("[" + result + "]", R"([{type_name=optional; item=string}; {type_name=list; item=utf8}])"); +} + +TEST(TypeSerialize, Binary) { + ASSERT_YSON_EQ( + NTi::NIo::SerializeYson( + NTi::Tagged(NTi::String(), "Url").Get(), + /* binary = */ true), + R"({type_name=tagged; tag=Url; item=string})"); +} diff --git a/library/cpp/type_info/ut/type_show.cpp b/library/cpp/type_info/ut/type_show.cpp new file mode 100644 index 0000000000..cde0d9371a --- /dev/null +++ b/library/cpp/type_info/ut/type_show.cpp @@ -0,0 +1,199 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include "utils.h" + +TEST_TF(TypeShow, Void) { + ASSERT_EQ(ToString(*f.Void()), "Void"); +} + +TEST_TF(TypeShow, Null) { + ASSERT_EQ(ToString(*f.Null()), "Null"); +} + +TEST_TF(TypeShow, Bool) { + ASSERT_EQ(ToString(*f.Bool()), "Bool"); +} + +TEST_TF(TypeShow, Int8) { + ASSERT_EQ(ToString(*f.Int8()), "Int8"); +} + +TEST_TF(TypeShow, Int16) { + ASSERT_EQ(ToString(*f.Int16()), "Int16"); +} + +TEST_TF(TypeShow, Int32) { + ASSERT_EQ(ToString(*f.Int32()), "Int32"); +} + +TEST_TF(TypeShow, Int64) { + ASSERT_EQ(ToString(*f.Int64()), "Int64"); +} + +TEST_TF(TypeShow, Uint8) { + ASSERT_EQ(ToString(*f.Uint8()), "Uint8"); +} + +TEST_TF(TypeShow, Uint16) { + ASSERT_EQ(ToString(*f.Uint16()), "Uint16"); +} + +TEST_TF(TypeShow, Uint32) { + ASSERT_EQ(ToString(*f.Uint32()), "Uint32"); +} + +TEST_TF(TypeShow, Uint64) { + ASSERT_EQ(ToString(*f.Uint64()), "Uint64"); +} + +TEST_TF(TypeShow, Float) { + ASSERT_EQ(ToString(*f.Float()), "Float"); +} + +TEST_TF(TypeShow, Double) { + ASSERT_EQ(ToString(*f.Double()), "Double"); +} + +TEST_TF(TypeShow, String) { + ASSERT_EQ(ToString(*f.String()), "String"); +} + +TEST_TF(TypeShow, Utf8) { + ASSERT_EQ(ToString(*f.Utf8()), "Utf8"); +} + +TEST_TF(TypeShow, Date) { + ASSERT_EQ(ToString(*f.Date()), "Date"); +} + +TEST_TF(TypeShow, Datetime) { + ASSERT_EQ(ToString(*f.Datetime()), "Datetime"); +} + +TEST_TF(TypeShow, Timestamp) { + ASSERT_EQ(ToString(*f.Timestamp()), "Timestamp"); +} + +TEST_TF(TypeShow, TzDate) { + ASSERT_EQ(ToString(*f.TzDate()), "TzDate"); +} + +TEST_TF(TypeShow, TzDatetime) { + ASSERT_EQ(ToString(*f.TzDatetime()), "TzDatetime"); +} + +TEST_TF(TypeShow, TzTimestamp) { + ASSERT_EQ(ToString(*f.TzTimestamp()), "TzTimestamp"); +} + +TEST_TF(TypeShow, Interval) { + ASSERT_EQ(ToString(*f.Interval()), "Interval"); +} + +TEST_TF(TypeShow, Decimal) { + ASSERT_EQ(ToString(*f.Decimal(20, 10)), "Decimal(20, 10)"); +} + +TEST_TF(TypeShow, Json) { + ASSERT_EQ(ToString(*f.Json()), "Json"); +} + +TEST_TF(TypeShow, Yson) { + ASSERT_EQ(ToString(*f.Yson()), "Yson"); +} + +TEST_TF(TypeShow, Uuid) { + ASSERT_EQ(ToString(*f.Uuid()), "Uuid"); +} + +TEST_TF(TypeShow, Optional) { + ASSERT_EQ(ToString(*f.Optional(f.String())), "Optional<String>"); +} + +TEST_TF(TypeShow, List) { + ASSERT_EQ(ToString(*f.List(f.String())), "List<String>"); +} + +TEST_TF(TypeShow, Dict) { + ASSERT_EQ(ToString(*f.Dict(f.String(), f.Int32())), "Dict<String, Int32>"); +} + +TEST_TF(TypeShow, Struct) { + ASSERT_EQ( + ToString(*f.Struct({})), + "Struct<>"); + ASSERT_EQ( + ToString(*f.Struct({{"x1", f.Void()}})), + "Struct<'x1': Void>"); + ASSERT_EQ( + ToString(*f.Struct({{"x1", f.Void()}, {"x2", f.String()}})), + "Struct<'x1': Void, 'x2': String>"); + ASSERT_EQ( + ToString(*f.Struct("Name", {})), + "Struct['Name']<>"); + ASSERT_EQ( + ToString(*f.Struct("Name", {{"x1", f.Void()}})), + "Struct['Name']<'x1': Void>"); + ASSERT_EQ( + ToString(*f.Struct("Name", {{"x1", f.Void()}, {"x2", f.String()}})), + "Struct['Name']<'x1': Void, 'x2': String>"); +} + +TEST_TF(TypeShow, Tuple) { + ASSERT_EQ( + ToString(*f.Tuple({})), + "Tuple<>"); + ASSERT_EQ( + ToString(*f.Tuple({{f.Void()}})), + "Tuple<Void>"); + ASSERT_EQ( + ToString(*f.Tuple({{f.Void()}, {f.String()}})), + "Tuple<Void, String>"); + ASSERT_EQ( + ToString(*f.Tuple("Name", {})), + "Tuple['Name']<>"); + ASSERT_EQ( + ToString(*f.Tuple("Name", {{f.Void()}})), + "Tuple['Name']<Void>"); + ASSERT_EQ( + ToString(*f.Tuple("Name", {{f.Void()}, {f.String()}})), + "Tuple['Name']<Void, String>"); +} + +TEST_TF(TypeShow, VariantStruct) { + ASSERT_EQ( + ToString(*f.Variant(f.Struct({{"x1", f.Void()}}))), + "Variant<'x1': Void>"); + ASSERT_EQ( + ToString(*f.Variant(f.Struct({{"x1", f.Void()}, {"x2", f.String()}}))), + "Variant<'x1': Void, 'x2': String>"); + ASSERT_EQ( + ToString(*f.Variant("Name", f.Struct({{"x1", f.Void()}}))), + "Variant['Name']<'x1': Void>"); + ASSERT_EQ( + ToString(*f.Variant("Name", f.Struct({{"x1", f.Void()}, {"x2", f.String()}}))), + "Variant['Name']<'x1': Void, 'x2': String>"); +} + +TEST_TF(TypeShow, VariantTuple) { + ASSERT_EQ( + ToString(*f.Variant(f.Tuple({{f.Void()}}))), + "Variant<Void>"); + ASSERT_EQ( + ToString(*f.Variant(f.Tuple({{f.Void()}, {f.String()}}))), + "Variant<Void, String>"); + ASSERT_EQ( + ToString(*f.Variant("Name", f.Tuple({{f.Void()}}))), + "Variant['Name']<Void>"); + ASSERT_EQ( + ToString(*f.Variant("Name", f.Tuple({{f.Void()}, {f.String()}}))), + "Variant['Name']<Void, String>"); +} + +TEST_TF(TypeShow, Tagged) { + ASSERT_EQ( + ToString(*f.Tagged(f.Void(), "Tag")), + "Tagged<Void, 'Tag'>"); +} diff --git a/library/cpp/type_info/ut/type_strip_tags.cpp b/library/cpp/type_info/ut/type_strip_tags.cpp new file mode 100644 index 0000000000..b3ef4c0825 --- /dev/null +++ b/library/cpp/type_info/ut/type_strip_tags.cpp @@ -0,0 +1,31 @@ +#include <library/cpp/testing/unittest/gtest.h> + +#include <library/cpp/type_info/type_info.h> + +#include "utils.h" + +TEST_TF(TypeStripTags, StripTags) { + auto t = f.List(f.Tagged(f.Void(), "Tag")); + + ASSERT_EQ(t->StripTags().Get(), t.Get()); + ASSERT_EQ(f.Tagged(t, "Tag")->StripTags().Get(), t.Get()); + ASSERT_EQ(f.Tagged(f.Tagged(t, "Tag"), "Tag2")->StripTags().Get(), t.Get()); +} + +TEST_TF(TypeStripTags, StripOptionals) { + auto t = f.Tagged(f.Optional(f.Void()), "Tag"); + + ASSERT_EQ(t->StripOptionals().Get(), t.Get()); + ASSERT_EQ(f.Optional(t)->StripOptionals().Get(), t.Get()); + ASSERT_EQ(f.Optional(f.Optional(t))->StripOptionals().Get(), t.Get()); +} + +TEST_TF(TypeStripTags, StripTagsAndOptionals) { + auto t = f.Void(); + + ASSERT_EQ(t->StripTagsAndOptionals().Get(), t.Get()); + ASSERT_EQ(f.Optional(t)->StripTagsAndOptionals().Get(), t.Get()); + ASSERT_EQ(f.Optional(f.Optional(t))->StripTagsAndOptionals().Get(), t.Get()); + ASSERT_EQ(f.Optional(f.Tagged(t, "Tag"))->StripTagsAndOptionals().Get(), t.Get()); + ASSERT_EQ(f.Optional(f.Optional(f.Optional(t)))->StripTagsAndOptionals().Get(), t.Get()); +} diff --git a/library/cpp/type_info/ut/utils.h b/library/cpp/type_info/ut/utils.h new file mode 100644 index 0000000000..b35316cb2f --- /dev/null +++ b/library/cpp/type_info/ut/utils.h @@ -0,0 +1,73 @@ +#pragma once + +//! @file utils.h +//! +//! Infrastructure for running type info tests with different factories and settings. + +#include <library/cpp/testing/unittest/registar.h> + +#include <library/cpp/yson/consumer.h> +#include <library/cpp/yson/node/node.h> +#include <library/cpp/yson/node/node_builder.h> +#include <library/cpp/yson/node/node_io.h> + +#include <library/cpp/type_info/fwd.h> + +template <typename T> +inline TString ToCanonicalYson(const T& value) { + return NYT::NodeToCanonicalYsonString(NYT::NodeFromYsonString(TStringBuf(value))); +} + +/// Assert that two YSON strings are equal. +#define ASSERT_YSON_EQ(L, R) ASSERT_EQ(ToCanonicalYson(L), ToCanonicalYson(R)) + +/// Assert that two types are strictly equal. +#define ASSERT_STRICT_EQ(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + ASSERT_TRUE(NTi::NEq::TStrictlyEqual()(lhs, rhs)); \ + ASSERT_TRUE(NTi::NEq::TStrictlyEqual().IgnoreHash(lhs, rhs)); \ + ASSERT_EQ(lhs->GetHash(), NTi::NEq::TStrictlyEqualHash()(lhs)); \ + ASSERT_EQ(rhs->GetHash(), NTi::NEq::TStrictlyEqualHash()(rhs)); \ + ASSERT_EQ(lhs->GetHash(), rhs->GetHash()); \ + } while (false) + +/// Assert that two types are strictly unequal. +/// +/// Note: we check that, if types are not equal, their hashes are also not equal. +/// While this is not guaranteed, we haven't seen any collisions so far. +/// If some collision happen, check if hashing isn't broken before removing the assert. +#define ASSERT_STRICT_NE(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + ASSERT_FALSE(NTi::NEq::TStrictlyEqual()(lhs, rhs)); \ + ASSERT_FALSE(NTi::NEq::TStrictlyEqual().IgnoreHash(lhs, rhs)); \ + ASSERT_EQ(lhs->GetHash(), NTi::NEq::TStrictlyEqualHash()(lhs)); \ + ASSERT_EQ(rhs->GetHash(), NTi::NEq::TStrictlyEqualHash()(rhs)); \ + ASSERT_NE(lhs->GetHash(), rhs->GetHash()); \ + } while (false) + +/// Assert that a type string is equal to the given type after deserialization. +#define ASSERT_DESERIALIZED_EQ(canonicalType, serializedType) \ + do { \ + auto reader = NYsonPull::TReader(NYsonPull::NInput::FromMemory(serializedType), NYsonPull::EStreamType::Node); \ + auto deserializedType = NTi::NIo::DeserializeYson(*NTi::HeapFactory(), reader); \ + ASSERT_STRICT_EQ(deserializedType, canonicalType); \ + ASSERT_YSON_EQ(NTi::NIo::SerializeYson(canonicalType.Get()), NTi::NIo::SerializeYson(deserializedType.Get())); \ + } while (false) + +/// Test parametrized over different type factories. +#define TEST_TF(N, NN) \ + void Test##N##NN(NTi::ITypeFactory& f); \ + TEST(N, NN##_Heap) { \ + Test##N##NN(*NTi::HeapFactory()); \ + } \ + TEST(N, NN##_Pool) { \ + Test##N##NN(*NTi::PoolFactory(false)); \ + } \ + TEST(N, NN##_PoolDedup) { \ + Test##N##NN(*NTi::PoolFactory(true)); \ + } \ + void Test##N##NN(NTi::ITypeFactory& f) diff --git a/library/cpp/type_info/ut/ya.make b/library/cpp/type_info/ut/ya.make new file mode 100644 index 0000000000..6b32a2a152 --- /dev/null +++ b/library/cpp/type_info/ut/ya.make @@ -0,0 +1,32 @@ +UNITTEST() + +SRCS( + builder.cpp + type_basics.cpp + type_complexity_ut.cpp + type_constraints.cpp + type_deserialize.cpp + type_equivalence.cpp + type_factory.cpp + type_factory_raw.cpp + type_io.cpp + type_list.cpp + type_serialize.cpp + type_show.cpp + type_strip_tags.cpp + test_data.cpp +) + +PEERDIR( + library/cpp/type_info + library/cpp/yson + library/cpp/yson/node + library/cpp/resource +) + +RESOURCE( + ${ARCADIA_ROOT}/library/cpp/type_info/ut/test-data/good-types.txt /good + ${ARCADIA_ROOT}/library/cpp/type_info/ut/test-data/bad-types.txt /bad +) + +END() diff --git a/library/cpp/type_info/ya.make b/library/cpp/type_info/ya.make new file mode 100644 index 0000000000..555f0d1433 --- /dev/null +++ b/library/cpp/type_info/ya.make @@ -0,0 +1,26 @@ +LIBRARY() + +SRCS( + type_info.cpp + + builder.cpp + error.cpp + type.cpp + type_complexity.cpp + type_equivalence.cpp + type_factory.cpp + type_io.cpp + type_list.cpp +) + +GENERATE_ENUM_SERIALIZATION( + type_list.h +) + +PEERDIR( + library/cpp/yson_pull +) + +END() + +RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/yson_pull/bridge.h b/library/cpp/yson_pull/bridge.h new file mode 100644 index 0000000000..ac767dcba0 --- /dev/null +++ b/library/cpp/yson_pull/bridge.h @@ -0,0 +1,34 @@ +#pragma once + +#include "consumer.h" +#include "event.h" +#include "writer.h" + +namespace NYsonPull { + //! \brief Connect YSON stream producer and consumer. + //! + //! Useful for writing YSON stream filters. + //! \p Producer must have a \p next_event() method (like \p NYsonPull::reader). + //! \p Consumer must be like \p NYsonPull::consumer interface. + template <typename Producer, typename Consumer> + inline void Bridge(Producer&& producer, Consumer&& consumer) { + for (;;) { + auto& event = producer.NextEvent(); + consumer.OnEvent(event); + if (event.Type() == EEventType::EndStream) { + break; + } + } + } + + template <typename Producer> + inline void Bridge(Producer&& producer, TWriter& writer_) { + Bridge(std::forward<Producer>(producer), writer_.GetConsumer()); + } + + template <typename Producer> + inline void Bridge(Producer&& producer, TWriter&& writer_) { + Bridge(std::forward<Producer>(producer), writer_.GetConsumer()); + } + +} diff --git a/library/cpp/yson_pull/yson.h b/library/cpp/yson_pull/yson.h new file mode 100644 index 0000000000..a77eaa5c94 --- /dev/null +++ b/library/cpp/yson_pull/yson.h @@ -0,0 +1,14 @@ +#pragma once + +#include "bridge.h" +#include "consumer.h" +#include "event.h" +#include "exceptions.h" +#include "input.h" +#include "output.h" +#include "position_info.h" +#include "range.h" +#include "reader.h" +#include "scalar.h" +#include "stream_type.h" +#include "writer.h" diff --git a/library/cpp/yt/CMakeLists.txt b/library/cpp/yt/CMakeLists.txt index eb0165800f..db789d02dd 100644 --- a/library/cpp/yt/CMakeLists.txt +++ b/library/cpp/yt/CMakeLists.txt @@ -7,12 +7,19 @@ add_subdirectory(assert) +add_subdirectory(backtrace) add_subdirectory(coding) +add_subdirectory(containers) +add_subdirectory(cpu_clock) add_subdirectory(exception) +add_subdirectory(logging) add_subdirectory(malloc) add_subdirectory(memory) add_subdirectory(misc) add_subdirectory(small_containers) add_subdirectory(string) +add_subdirectory(system) +add_subdirectory(threading) +add_subdirectory(user_job_statistics) add_subdirectory(yson) add_subdirectory(yson_string) diff --git a/library/cpp/yt/backtrace/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/backtrace/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..dbc5fa609f --- /dev/null +++ b/library/cpp/yt/backtrace/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(cursors) + +add_library(cpp-yt-backtrace) +target_compile_options(cpp-yt-backtrace PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-backtrace PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-string +) +target_sources(cpp-yt-backtrace PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/backtrace.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp +) diff --git a/library/cpp/yt/backtrace/CMakeLists.linux-aarch64.txt b/library/cpp/yt/backtrace/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..358ab6a86f --- /dev/null +++ b/library/cpp/yt/backtrace/CMakeLists.linux-aarch64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(cursors) + +add_library(cpp-yt-backtrace) +target_compile_options(cpp-yt-backtrace PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-backtrace PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-string +) +target_sources(cpp-yt-backtrace PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/backtrace.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp +) diff --git a/library/cpp/yt/backtrace/CMakeLists.linux-x86_64.txt b/library/cpp/yt/backtrace/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..358ab6a86f --- /dev/null +++ b/library/cpp/yt/backtrace/CMakeLists.linux-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(cursors) + +add_library(cpp-yt-backtrace) +target_compile_options(cpp-yt-backtrace PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-backtrace PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-string +) +target_sources(cpp-yt-backtrace PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/backtrace.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp +) diff --git a/library/cpp/yt/backtrace/CMakeLists.txt b/library/cpp/yt/backtrace/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/yt/backtrace/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/backtrace/CMakeLists.windows-x86_64.txt b/library/cpp/yt/backtrace/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..8b4a651f33 --- /dev/null +++ b/library/cpp/yt/backtrace/CMakeLists.windows-x86_64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(cursors) + +add_library(cpp-yt-backtrace) +target_link_libraries(cpp-yt-backtrace PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-string +) +target_sources(cpp-yt-backtrace PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/backtrace.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp +) diff --git a/library/cpp/yt/backtrace/backtrace-inl.h b/library/cpp/yt/backtrace/backtrace-inl.h new file mode 100644 index 0000000000..b78eeffd75 --- /dev/null +++ b/library/cpp/yt/backtrace/backtrace-inl.h @@ -0,0 +1,36 @@ +#pragma once +#ifndef BACKTRACE_INL_H_ +#error "Direct inclusion of this file is not allowed, include backtrace.h" +// For the sake of sane code completion. +#include "backtrace.h" +#endif + +#include <util/system/compiler.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +template <class TCursor> +Y_NO_INLINE TBacktrace GetBacktrace( + TCursor* cursor, + TBacktraceBuffer buffer, + int framesToSkip) +{ + // Account for the current frame. + ++framesToSkip; + size_t frameCount = 0; + while (frameCount < buffer.size() && !cursor->IsFinished()) { + if (framesToSkip > 0) { + --framesToSkip; + } else { + buffer[frameCount++] = cursor->GetCurrentIP(); + } + cursor->MoveNext(); + } + return {buffer.begin(), frameCount}; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/backtrace.cpp b/library/cpp/yt/backtrace/backtrace.cpp new file mode 100644 index 0000000000..153a0a5dd0 --- /dev/null +++ b/library/cpp/yt/backtrace/backtrace.cpp @@ -0,0 +1,18 @@ +#include "backtrace.h" + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TString SymbolizeBacktrace(TBacktrace backtrace) +{ + TString result; + SymbolizeBacktrace( + backtrace, + [&] (TStringBuf str) { result += str; }); + return result; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/backtrace.h b/library/cpp/yt/backtrace/backtrace.h new file mode 100644 index 0000000000..ea70d9558c --- /dev/null +++ b/library/cpp/yt/backtrace/backtrace.h @@ -0,0 +1,45 @@ +#pragma once + +#include <library/cpp/yt/memory/range.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +using TBacktrace = TRange<const void*>; +using TBacktraceBuffer = TMutableRange<const void*>; + +//! Obtains a backtrace via a given cursor. +/*! + * \param buffer is the buffer where the backtrace is written to + * \param framesToSkip is the number of top frames to skip + * \returns the portion of #buffer that has actually been filled + */ +template <class TCursor> +TBacktrace GetBacktrace( + TCursor* cursor, + TBacktraceBuffer buffer, + int framesToSkip); + +//! Symbolizes a backtrace invoking a given callback for each frame. +/*! + * \param backtrace Backtrace to symbolize + * \param frameCallback Callback to invoke per each frame + */ +void SymbolizeBacktrace( + TBacktrace backtrace, + const std::function<void(TStringBuf)>& frameCallback); + +//! Symbolizes a backtrace to a string. +/*! + * \param backtrace Backtrace to symbolize + */ +TString SymbolizeBacktrace(TBacktrace backtrace); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace + +#define BACKTRACE_INL_H_ +#include "backtrace-inl.h" +#undef BACKTRACE_INL_H_ diff --git a/library/cpp/yt/backtrace/cursors/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/backtrace/cursors/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..6c6f5d1c50 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,9 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(libunwind) diff --git a/library/cpp/yt/backtrace/cursors/CMakeLists.linux-aarch64.txt b/library/cpp/yt/backtrace/cursors/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..6c6f5d1c50 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/CMakeLists.linux-aarch64.txt @@ -0,0 +1,9 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(libunwind) diff --git a/library/cpp/yt/backtrace/cursors/CMakeLists.linux-x86_64.txt b/library/cpp/yt/backtrace/cursors/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..6c6f5d1c50 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/CMakeLists.linux-x86_64.txt @@ -0,0 +1,9 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(libunwind) diff --git a/library/cpp/yt/backtrace/cursors/CMakeLists.txt b/library/cpp/yt/backtrace/cursors/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/backtrace/cursors/CMakeLists.windows-x86_64.txt b/library/cpp/yt/backtrace/cursors/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..961a9a908b --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/CMakeLists.windows-x86_64.txt @@ -0,0 +1,9 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(dummy) diff --git a/library/cpp/yt/backtrace/cursors/dummy/CMakeLists.txt b/library/cpp/yt/backtrace/cursors/dummy/CMakeLists.txt new file mode 100644 index 0000000000..03d4a7153c --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/dummy/CMakeLists.txt @@ -0,0 +1,11 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +endif() diff --git a/library/cpp/yt/backtrace/cursors/dummy/CMakeLists.windows-x86_64.txt b/library/cpp/yt/backtrace/cursors/dummy/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..40a6e7d0a8 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/dummy/CMakeLists.windows-x86_64.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(backtrace-cursors-dummy) +target_link_libraries(backtrace-cursors-dummy PUBLIC + contrib-libs-cxxsupp + yutil +) +target_sources(backtrace-cursors-dummy PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp +) diff --git a/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp new file mode 100644 index 0000000000..ea6e0bc08e --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp @@ -0,0 +1,22 @@ +#include "dummy_cursor.h" + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +bool TDummyCursor::IsFinished() const +{ + return true; +} + +const void* TDummyCursor::GetCurrentIP() const +{ + return nullptr; +} + +void TDummyCursor::MoveNext() +{ } + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h new file mode 100644 index 0000000000..b47d7d2aba --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h @@ -0,0 +1,17 @@ +#pragma once + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +class TDummyCursor +{ +public: + bool IsFinished() const; + const void* GetCurrentIP() const; + void MoveNext(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/dummy/ya.make b/library/cpp/yt/backtrace/cursors/dummy/ya.make new file mode 100644 index 0000000000..49fd7be050 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/dummy/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + dummy_cursor.cpp +) + +END() diff --git a/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp new file mode 100644 index 0000000000..290d30c3ce --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp @@ -0,0 +1,146 @@ +#include "frame_pointer_cursor.h" + +#include <util/generic/size_literals.h> + +#include <array> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TFramePointerCursor::TFramePointerCursor( + TSafeMemoryReader* memoryReader, + const TFramePointerCursorContext& context) + : MemoryReader_(memoryReader) + , Rip_(reinterpret_cast<const void*>(context.Rip)) + , Rbp_(reinterpret_cast<const void*>(context.Rbp)) + , StartRsp_(reinterpret_cast<const void*>(context.Rsp)) +{ } + +bool TFramePointerCursor::IsFinished() const +{ + return Finished_; +} + +const void* TFramePointerCursor::GetCurrentIP() const +{ + return Rip_; +} + +void TFramePointerCursor::MoveNext() +{ + if (Finished_) { + return; + } + + auto add = [] (auto ptr, auto delta) { + return reinterpret_cast<void*>(reinterpret_cast<intptr_t>(ptr) + delta); + }; + + auto checkPtr = [&] (auto ptr) { + ui8 data; + return MemoryReader_->Read(ptr, &data); + }; + + // We try unwinding stack manually by following frame pointers. + // + // We assume that stack does not span more than 4mb. + + if (First_) { + First_ = false; + + // For the first frame there are three special cases where naive + // unwinding would skip the caller frame. + // + // 1) Right after call instruction, rbp points to frame of a caller. + // 2) Right after "push rbp" instruction. + // 3) Right before ret instruction, rbp points to frame of a caller. + // + // We read current instruction and try to detect such cases. + // + // 55 push %rbp + // 48 89 e5 mov %rsp, %rbp + // c3 retq + + std::array<ui8, 3> data; + if (!MemoryReader_->Read(Rip_, &data)) { + Finished_ = true; + return; + } + + if (data[0] == 0xc3 || data[0] == 0x55) { + void* savedRip; + if (!MemoryReader_->Read(StartRsp_, &savedRip)) { + Finished_ = true; + return; + } + + // Avoid infinite loop. + if (Rip_ == savedRip) { + Finished_ = true; + return; + } + + // Detect garbage pointer. + if (!checkPtr(savedRip)) { + Finished_ = true; + return; + } + + Rip_ = savedRip; + return; + } + + if (data[0] == 0x48 && data[1] == 0x89 && data[2] == 0xe5) { + void* savedRip; + if (!MemoryReader_->Read(add(StartRsp_, 8), &savedRip)) { + Finished_ = true; + return; + } + + // Avoid infinite loop. + if (Rip_ == savedRip) { + Finished_ = true; + return; + } + + // Detect garbage pointer. + if (!checkPtr(savedRip)) { + Finished_ = true; + return; + } + + Rip_ = savedRip; + return; + } + } + + const void* savedRbp; + const void* savedRip; + if (!MemoryReader_->Read(Rbp_, &savedRbp) || !MemoryReader_->Read(add(Rbp_, 8), &savedRip)) { + Finished_ = true; + return; + } + + if (!checkPtr(savedRbp)) { + Finished_ = true; + return; + } + + if (!checkPtr(savedRip)) { + Finished_ = true; + return; + } + + if (savedRbp < StartRsp_ || savedRbp > add(StartRsp_, 4_MB)) { + Finished_ = true; + return; + } + + Rip_ = savedRip; + Rbp_ = savedRbp; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h new file mode 100644 index 0000000000..7a6eaf431b --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h @@ -0,0 +1,39 @@ +#pragma once + +#include <library/cpp/yt/memory/safe_memory_reader.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TFramePointerCursorContext +{ + ui64 Rip; + ui64 Rsp; + ui64 Rbp; +}; + +class TFramePointerCursor +{ +public: + TFramePointerCursor( + TSafeMemoryReader* memoryReader, + const TFramePointerCursorContext& context); + + bool IsFinished() const; + const void* GetCurrentIP() const; + void MoveNext(); + +private: + TSafeMemoryReader* MemoryReader_; + bool Finished_ = false; + bool First_ = true; + + const void* Rip_ = nullptr; + const void* Rbp_ = nullptr; + const void* StartRsp_ = nullptr; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/frame_pointer/ya.make b/library/cpp/yt/backtrace/cursors/frame_pointer/ya.make new file mode 100644 index 0000000000..cb85d70315 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/frame_pointer/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + frame_pointer_cursor.cpp +) + +END() diff --git a/library/cpp/yt/backtrace/cursors/interop/interop.cpp b/library/cpp/yt/backtrace/cursors/interop/interop.cpp new file mode 100644 index 0000000000..b4e6cfbe6e --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/interop/interop.cpp @@ -0,0 +1,102 @@ +#include "interop.h" + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TFramePointerCursorContext FramePointerCursorContextFromUcontext(const ucontext_t& ucontext) +{ +#if defined(_linux_) + return { + .Rip = static_cast<ui64>(ucontext.uc_mcontext.gregs[REG_RIP]), + .Rsp = static_cast<ui64>(ucontext.uc_mcontext.gregs[REG_RSP]), + .Rbp = static_cast<ui64>(ucontext.uc_mcontext.gregs[REG_RBP]), + }; +#elif defined(_darwin_) + return { + .Rip = static_cast<ui64>(ucontext.uc_mcontext->__ss.__rip), + .Rsp = static_cast<ui64>(ucontext.uc_mcontext->__ss.__rsp), + .Rbp = static_cast<ui64>(ucontext.uc_mcontext->__ss.__rbp), + }; +#else + #error Unsupported platform +#endif +} + +std::optional<unw_context_t> TrySynthesizeLibunwindContextFromMachineContext( + const TContMachineContext& machineContext) +{ + unw_context_t unwindContext; + if (unw_getcontext(&unwindContext) != 0) { + return {}; + } + + // Some dirty hacks follow. + struct TUnwindContextRegisters + { + ui64 Rax; + ui64 Rbx; + ui64 Rcx; + ui64 Rdx; + ui64 Rdi; + ui64 Rsi; + ui64 Rbp; + ui64 Rsp; + ui64 R8; + ui64 R9; + ui64 R10; + ui64 R11; + ui64 R12; + ui64 R13; + ui64 R14; + ui64 R15; + ui64 Rip; + ui64 Rflags; + ui64 CS; + ui64 FS; + ui64 GS; + }; + + struct TMachineContextRegisters + { + ui64 Rbx; + ui64 Rbp; + ui64 R12; + ui64 R13; + ui64 R14; + ui64 R15; + ui64 Rsp; + ui64 Rip; + }; + + static_assert(sizeof(TContMachineContext) >= sizeof(TMachineContextRegisters)); + static_assert(sizeof(unw_context_t) >= sizeof(TUnwindContextRegisters)); + const auto* machineContextRegisters = reinterpret_cast<const TMachineContextRegisters*>(&machineContext); + auto* unwindContextRegisters = reinterpret_cast<TUnwindContextRegisters*>(&unwindContext); + #define XX(register) unwindContextRegisters->register = machineContextRegisters->register; + XX(Rbx) + XX(Rbp) + XX(R12) + XX(R13) + XX(R14) + XX(R15) + XX(Rsp) + XX(Rip) + #undef XX + return unwindContext; +} + +TFramePointerCursorContext FramePointerCursorContextFromLibunwindCursor( + const unw_cursor_t& cursor) +{ + TFramePointerCursorContext context{}; + auto& mutableCursor = const_cast<unw_cursor_t&>(cursor); + YT_VERIFY(unw_get_reg(&mutableCursor, UNW_REG_IP, &context.Rip) == 0); + YT_VERIFY(unw_get_reg(&mutableCursor, UNW_X86_64_RSP, &context.Rsp) == 0); + YT_VERIFY(unw_get_reg(&mutableCursor, UNW_X86_64_RBP, &context.Rbp) == 0); + return context; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/interop/interop.h b/library/cpp/yt/backtrace/cursors/interop/interop.h new file mode 100644 index 0000000000..62e7177107 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/interop/interop.h @@ -0,0 +1,25 @@ +#pragma once + +#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h> + +#include <contrib/libs/libunwind/include/libunwind.h> + +#include <util/system/context.h> + +#include <optional> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TFramePointerCursorContext FramePointerCursorContextFromUcontext(const ucontext_t& ucontext); + +std::optional<unw_context_t> TrySynthesizeLibunwindContextFromMachineContext( + const TContMachineContext& machineContext); + +TFramePointerCursorContext FramePointerCursorContextFromLibunwindCursor( + const unw_cursor_t& uwCursor); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/interop/ya.make b/library/cpp/yt/backtrace/cursors/interop/ya.make new file mode 100644 index 0000000000..6637f6a9b4 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/interop/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + interop.cpp +) + +PEERDIR( + library/cpp/yt/backtrace/cursors/frame_pointer + contrib/libs/libunwind +) + +END() diff --git a/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..fdea07f78c --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(backtrace-cursors-libunwind) +target_compile_options(backtrace-cursors-libunwind PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(backtrace-cursors-libunwind PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-libunwind +) +target_sources(backtrace-cursors-libunwind PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp +) diff --git a/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.linux-aarch64.txt b/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..9d7858cc27 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.linux-aarch64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(backtrace-cursors-libunwind) +target_compile_options(backtrace-cursors-libunwind PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(backtrace-cursors-libunwind PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-libunwind +) +target_sources(backtrace-cursors-libunwind PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp +) diff --git a/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.linux-x86_64.txt b/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..9d7858cc27 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.linux-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(backtrace-cursors-libunwind) +target_compile_options(backtrace-cursors-libunwind PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(backtrace-cursors-libunwind PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-libunwind +) +target_sources(backtrace-cursors-libunwind PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp +) diff --git a/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.txt b/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.txt new file mode 100644 index 0000000000..606ff46b4b --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/CMakeLists.txt @@ -0,0 +1,15 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp new file mode 100644 index 0000000000..f814753034 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp @@ -0,0 +1,70 @@ +#include "libunwind_cursor.h" + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TLibunwindCursor::TLibunwindCursor() +{ + if (unw_getcontext(&Context_) != 0) { + Finished_ = true; + return; + } + + Initialize(); +} + +TLibunwindCursor::TLibunwindCursor(const unw_context_t& context) + : Context_(context) +{ + Initialize(); +} + +void TLibunwindCursor::Initialize() +{ + if (unw_init_local(&Cursor_, &Context_) != 0) { + Finished_ = true; + return; + } + + ReadCurrentIP(); +} + +bool TLibunwindCursor::IsFinished() const +{ + return Finished_; +} + +const void* TLibunwindCursor::GetCurrentIP() const +{ + return CurrentIP_; +} + +void TLibunwindCursor::MoveNext() +{ + if (Finished_) { + return; + } + + if (unw_step(&Cursor_) <= 0) { + Finished_ = true; + return; + } + + ReadCurrentIP(); +} + +void TLibunwindCursor::ReadCurrentIP() +{ + unw_word_t ip = 0; + if (unw_get_reg(&Cursor_, UNW_REG_IP, &ip) < 0) { + Finished_ = true; + return; + } + + CurrentIP_ = reinterpret_cast<const void*>(ip); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h new file mode 100644 index 0000000000..08b01d07ef --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h @@ -0,0 +1,33 @@ +#pragma once + +#include <contrib/libs/libunwind/include/libunwind.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +class TLibunwindCursor +{ +public: + TLibunwindCursor(); + explicit TLibunwindCursor(const unw_context_t& context); + + bool IsFinished() const; + const void* GetCurrentIP() const; + void MoveNext(); + +private: + unw_context_t Context_; + unw_cursor_t Cursor_; + + bool Finished_ = false; + + const void* CurrentIP_ = nullptr; + + void Initialize(); + void ReadCurrentIP(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/libunwind/ya.make b/library/cpp/yt/backtrace/cursors/libunwind/ya.make new file mode 100644 index 0000000000..8f3a8c5284 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + libunwind_cursor.cpp +) + +PEERDIR( + contrib/libs/libunwind +) + +END() diff --git a/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp b/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp new file mode 100644 index 0000000000..19cb41e795 --- /dev/null +++ b/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp @@ -0,0 +1,25 @@ +#include <library/cpp/yt/backtrace/backtrace.h> + +#include <library/cpp/yt/string/raw_formatter.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +void SymbolizeBacktrace( + TBacktrace backtrace, + const std::function<void(TStringBuf)>& frameCallback) +{ + for (int index = 0; index < std::ssize(backtrace); ++index) { + TRawFormatter<1024> formatter; + formatter.AppendNumber(index + 1, 10, 2); + formatter.AppendString(". "); + formatter.AppendNumberAsHexWithPadding(reinterpret_cast<uintptr_t>(backtrace[index]), 12); + formatter.AppendString("\n"); + frameCallback(formatter.GetBuffer()); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp b/library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp new file mode 100644 index 0000000000..f5d02aaa33 --- /dev/null +++ b/library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp @@ -0,0 +1,64 @@ +#include <library/cpp/yt/backtrace/backtrace.h> + +#include <library/cpp/dwarf_backtrace/backtrace.h> + +#include <library/cpp/yt/string/raw_formatter.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +void SymbolizeBacktrace( + TBacktrace backtrace, + const std::function<void(TStringBuf)>& frameCallback) +{ + auto error = NDwarf::ResolveBacktrace({backtrace.begin(), backtrace.size()}, [&] (const NDwarf::TLineInfo& info) { + TRawFormatter<1024> formatter; + formatter.AppendNumber(info.Index + 1, 10, 2); + formatter.AppendString(". "); + formatter.AppendString("0x"); + const int width = (sizeof(void*) == 8 ? 12 : 8); + // 12 for x86_64 because higher bits are always zeroed. + formatter.AppendNumber(info.Address, 16, width, '0'); + formatter.AppendString(" in "); + formatter.AppendString(info.FunctionName); + const int bytesToAppendEstimate = 4 + info.FileName.Size() + 1 + 4 /* who cares about line numbers > 9999 */ + 1; + if (formatter.GetBytesRemaining() < bytesToAppendEstimate) { + const int offset = formatter.GetBytesRemaining() - bytesToAppendEstimate; + if (formatter.GetBytesWritten() + offset >= 0) { + formatter.Advance(offset); + } + } + formatter.AppendString(" at "); + formatter.AppendString(info.FileName); + formatter.AppendChar(':'); + formatter.AppendNumber(info.Line); + if (formatter.GetBytesRemaining() == 0) { + formatter.Revert(1); + } + formatter.AppendString("\n"); + frameCallback(formatter.GetBuffer()); + // Call the callback exactly `frameCount` times, + // even if there are inline functions and one frame resolved to several lines. + // It needs for case when caller uses `frameCount` less than 100 for pretty formatting. + if (info.Index + 1 == std::ssize(backtrace)) { + return NDwarf::EResolving::Break; + } + return NDwarf::EResolving::Continue; + }); + if (error) { + TRawFormatter<1024> formatter; + formatter.AppendString("*** Error symbolizing backtrace via Dwarf\n"); + formatter.AppendString("*** Code: "); + formatter.AppendNumber(error->Code); + formatter.AppendString("\n"); + formatter.AppendString("*** Message: "); + formatter.AppendString(error->Message); + formatter.AppendString("\n"); + frameCallback(formatter.GetBuffer()); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/symbolizers/dwarf/ya.make b/library/cpp/yt/backtrace/symbolizers/dwarf/ya.make new file mode 100644 index 0000000000..bffeb676d8 --- /dev/null +++ b/library/cpp/yt/backtrace/symbolizers/dwarf/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +SRCS( + GLOBAL dwarf_symbolizer.cpp +) + +PEERDIR( + library/cpp/dwarf_backtrace + library/cpp/yt/backtrace +) + +END() + +IF (BUILD_TYPE == "DEBUG" OR BUILD_TYPE == "PROFILE") + RECURSE_FOR_TESTS( + unittests + ) +ENDIF() diff --git a/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp b/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp new file mode 100644 index 0000000000..37ebda8e48 --- /dev/null +++ b/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp @@ -0,0 +1,113 @@ +#include <library/cpp/yt/backtrace/backtrace.h> + +#include <library/cpp/yt/string/raw_formatter.h> + +#include <util/system/compiler.h> + +#include <dlfcn.h> +#include <cxxabi.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +int GetSymbolInfo(const void* pc, char* buffer, int length) +{ + TBaseFormatter formatter(buffer, length); + + // See http://www.codesourcery.com/cxx-abi/abi.html#mangling + // And, yes, dladdr() is not async signal safe. We can substitute it + // with hand-written symbolization code from google-glog in case of any trouble. + Dl_info info; + if (!dladdr(pc, &info)) { + return 0; + } + + /* + * typedef struct { + * const char *dli_fname; // Pathname of shared object that + * // contains address + * void *dli_fbase; // Address at which shared object + * // is loaded + * const char *dli_sname; // Name of nearest symbol with address + * // lower than addr + * void *dli_saddr; // Exact address of symbol named + * // in dli_sname + * } Dl_info; + * + * If no symbol matching addr could be found, then dli_sname and dli_saddr are set to NULL. + */ + + if (info.dli_sname && info.dli_saddr) { + formatter.AppendString("<"); + int demangleStatus = 0; + + if (info.dli_sname[0] == '_' && info.dli_sname[1] == 'Z') { + // This is also not async signal safe. + // But (ta-dah!) we can replace it with symbolization code from google-glob. + char* demangledName = abi::__cxa_demangle(info.dli_sname, 0, 0, &demangleStatus); + if (demangleStatus == 0) { + formatter.AppendString(demangledName); + } else { + formatter.AppendString(info.dli_sname); + } + free(demangledName); + } else { + formatter.AppendString(info.dli_sname); + } + formatter.AppendString("+"); + formatter.AppendNumber((char*)pc - (char*)info.dli_saddr); + formatter.AppendString(">"); + formatter.AppendString(" "); + } + + if (info.dli_fname && info.dli_fbase) { + formatter.AppendString("("); + formatter.AppendString(info.dli_fname); + formatter.AppendString("+"); + formatter.AppendNumber((char*)pc - (char*)info.dli_fbase); + formatter.AppendString(")"); + } + return formatter.GetBytesWritten(); +} + +void DumpStackFrameInfo(TBaseFormatter* formatter, const void* pc) +{ + formatter->AppendString("@ "); + const int width = (sizeof(void*) == 8 ? 12 : 8) + 2; + // +2 for "0x"; 12 for x86_64 because higher bits are always zeroed. + formatter->AppendNumberAsHexWithPadding(reinterpret_cast<uintptr_t>(pc), width); + formatter->AppendString(" "); + // Get the symbol from the previous address of PC, + // because PC may be in the next function. + formatter->Advance(GetSymbolInfo( + reinterpret_cast<const char*>(pc) - 1, + formatter->GetCursor(), + formatter->GetBytesRemaining())); + if (formatter->GetBytesRemaining() == 0) { + formatter->Revert(1); + } + formatter->AppendString("\n"); +} + +} // namespace + +Y_WEAK void SymbolizeBacktrace( + TBacktrace backtrace, + const std::function<void(TStringBuf)>& frameCallback) +{ + for (int i = 0; i < std::ssize(backtrace); ++i) { + TRawFormatter<1024> formatter; + formatter.Reset(); + formatter.AppendNumber(i + 1, 10, 2); + formatter.AppendString(". "); + DumpStackFrameInfo(&formatter, backtrace[i]); + frameCallback(formatter.GetBuffer()); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/unittests/backtrace_ut.cpp b/library/cpp/yt/backtrace/unittests/backtrace_ut.cpp new file mode 100644 index 0000000000..5992b69277 --- /dev/null +++ b/library/cpp/yt/backtrace/unittests/backtrace_ut.cpp @@ -0,0 +1,61 @@ +#include <gtest/gtest.h> + +#include <gmock/gmock.h> + +#include <library/cpp/yt/memory/safe_memory_reader.h> + +#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h> + +#include <library/cpp/yt/backtrace/cursors/interop/interop.h> + +#include <util/system/compiler.h> + +#include <contrib/libs/libunwind/include/libunwind.h> + +namespace NYT::NBacktrace { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +template <int Depth, class TFn> +Y_NO_INLINE void RunInDeepStack(TFn cb) +{ + if constexpr (Depth == 0) { + cb(); + } else { + std::vector<int> touchMem; + touchMem.push_back(0); + + RunInDeepStack<Depth-1>(cb); + + DoNotOptimizeAway(touchMem); + } +} + +TEST(TFramePointerCursor, FramePointerCursor) +{ + std::vector<const void*> backtrace; + RunInDeepStack<64>([&] { + unw_context_t unwContext; + ASSERT_TRUE(unw_getcontext(&unwContext) == 0); + + unw_cursor_t unwCursor; + ASSERT_TRUE(unw_init_local(&unwCursor, &unwContext) == 0); + + TSafeMemoryReader reader; + auto fpCursorContext = NBacktrace::FramePointerCursorContextFromLibunwindCursor(unwCursor); + NBacktrace::TFramePointerCursor fpCursor(&reader, fpCursorContext); + + while (!fpCursor.IsFinished()) { + backtrace.push_back(fpCursor.GetCurrentIP()); + fpCursor.MoveNext(); + } + }); + + ASSERT_THAT(backtrace, testing::SizeIs(testing::Ge(64u))); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/unittests/ya.make b/library/cpp/yt/backtrace/unittests/ya.make new file mode 100644 index 0000000000..89e55a95ef --- /dev/null +++ b/library/cpp/yt/backtrace/unittests/ya.make @@ -0,0 +1,20 @@ +GTEST() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +PEERDIR( + library/cpp/testing/gtest + library/cpp/yt/backtrace + library/cpp/yt/backtrace/cursors/interop + library/cpp/yt/backtrace/cursors/frame_pointer + library/cpp/yt/backtrace/cursors/libunwind + library/cpp/yt/memory +) + +IF (BUILD_TYPE == "DEBUG" OR BUILD_TYPE == "PROFILE") + SRCS( + backtrace_ut.cpp + ) +ENDIF() + +END() diff --git a/library/cpp/yt/backtrace/ya.make b/library/cpp/yt/backtrace/ya.make new file mode 100644 index 0000000000..d294082e06 --- /dev/null +++ b/library/cpp/yt/backtrace/ya.make @@ -0,0 +1,44 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + backtrace.cpp +) + +IF (OS_WINDOWS) + SRCS( + symbolizers/dummy/dummy_symbolizer.cpp + ) +ELSE() + SRCS( + symbolizers/dynload/dynload_symbolizer.cpp + ) +ENDIF() + +PEERDIR( + library/cpp/yt/string +) + +END() + +RECURSE( + cursors/dummy + cursors/frame_pointer +) + +IF (NOT OS_WINDOWS) + RECURSE( + cursors/libunwind + ) +ENDIF() + +IF (OS_LINUX) + RECURSE( + symbolizers/dwarf + ) + + RECURSE_FOR_TESTS( + unittests + ) +ENDIF() diff --git a/library/cpp/yt/containers/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/containers/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..e570a542e4 --- /dev/null +++ b/library/cpp/yt/containers/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,15 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-containers INTERFACE) +target_link_libraries(cpp-yt-containers INTERFACE + contrib-libs-cxxsupp + yutil + cpp-yt-assert +) diff --git a/library/cpp/yt/containers/CMakeLists.linux-aarch64.txt b/library/cpp/yt/containers/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..b0ce8a3922 --- /dev/null +++ b/library/cpp/yt/containers/CMakeLists.linux-aarch64.txt @@ -0,0 +1,16 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-containers INTERFACE) +target_link_libraries(cpp-yt-containers INTERFACE + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert +) diff --git a/library/cpp/yt/containers/CMakeLists.linux-x86_64.txt b/library/cpp/yt/containers/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..b0ce8a3922 --- /dev/null +++ b/library/cpp/yt/containers/CMakeLists.linux-x86_64.txt @@ -0,0 +1,16 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-containers INTERFACE) +target_link_libraries(cpp-yt-containers INTERFACE + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert +) diff --git a/library/cpp/yt/containers/CMakeLists.txt b/library/cpp/yt/containers/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/yt/containers/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/containers/CMakeLists.windows-x86_64.txt b/library/cpp/yt/containers/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..e570a542e4 --- /dev/null +++ b/library/cpp/yt/containers/CMakeLists.windows-x86_64.txt @@ -0,0 +1,15 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-containers INTERFACE) +target_link_libraries(cpp-yt-containers INTERFACE + contrib-libs-cxxsupp + yutil + cpp-yt-assert +) diff --git a/library/cpp/yt/containers/sharded_set-inl.h b/library/cpp/yt/containers/sharded_set-inl.h new file mode 100644 index 0000000000..67d5be58c6 --- /dev/null +++ b/library/cpp/yt/containers/sharded_set-inl.h @@ -0,0 +1,217 @@ +#ifndef SHARDED_SET_INL_H_ +#error "Direct inclusion of this file is not allowed, include sharded_set.h" +// For the sake of sane code completion. +#include "sharded_set.h" +#endif + +#include <library/cpp/yt/assert/assert.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, int N, class F, class S> +class TShardedSet<T, N, F, S>::const_iterator +{ +private: + friend class TShardedSet<T, N, F, S>; + + using TOwner = TShardedSet<T, N, F, S>; + using TShardIterator = typename S::const_iterator; + + const TOwner* const Owner_; + + int ShardIndex_; + TShardIterator ShardIterator_; + + const_iterator( + const TOwner* owner, + int shardIndex, + TShardIterator shardIterator) + : Owner_(owner) + , ShardIndex_(shardIndex) + , ShardIterator_(shardIterator) + { } + + bool IsValid() const + { + return ShardIterator_ != Owner_->Shards_[ShardIndex_].end(); + } + + void FastForward() + { + while (ShardIndex_ != N - 1 && !IsValid()) { + ++ShardIndex_; + ShardIterator_ = Owner_->Shards_[ShardIndex_].begin(); + } + } + +public: + using difference_type = typename std::iterator_traits<TShardIterator>::difference_type; + using value_type = typename std::iterator_traits<TShardIterator>::value_type; + using pointer = typename std::iterator_traits<TShardIterator>::pointer; + using reference = typename std::iterator_traits<TShardIterator>::reference; + using iterator_category = std::forward_iterator_tag; + + const_iterator& operator++() + { + ++ShardIterator_; + FastForward(); + + return *this; + } + + const_iterator operator++(int) + { + auto result = *this; + + ++ShardIterator_; + FastForward(); + + return result; + } + + bool operator==(const const_iterator& rhs) const + { + return + ShardIndex_ == rhs.ShardIndex_ && + ShardIterator_ == rhs.ShardIterator_; + } + + bool operator!=(const const_iterator& rhs) const + { + return !(*this == rhs); + } + + const T& operator*() const + { + return *ShardIterator_; + } + + const T* operator->() const + { + return &operator*(); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, int N, class F, class S> +TShardedSet<T, N, F, S>::TShardedSet(F elementToShard) + : ElementToShard_(elementToShard) +{ } + +template <class T, int N, class F, class S> +bool TShardedSet<T, N, F, S>::empty() const +{ + return size() == 0; +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::size_type TShardedSet<T, N, F, S>::size() const +{ + size_type result = 0; + for (const auto& shard : Shards_) { + result += shard.size(); + } + + return result; +} + +template <class T, int N, class F, class S> +const T& TShardedSet<T, N, F, S>::front() const +{ + return *begin(); +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::size_type TShardedSet<T, N, F, S>::count(const T& value) const +{ + return GetShard(value).count(value); +} + +template <class T, int N, class F, class S> +bool TShardedSet<T, N, F, S>::contains(const T& value) const +{ + return GetShard(value).contains(value); +} + +template <class T, int N, class F, class S> +std::pair<typename TShardedSet<T, N, F, S>::const_iterator, bool> TShardedSet<T, N, F, S>::insert(const T& value) +{ + auto shardIndex = ElementToShard_(value); + auto& shard = Shards_[shardIndex]; + auto [shardIterator, inserted] = shard.insert(value); + + const_iterator iterator(this, shardIndex, shardIterator); + return {iterator, inserted}; +} + +template <class T, int N, class F, class S> +bool TShardedSet<T, N, F, S>::erase(const T& value) +{ + return GetShard(value).erase(value); +} + +template <class T, int N, class F, class S> +void TShardedSet<T, N, F, S>::clear() +{ + for (auto& shard : Shards_) { + shard.clear(); + } +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::begin() const +{ + const_iterator iterator(this, /*shardIndex*/ 0, /*shardIterator*/ Shards_[0].begin()); + iterator.FastForward(); + + return iterator; +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::cbegin() const +{ + return begin(); +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::end() const +{ + return const_iterator(this, /*shardIndex*/ N - 1, /*shardIterator*/ Shards_[N - 1].end()); +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::cend() const +{ + return end(); +} + +template <class T, int N, class F, class S> +const S& TShardedSet<T, N, F, S>::Shard(int shardIndex) const +{ + return Shards_[shardIndex]; +} + +template <class T, int N, class F, class S> +S& TShardedSet<T, N, F, S>::MutableShard(int shardIndex) +{ + return Shards_[shardIndex]; +} + +template <class T, int N, class F, class S> +S& TShardedSet<T, N, F, S>::GetShard(const T& value) +{ + return Shards_[ElementToShard_(value)]; +} + +template <class T, int N, class F, class S> +const S& TShardedSet<T, N, F, S>::GetShard(const T& value) const +{ + return Shards_[ElementToShard_(value)]; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/containers/sharded_set.h b/library/cpp/yt/containers/sharded_set.h new file mode 100644 index 0000000000..fa24893aa4 --- /dev/null +++ b/library/cpp/yt/containers/sharded_set.h @@ -0,0 +1,69 @@ +#pragma once + +#include <util/generic/hash_set.h> + +#include <array> +#include <cstddef> +#include <utility> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +//! A set that stores elements divided into fixed amount of shards. +//! Provides access to whole set and particular shards. +//! The interface is pretty minimalistic, feel free to extend it when needed. +template <class T, int N, class F, class S = THashSet<T>> +class TShardedSet +{ +public: + using size_type = size_t; + using difference_type = ptrdiff_t; + + using value_type = T; + + class const_iterator; + + explicit TShardedSet(F elementToShard = F()); + + [[nodiscard]] bool empty() const; + + size_type size() const; + + const T& front() const; + + size_type count(const T& value) const; + + bool contains(const T& value) const; + + std::pair<const_iterator, bool> insert(const T& value); + + bool erase(const T& value); + + void clear(); + + const_iterator begin() const; + const_iterator cbegin() const; + + const_iterator end() const; + const_iterator cend() const; + + const S& Shard(int shardIndex) const; + S& MutableShard(int shardIndex); + +private: + std::array<S, N> Shards_; + + const F ElementToShard_; + + S& GetShard(const T& value); + const S& GetShard(const T& value) const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define SHARDED_SET_INL_H_ +#include "sharded_set-inl.h" +#undef SHARDED_SET_INL_H_ diff --git a/library/cpp/yt/containers/unittests/sharded_set_ut.cpp b/library/cpp/yt/containers/unittests/sharded_set_ut.cpp new file mode 100644 index 0000000000..2c4f8c5935 --- /dev/null +++ b/library/cpp/yt/containers/unittests/sharded_set_ut.cpp @@ -0,0 +1,121 @@ +#include <library/cpp/yt/containers/sharded_set.h> + +#include <library/cpp/testing/gtest/gtest.h> + +#include <random> + +namespace NYT { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TIntToShard +{ + int operator()(int value) const + { + return value % 16; + } +}; + +using TSet = TShardedSet<int, 16, TIntToShard>; + +//////////////////////////////////////////////////////////////////////////////// + +TEST(CompactSetTest, Insert) +{ + TSet set; + + for (int i = 0; i < 4; i++) { + set.insert(i); + } + + for (int i = 0; i < 4; i++) { + set.insert(i); + } + + EXPECT_EQ(4u, set.size()); + + for (int i = 0; i < 4; i++) + EXPECT_EQ(1u, set.count(i)); + + EXPECT_EQ(0u, set.count(4)); +} + +TEST(CompactSetTest, Erase) +{ + TSet set; + + for (int i = 0; i < 8; i++) { + set.insert(i); + } + + EXPECT_EQ(8u, set.size()); + + // Remove elements one by one and check if all other elements are still there. + for (int i = 0; i < 8; i++) { + EXPECT_EQ(1u, set.count(i)); + EXPECT_TRUE(set.erase(i)); + EXPECT_EQ(0u, set.count(i)); + EXPECT_EQ(8u - i - 1, set.size()); + for (int j = i + 1; j < 8; j++) { + EXPECT_EQ(1u, set.count(j)); + } + } + + EXPECT_EQ(0u, set.count(8)); +} + +TEST(CompactSetTest, StressTest) +{ + TSet set; + + constexpr int Iterations = 1'000'000; + constexpr int Values = 128; + + THashSet<int> values; + + auto checkEverything = [&] { + EXPECT_EQ(values.size(), set.size()); + EXPECT_EQ(values.empty(), set.empty()); + EXPECT_EQ(values, THashSet<int>(set.begin(), set.end())); + + std::array<THashSet<int>, 16> shards; + for (int value : values) { + shards[value % 16].insert(value); + } + for (int shardIndex = 0; shardIndex < 16; ++shardIndex) { + EXPECT_EQ(shards[shardIndex], set.Shard(shardIndex)); + } + + for (int value = 0; value < Values; ++value) { + EXPECT_EQ(values.contains(value), set.contains(value)); + EXPECT_EQ(values.count(value), set.count(value)); + } + }; + + std::mt19937_64 rng(42); + + for (int iteration = 0; iteration < Iterations; ++iteration) { + if (rng() % 100 == 0) { + set.clear(); + values.clear(); + checkEverything(); + } + + int value = rng() % Values; + if (rng() % 2 == 0) { + set.insert(value); + values.insert(value); + } else { + set.erase(value); + values.erase(value); + } + + checkEverything(); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/library/cpp/yt/containers/unittests/ya.make b/library/cpp/yt/containers/unittests/ya.make new file mode 100644 index 0000000000..3e7cfd4311 --- /dev/null +++ b/library/cpp/yt/containers/unittests/ya.make @@ -0,0 +1,15 @@ +GTEST(unittester-containers) + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + sharded_set_ut.cpp +) + +PEERDIR( + library/cpp/yt/containers + + library/cpp/testing/gtest +) + +END() diff --git a/library/cpp/yt/cpu_clock/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/cpu_clock/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..b9afea23f7 --- /dev/null +++ b/library/cpp/yt/cpu_clock/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-cpu_clock) +target_compile_options(cpp-yt-cpu_clock PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-cpu_clock PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-assert +) +target_sources(cpp-yt-cpu_clock PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/cpu_clock/clock.cpp +) diff --git a/library/cpp/yt/cpu_clock/CMakeLists.linux-aarch64.txt b/library/cpp/yt/cpu_clock/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..89fe774bc0 --- /dev/null +++ b/library/cpp/yt/cpu_clock/CMakeLists.linux-aarch64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-cpu_clock) +target_compile_options(cpp-yt-cpu_clock PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-cpu_clock PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert +) +target_sources(cpp-yt-cpu_clock PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/cpu_clock/clock.cpp +) diff --git a/library/cpp/yt/cpu_clock/CMakeLists.linux-x86_64.txt b/library/cpp/yt/cpu_clock/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..89fe774bc0 --- /dev/null +++ b/library/cpp/yt/cpu_clock/CMakeLists.linux-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-cpu_clock) +target_compile_options(cpp-yt-cpu_clock PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-cpu_clock PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert +) +target_sources(cpp-yt-cpu_clock PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/cpu_clock/clock.cpp +) diff --git a/library/cpp/yt/cpu_clock/CMakeLists.txt b/library/cpp/yt/cpu_clock/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/yt/cpu_clock/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/cpu_clock/CMakeLists.windows-x86_64.txt b/library/cpp/yt/cpu_clock/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..cbe906d57f --- /dev/null +++ b/library/cpp/yt/cpu_clock/CMakeLists.windows-x86_64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-cpu_clock) +target_link_libraries(cpp-yt-cpu_clock PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-assert +) +target_sources(cpp-yt-cpu_clock PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/cpu_clock/clock.cpp +) diff --git a/library/cpp/yt/cpu_clock/benchmark/benchmark.cpp b/library/cpp/yt/cpu_clock/benchmark/benchmark.cpp new file mode 100644 index 0000000000..9d300b6726 --- /dev/null +++ b/library/cpp/yt/cpu_clock/benchmark/benchmark.cpp @@ -0,0 +1,41 @@ +#include "benchmark/benchmark.h" +#include <benchmark/benchmark.h> + +#include <library/cpp/yt/cpu_clock/clock.h> + +namespace NYT { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +void BM_GetCpuInstant(benchmark::State& state) +{ + for (auto _ : state) { + benchmark::DoNotOptimize(GetCpuInstant()); + } +} + +BENCHMARK(BM_GetCpuInstant); + +void BM_GetCpuApproximateInstant(benchmark::State& state) +{ + for (auto _ : state) { + benchmark::DoNotOptimize(GetApproximateCpuInstant()); + } +} + +BENCHMARK(BM_GetCpuApproximateInstant); + +void BM_InstantNow(benchmark::State& state) +{ + for (auto _ : state) { + benchmark::DoNotOptimize(TInstant::Now()); + } +} + +BENCHMARK(BM_InstantNow); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/library/cpp/yt/cpu_clock/benchmark/ya.make b/library/cpp/yt/cpu_clock/benchmark/ya.make new file mode 100644 index 0000000000..4550bf5934 --- /dev/null +++ b/library/cpp/yt/cpu_clock/benchmark/ya.make @@ -0,0 +1,11 @@ +G_BENCHMARK() + +SRCS( + benchmark.cpp +) + +PEERDIR( + library/cpp/yt/cpu_clock +) + +END() diff --git a/library/cpp/yt/cpu_clock/unittests/clock_ut.cpp b/library/cpp/yt/cpu_clock/unittests/clock_ut.cpp new file mode 100644 index 0000000000..bd9cb6d4be --- /dev/null +++ b/library/cpp/yt/cpu_clock/unittests/clock_ut.cpp @@ -0,0 +1,46 @@ +#include <gtest/gtest.h> + +#include <library/cpp/yt/cpu_clock/clock.h> + +namespace NYT { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +i64 DiffMS(T a, T b) +{ + return a >= b + ? static_cast<i64>(a.MilliSeconds()) - static_cast<i64>(b.MilliSeconds()) + : DiffMS(b, a); +} + +TEST(TTimingTest, GetInstant) +{ + GetInstant(); + + EXPECT_LE(DiffMS(GetInstant(), TInstant::Now()), 10); +} + +TEST(TTimingTest, InstantVSCpuInstant) +{ + auto instant1 = TInstant::Now(); + auto cpuInstant = InstantToCpuInstant(instant1); + auto instant2 = CpuInstantToInstant(cpuInstant); + EXPECT_LE(DiffMS(instant1, instant2), 10); +} + +TEST(TTimingTest, DurationVSCpuDuration) +{ + auto cpuInstant1 = GetCpuInstant(); + constexpr auto duration1 = TDuration::MilliSeconds(100); + Sleep(duration1); + auto cpuInstant2 = GetCpuInstant(); + auto duration2 = CpuDurationToDuration(cpuInstant2 - cpuInstant1); + EXPECT_LE(DiffMS(duration1, duration2), 10); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/library/cpp/yt/cpu_clock/unittests/ya.make b/library/cpp/yt/cpu_clock/unittests/ya.make new file mode 100644 index 0000000000..921087c295 --- /dev/null +++ b/library/cpp/yt/cpu_clock/unittests/ya.make @@ -0,0 +1,13 @@ +GTEST() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + clock_ut.cpp +) + +PEERDIR( + library/cpp/yt/cpu_clock +) + +END() diff --git a/library/cpp/yt/farmhash/farm_hash.h b/library/cpp/yt/farmhash/farm_hash.h new file mode 100644 index 0000000000..fe4c8193a0 --- /dev/null +++ b/library/cpp/yt/farmhash/farm_hash.h @@ -0,0 +1,63 @@ +#pragma once + +#include <contrib/libs/farmhash/farmhash.h> + +#include <util/system/types.h> + +#include <util/generic/strbuf.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +using TFingerprint = ui64; + +static inline TFingerprint FarmHash(ui64 value) +{ + return ::util::Fingerprint(value); +} + +static inline TFingerprint FarmHash(const void* buf, size_t len) +{ + return ::util::Hash64(static_cast<const char*>(buf), len); +} + +static inline TFingerprint FarmHash(const void* buf, size_t len, ui64 seed) +{ + return ::util::Hash64WithSeed(static_cast<const char*>(buf), len, seed); +} + +static inline TFingerprint FarmFingerprint(ui64 value) +{ + return ::util::Fingerprint(value); +} + +static inline TFingerprint FarmFingerprint(const void* buf, size_t len) +{ + return ::util::Fingerprint64(static_cast<const char*>(buf), len); +} + +static inline TFingerprint FarmFingerprint(TStringBuf buf) +{ + return FarmFingerprint(buf.Data(), buf.Size()); +} + +static inline TFingerprint FarmFingerprint(ui64 first, ui64 second) +{ + return ::util::Fingerprint(::util::Uint128(first, second)); +} + +// Forever-fixed Google FarmHash fingerprint. +template <class T> +TFingerprint FarmFingerprint(const T* begin, const T* end) +{ + ui64 result = 0xdeadc0de; + for (const auto* value = begin; value < end; ++value) { + result = FarmFingerprint(result, FarmFingerprint(*value)); + } + return result ^ (end - begin); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/logging/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/logging/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..b9c4a4c7db --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(plain_text_formatter) + +add_library(cpp-yt-logging) +target_compile_options(cpp-yt-logging PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-logging PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-memory + cpp-yt-misc + cpp-yt-yson_string +) +target_sources(cpp-yt-logging PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/logger.cpp +) diff --git a/library/cpp/yt/logging/CMakeLists.linux-aarch64.txt b/library/cpp/yt/logging/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..bd29994891 --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.linux-aarch64.txt @@ -0,0 +1,26 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(plain_text_formatter) + +add_library(cpp-yt-logging) +target_compile_options(cpp-yt-logging PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-logging PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-memory + cpp-yt-misc + cpp-yt-yson_string +) +target_sources(cpp-yt-logging PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/logger.cpp +) diff --git a/library/cpp/yt/logging/CMakeLists.linux-x86_64.txt b/library/cpp/yt/logging/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..bd29994891 --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.linux-x86_64.txt @@ -0,0 +1,26 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(plain_text_formatter) + +add_library(cpp-yt-logging) +target_compile_options(cpp-yt-logging PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-logging PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-memory + cpp-yt-misc + cpp-yt-yson_string +) +target_sources(cpp-yt-logging PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/logger.cpp +) diff --git a/library/cpp/yt/logging/CMakeLists.txt b/library/cpp/yt/logging/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/logging/CMakeLists.windows-x86_64.txt b/library/cpp/yt/logging/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..5e766966b0 --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.windows-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(plain_text_formatter) + +add_library(cpp-yt-logging) +target_link_libraries(cpp-yt-logging PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-memory + cpp-yt-misc + cpp-yt-yson_string +) +target_sources(cpp-yt-logging PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/logger.cpp +) diff --git a/library/cpp/yt/logging/backends/arcadia/backend.cpp b/library/cpp/yt/logging/backends/arcadia/backend.cpp new file mode 100644 index 0000000000..3c6ff9f5f5 --- /dev/null +++ b/library/cpp/yt/logging/backends/arcadia/backend.cpp @@ -0,0 +1,86 @@ +#include "backend.h" + +#include <library/cpp/logger/backend.h> +#include <library/cpp/logger/record.h> + +#include <library/cpp/yt/assert/assert.h> + +#include <library/cpp/yt/logging/logger.h> + +namespace NYT::NLogging { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +ELogLevel ConvertToLogLevel(ELogPriority priority) +{ + switch (priority) { + case ELogPriority::TLOG_DEBUG: + return ELogLevel::Debug; + case ELogPriority::TLOG_INFO: + [[fallthrough]]; + case ELogPriority::TLOG_NOTICE: + return ELogLevel::Info; + case ELogPriority::TLOG_WARNING: + return ELogLevel::Warning; + case ELogPriority::TLOG_ERR: + return ELogLevel::Error; + case ELogPriority::TLOG_CRIT: + case ELogPriority::TLOG_ALERT: + return ELogLevel::Alert; + case ELogPriority::TLOG_EMERG: + return ELogLevel::Fatal; + case ELogPriority::TLOG_RESOURCES: + return ELogLevel::Maximum; + } + YT_ABORT(); +} + +class TLogBackendBridge + : public TLogBackend +{ +public: + TLogBackendBridge(const TLogger& logger) + : Logger_(logger) + { } + + void WriteData(const TLogRecord& rec) override + { + const auto logLevel = ConvertToLogLevel(rec.Priority); + if (!Logger_.IsLevelEnabled(logLevel)) { + return; + } + + // Remove trailing \n, because it will add it. + TStringBuf message(rec.Data, rec.Len); + message.ChopSuffix(TStringBuf("\n")); + // Use low-level api, because it is more convinient here. + auto loggingContext = GetLoggingContext(); + auto event = NDetail::CreateLogEvent(loggingContext, Logger_, logLevel); + event.MessageRef = NDetail::BuildLogMessage(loggingContext, Logger_, message).MessageRef; + event.Family = ELogFamily::PlainText; + Logger_.Write(std::move(event)); + } + + void ReopenLog() override + { } + + ELogPriority FiltrationLevel() const override + { + return LOG_MAX_PRIORITY; + } + +private: + const TLogger Logger_; +}; + +} // namespace + +THolder<TLogBackend> CreateArcadiaLogBackend(const TLogger& logger) +{ + return MakeHolder<TLogBackendBridge>(logger); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/arcadia/backend.h b/library/cpp/yt/logging/backends/arcadia/backend.h new file mode 100644 index 0000000000..251918c972 --- /dev/null +++ b/library/cpp/yt/logging/backends/arcadia/backend.h @@ -0,0 +1,18 @@ +#pragma once + +#include <library/cpp/yt/logging/public.h> + +#include <util/generic/ptr.h> + +class TLogBackend; + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +//! Create TLogBackend which redirects log messages to |logger|. +THolder<TLogBackend> CreateArcadiaLogBackend(const TLogger& logger); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/arcadia/ya.make b/library/cpp/yt/logging/backends/arcadia/ya.make new file mode 100644 index 0000000000..ee90be8108 --- /dev/null +++ b/library/cpp/yt/logging/backends/arcadia/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + backend.cpp +) + +PEERDIR( + library/cpp/yt/assert + library/cpp/yt/logging + + library/cpp/logger +) + +END() diff --git a/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp b/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp new file mode 100644 index 0000000000..62269dc0c0 --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp @@ -0,0 +1,87 @@ +#include "stream_log_manager.h" + +#include <library/cpp/yt/logging/logger.h> + +#include <library/cpp/yt/logging/plain_text_formatter/formatter.h> + +#include <library/cpp/yt/string/raw_formatter.h> + +#include <library/cpp/yt/threading/fork_aware_spin_lock.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +class TStreamLogManager + : public ILogManager +{ +public: + explicit TStreamLogManager(IOutputStream* output) + : Output_(output) + { } + + void RegisterStaticAnchor( + TLoggingAnchor* anchor, + ::TSourceLocation /*sourceLocation*/, + TStringBuf /*anchorMessage*/) override + { + anchor->Registered = true; + } + + virtual void UpdateAnchor(TLoggingAnchor* anchor) override + { + anchor->Enabled = true; + } + + virtual void Enqueue(TLogEvent&& event) override + { + Buffer_.Reset(); + EventFormatter_.Format(&Buffer_, event); + *Output_ << Buffer_.GetBuffer() << Endl; + } + + virtual const TLoggingCategory* GetCategory(TStringBuf categoryName) override + { + if (!categoryName) { + return nullptr; + } + + auto guard = Guard(SpinLock_); + auto it = NameToCategory_.find(categoryName); + if (it == NameToCategory_.end()) { + auto category = std::make_unique<TLoggingCategory>(); + category->Name = categoryName; + category->ActualVersion = &Version_; + category->CurrentVersion = Version_.load(); + it = NameToCategory_.emplace(categoryName, std::move(category)).first; + } + return it->second.get(); + } + + virtual void UpdateCategory(TLoggingCategory* /*category*/) override + { } + + virtual bool GetAbortOnAlert() const override + { + return false; + } + +private: + IOutputStream* const Output_; + + NThreading::TForkAwareSpinLock SpinLock_; + THashMap<TString, std::unique_ptr<TLoggingCategory>> NameToCategory_; + std::atomic<int> Version_ = 1; + + TPlainTextEventFormatter EventFormatter_{/*enableSourceLocation*/ false}; + TRawFormatter<MessageBufferSize> Buffer_; +}; + +std::unique_ptr<ILogManager> CreateStreamLogManager(IOutputStream* output) +{ + return std::make_unique<TStreamLogManager>(output); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/stream/stream_log_manager.h b/library/cpp/yt/logging/backends/stream/stream_log_manager.h new file mode 100644 index 0000000000..2f6794e587 --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/stream_log_manager.h @@ -0,0 +1,15 @@ +#pragma once + +#include <library/cpp/yt/logging/public.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +//! Creates a dead-simple implementation that synchronously logs +//! all events to #output. +std::unique_ptr<ILogManager> CreateStreamLogManager(IOutputStream* output); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp new file mode 100644 index 0000000000..cb3e244e3b --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp @@ -0,0 +1,37 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/logging/logger.h> + +#include <library/cpp/yt/logging/backends/stream/stream_log_manager.h> + +#include <util/stream/str.h> + +#include <util/string/split.h> + +namespace NYT::NLogging { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TStreamLogManagerTest, Simple) +{ + TString str; + { + TStringOutput output(str); + auto logManager = CreateStreamLogManager(&output); + TLogger Logger(logManager.get(), "Test"); + YT_LOG_INFO("Hello world"); + } + + TVector<TStringBuf> tokens; + Split(str, "\t", tokens); + EXPECT_GE(std::ssize(tokens), 4); + EXPECT_EQ(tokens[1], "I"); + EXPECT_EQ(tokens[2], "Test"); + EXPECT_EQ(tokens[3], "Hello world"); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/stream/unittests/ya.make b/library/cpp/yt/logging/backends/stream/unittests/ya.make new file mode 100644 index 0000000000..29270459fa --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/unittests/ya.make @@ -0,0 +1,14 @@ +GTEST(unittester-library-logging) + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + stream_log_manager_ut.cpp +) + +PEERDIR( + library/cpp/testing/gtest + library/cpp/yt/logging/backends/stream +) + +END() diff --git a/library/cpp/yt/logging/backends/stream/ya.make b/library/cpp/yt/logging/backends/stream/ya.make new file mode 100644 index 0000000000..86e3aa046b --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + stream_log_manager.cpp +) + +PEERDIR( + library/cpp/yt/logging + library/cpp/yt/logging/plain_text_formatter + library/cpp/yt/string + library/cpp/yt/threading +) + +END() + +RECURSE_FOR_TESTS( + unittests +) diff --git a/library/cpp/yt/logging/backends/ya.make b/library/cpp/yt/logging/backends/ya.make new file mode 100644 index 0000000000..6ee4b72bfe --- /dev/null +++ b/library/cpp/yt/logging/backends/ya.make @@ -0,0 +1,4 @@ +RECURSE( + arcadia + stream +) diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h new file mode 100644 index 0000000000..6f489da82d --- /dev/null +++ b/library/cpp/yt/logging/logger-inl.h @@ -0,0 +1,303 @@ +#ifndef LOGGER_INL_H_ +#error "Direct inclusion of this file is not allowed, include logger.h" +// For the sake of sane code completion. +#include "logger.h" +#endif +#undef LOGGER_INL_H_ + +#include <library/cpp/yt/yson_string/convert.h> +#include <library/cpp/yt/yson_string/string.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +inline bool TLogger::IsAnchorUpToDate(const TLoggingAnchor& position) const +{ + return + !Category_ || + position.CurrentVersion == Category_->ActualVersion->load(std::memory_order::relaxed); +} + +template <class... TArgs> +void TLogger::AddTag(const char* format, TArgs&&... args) +{ + AddRawTag(Format(format, std::forward<TArgs>(args)...)); +} + +template <class TType> +void TLogger::AddStructuredTag(TStringBuf key, TType value) +{ + StructuredTags_.emplace_back(key, NYson::ConvertToYsonString(value)); +} + +template <class... TArgs> +TLogger TLogger::WithTag(const char* format, TArgs&&... args) const +{ + auto result = *this; + result.AddTag(format, std::forward<TArgs>(args)...); + return result; +} + +template <class TType> +TLogger TLogger::WithStructuredTag(TStringBuf key, TType value) const +{ + auto result = *this; + result.AddStructuredTag(key, value); + return result; +} + +Y_FORCE_INLINE bool TLogger::IsLevelEnabled(ELogLevel level) const +{ + // This is the first check which is intended to be inlined next to + // logging invocation point. Check below is almost zero-cost due + // to branch prediction (which requires inlining for proper work). + if (level < MinLevel_) { + return false; + } + + // Next check is heavier and requires full log manager definition which + // is undesirable in -inl.h header file. This is why we extract it + // to a separate method which is implemented in cpp file. + return IsLevelEnabledHeavy(level); +} + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +struct TMessageStringBuilderContext +{ + TSharedMutableRef Chunk; +}; + +struct TMessageBufferTag +{ }; + +class TMessageStringBuilder + : public TStringBuilderBase +{ +public: + TSharedRef Flush(); + + // For testing only. + static void DisablePerThreadCache(); + +protected: + void DoReset() override; + void DoReserve(size_t newLength) override; + +private: + struct TPerThreadCache + { + ~TPerThreadCache(); + + TSharedMutableRef Chunk; + size_t ChunkOffset = 0; + }; + + TSharedMutableRef Buffer_; + + static thread_local TPerThreadCache* Cache_; + static thread_local bool CacheDestroyed_; + static TPerThreadCache* GetCache(); + + static constexpr size_t ChunkSize = 128_KB - 64; +}; + +inline bool HasMessageTags( + const TLoggingContext& loggingContext, + const TLogger& logger) +{ + if (logger.GetTag()) { + return true; + } + if (loggingContext.TraceLoggingTag) { + return true; + } + return false; +} + +inline void AppendMessageTags( + TStringBuilderBase* builder, + const TLoggingContext& loggingContext, + const TLogger& logger) +{ + bool printComma = false; + if (const auto& loggerTag = logger.GetTag()) { + builder->AppendString(loggerTag); + printComma = true; + } + if (auto traceLoggingTag = loggingContext.TraceLoggingTag) { + if (printComma) { + builder->AppendString(TStringBuf(", ")); + } + builder->AppendString(traceLoggingTag); + } +} + +inline void AppendLogMessage( + TStringBuilderBase* builder, + const TLoggingContext& loggingContext, + const TLogger& logger, + TRef message) +{ + if (HasMessageTags(loggingContext, logger)) { + if (message.Size() >= 1 && message[message.Size() - 1] == ')') { + builder->AppendString(TStringBuf(message.Begin(), message.Size() - 1)); + builder->AppendString(TStringBuf(", ")); + } else { + builder->AppendString(TStringBuf(message.Begin(), message.Size())); + builder->AppendString(TStringBuf(" (")); + } + AppendMessageTags(builder, loggingContext, logger); + builder->AppendChar(')'); + } else { + builder->AppendString(TStringBuf(message.Begin(), message.Size())); + } +} + +template <class... TArgs> +void AppendLogMessageWithFormat( + TStringBuilderBase* builder, + const TLoggingContext& loggingContext, + const TLogger& logger, + TStringBuf format, + TArgs&&... args) +{ + if (HasMessageTags(loggingContext, logger)) { + if (format.size() >= 2 && format[format.size() - 1] == ')') { + builder->AppendFormat(format.substr(0, format.size() - 1), std::forward<TArgs>(args)...); + builder->AppendString(TStringBuf(", ")); + } else { + builder->AppendFormat(format, std::forward<TArgs>(args)...); + builder->AppendString(TStringBuf(" (")); + } + AppendMessageTags(builder, loggingContext, logger); + builder->AppendChar(')'); + } else { + builder->AppendFormat(format, std::forward<TArgs>(args)...); + } +} + +struct TLogMessage +{ + TSharedRef MessageRef; + TStringBuf Anchor; +}; + +template <size_t Length, class... TArgs> +TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + const char (&format)[Length], + TArgs&&... args) +{ + TMessageStringBuilder builder; + AppendLogMessageWithFormat(&builder, loggingContext, logger, format, std::forward<TArgs>(args)...); + return {builder.Flush(), format}; +} + +template <class T> +TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + const T& obj) +{ + TMessageStringBuilder builder; + FormatValue(&builder, obj, TStringBuf()); + if (HasMessageTags(loggingContext, logger)) { + builder.AppendString(TStringBuf(" (")); + AppendMessageTags(&builder, loggingContext, logger); + builder.AppendChar(')'); + } + return {builder.Flush(), TStringBuf()}; +} + +inline TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + TStringBuf message) +{ + TMessageStringBuilder builder; + builder.AppendString(message); + if (HasMessageTags(loggingContext, logger)) { + builder.AppendString(TStringBuf(" (")); + AppendMessageTags(&builder, loggingContext, logger); + builder.AppendChar(')'); + } + return {builder.Flush(), message}; +} + +template <size_t Length> +TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + const char (&message)[Length]) +{ + return BuildLogMessage( + loggingContext, + logger, + TStringBuf(message)); +} + +inline TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + TSharedRef&& message) +{ + if (HasMessageTags(loggingContext, logger)) { + TMessageStringBuilder builder; + AppendLogMessage(&builder, loggingContext, logger, message); + return {builder.Flush(), TStringBuf()}; + } else { + return {std::move(message), TStringBuf()}; + } +} + +inline TLogEvent CreateLogEvent( + const TLoggingContext& loggingContext, + const TLogger& logger, + ELogLevel level) +{ + TLogEvent event; + event.Instant = loggingContext.Instant; + event.Category = logger.GetCategory(); + event.Essential = logger.IsEssential(); + event.Level = level; + event.ThreadId = loggingContext.ThreadId; + event.ThreadName = loggingContext.ThreadName; + event.FiberId = loggingContext.FiberId; + event.TraceId = loggingContext.TraceId; + event.RequestId = loggingContext.RequestId; + return event; +} + +void OnCriticalLogEvent( + const TLogger& logger, + const TLogEvent& event); + +inline void LogEventImpl( + const TLoggingContext& loggingContext, + const TLogger& logger, + ELogLevel level, + ::TSourceLocation sourceLocation, + TSharedRef message) +{ + auto event = CreateLogEvent(loggingContext, logger, level); + event.MessageKind = ELogMessageKind::Unstructured; + event.MessageRef = std::move(message); + event.Family = ELogFamily::PlainText; + event.SourceFile = sourceLocation.File; + event.SourceLine = sourceLocation.Line; + logger.Write(std::move(event)); + if (Y_UNLIKELY(event.Level >= ELogLevel::Alert)) { + OnCriticalLogEvent(logger, event); + } +} + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/logger.cpp b/library/cpp/yt/logging/logger.cpp new file mode 100644 index 0000000000..4ee5c1a01b --- /dev/null +++ b/library/cpp/yt/logging/logger.cpp @@ -0,0 +1,289 @@ +#include "logger.h" + +#include <library/cpp/yt/assert/assert.h> + +#include <library/cpp/yt/cpu_clock/clock.h> + +#include <library/cpp/yt/misc/thread_name.h> + +#include <util/system/compiler.h> +#include <util/system/thread.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +void OnCriticalLogEvent( + const TLogger& logger, + const TLogEvent& event) +{ + if (event.Level == ELogLevel::Fatal || + event.Level == ELogLevel::Alert && logger.GetAbortOnAlert()) + { + fprintf(stderr, "*** Aborting on critical log event\n"); + fwrite(event.MessageRef.begin(), 1, event.MessageRef.size(), stderr); + fprintf(stderr, "\n"); + YT_ABORT(); + } +} + +TSharedRef TMessageStringBuilder::Flush() +{ + return Buffer_.Slice(0, GetLength()); +} + +void TMessageStringBuilder::DisablePerThreadCache() +{ + Cache_ = nullptr; + CacheDestroyed_ = true; +} + +void TMessageStringBuilder::DoReset() +{ + Buffer_.Reset(); +} + +void TMessageStringBuilder::DoReserve(size_t newCapacity) +{ + auto oldLength = GetLength(); + newCapacity = FastClp2(newCapacity); + + auto newChunkSize = std::max(ChunkSize, newCapacity); + // Hold the old buffer until the data is copied. + auto oldBuffer = std::move(Buffer_); + auto* cache = GetCache(); + if (Y_LIKELY(cache)) { + auto oldCapacity = End_ - Begin_; + auto deltaCapacity = newCapacity - oldCapacity; + if (End_ == cache->Chunk.Begin() + cache->ChunkOffset && + cache->ChunkOffset + deltaCapacity <= cache->Chunk.Size()) + { + // Resize inplace. + Buffer_ = cache->Chunk.Slice(cache->ChunkOffset - oldCapacity, cache->ChunkOffset + deltaCapacity); + cache->ChunkOffset += deltaCapacity; + End_ = Begin_ + newCapacity; + return; + } + + if (Y_UNLIKELY(cache->ChunkOffset + newCapacity > cache->Chunk.Size())) { + cache->Chunk = TSharedMutableRef::Allocate<TMessageBufferTag>(newChunkSize, {.InitializeStorage = false}); + cache->ChunkOffset = 0; + } + + Buffer_ = cache->Chunk.Slice(cache->ChunkOffset, cache->ChunkOffset + newCapacity); + cache->ChunkOffset += newCapacity; + } else { + Buffer_ = TSharedMutableRef::Allocate<TMessageBufferTag>(newChunkSize, {.InitializeStorage = false}); + newCapacity = newChunkSize; + } + if (oldLength > 0) { + ::memcpy(Buffer_.Begin(), Begin_, oldLength); + } + Begin_ = Buffer_.Begin(); + End_ = Begin_ + newCapacity; +} + +TMessageStringBuilder::TPerThreadCache* TMessageStringBuilder::GetCache() +{ + if (Y_LIKELY(Cache_)) { + return Cache_; + } + if (CacheDestroyed_) { + return nullptr; + } + static thread_local TPerThreadCache Cache; + Cache_ = &Cache; + return Cache_; +} + +TMessageStringBuilder::TPerThreadCache::~TPerThreadCache() +{ + TMessageStringBuilder::DisablePerThreadCache(); +} + +thread_local TMessageStringBuilder::TPerThreadCache* TMessageStringBuilder::Cache_; +thread_local bool TMessageStringBuilder::CacheDestroyed_; + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK TLoggingContext GetLoggingContext() +{ + return { + .Instant = GetCpuInstant(), + .ThreadId = TThread::CurrentThreadId(), + .ThreadName = GetCurrentThreadName(), + }; +} + +Y_WEAK ILogManager* GetDefaultLogManager() +{ + return nullptr; +} + +//////////////////////////////////////////////////////////////////////////////// + +thread_local ELogLevel ThreadMinLogLevel = ELogLevel::Minimum; + +void SetThreadMinLogLevel(ELogLevel minLogLevel) +{ + ThreadMinLogLevel = minLogLevel; +} + +ELogLevel GetThreadMinLogLevel() +{ + return ThreadMinLogLevel; +} + +//////////////////////////////////////////////////////////////////////////////// + +TLogger::TLogger(ILogManager* logManager, TStringBuf categoryName) + : LogManager_(logManager) + , Category_(LogManager_ ? LogManager_->GetCategory(categoryName) : nullptr) + , MinLevel_(LogManager_ ? LoggerDefaultMinLevel : NullLoggerMinLevel) +{ } + +TLogger::TLogger(TStringBuf categoryName) + : TLogger(GetDefaultLogManager(), categoryName) +{ } + +TLogger::operator bool() const +{ + return LogManager_; +} + +const TLoggingCategory* TLogger::GetCategory() const +{ + return Category_; +} + +bool TLogger::IsLevelEnabledHeavy(ELogLevel level) const +{ + // Note that we managed to reach this point, i.e. level >= MinLevel_, + // which implies that MinLevel_ != ELogLevel::Maximum, so this logger was not + // default constructed, thus it has non-trivial category. + YT_ASSERT(Category_); + + if (Category_->CurrentVersion != Category_->ActualVersion->load(std::memory_order::relaxed)) { + LogManager_->UpdateCategory(const_cast<TLoggingCategory*>(Category_)); + } + + return + level >= Category_->MinPlainTextLevel && + level >= ThreadMinLogLevel; +} + +bool TLogger::GetAbortOnAlert() const +{ + return LogManager_->GetAbortOnAlert(); +} + +bool TLogger::IsEssential() const +{ + return Essential_; +} + +void TLogger::UpdateAnchor(TLoggingAnchor* anchor) const +{ + LogManager_->UpdateAnchor(anchor); +} + +void TLogger::RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const +{ + LogManager_->RegisterStaticAnchor(anchor, sourceLocation, message); +} + +void TLogger::Write(TLogEvent&& event) const +{ + LogManager_->Enqueue(std::move(event)); +} + +void TLogger::AddRawTag(const TString& tag) +{ + if (!Tag_.empty()) { + Tag_ += ", "; + } + Tag_ += tag; +} + +TLogger TLogger::WithRawTag(const TString& tag) const +{ + auto result = *this; + result.AddRawTag(tag); + return result; +} + +TLogger TLogger::WithEssential(bool essential) const +{ + auto result = *this; + result.Essential_ = essential; + return result; +} + +TLogger TLogger::WithStructuredValidator(TStructuredValidator validator) const +{ + auto result = *this; + result.StructuredValidators_.push_back(std::move(validator)); + return result; +} + +TLogger TLogger::WithMinLevel(ELogLevel minLevel) const +{ + auto result = *this; + if (result) { + result.MinLevel_ = minLevel; + } + return result; +} + +const TString& TLogger::GetTag() const +{ + return Tag_; +} + +const TLogger::TStructuredTags& TLogger::GetStructuredTags() const +{ + return StructuredTags_; +} + +const TLogger::TStructuredValidators& TLogger::GetStructuredValidators() const +{ + return StructuredValidators_; +} + +//////////////////////////////////////////////////////////////////////////////// + +void LogStructuredEvent( + const TLogger& logger, + NYson::TYsonString message, + ELogLevel level) +{ + YT_VERIFY(message.GetType() == NYson::EYsonType::MapFragment); + + if (!logger.GetStructuredValidators().empty()) { + auto samplingRate = logger.GetCategory()->StructuredValidationSamplingRate.load(); + auto p = RandomNumber<double>(); + if (p < samplingRate) { + for (const auto& validator : logger.GetStructuredValidators()) { + validator(message); + } + } + } + + auto loggingContext = GetLoggingContext(); + auto event = NDetail::CreateLogEvent( + loggingContext, + logger, + level); + event.MessageKind = ELogMessageKind::Structured; + event.MessageRef = message.ToSharedRef(); + event.Family = ELogFamily::Structured; + logger.Write(std::move(event)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/logger.h b/library/cpp/yt/logging/logger.h new file mode 100644 index 0000000000..686ce9251c --- /dev/null +++ b/library/cpp/yt/logging/logger.h @@ -0,0 +1,351 @@ +#pragma once + +#include "public.h" + +#include <library/cpp/yt/string/format.h> + +#include <library/cpp/yt/memory/ref.h> + +#include <library/cpp/yt/cpu_clock/public.h> + +#include <library/cpp/yt/yson_string/string.h> + +#include <library/cpp/yt/misc/guid.h> + +#include <library/cpp/yt/misc/thread_name.h> + +#include <library/cpp/yt/memory/leaky_singleton.h> + +#include <util/system/src_location.h> + +#include <util/generic/size_literals.h> + +#include <atomic> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +constexpr double DefaultStructuredValidationSamplingRate = 0.01; + +struct TLoggingCategory +{ + TString Name; + //! This value is used for early dropping of plaintext events in order + //! to reduce load on logging thread for events which are definitely going + //! to be dropped due to rule setup. + //! NB: this optimization is used only for plaintext events since structured + //! logging rate is negligible comparing to the plaintext logging rate. + std::atomic<ELogLevel> MinPlainTextLevel; + std::atomic<int> CurrentVersion; + std::atomic<int>* ActualVersion; + std::atomic<double> StructuredValidationSamplingRate = DefaultStructuredValidationSamplingRate; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TLoggingAnchor +{ + std::atomic<bool> Registered = false; + ::TSourceLocation SourceLocation = {TStringBuf{}, 0}; + TString AnchorMessage; + TLoggingAnchor* NextAnchor = nullptr; + + std::atomic<int> CurrentVersion = 0; + std::atomic<bool> Enabled = false; + + struct TCounter + { + std::atomic<i64> Current = 0; + i64 Previous = 0; + }; + + TCounter MessageCounter; + TCounter ByteCounter; +}; + +//////////////////////////////////////////////////////////////////////////////// +// Declare some type aliases to avoid circular dependencies. +using TThreadId = size_t; +using TFiberId = size_t; +using TTraceId = TGuid; +using TRequestId = TGuid; + +//////////////////////////////////////////////////////////////////////////////// + +DEFINE_ENUM(ELogMessageKind, + (Unstructured) + (Structured) +); + +struct TLogEvent +{ + const TLoggingCategory* Category = nullptr; + ELogLevel Level = ELogLevel::Minimum; + ELogFamily Family = ELogFamily::PlainText; + bool Essential = false; + + ELogMessageKind MessageKind = ELogMessageKind::Unstructured; + TSharedRef MessageRef; + + TCpuInstant Instant = 0; + + TThreadId ThreadId = {}; + TThreadName ThreadName = {}; + + TFiberId FiberId = {}; + + TTraceId TraceId; + TRequestId RequestId; + + TStringBuf SourceFile; + int SourceLine = -1; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct ILogManager +{ + virtual ~ILogManager() = default; + + virtual void RegisterStaticAnchor( + TLoggingAnchor* anchor, + ::TSourceLocation sourceLocation, + TStringBuf anchorMessage) = 0; + virtual void UpdateAnchor(TLoggingAnchor* anchor) = 0; + + virtual void Enqueue(TLogEvent&& event) = 0; + + virtual const TLoggingCategory* GetCategory(TStringBuf categoryName) = 0; + virtual void UpdateCategory(TLoggingCategory* category) = 0; + + virtual bool GetAbortOnAlert() const = 0; +}; + +ILogManager* GetDefaultLogManager(); + +//////////////////////////////////////////////////////////////////////////////// + +struct TLoggingContext +{ + TCpuInstant Instant; + TThreadId ThreadId; + TThreadName ThreadName; + TFiberId FiberId; + TTraceId TraceId; + TRequestId RequestId; + TStringBuf TraceLoggingTag; +}; + +TLoggingContext GetLoggingContext(); + +//////////////////////////////////////////////////////////////////////////////// + +//! Sets the minimum logging level for messages in current thread. +// NB: In fiber environment, min log level is attached to a fiber, +// so after context switch thread min log level might change. +void SetThreadMinLogLevel(ELogLevel minLogLevel); +ELogLevel GetThreadMinLogLevel(); + +//////////////////////////////////////////////////////////////////////////////// + +static constexpr auto NullLoggerMinLevel = ELogLevel::Maximum; + +// Min level for non-null logger depends on whether we are in debug or release build. +// - For release mode default behavior is to omit trace logging, +// this is done by setting logger min level to Debug by default. +// - For debug mode logger min level is set to trace by default, so that trace logging is +// allowed by logger, but still may be discarded by category min level. +#ifdef NDEBUG +static constexpr auto LoggerDefaultMinLevel = ELogLevel::Debug; +#else +static constexpr auto LoggerDefaultMinLevel = ELogLevel::Trace; +#endif + +class TLogger +{ +public: + using TStructuredValidator = std::function<void(const NYson::TYsonString&)>; + using TStructuredValidators = std::vector<TStructuredValidator>; + + using TStructuredTag = std::pair<TString, NYson::TYsonString>; + // TODO(max42): switch to TCompactVector after YT-15430. + using TStructuredTags = std::vector<TStructuredTag>; + + TLogger() = default; + TLogger(const TLogger& other) = default; + TLogger& operator=(const TLogger& other) = default; + + TLogger(ILogManager* logManager, TStringBuf categoryName); + explicit TLogger(TStringBuf categoryName); + + explicit operator bool() const; + + const TLoggingCategory* GetCategory() const; + + //! Validate that level is admitted by logger's own min level + //! and by category's min level. + bool IsLevelEnabled(ELogLevel level) const; + + bool GetAbortOnAlert() const; + + bool IsEssential() const; + + bool IsAnchorUpToDate(const TLoggingAnchor& anchor) const; + void UpdateAnchor(TLoggingAnchor* anchor) const; + void RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const; + + void Write(TLogEvent&& event) const; + + void AddRawTag(const TString& tag); + template <class... TArgs> + void AddTag(const char* format, TArgs&&... args); + + template <class TType> + void AddStructuredTag(TStringBuf key, TType value); + + TLogger WithRawTag(const TString& tag) const; + template <class... TArgs> + TLogger WithTag(const char* format, TArgs&&... args) const; + + template <class TType> + TLogger WithStructuredTag(TStringBuf key, TType value) const; + + TLogger WithStructuredValidator(TStructuredValidator validator) const; + + TLogger WithMinLevel(ELogLevel minLevel) const; + + TLogger WithEssential(bool essential = true) const; + + const TString& GetTag() const; + const TStructuredTags& GetStructuredTags() const; + + const TStructuredValidators& GetStructuredValidators() const; + +protected: + // These fields are set only during logger creation, so they are effectively const + // and accessing them is thread-safe. + ILogManager* LogManager_ = nullptr; + const TLoggingCategory* Category_ = nullptr; + bool Essential_ = false; + ELogLevel MinLevel_ = NullLoggerMinLevel; + TString Tag_; + TStructuredTags StructuredTags_; + TStructuredValidators StructuredValidators_; + +private: + //! This method checks level against category's min level. + //! Refer to comment in TLogger::IsLevelEnabled for more details. + bool IsLevelEnabledHeavy(ELogLevel level) const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +void LogStructuredEvent( + const TLogger& logger, + NYson::TYsonString message, + ELogLevel level); + +//////////////////////////////////////////////////////////////////////////////// + +#ifdef YT_ENABLE_TRACE_LOGGING +#define YT_LOG_TRACE(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Trace, __VA_ARGS__) +#define YT_LOG_TRACE_IF(condition, ...) if (condition) YT_LOG_TRACE(__VA_ARGS__) +#define YT_LOG_TRACE_UNLESS(condition, ...) if (!(condition)) YT_LOG_TRACE(__VA_ARGS__) +#else +#define YT_LOG_UNUSED(...) if (true) { } else { YT_LOG_DEBUG(__VA_ARGS__); } +#define YT_LOG_TRACE(...) YT_LOG_UNUSED(__VA_ARGS__) +#define YT_LOG_TRACE_IF(condition, ...) YT_LOG_UNUSED(__VA_ARGS__) +#define YT_LOG_TRACE_UNLESS(condition, ...) YT_LOG_UNUSED(__VA_ARGS__) +#endif + +#define YT_LOG_DEBUG(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Debug, __VA_ARGS__) +#define YT_LOG_DEBUG_IF(condition, ...) if (condition) YT_LOG_DEBUG(__VA_ARGS__) +#define YT_LOG_DEBUG_UNLESS(condition, ...) if (!(condition)) YT_LOG_DEBUG(__VA_ARGS__) + +#define YT_LOG_INFO(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Info, __VA_ARGS__) +#define YT_LOG_INFO_IF(condition, ...) if (condition) YT_LOG_INFO(__VA_ARGS__) +#define YT_LOG_INFO_UNLESS(condition, ...) if (!(condition)) YT_LOG_INFO(__VA_ARGS__) + +#define YT_LOG_WARNING(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Warning, __VA_ARGS__) +#define YT_LOG_WARNING_IF(condition, ...) if (condition) YT_LOG_WARNING(__VA_ARGS__) +#define YT_LOG_WARNING_UNLESS(condition, ...) if (!(condition)) YT_LOG_WARNING(__VA_ARGS__) + +#define YT_LOG_ERROR(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Error, __VA_ARGS__) +#define YT_LOG_ERROR_IF(condition, ...) if (condition) YT_LOG_ERROR(__VA_ARGS__) +#define YT_LOG_ERROR_UNLESS(condition, ...) if (!(condition)) YT_LOG_ERROR(__VA_ARGS__) + +#define YT_LOG_ALERT(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Alert, __VA_ARGS__); +#define YT_LOG_ALERT_IF(condition, ...) if (condition) YT_LOG_ALERT(__VA_ARGS__) +#define YT_LOG_ALERT_UNLESS(condition, ...) if (!(condition)) YT_LOG_ALERT(__VA_ARGS__) + +#define YT_LOG_FATAL(...) \ + do { \ + YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Fatal, __VA_ARGS__); \ + Y_UNREACHABLE(); \ + } while(false) +#define YT_LOG_FATAL_IF(condition, ...) if (Y_UNLIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__) +#define YT_LOG_FATAL_UNLESS(condition, ...) if (!Y_LIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__) + +#define YT_LOG_EVENT(logger, level, ...) \ + YT_LOG_EVENT_WITH_ANCHOR(logger, level, nullptr, __VA_ARGS__) + +#define YT_LOG_EVENT_WITH_ANCHOR(logger, level, anchor, ...) \ + do { \ + const auto& logger__##__LINE__ = (logger); \ + auto level__##__LINE__ = (level); \ + \ + if (!logger__##__LINE__.IsLevelEnabled(level__##__LINE__)) { \ + break; \ + } \ + \ + auto location__##__LINE__ = __LOCATION__; \ + \ + ::NYT::NLogging::TLoggingAnchor* anchor__##__LINE__ = (anchor); \ + if (!anchor__##__LINE__) { \ + static ::NYT::TLeakyStorage<::NYT::NLogging::TLoggingAnchor> staticAnchor__##__LINE__; \ + anchor__##__LINE__ = staticAnchor__##__LINE__.Get(); \ + } \ + \ + bool anchorUpToDate__##__LINE__ = logger__##__LINE__.IsAnchorUpToDate(*anchor__##__LINE__); \ + if (anchorUpToDate__##__LINE__ && !anchor__##__LINE__->Enabled.load(std::memory_order::relaxed)) { \ + break; \ + } \ + \ + auto loggingContext__##__LINE__ = ::NYT::NLogging::GetLoggingContext(); \ + auto message__##__LINE__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__##__LINE__, logger__##__LINE__, __VA_ARGS__); \ + \ + if (!anchorUpToDate__##__LINE__) { \ + logger__##__LINE__.RegisterStaticAnchor(anchor__##__LINE__, location__##__LINE__, message__##__LINE__.Anchor); \ + logger__##__LINE__.UpdateAnchor(anchor__##__LINE__); \ + } \ + \ + if (!anchor__##__LINE__->Enabled.load(std::memory_order::relaxed)) { \ + break; \ + } \ + \ + static thread_local i64 localByteCounter__##__LINE__; \ + static thread_local ui8 localMessageCounter__##__LINE__; \ + \ + localByteCounter__##__LINE__ += message__##__LINE__.MessageRef.Size(); \ + if (Y_UNLIKELY(++localMessageCounter__##__LINE__ == 0)) { \ + anchor__##__LINE__->MessageCounter.Current += 256; \ + anchor__##__LINE__->ByteCounter.Current += localByteCounter__##__LINE__; \ + localByteCounter__##__LINE__ = 0; \ + } \ + \ + ::NYT::NLogging::NDetail::LogEventImpl( \ + loggingContext__##__LINE__, \ + logger__##__LINE__, \ + level__##__LINE__, \ + location__##__LINE__, \ + std::move(message__##__LINE__.MessageRef)); \ + } while (false) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging + +#define LOGGER_INL_H_ +#include "logger-inl.h" +#undef LOGGER_INL_H_ diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..3be59e00af --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-logging-plain_text_formatter) +target_compile_options(yt-logging-plain_text_formatter PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-logging-plain_text_formatter PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-cpu_clock + cpp-yt-logging + cpp-yt-string + cpp-yt-misc +) +target_sources(yt-logging-plain_text_formatter PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/plain_text_formatter/formatter.cpp +) diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-aarch64.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..2771fdd74f --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-aarch64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-logging-plain_text_formatter) +target_compile_options(yt-logging-plain_text_formatter PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-logging-plain_text_formatter PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-cpu_clock + cpp-yt-logging + cpp-yt-string + cpp-yt-misc +) +target_sources(yt-logging-plain_text_formatter PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/plain_text_formatter/formatter.cpp +) diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-x86_64.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..2771fdd74f --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-logging-plain_text_formatter) +target_compile_options(yt-logging-plain_text_formatter PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-logging-plain_text_formatter PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-cpu_clock + cpp-yt-logging + cpp-yt-string + cpp-yt-misc +) +target_sources(yt-logging-plain_text_formatter PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/plain_text_formatter/formatter.cpp +) diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.windows-x86_64.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..d78a3651a0 --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.windows-x86_64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-logging-plain_text_formatter) +target_link_libraries(yt-logging-plain_text_formatter PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-cpu_clock + cpp-yt-logging + cpp-yt-string + cpp-yt-misc +) +target_sources(yt-logging-plain_text_formatter PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/plain_text_formatter/formatter.cpp +) diff --git a/library/cpp/yt/logging/plain_text_formatter/formatter.cpp b/library/cpp/yt/logging/plain_text_formatter/formatter.cpp new file mode 100644 index 0000000000..ab2181113d --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/formatter.cpp @@ -0,0 +1,226 @@ +#include "formatter.h" + +#include <library/cpp/yt/cpu_clock/clock.h> + +#include <library/cpp/yt/misc/port.h> + +#ifdef YT_USE_SSE42 + #include <emmintrin.h> + #include <pmmintrin.h> +#endif + +namespace NYT::NLogging { + +constexpr int MessageBufferWatermarkSize = 256; + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +// Ultra-fast specialized versions of AppendNumber. +void AppendDigit(TBaseFormatter* out, ui32 value) +{ + out->AppendChar('0' + value); +} + +void AppendNumber2(TBaseFormatter* out, ui32 value) +{ + AppendDigit(out, value / 10); + AppendDigit(out, value % 10); +} + +void AppendNumber3(TBaseFormatter* out, ui32 value) +{ + AppendDigit(out, value / 100); + AppendDigit(out, (value / 10) % 10); + AppendDigit(out, value % 10); +} + +void AppendNumber4(TBaseFormatter* out, ui32 value) +{ + AppendDigit(out, value / 1000); + AppendDigit(out, (value / 100) % 10); + AppendDigit(out, (value / 10) % 10); + AppendDigit(out, value % 10); +} + +void AppendNumber6(TBaseFormatter* out, ui32 value) +{ + AppendDigit(out, value / 100000); + AppendDigit(out, (value / 10000) % 10); + AppendDigit(out, (value / 1000) % 10); + AppendDigit(out, (value / 100) % 10); + AppendDigit(out, (value / 10) % 10); + AppendDigit(out, value % 10); +} + +} // namespace + +void FormatDateTime(TBaseFormatter* out, TInstant dateTime) +{ + tm localTime; + dateTime.LocalTime(&localTime); + AppendNumber4(out, localTime.tm_year + 1900); + out->AppendChar('-'); + AppendNumber2(out, localTime.tm_mon + 1); + out->AppendChar('-'); + AppendNumber2(out, localTime.tm_mday); + out->AppendChar(' '); + AppendNumber2(out, localTime.tm_hour); + out->AppendChar(':'); + AppendNumber2(out, localTime.tm_min); + out->AppendChar(':'); + AppendNumber2(out, localTime.tm_sec); +} + +void FormatMilliseconds(TBaseFormatter* out, TInstant dateTime) +{ + AppendNumber3(out, dateTime.MilliSecondsOfSecond()); +} + +void FormatMicroseconds(TBaseFormatter* out, TInstant dateTime) +{ + AppendNumber6(out, dateTime.MicroSecondsOfSecond()); +} + +void FormatLevel(TBaseFormatter* out, ELogLevel level) +{ + static char chars[] = "?TDIWEAF?"; + out->AppendChar(chars[static_cast<int>(level)]); +} + +void FormatMessage(TBaseFormatter* out, TStringBuf message) +{ + auto current = message.begin(); + +#ifdef YT_USE_SSE42 + auto vectorLow = _mm_set1_epi8(PrintableASCIILow); + auto vectorHigh = _mm_set1_epi8(PrintableASCIIHigh); +#endif + + auto appendChar = [&] { + char ch = *current; + if (ch == '\n') { + out->AppendString("\\n"); + } else if (ch == '\t') { + out->AppendString("\\t"); + } else if (ch < PrintableASCIILow || ch > PrintableASCIIHigh) { + unsigned char unsignedCh = ch; + out->AppendString("\\x"); + out->AppendChar(IntToHexLowercase[unsignedCh >> 4]); + out->AppendChar(IntToHexLowercase[unsignedCh & 15]); + } else { + out->AppendChar(ch); + } + ++current; + }; + + while (current < message.end()) { + if (out->GetBytesRemaining() < MessageBufferWatermarkSize) { + out->AppendString(TStringBuf("...<message truncated>")); + break; + } +#ifdef YT_USE_SSE42 + // Use SSE for optimization. + if (current + 16 > message.end()) { + appendChar(); + } else { + const void* inPtr = &(*current); + void* outPtr = out->GetCursor(); + auto value = _mm_lddqu_si128(static_cast<const __m128i*>(inPtr)); + if (_mm_movemask_epi8(_mm_cmplt_epi8(value, vectorLow)) || + _mm_movemask_epi8(_mm_cmpgt_epi8(value, vectorHigh))) { + for (int index = 0; index < 16; ++index) { + appendChar(); + } + } else { + _mm_storeu_si128(static_cast<__m128i*>(outPtr), value); + out->Advance(16); + current += 16; + } + } +#else + // Unoptimized version. + appendChar(); +#endif + } +} + +//////////////////////////////////////////////////////////////////////////////// + +void TCachingDateFormatter::Format(TBaseFormatter* buffer, TInstant dateTime, bool printMicroseconds) +{ + auto currentSecond = dateTime.Seconds(); + if (CachedSecond_ != currentSecond) { + Cached_.Reset(); + FormatDateTime(&Cached_, dateTime); + CachedSecond_ = currentSecond; + } + + buffer->AppendString(Cached_.GetBuffer()); + buffer->AppendChar(','); + if (printMicroseconds) { + FormatMicroseconds(buffer, dateTime); + } else { + FormatMilliseconds(buffer, dateTime); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TPlainTextEventFormatter::TPlainTextEventFormatter(bool enableSourceLocation) + : EnableSourceLocation_(enableSourceLocation) +{ } + +void TPlainTextEventFormatter::Format(TBaseFormatter* buffer, const TLogEvent& event) +{ + CachingDateFormatter_.Format(buffer, CpuInstantToInstant(event.Instant), true); + + buffer->AppendChar('\t'); + + FormatLevel(buffer, event.Level); + + buffer->AppendChar('\t'); + + buffer->AppendString(event.Category->Name); + + buffer->AppendChar('\t'); + + FormatMessage(buffer, event.MessageRef.ToStringBuf()); + + buffer->AppendChar('\t'); + + if (event.ThreadName.Length > 0) { + buffer->AppendString(TStringBuf(event.ThreadName.Buffer.data(), event.ThreadName.Length)); + } else if (event.ThreadId != TThreadId()) { + buffer->AppendNumber(event.ThreadId, 16); + } + + buffer->AppendChar('\t'); + + if (event.FiberId != TFiberId()) { + buffer->AppendNumber(event.FiberId, 16); + } + + buffer->AppendChar('\t'); + + if (event.TraceId != TTraceId()) { + buffer->AppendGuid(event.TraceId); + } + + if (EnableSourceLocation_) { + buffer->AppendChar('\t'); + if (event.SourceFile) { + auto sourceFile = event.SourceFile; + buffer->AppendString(sourceFile.RNextTok(LOCSLASH_C)); + buffer->AppendChar(':'); + buffer->AppendNumber(event.SourceLine); + } + } + + buffer->AppendChar('\n'); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/plain_text_formatter/formatter.h b/library/cpp/yt/logging/plain_text_formatter/formatter.h new file mode 100644 index 0000000000..1c35c7c5ee --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/formatter.h @@ -0,0 +1,48 @@ +#pragma once + +#include <library/cpp/yt/string/raw_formatter.h> + +#include <library/cpp/yt/logging//logger.h> + +namespace NYT::NLogging { + +/////////////////////////////////////////////////////////////π/////////////////// + +constexpr int DateTimeBufferSize = 64; +constexpr int MessageBufferSize = 64_KB; + +void FormatMilliseconds(TBaseFormatter* out, TInstant dateTime); +void FormatMicroseconds(TBaseFormatter* out, TInstant dateTime); +void FormatLevel(TBaseFormatter* out, ELogLevel level); +void FormatMessage(TBaseFormatter* out, TStringBuf message); + +/////////////////////////////////////////////////////////////π/////////////////// + +class TCachingDateFormatter +{ +public: + void Format(TBaseFormatter* buffer, TInstant dateTime, bool printMicroseconds = false); + +private: + ui64 CachedSecond_ = 0; + TRawFormatter<DateTimeBufferSize> Cached_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TPlainTextEventFormatter +{ +public: + explicit TPlainTextEventFormatter(bool enableSourceLocation); + + void Format(TBaseFormatter* buffer, const TLogEvent& event); + +private: + const bool EnableSourceLocation_; + + TCachingDateFormatter CachingDateFormatter_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/plain_text_formatter/ya.make b/library/cpp/yt/logging/plain_text_formatter/ya.make new file mode 100644 index 0000000000..cb31c29d11 --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + formatter.cpp +) + +PEERDIR( + library/cpp/yt/cpu_clock + library/cpp/yt/logging + library/cpp/yt/string + library/cpp/yt/misc +) + +END() diff --git a/library/cpp/yt/logging/public.h b/library/cpp/yt/logging/public.h new file mode 100644 index 0000000000..1e2b59ca0d --- /dev/null +++ b/library/cpp/yt/logging/public.h @@ -0,0 +1,39 @@ +#pragma once + +#include <library/cpp/yt/misc/enum.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +// Any change to this enum must be also propagated to FormatLevel. +DEFINE_ENUM(ELogLevel, + (Minimum) + (Trace) + (Debug) + (Info) + (Warning) + (Error) + (Alert) + (Fatal) + (Maximum) +); + +DEFINE_ENUM(ELogFamily, + (PlainText) + (Structured) +); + +//////////////////////////////////////////////////////////////////////////////// + +struct TLoggingCategory; +struct TLoggingAnchor; +struct TLogEvent; +struct TLoggingContext; + +class TLogger; +struct ILogManager; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/unittests/logger_ut.cpp b/library/cpp/yt/logging/unittests/logger_ut.cpp new file mode 100644 index 0000000000..7696ea4a83 --- /dev/null +++ b/library/cpp/yt/logging/unittests/logger_ut.cpp @@ -0,0 +1,38 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/logging/logger.h> + +namespace NYT::NLogging { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TLogger, NullByDefault) +{ + { + TLogger logger; + EXPECT_FALSE(logger); + EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal)); + } + { + TLogger logger{"Category"}; + EXPECT_FALSE(logger); + EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal)); + } +} + +TEST(TLogger, CopyOfNullLogger) +{ + TLogger nullLogger{/*logManager*/ nullptr, "Category"}; + ASSERT_FALSE(nullLogger); + + auto logger = nullLogger.WithMinLevel(ELogLevel::Debug); + + EXPECT_FALSE(logger); + EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/unittests/ya.make b/library/cpp/yt/logging/unittests/ya.make new file mode 100644 index 0000000000..42268d3db2 --- /dev/null +++ b/library/cpp/yt/logging/unittests/ya.make @@ -0,0 +1,18 @@ +GTEST(unittester-library-logging) + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +IF (NOT OS_WINDOWS) + ALLOCATOR(YT) +ENDIF() + +SRCS( + logger_ut.cpp +) + +PEERDIR( + library/cpp/testing/gtest + library/cpp/yt/logging +) + +END() diff --git a/library/cpp/yt/logging/ya.make b/library/cpp/yt/logging/ya.make new file mode 100644 index 0000000000..e611c2e554 --- /dev/null +++ b/library/cpp/yt/logging/ya.make @@ -0,0 +1,25 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + logger.cpp +) + +PEERDIR( + library/cpp/yt/assert + library/cpp/yt/memory + library/cpp/yt/misc + library/cpp/yt/yson_string +) + +END() + +RECURSE( + backends + plain_text_formatter +) + +RECURSE_FOR_TESTS( + unittests +) diff --git a/library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h b/library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h new file mode 100644 index 0000000000..1fba63c427 --- /dev/null +++ b/library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h @@ -0,0 +1,43 @@ +#ifndef LEAKY_REF_COUNTED_SINGLETON_INL_H_ +#error "Direct inclusion of this file is not allowed, include leaky_ref_counted_singleton.h" +// For the sake of sane code completion. +#include "leaky_ref_counted_singleton.h" +#endif + +#include "new.h" + +#include <atomic> +#include <mutex> + +#include <util/system/compiler.h> +#include <util/system/sanitizers.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class... TArgs> +TIntrusivePtr<T> LeakyRefCountedSingleton(TArgs&&... args) +{ + static std::atomic<T*> Ptr; + auto* ptr = Ptr.load(std::memory_order::acquire); + if (Y_LIKELY(ptr)) { + return ptr; + } + + static std::once_flag Initialized; + std::call_once(Initialized, [&] { + auto ptr = New<T>(std::forward<TArgs>(args)...); + Ref(ptr.Get()); + Ptr.store(ptr.Get()); +#if defined(_asan_enabled_) + NSan::MarkAsIntentionallyLeaked(ptr.Get()); +#endif + }); + + return Ptr.load(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/memory/leaky_ref_counted_singleton.h b/library/cpp/yt/memory/leaky_ref_counted_singleton.h new file mode 100644 index 0000000000..d77c3c9829 --- /dev/null +++ b/library/cpp/yt/memory/leaky_ref_counted_singleton.h @@ -0,0 +1,22 @@ +#pragma once + +#include "intrusive_ptr.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +#define DECLARE_LEAKY_REF_COUNTED_SINGLETON_FRIEND() \ + template <class T> \ + friend struct ::NYT::TRefCountedWrapper; + +template <class T, class... TArgs> +TIntrusivePtr<T> LeakyRefCountedSingleton(TArgs&&... args); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define LEAKY_REF_COUNTED_SINGLETON_INL_H_ +#include "leaky_ref_counted_singleton-inl.h" +#undef LEAKY_REF_COUNTED_SINGLETON_INL_H_ diff --git a/library/cpp/yt/misc/arcadia_enum-inl.h b/library/cpp/yt/misc/arcadia_enum-inl.h new file mode 100644 index 0000000000..17a10bb3b2 --- /dev/null +++ b/library/cpp/yt/misc/arcadia_enum-inl.h @@ -0,0 +1,49 @@ +#pragma once +#ifndef ARCADIA_ENUM_INL_H_ +#error "Direct inclusion of this file is not allowed, include arcadia_enum.h" +// For the sake of sane code completion. +#include "arcadia_enum.h" +#endif + +#include <util/system/type_name.h> + +namespace NYT::NDetail { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +struct TArcadiaEnumTraitsImpl +{ + static constexpr bool IsBitEnum = false; + static constexpr bool IsStringSerializableEnum = false; + + static TStringBuf GetTypeName() + { + static const auto Result = TypeName<T>(); + return Result; + } + + static std::optional<TStringBuf> FindLiteralByValue(T value) + { + auto names = GetEnumNames<T>(); + auto it = names.find(value); + return it == names.end() ? std::nullopt : std::make_optional(TStringBuf(it->second)); + } + + static std::optional<T> FindValueByLiteral(TStringBuf literal) + { + static const auto LiteralToValue = [] { + THashMap<TString, T> result; + for (const auto& [value, name] : GetEnumNames<T>()) { + result.emplace(name, value); + } + return result; + }(); + auto it = LiteralToValue.find(literal); + return it == LiteralToValue.end() ? std::nullopt : std::make_optional(it->second); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NDetail diff --git a/library/cpp/yt/misc/arcadia_enum.h b/library/cpp/yt/misc/arcadia_enum.h new file mode 100644 index 0000000000..85ad182a6c --- /dev/null +++ b/library/cpp/yt/misc/arcadia_enum.h @@ -0,0 +1,18 @@ +#pragma once + +#include <util/generic/serialized_enum.h> + +//////////////////////////////////////////////////////////////////////////////// +// TEnumTraits interop for Arcadia enums + +#define YT_DEFINE_ARCADIA_ENUM_TRAITS(enumType) \ + [[maybe_unused]] inline ::NYT::NDetail::TArcadiaEnumTraitsImpl<enumType> GetEnumTraitsImpl(enumType) \ + { \ + return {}; \ + } + +//////////////////////////////////////////////////////////////////////////////// + +#define ARCADIA_ENUM_INL_H_ +#include "arcadia_enum-inl.h" +#undef ARCADIA_ENUM_INL_H_ diff --git a/library/cpp/yt/misc/property.h b/library/cpp/yt/misc/property.h new file mode 100644 index 0000000000..eb8264968e --- /dev/null +++ b/library/cpp/yt/misc/property.h @@ -0,0 +1,308 @@ +#pragma once + +#include <util/system/compiler.h> + +//////////////////////////////////////////////////////////////////////////////// + +//! Declares a trivial public read-write property that is passed by reference. +#define DECLARE_BYREF_RW_PROPERTY(type, name) \ +public: \ + type& name(); \ + const type& name() const + +//! Defines a trivial public read-write property that is passed by reference. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYREF_RW_PROPERTY(type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE type& name() \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE const type& name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Defines a trivial public read-write property that is passed by reference +//! and is not inline-initialized. +#define DEFINE_BYREF_RW_PROPERTY_NO_INIT(type, name) \ +protected: \ + type name##_; \ + \ +public: \ + Y_FORCE_INLINE type& name() \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE const type& name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Forwards a trivial public read-write property that is passed by reference. +#define DELEGATE_BYREF_RW_PROPERTY(declaringType, type, name, delegateTo) \ + type& declaringType::name() \ + { \ + return (delegateTo).name(); \ + } \ + \ + const type& declaringType::name() const \ + { \ + return (delegateTo).name(); \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// + +//! Declares a trivial public read-only property that is passed by reference. +#define DECLARE_BYREF_RO_PROPERTY(type, name) \ +public: \ + const type& name() const + +//! Defines a trivial public read-only property that is passed by reference. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYREF_RO_PROPERTY(type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE const type& name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Defines a trivial public read-only property that is passed by reference +//! and is not inline-initialized. +#define DEFINE_BYREF_RO_PROPERTY_NO_INIT(type, name) \ +protected: \ + type name##_; \ + \ +public: \ + Y_FORCE_INLINE const type& name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Forwards a trivial public read-only property that is passed by reference. +#define DELEGATE_BYREF_RO_PROPERTY(declaringType, type, name, delegateTo) \ + const type& declaringType::name() const \ + { \ + return (delegateTo).name(); \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// + +//! Declares a trivial public read-write property that is passed by value. +#define DECLARE_BYVAL_RW_PROPERTY(type, name) \ +public: \ + type Get##name() const; \ + void Set##name(type value) + +//! Defines a trivial public read-write property that is passed by value. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYVAL_RW_PROPERTY(type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE void Set##name(type value) \ + { \ + name##_ = value; \ + } \ + static_assert(true) + + +//! Defines a trivial public read-write property that is passed by value. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYVAL_RW_PROPERTY_WITH_FLUENT_SETTER(declaringType, type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE void Set##name(type value) &\ + { \ + name##_ = value; \ + } \ + \ + Y_FORCE_INLINE declaringType&& Set##name(type value) &&\ + { \ + name##_ = value; \ + return std::move(*this); \ + } \ + static_assert(true) + +//! Defines a trivial public read-write property that is passed by value +//! and is not inline-initialized. +#define DEFINE_BYVAL_RW_PROPERTY_NO_INIT(type, name, ...) \ +protected: \ + type name##_; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE void Set##name(type value) \ + { \ + name##_ = value; \ + } \ + static_assert(true) + +//! Forwards a trivial public read-write property that is passed by value. +#define DELEGATE_BYVAL_RW_PROPERTY(declaringType, type, name, delegateTo) \ + type declaringType::Get##name() const \ + { \ + return (delegateTo).Get##name(); \ + } \ + \ + void declaringType::Set##name(type value) \ + { \ + (delegateTo).Set##name(value); \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// + +//! Declares a trivial public read-only property that is passed by value. +#define DECLARE_BYVAL_RO_PROPERTY(type, name) \ +public: \ + type Get##name() const + +//! Defines a trivial public read-only property that is passed by value. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYVAL_RO_PROPERTY(type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + + +//! Defines a trivial public read-only property that is passed by value +//! and is not inline-initialized. +#define DEFINE_BYVAL_RO_PROPERTY_NO_INIT(type, name) \ +protected: \ + type name##_; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Forwards a trivial public read-only property that is passed by value. +#define DELEGATE_BYVAL_RO_PROPERTY(declaringType, type, name, delegateTo) \ + type declaringType::Get##name() \ + { \ + return (delegateTo).Get##name(); \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// + +//! Below are macro helpers for extra properties. +//! Extra properties should be used for lazy memory allocation for properties that +//! hold default values for the majority of objects. + +//! Initializes extra property holder if it is not initialized. +#define INITIALIZE_EXTRA_PROPERTY_HOLDER(holder) \ + if (!holder##_) { \ + holder##_.reset(new decltype(holder##_)::element_type()); \ + } \ + static_assert(true) + +//! Declares an extra property holder. Holder contains extra properties values. +//! Holder is not created until some property is set with a non-default value. +//! If there is no holder property getter returns default value. +#define DECLARE_EXTRA_PROPERTY_HOLDER(type, holder) \ +public: \ + Y_FORCE_INLINE bool HasCustom##holder() const \ + { \ + return static_cast<bool>(holder##_); \ + } \ + Y_FORCE_INLINE const type* GetCustom##holder() const \ + { \ + return holder##_.get(); \ + } \ + Y_FORCE_INLINE type* GetCustom##holder() \ + { \ + return holder##_.get(); \ + } \ + Y_FORCE_INLINE void InitializeCustom##holder() \ + { \ + INITIALIZE_EXTRA_PROPERTY_HOLDER(holder); \ + } \ +private: \ + std::unique_ptr<type> holder##_; \ + static const type Default##holder##_ + +//! Defines a storage for extra properties default values. +#define DEFINE_EXTRA_PROPERTY_HOLDER(class, type, holder) \ + const type class::Default##holder##_ + +//! Defines a public read-write extra property that is passed by value. +#define DEFINE_BYVAL_RW_EXTRA_PROPERTY(holder, name) \ +public: \ + Y_FORCE_INLINE decltype(holder##_->name) Get##name() const \ + { \ + if (!holder##_) { \ + return Default##holder##_.name; \ + } \ + return holder##_->name; \ + } \ + Y_FORCE_INLINE void Set##name(decltype(holder##_->name) val) \ + { \ + if (!holder##_) { \ + if (val == Default##holder##_.name) { \ + return; \ + } \ + INITIALIZE_EXTRA_PROPERTY_HOLDER(holder); \ + } \ + holder##_->name = val; \ + } \ + static_assert(true) + +//! Defines a public read-write extra property that is passed by reference. +#define DEFINE_BYREF_RW_EXTRA_PROPERTY(holder, name) \ +public: \ + Y_FORCE_INLINE const decltype(holder##_->name)& name() const \ + { \ + if (!holder##_) { \ + return Default##holder##_.name; \ + } \ + return holder##_->name; \ + } \ + Y_FORCE_INLINE decltype(holder##_->name)& Mutable##name() \ + { \ + INITIALIZE_EXTRA_PROPERTY_HOLDER(holder); \ + return holder##_->name; \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/string/raw_formatter.h b/library/cpp/yt/string/raw_formatter.h new file mode 100644 index 0000000000..6956330883 --- /dev/null +++ b/library/cpp/yt/string/raw_formatter.h @@ -0,0 +1,212 @@ +#pragma once + +#include "guid.h" + +#include <algorithm> +#include <array> + +#include <util/generic/strbuf.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +//! A dead-simple string formatter. +/*! + * This formatter is intended to be as simple as possible and async signal safe. + * This is the reason we do not use printf(): it does not meet signal-safety + * requirements. + */ + +class TBaseFormatter +{ +public: + TBaseFormatter(char* buffer, int length) + : Begin_(buffer) + , Cursor_(buffer) + , End_(buffer + length) + { } + + //! Returns an underlying cursor. + char* GetCursor() + { + return Cursor_; + } + + //! Returns an pointer to the underlying buffer. + const char* GetData() const + { + return Begin_; + } + + //! Returns the number of bytes written in the buffer. + int GetBytesWritten() const + { + return Cursor_ - Begin_; + } + + //! Returns the number of bytes available in the buffer. + int GetBytesRemaining() const + { + return End_ - Cursor_; + } + + //! Advances the internal cursor #count symbols forward (assuming the data is already present). + void Advance(int count) + { + Cursor_ += count; + + if (Cursor_ > End_) { + Cursor_ = End_; + } + } + + //! Drops trailing #count symbols (assuming these are present). + void Revert(int count) + { + Cursor_ -= count; + } + + //! Appends the string and updates the internal cursor. + void AppendString(const char* string) + { + while (*string != '\0' && Cursor_ < End_) { + *Cursor_++ = *string++; + } + } + + //! Appends the string and updates the internal cursor. + void AppendString(TStringBuf string) + { + size_t position = 0; + while (position < string.length() && Cursor_ < End_) { + *Cursor_++ = string[position++]; + } + } + + //! Appends a single character and updates the internal cursor. + void AppendChar(char ch) + { + if (Cursor_ < End_) { + *Cursor_++ = ch; + } + } + + //! Formats |number| in base |radix| and updates the internal cursor. + void AppendNumber(uintptr_t number, int radix = 10, int width = 0, char ch = ' ') + { + int digits = 0; + + if (radix == 16) { + // Optimize output of hex numbers. + + uintptr_t reverse = 0; + int length = 0; + do { + reverse <<= 4; + reverse |= number & 0xf; + number >>= 4; + ++length; + } while (number > 0); + + for (int index = 0; index < length && Cursor_ + digits < End_; ++index) { + unsigned int modulus = reverse & 0xf; + Cursor_[digits] = (modulus < 10 ? '0' + modulus : 'a' + modulus - 10); + ++digits; + reverse >>= 4; + } + } else { + while (Cursor_ + digits < End_) { + const int modulus = number % radix; + number /= radix; + Cursor_[digits] = (modulus < 10 ? '0' + modulus : 'a' + modulus - 10); + ++digits; + if (number == 0) { + break; + } + } + + // Reverse the bytes written. + std::reverse(Cursor_, Cursor_ + digits); + } + + if (digits < width) { + auto delta = width - digits; + std::copy(Cursor_, Cursor_ + digits, Cursor_ + delta); + std::fill(Cursor_, Cursor_ + delta, ch); + Cursor_ += width; + } else { + Cursor_ += digits; + } + } + + //! Formats |number| as hexadecimal number and updates the internal cursor. + //! Padding will be added in front if needed. + void AppendNumberAsHexWithPadding(uintptr_t number, int width) + { + char* begin = Cursor_; + AppendString("0x"); + AppendNumber(number, 16); + // Move to right and add padding in front if needed. + if (Cursor_ < begin + width) { + auto delta = begin + width - Cursor_; + std::copy(begin, Cursor_, begin + delta); + std::fill(begin, begin + delta, ' '); + Cursor_ = begin + width; + } + } + + //! Formats |guid| and updates the internal cursor. + void AppendGuid(TGuid guid) + { + if (Y_LIKELY(End_ - Cursor_ >= MaxGuidStringSize)) { + // Fast path. + Cursor_ = WriteGuidToBuffer(Cursor_, guid); + } else { + // Slow path. + std::array<char, MaxGuidStringSize> buffer; + auto* end = WriteGuidToBuffer(buffer.data(), guid); + AppendString(TStringBuf(buffer.data(), end)); + } + } + + //! Resets the underlying cursor. + void Reset() + { + Cursor_ = Begin_; + } + + TStringBuf GetBuffer() const + { + return {Begin_, Cursor_}; + } + +private: + char* const Begin_; + char* Cursor_; + char* const End_; + +}; + +template <size_t N> +class TRawFormatter + : public TBaseFormatter +{ +public: + TRawFormatter() + : TBaseFormatter(Buffer_, N) + { } + + TRawFormatter(char* buffer, int length) + : TBaseFormatter(buffer, length) + { } + +private: + char Buffer_[N]; + +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + diff --git a/library/cpp/yt/system/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/system/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..ad24a0da29 --- /dev/null +++ b/library/cpp/yt/system/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-system) +target_compile_options(cpp-yt-system PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-system PUBLIC + contrib-libs-cxxsupp + yutil +) +target_sources(cpp-yt-system PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/system/thread_id.cpp +) diff --git a/library/cpp/yt/system/CMakeLists.linux-aarch64.txt b/library/cpp/yt/system/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..6dc2a07499 --- /dev/null +++ b/library/cpp/yt/system/CMakeLists.linux-aarch64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-system) +target_compile_options(cpp-yt-system PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-system PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_sources(cpp-yt-system PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/system/thread_id.cpp +) diff --git a/library/cpp/yt/system/CMakeLists.linux-x86_64.txt b/library/cpp/yt/system/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..6dc2a07499 --- /dev/null +++ b/library/cpp/yt/system/CMakeLists.linux-x86_64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-system) +target_compile_options(cpp-yt-system PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-system PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_sources(cpp-yt-system PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/system/thread_id.cpp +) diff --git a/library/cpp/yt/system/CMakeLists.txt b/library/cpp/yt/system/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/yt/system/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/system/CMakeLists.windows-x86_64.txt b/library/cpp/yt/system/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..338956fa70 --- /dev/null +++ b/library/cpp/yt/system/CMakeLists.windows-x86_64.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-system) +target_link_libraries(cpp-yt-system PUBLIC + contrib-libs-cxxsupp + yutil +) +target_sources(cpp-yt-system PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/system/thread_id.cpp +) diff --git a/library/cpp/yt/threading/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/threading/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..7dbacb9da4 --- /dev/null +++ b/library/cpp/yt/threading/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,37 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-threading) +target_compile_options(cpp-yt-threading PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-threading PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-cpu_clock + cpp-yt-system + cpp-yt-memory +) +target_sources(cpp-yt-threading PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/at_fork.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/count_down_latch.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/event_count.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/fork_aware_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/fork_aware_rw_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/futex.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/notification_handle.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/public.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/recursive_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/rw_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_lock_base.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_wait.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_wait_hook.cpp +) diff --git a/library/cpp/yt/threading/CMakeLists.linux-aarch64.txt b/library/cpp/yt/threading/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..644ee262f0 --- /dev/null +++ b/library/cpp/yt/threading/CMakeLists.linux-aarch64.txt @@ -0,0 +1,38 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-threading) +target_compile_options(cpp-yt-threading PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-threading PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-cpu_clock + cpp-yt-system + cpp-yt-memory +) +target_sources(cpp-yt-threading PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/at_fork.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/count_down_latch.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/event_count.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/fork_aware_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/fork_aware_rw_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/futex.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/notification_handle.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/public.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/recursive_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/rw_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_lock_base.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_wait.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_wait_hook.cpp +) diff --git a/library/cpp/yt/threading/CMakeLists.linux-x86_64.txt b/library/cpp/yt/threading/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..644ee262f0 --- /dev/null +++ b/library/cpp/yt/threading/CMakeLists.linux-x86_64.txt @@ -0,0 +1,38 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-threading) +target_compile_options(cpp-yt-threading PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-threading PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-cpu_clock + cpp-yt-system + cpp-yt-memory +) +target_sources(cpp-yt-threading PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/at_fork.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/count_down_latch.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/event_count.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/fork_aware_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/fork_aware_rw_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/futex.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/notification_handle.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/public.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/recursive_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/rw_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_lock_base.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_wait.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_wait_hook.cpp +) diff --git a/library/cpp/yt/threading/CMakeLists.txt b/library/cpp/yt/threading/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/yt/threading/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/threading/CMakeLists.windows-x86_64.txt b/library/cpp/yt/threading/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..eedf3ebee2 --- /dev/null +++ b/library/cpp/yt/threading/CMakeLists.windows-x86_64.txt @@ -0,0 +1,34 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-threading) +target_link_libraries(cpp-yt-threading PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-cpu_clock + cpp-yt-system + cpp-yt-memory +) +target_sources(cpp-yt-threading PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/at_fork.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/count_down_latch.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/event_count.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/fork_aware_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/fork_aware_rw_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/futex.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/notification_handle.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/public.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/recursive_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/rw_spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_lock_base.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_lock.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_wait.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/yt/threading/spin_wait_hook.cpp +) diff --git a/library/cpp/yt/threading/unittests/count_down_latch_ut.cpp b/library/cpp/yt/threading/unittests/count_down_latch_ut.cpp new file mode 100644 index 0000000000..894bdab22a --- /dev/null +++ b/library/cpp/yt/threading/unittests/count_down_latch_ut.cpp @@ -0,0 +1,78 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/threading/count_down_latch.h> + +#include <thread> + +namespace NYT::NThreading { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +void WaitForLatch(const TCountDownLatch& latch) +{ + latch.Wait(); + EXPECT_EQ(0, latch.GetCount()); +} + +TEST(TCountDownLatch, TwoThreads) +{ + TCountDownLatch latch(2); + + std::thread t1(std::bind(&WaitForLatch, std::cref(latch))); + std::thread t2(std::bind(&WaitForLatch, std::cref(latch))); + + EXPECT_EQ(2, latch.GetCount()); + latch.CountDown(); + EXPECT_EQ(1, latch.GetCount()); + latch.CountDown(); + EXPECT_EQ(0, latch.GetCount()); + + t1.join(); + t2.join(); +} + +TEST(TCountDownLatch, TwoThreadsPredecremented) +{ + TCountDownLatch latch(2); + + EXPECT_EQ(2, latch.GetCount()); + latch.CountDown(); + EXPECT_EQ(1, latch.GetCount()); + latch.CountDown(); + EXPECT_EQ(0, latch.GetCount()); + + std::thread t1(std::bind(&WaitForLatch, std::cref(latch))); + std::thread t2(std::bind(&WaitForLatch, std::cref(latch))); + + t1.join(); + t2.join(); +} + +TEST(TCountDownLatch, TwoThreadsTwoLatches) +{ + TCountDownLatch first(1); + TCountDownLatch second(1); + + std::thread t1([&] () { + first.Wait(); + second.CountDown(); + EXPECT_EQ(0, first.GetCount()); + EXPECT_EQ(0, second.GetCount()); + }); + + std::thread t2([&] () { + first.CountDown(); + second.Wait(); + EXPECT_EQ(0, first.GetCount()); + EXPECT_EQ(0, second.GetCount()); + }); + + t1.join(); + t2.join(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NThreading diff --git a/library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp b/library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp new file mode 100644 index 0000000000..9c2d8f16cb --- /dev/null +++ b/library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp @@ -0,0 +1,88 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/threading/recursive_spin_lock.h> +#include <library/cpp/yt/threading/event_count.h> + +#include <thread> + +namespace NYT::NThreading { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TRecursiveSpinLockTest, SingleThread) +{ + TRecursiveSpinLock lock; + EXPECT_FALSE(lock.IsLocked()); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLocked()); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLocked()); + lock.Release(); + EXPECT_TRUE(lock.IsLocked()); + lock.Release(); + EXPECT_FALSE(lock.IsLocked()); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLocked()); + lock.Release(); + lock.Acquire(); + lock.Release(); +} + +TEST(TRecursiveSpinLockTest, TwoThreads) +{ + TRecursiveSpinLock lock; + TEvent e1, e2, e3, e4, e5, e6, e7; + + std::thread t1([&] { + e1.Wait(); + EXPECT_TRUE(lock.IsLocked()); + EXPECT_FALSE(lock.IsLockedByCurrentThread()); + EXPECT_FALSE(lock.TryAcquire()); + e2.NotifyOne(); + e3.Wait(); + EXPECT_TRUE(lock.IsLocked()); + EXPECT_FALSE(lock.IsLockedByCurrentThread()); + EXPECT_FALSE(lock.TryAcquire()); + e4.NotifyOne(); + e5.Wait(); + EXPECT_FALSE(lock.IsLocked()); + EXPECT_FALSE(lock.IsLockedByCurrentThread()); + EXPECT_TRUE(lock.TryAcquire()); + e6.NotifyOne(); + e7.Wait(); + lock.Release(); + }); + + std::thread t2([&] { + EXPECT_FALSE(lock.IsLocked()); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLockedByCurrentThread()); + e1.NotifyOne(); + e2.Wait(); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLockedByCurrentThread()); + e3.NotifyOne(); + e4.Wait(); + lock.Release(); + lock.Release(); + EXPECT_FALSE(lock.IsLocked()); + e5.NotifyOne(); + e6.Wait(); + EXPECT_TRUE(lock.IsLocked()); + EXPECT_FALSE(lock.IsLockedByCurrentThread()); + e7.NotifyOne(); + lock.Acquire(); + lock.Acquire(); + lock.Release(); + lock.Release(); + }); + + t1.join(); + t2.join(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NThreading diff --git a/library/cpp/yt/threading/unittests/spin_wait_ut.cpp b/library/cpp/yt/threading/unittests/spin_wait_ut.cpp new file mode 100644 index 0000000000..8469634f34 --- /dev/null +++ b/library/cpp/yt/threading/unittests/spin_wait_ut.cpp @@ -0,0 +1,48 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/threading/spin_wait.h> +#include <library/cpp/yt/threading/spin_wait_hook.h> + +#include <thread> +#include <mutex> + +namespace NYT::NThreading { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +bool SpinWaitSlowPathHookInvoked; + +void SpinWaitSlowPathHook( + TCpuDuration cpuDelay, + const TSourceLocation& /*location*/, + ESpinLockActivityKind /*activityKind*/) +{ + SpinWaitSlowPathHookInvoked = true; + auto delay = CpuDurationToDuration(cpuDelay); + EXPECT_GE(delay, TDuration::Seconds(1)); + EXPECT_LE(delay, TDuration::Seconds(5)); +} + +TEST(TSpinWaitTest, SlowPathHook) +{ + static std::once_flag registerFlag; + std::call_once( + registerFlag, + [] { + RegisterSpinWaitSlowPathHook(SpinWaitSlowPathHook); + }); + SpinWaitSlowPathHookInvoked = false; + { + TSpinWait spinWait(__LOCATION__, ESpinLockActivityKind::ReadWrite); + for (int i = 0; i < 1'000'000; ++i) { + spinWait.Wait(); + } + } + EXPECT_TRUE(SpinWaitSlowPathHookInvoked); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NThreading diff --git a/library/cpp/yt/threading/unittests/ya.make b/library/cpp/yt/threading/unittests/ya.make new file mode 100644 index 0000000000..ef9b5d2995 --- /dev/null +++ b/library/cpp/yt/threading/unittests/ya.make @@ -0,0 +1,17 @@ +GTEST() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + count_down_latch_ut.cpp + recursive_spin_lock_ut.cpp + spin_wait_ut.cpp +) + +PEERDIR( + library/cpp/yt/assert + library/cpp/yt/threading + library/cpp/testing/gtest +) + +END() diff --git a/library/cpp/yt/user_job_statistics/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/user_job_statistics/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..6ad2fc33f1 --- /dev/null +++ b/library/cpp/yt/user_job_statistics/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-user_job_statistics) +target_link_libraries(cpp-yt-user_job_statistics PUBLIC + contrib-libs-cxxsupp + yutil + cpp-mapreduce-common +) +target_sources(cpp-yt-user_job_statistics PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/user_job_statistics/user_job_statistics.cpp +) diff --git a/library/cpp/yt/user_job_statistics/CMakeLists.linux-aarch64.txt b/library/cpp/yt/user_job_statistics/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..33fb6e94cc --- /dev/null +++ b/library/cpp/yt/user_job_statistics/CMakeLists.linux-aarch64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-user_job_statistics) +target_link_libraries(cpp-yt-user_job_statistics PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-mapreduce-common +) +target_sources(cpp-yt-user_job_statistics PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/user_job_statistics/user_job_statistics.cpp +) diff --git a/library/cpp/yt/user_job_statistics/CMakeLists.linux-x86_64.txt b/library/cpp/yt/user_job_statistics/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..33fb6e94cc --- /dev/null +++ b/library/cpp/yt/user_job_statistics/CMakeLists.linux-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-user_job_statistics) +target_link_libraries(cpp-yt-user_job_statistics PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-mapreduce-common +) +target_sources(cpp-yt-user_job_statistics PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/user_job_statistics/user_job_statistics.cpp +) diff --git a/library/cpp/yt/user_job_statistics/CMakeLists.txt b/library/cpp/yt/user_job_statistics/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/yt/user_job_statistics/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/user_job_statistics/CMakeLists.windows-x86_64.txt b/library/cpp/yt/user_job_statistics/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..6ad2fc33f1 --- /dev/null +++ b/library/cpp/yt/user_job_statistics/CMakeLists.windows-x86_64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yt-user_job_statistics) +target_link_libraries(cpp-yt-user_job_statistics PUBLIC + contrib-libs-cxxsupp + yutil + cpp-mapreduce-common +) +target_sources(cpp-yt-user_job_statistics PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/user_job_statistics/user_job_statistics.cpp +) diff --git a/library/cpp/yt/user_job_statistics/user_job_statistics.cpp b/library/cpp/yt/user_job_statistics/user_job_statistics.cpp new file mode 100644 index 0000000000..b7fd71503d --- /dev/null +++ b/library/cpp/yt/user_job_statistics/user_job_statistics.cpp @@ -0,0 +1,133 @@ +#include "user_job_statistics.h" +#include <yt/cpp/mapreduce/common/helpers.h> +#include <util/stream/null.h> +#include <util/string/builder.h> +#include <util/system/mutex.h> +#include <util/system/env.h> + +using namespace NYtTools; + +static TMutex GlobalStatsWritingMutex; + +#if defined(_unix_) +const FHANDLE TUserJobStatsProxy::JobStatisticsHandle = 5; +#elif defined(_win_) +const FHANDLE TUserJobStatsProxy::JobStatisticsHandle = nullptr; +#endif + +static IOutputStream* CorrectHandle(const FHANDLE h) { +#if defined(_unix_) + if (fcntl(h, F_GETFD) == -1) { + return &Cerr; + } + return nullptr; +#elif defined(_win_) + return &Cerr; +#endif +} + +static TString PrintNodeSimple(const NYT::TNode& n) { + return NYT::NodeToYsonString(n, NYson::EYsonFormat::Text); +} + +void TUserJobStatsProxy::Init(IOutputStream * usingStream) { + if (usingStream == nullptr) { + usingStream = CorrectHandle(JobStatisticsHandle); + } + + if(usingStream == nullptr && GetEnv("YT_JOB_ID").empty()) { + usingStream = &Cerr; + } + + + if (usingStream == nullptr) { + TFileHandle fixedDesrc(JobStatisticsHandle); + FetchedOut = MakeHolder<TFixedBufferFileOutput>(TFile(fixedDesrc.Duplicate())); + UsingStream = FetchedOut.Get(); + fixedDesrc.Release(); + } else { + UsingStream = usingStream; + } +} + +void TUserJobStatsProxy::InitChecked(IOutputStream* def) { + IOutputStream* usingStream = CorrectHandle(JobStatisticsHandle); + + if (usingStream == nullptr && !GetEnv("YT_JOB_ID").empty()) { + TFileHandle fixedDesrc(JobStatisticsHandle); + FetchedOut = MakeHolder<TFixedBufferFileOutput>(TFile(fixedDesrc.Duplicate())); + UsingStream = FetchedOut.Get(); + fixedDesrc.Release(); + } else { + UsingStream = def; + } +} + +void TUserJobStatsProxy::InitIfNotInited(IOutputStream * usingStream) { + if (UsingStream == nullptr) { + Init(usingStream); + } +} + +void TUserJobStatsProxy::CommitStats() { + if (Stats.empty()) { + return; + } + + auto res = NYT::TNode::CreateMap(); + for (auto& p : Stats) { + res[p.first] = p.second; + } + for (auto& p : TimeStats) { + res[p.first] = p.second.MilliSeconds(); + } + with_lock(GlobalStatsWritingMutex) { + *UsingStream << PrintNodeSimple(res) << ";" << Endl; + } + Stats.clear(); +} + + +TTimeStatHolder TUserJobStatsProxy::TimerStart(TString name, bool commitOnFinish) { + return THolder(new TTimeStat(this, name, commitOnFinish)); +} + +void TUserJobStatsProxy::WriteStat(TString name, i64 val) { + auto res = NYT::TNode {} (name, val); + with_lock(GlobalStatsWritingMutex) { + *UsingStream << PrintNodeSimple(res) << ";" << Endl; + } +} + +void TUserJobStatsProxy::WriteStatNoFlush(TString name, i64 val) { + auto res = NYT::TNode {} (name, val); + with_lock(GlobalStatsWritingMutex) { + *UsingStream << (TStringBuilder{} << PrintNodeSimple(res) << ";\n"); + } +} + +TTimeStat::TTimeStat(TUserJobStatsProxy* parent, TString name, bool commit) + : Parent(parent) + , Name(name) + , Commit(commit) {} + +TTimeStat::~TTimeStat() { + Finish(); +} + +void TTimeStat::Cancel() { + Parent = nullptr; +} + +void TTimeStat::Finish() { + if (!Parent) { + return; + } + + if (Commit) { + Parent->WriteStatNoFlush(Name, Timer.Get().MilliSeconds()); + } else { + Parent->TimeStats[Name] += Timer.Get(); + } + Cancel(); +} diff --git a/library/cpp/yt/user_job_statistics/user_job_statistics.h b/library/cpp/yt/user_job_statistics/user_job_statistics.h new file mode 100644 index 0000000000..6939d20417 --- /dev/null +++ b/library/cpp/yt/user_job_statistics/user_job_statistics.h @@ -0,0 +1,58 @@ +#pragma once + +#include <util/stream/file.h> +#include <util/generic/hash.h> +#include <util/datetime/cputimer.h> + +namespace NYtTools { + class TTimeStat; + using TTimeStatHolder = THolder<TTimeStat>; + + class TUserJobStatsProxy { + public: + static const FHANDLE JobStatisticsHandle; + private: + THolder<IOutputStream> FetchedOut; + IOutputStream* UsingStream = &Cerr; + public: + // TODO: add inheritance + THashMap<TString, i64> Stats;//will be dumped in CommitStats or desctructor + THashMap<TString, TDuration> TimeStats;//will be dumped in CommitStats or desctructor + + TUserJobStatsProxy() { Init(nullptr); } + ~TUserJobStatsProxy() { + CommitStats(); + } + TUserJobStatsProxy (IOutputStream* usingStream) {Init(usingStream);} + + void Init(IOutputStream* usingStream); + void InitChecked(IOutputStream* ifNotInJob); + void InitIfNotInited(IOutputStream* usingStream); + IOutputStream* GetStream() const { return UsingStream; } + void CommitStats(); + void WriteStat(TString name, i64 val); //immidiatly wirtes stat + void WriteStatNoFlush(TString name, i64 val); //immidiatly wirtes stat but do not flush it + + //@param name name of statistic to be written in millisecs from creation to destruction + //@param commitOnFinish if false: will update state/write on job finish; if true: write stat in destructor + TTimeStatHolder TimerStart(TString name, bool commitOnFinish = false); + }; + + class TTimeStat { + TUserJobStatsProxy* Parent; + TString Name; + bool Commit; + + TTimeStat(TUserJobStatsProxy* parent, TString name, bool commit); + friend class TUserJobStatsProxy; + + TSimpleTimer Timer; + public: + ~TTimeStat(); + TDuration Get() const { + return Timer.Get(); + } + void Cancel(); + void Finish(); + }; +} diff --git a/library/cpp/yt/user_job_statistics/ya.make b/library/cpp/yt/user_job_statistics/ya.make new file mode 100644 index 0000000000..7179660b31 --- /dev/null +++ b/library/cpp/yt/user_job_statistics/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + user_job_statistics.cpp +) + +PEERDIR( + yt/cpp/mapreduce/common +) + +END() |