aboutsummaryrefslogtreecommitdiffstats
path: root/library
diff options
context:
space:
mode:
authormax42 <max42@yandex-team.com>2023-06-30 03:37:03 +0300
committermax42 <max42@yandex-team.com>2023-06-30 03:37:03 +0300
commitfac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a (patch)
treeb8cbc1deb00309c7f1a7ab6df520a76cf0b5c6d7 /library
parent7bf166b1a7ed0af927f230022b245af618e998c1 (diff)
downloadydb-fac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a.tar.gz
YT-19324: move YT provider to ydb/library/yql
This commit is formed by the following script: https://paste.yandex-team.ru/6f92e4b8-efc5-4d34-948b-15ee2accd7e7/text. This commit has zero effect on all projects that depend on YQL. The summary of changes: - `yql/providers/yt -> ydb/library/yql/providers/yt `- the whole implementation of YT provider is moved into YDB code base for further export as a part of YT YQL plugin shared library; - `yql/providers/stat/{expr_nodes,uploader} -> ydb/library/yql/providers/stat/{expr_nodes,uploader}` - a small interface without implementation and the description of stat expr nodes; - `yql/core/extract_predicate/ut -> ydb/library/yql/core/extract_predicate/ut`; - `yql/core/{ut,ut_common} -> ydb/library/yql/core/{ut,ut_common}`; - `yql/core` is gone; - `yql/library/url_preprocessing -> ydb/library/yql/core/url_preprocessing`. **NB**: all new targets inside `ydb/` are under `IF (NOT CMAKE_EXPORT)` clause which disables them from open-source cmake generation and ya make build. They will be enabled in the subsequent commits.
Diffstat (limited to 'library')
-rw-r--r--library/cpp/containers/concurrent_hash/concurrent_hash.h128
-rw-r--r--library/cpp/disjoint_sets/disjoint_sets.cpp1
-rw-r--r--library/cpp/disjoint_sets/disjoint_sets.h81
-rw-r--r--library/cpp/disjoint_sets/ya.make7
-rw-r--r--library/cpp/dwarf_backtrace/backtrace.cpp62
-rw-r--r--library/cpp/dwarf_backtrace/backtrace.h36
-rw-r--r--library/cpp/dwarf_backtrace/ya.make19
-rw-r--r--library/cpp/erasure/README.md9
-rw-r--r--library/cpp/erasure/codec.cpp1
-rw-r--r--library/cpp/erasure/codec.h80
-rw-r--r--library/cpp/erasure/helpers.cpp80
-rw-r--r--library/cpp/erasure/helpers.h30
-rw-r--r--library/cpp/erasure/isa_erasure.cpp1
-rw-r--r--library/cpp/erasure/isa_erasure.h170
-rw-r--r--library/cpp/erasure/lrc.cpp1
-rw-r--r--library/cpp/erasure/lrc.h326
-rw-r--r--library/cpp/erasure/lrc_isa.cpp1
-rw-r--r--library/cpp/erasure/lrc_isa.h77
-rw-r--r--library/cpp/erasure/public.cpp1
-rw-r--r--library/cpp/erasure/public.h57
-rw-r--r--library/cpp/erasure/reed_solomon.cpp1
-rw-r--r--library/cpp/erasure/reed_solomon.h60
-rw-r--r--library/cpp/erasure/reed_solomon_isa.cpp1
-rw-r--r--library/cpp/erasure/reed_solomon_isa.h69
-rw-r--r--library/cpp/erasure/ya.make35
-rw-r--r--library/cpp/skiff/public.h63
-rw-r--r--library/cpp/skiff/skiff-inl.h39
-rw-r--r--library/cpp/skiff/skiff.cpp591
-rw-r--r--library/cpp/skiff/skiff.h259
-rw-r--r--library/cpp/skiff/skiff_schema-inl.h61
-rw-r--r--library/cpp/skiff/skiff_schema.cpp164
-rw-r--r--library/cpp/skiff/skiff_schema.h121
-rw-r--r--library/cpp/skiff/skiff_validator.cpp396
-rw-r--r--library/cpp/skiff/skiff_validator.h39
-rw-r--r--library/cpp/skiff/unittests/skiff_schema_ut.cpp148
-rw-r--r--library/cpp/skiff/unittests/skiff_ut.cpp627
-rw-r--r--library/cpp/skiff/unittests/ya.make12
-rw-r--r--library/cpp/skiff/ya.make16
-rw-r--r--library/cpp/skiff/zerocopy_output_writer-inl.h51
-rw-r--r--library/cpp/skiff/zerocopy_output_writer.cpp38
-rw-r--r--library/cpp/skiff/zerocopy_output_writer.h41
-rw-r--r--library/cpp/testing/gtest/friend.h5
-rw-r--r--library/cpp/threading/blocking_queue/blocking_queue.cpp3
-rw-r--r--library/cpp/threading/blocking_queue/blocking_queue.h158
-rw-r--r--library/cpp/threading/blocking_queue/blocking_queue_ut.cpp211
-rw-r--r--library/cpp/threading/blocking_queue/ut/ya.make7
-rw-r--r--library/cpp/threading/blocking_queue/ya.make9
-rw-r--r--library/cpp/threading/cron/cron.cpp69
-rw-r--r--library/cpp/threading/cron/cron.h18
-rw-r--r--library/cpp/threading/cron/ya.make11
-rw-r--r--library/cpp/type_info/Readme.md9
-rw-r--r--library/cpp/type_info/builder.cpp458
-rw-r--r--library/cpp/type_info/builder.h346
-rw-r--r--library/cpp/type_info/error.cpp1
-rw-r--r--library/cpp/type_info/error.h33
-rw-r--r--library/cpp/type_info/fwd.h123
-rw-r--r--library/cpp/type_info/test-data/bad-types.txt229
-rw-r--r--library/cpp/type_info/test-data/good-types.txt478
-rw-r--r--library/cpp/type_info/type.cpp1662
-rw-r--r--library/cpp/type_info/type.h2421
-rw-r--r--library/cpp/type_info/type_complexity.cpp78
-rw-r--r--library/cpp/type_info/type_complexity.h18
-rw-r--r--library/cpp/type_info/type_constructors.h110
-rw-r--r--library/cpp/type_info/type_equivalence.cpp286
-rw-r--r--library/cpp/type_info/type_equivalence.h36
-rw-r--r--library/cpp/type_info/type_factory.cpp495
-rw-r--r--library/cpp/type_info/type_factory.h906
-rw-r--r--library/cpp/type_info/type_info.cpp1
-rw-r--r--library/cpp/type_info/type_info.h10
-rw-r--r--library/cpp/type_info/type_io.cpp1186
-rw-r--r--library/cpp/type_info/type_io.h115
-rw-r--r--library/cpp/type_info/type_list.cpp1
-rw-r--r--library/cpp/type_info/type_list.h183
-rw-r--r--library/cpp/type_info/ut/builder.cpp125
-rw-r--r--library/cpp/type_info/ut/test_data.cpp91
-rw-r--r--library/cpp/type_info/ut/type_basics.cpp381
-rw-r--r--library/cpp/type_info/ut/type_complexity_ut.cpp33
-rw-r--r--library/cpp/type_info/ut/type_constraints.cpp53
-rw-r--r--library/cpp/type_info/ut/type_deserialize.cpp528
-rw-r--r--library/cpp/type_info/ut/type_equivalence.cpp394
-rw-r--r--library/cpp/type_info/ut/type_factory.cpp121
-rw-r--r--library/cpp/type_info/ut/type_factory_raw.cpp365
-rw-r--r--library/cpp/type_info/ut/type_io.cpp535
-rw-r--r--library/cpp/type_info/ut/type_list.cpp64
-rw-r--r--library/cpp/type_info/ut/type_serialize.cpp251
-rw-r--r--library/cpp/type_info/ut/type_show.cpp199
-rw-r--r--library/cpp/type_info/ut/type_strip_tags.cpp31
-rw-r--r--library/cpp/type_info/ut/utils.h73
-rw-r--r--library/cpp/type_info/ut/ya.make32
-rw-r--r--library/cpp/type_info/ya.make26
-rw-r--r--library/cpp/yson_pull/bridge.h34
-rw-r--r--library/cpp/yson_pull/yson.h14
-rw-r--r--library/cpp/yt/backtrace/backtrace-inl.h36
-rw-r--r--library/cpp/yt/backtrace/backtrace.cpp18
-rw-r--r--library/cpp/yt/backtrace/backtrace.h45
-rw-r--r--library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp22
-rw-r--r--library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h17
-rw-r--r--library/cpp/yt/backtrace/cursors/dummy/ya.make9
-rw-r--r--library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp146
-rw-r--r--library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h39
-rw-r--r--library/cpp/yt/backtrace/cursors/frame_pointer/ya.make9
-rw-r--r--library/cpp/yt/backtrace/cursors/interop/interop.cpp102
-rw-r--r--library/cpp/yt/backtrace/cursors/interop/interop.h25
-rw-r--r--library/cpp/yt/backtrace/cursors/interop/ya.make14
-rw-r--r--library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp70
-rw-r--r--library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h33
-rw-r--r--library/cpp/yt/backtrace/cursors/libunwind/ya.make13
-rw-r--r--library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp25
-rw-r--r--library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp64
-rw-r--r--library/cpp/yt/backtrace/symbolizers/dwarf/ya.make18
-rw-r--r--library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp113
-rw-r--r--library/cpp/yt/backtrace/unittests/backtrace_ut.cpp61
-rw-r--r--library/cpp/yt/backtrace/unittests/ya.make20
-rw-r--r--library/cpp/yt/backtrace/ya.make44
-rw-r--r--library/cpp/yt/containers/sharded_set-inl.h217
-rw-r--r--library/cpp/yt/containers/sharded_set.h69
-rw-r--r--library/cpp/yt/containers/unittests/sharded_set_ut.cpp121
-rw-r--r--library/cpp/yt/containers/unittests/ya.make15
-rw-r--r--library/cpp/yt/cpu_clock/benchmark/benchmark.cpp41
-rw-r--r--library/cpp/yt/cpu_clock/benchmark/ya.make11
-rw-r--r--library/cpp/yt/cpu_clock/unittests/clock_ut.cpp46
-rw-r--r--library/cpp/yt/cpu_clock/unittests/ya.make13
-rw-r--r--library/cpp/yt/farmhash/farm_hash.h63
-rw-r--r--library/cpp/yt/logging/logger-inl.h303
-rw-r--r--library/cpp/yt/logging/logger.cpp289
-rw-r--r--library/cpp/yt/logging/logger.h351
-rw-r--r--library/cpp/yt/logging/public.h39
-rw-r--r--library/cpp/yt/logging/unittests/logger_ut.cpp38
-rw-r--r--library/cpp/yt/logging/unittests/ya.make18
-rw-r--r--library/cpp/yt/logging/ya.make20
-rw-r--r--library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h43
-rw-r--r--library/cpp/yt/memory/leaky_ref_counted_singleton.h22
-rw-r--r--library/cpp/yt/misc/arcadia_enum-inl.h49
-rw-r--r--library/cpp/yt/misc/arcadia_enum.h18
-rw-r--r--library/cpp/yt/misc/property.h306
-rw-r--r--library/cpp/yt/string/raw_formatter.h212
-rw-r--r--library/cpp/yt/threading/unittests/count_down_latch_ut.cpp78
-rw-r--r--library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp88
-rw-r--r--library/cpp/yt/threading/unittests/spin_wait_ut.cpp48
-rw-r--r--library/cpp/yt/threading/unittests/ya.make17
-rw-r--r--library/cpp/yt/user_job_statistics/user_job_statistics.cpp133
-rw-r--r--library/cpp/yt/user_job_statistics/user_job_statistics.h58
-rw-r--r--library/cpp/yt/user_job_statistics/ya.make11
143 files changed, 20706 insertions, 0 deletions
diff --git a/library/cpp/containers/concurrent_hash/concurrent_hash.h b/library/cpp/containers/concurrent_hash/concurrent_hash.h
new file mode 100644
index 0000000000..f15a1c3d6e
--- /dev/null
+++ b/library/cpp/containers/concurrent_hash/concurrent_hash.h
@@ -0,0 +1,128 @@
+#pragma once
+
+#include <util/generic/hash.h>
+#include <util/system/spinlock.h>
+
+#include <array>
+
+template <typename K, typename V, size_t BucketCount = 64, typename L = TAdaptiveLock>
+class TConcurrentHashMap {
+public:
+ using TActualMap = THashMap<K, V>;
+ using TLock = L;
+
+ struct TBucket {
+ friend class TConcurrentHashMap;
+
+ private:
+ TActualMap Map;
+ mutable TLock Mutex;
+
+ public:
+ TLock& GetMutex() const {
+ return Mutex;
+ }
+
+ TActualMap& GetMap() {
+ return Map;
+ }
+ const TActualMap& GetMap() const {
+ return Map;
+ }
+
+ const V& GetUnsafe(const K& key) const {
+ typename TActualMap::const_iterator it = Map.find(key);
+ Y_VERIFY(it != Map.end(), "not found by key");
+ return it->second;
+ }
+
+ V& GetUnsafe(const K& key) {
+ typename TActualMap::iterator it = Map.find(key);
+ Y_VERIFY(it != Map.end(), "not found by key");
+ return it->second;
+ }
+
+ V RemoveUnsafe(const K& key) {
+ typename TActualMap::iterator it = Map.find(key);
+ Y_VERIFY(it != Map.end(), "removing non-existent key");
+ V r = std::move(it->second);
+ Map.erase(it);
+ return r;
+ }
+
+ bool HasUnsafe(const K& key) const {
+ typename TActualMap::const_iterator it = Map.find(key);
+ return (it != Map.end());
+ }
+ };
+
+ std::array<TBucket, BucketCount> Buckets;
+
+public:
+ TBucket& GetBucketForKey(const K& key) {
+ return Buckets[THash<K>()(key) % BucketCount];
+ }
+
+ const TBucket& GetBucketForKey(const K& key) const {
+ return Buckets[THash<K>()(key) % BucketCount];
+ }
+
+ void Insert(const K& key, const V& value) {
+ TBucket& bucket = GetBucketForKey(key);
+ TGuard<TLock> guard(bucket.Mutex);
+ bucket.Map[key] = value;
+ }
+
+ void InsertUnique(const K& key, const V& value) {
+ TBucket& bucket = GetBucketForKey(key);
+ TGuard<TLock> guard(bucket.Mutex);
+ if (!bucket.Map.insert(std::make_pair(key, value)).second) {
+ Y_FAIL("non-unique key");
+ }
+ }
+
+ V& InsertIfAbsent(const K& key, const V& value) {
+ TBucket& bucket = GetBucketForKey(key);
+ TGuard<TLock> guard(bucket.Mutex);
+ return bucket.Map.insert(std::make_pair(key, value)).first->second;
+ }
+
+ template <typename Callable>
+ V& InsertIfAbsentWithInit(const K& key, Callable initFunc) {
+ TBucket& bucket = GetBucketForKey(key);
+ TGuard<TLock> guard(bucket.Mutex);
+ if (bucket.HasUnsafe(key)) {
+ return bucket.GetUnsafe(key);
+ }
+
+ return bucket.Map.insert(std::make_pair(key, initFunc())).first->second;
+ }
+
+ V Get(const K& key) const {
+ const TBucket& bucket = GetBucketForKey(key);
+ TGuard<TLock> guard(bucket.Mutex);
+ return bucket.GetUnsafe(key);
+ }
+
+ bool Get(const K& key, V& result) const {
+ const TBucket& bucket = GetBucketForKey(key);
+ TGuard<TLock> guard(bucket.Mutex);
+ if (bucket.HasUnsafe(key)) {
+ result = bucket.GetUnsafe(key);
+ return true;
+ }
+ return false;
+ }
+
+ V Remove(const K& key) {
+ TBucket& bucket = GetBucketForKey(key);
+ TGuard<TLock> guard(bucket.Mutex);
+ return bucket.RemoveUnsafe(key);
+ }
+
+ bool Has(const K& key) const {
+ const TBucket& bucket = GetBucketForKey(key);
+ TGuard<TLock> guard(bucket.Mutex);
+ return bucket.HasUnsafe(key);
+ }
+};
diff --git a/library/cpp/disjoint_sets/disjoint_sets.cpp b/library/cpp/disjoint_sets/disjoint_sets.cpp
new file mode 100644
index 0000000000..5720e6c41a
--- /dev/null
+++ b/library/cpp/disjoint_sets/disjoint_sets.cpp
@@ -0,0 +1 @@
+#include "disjoint_sets.h"
diff --git a/library/cpp/disjoint_sets/disjoint_sets.h b/library/cpp/disjoint_sets/disjoint_sets.h
new file mode 100644
index 0000000000..5768886704
--- /dev/null
+++ b/library/cpp/disjoint_sets/disjoint_sets.h
@@ -0,0 +1,81 @@
+#pragma once
+
+#include <util/system/yassert.h>
+#include <util/generic/vector.h>
+
+// Implementation of disjoint-set data structure with union by rank and path compression.
+// See http://en.wikipedia.org/wiki/Disjoint-set_data_structure
+class TDisjointSets {
+public:
+ using TElement = size_t;
+
+private:
+ mutable TVector<TElement> Parents;
+ TVector<size_t> Ranks;
+ TVector<size_t> Sizes;
+ size_t NumberOfSets;
+
+public:
+ TDisjointSets(size_t setCount)
+ : Parents(setCount)
+ , Ranks(setCount, 0)
+ , Sizes(setCount, 1)
+ , NumberOfSets(setCount)
+ {
+ for (size_t i = 0; i < setCount; ++i)
+ Parents[i] = i;
+ }
+
+ TElement CanonicSetElement(TElement item) const {
+ if (Parents[item] != item)
+ Parents[item] = CanonicSetElement(Parents[item]);
+ return Parents[item];
+ }
+
+ size_t SizeOfSet(TElement item) const {
+ return Sizes[CanonicSetElement(item)];
+ }
+
+ size_t InitialSetCount() const {
+ return Parents.size();
+ }
+
+ size_t SetCount() const {
+ return NumberOfSets;
+ }
+
+ void UnionSets(TElement item1, TElement item2) {
+ TElement canonic1 = CanonicSetElement(item1);
+ TElement canonic2 = CanonicSetElement(item2);
+ if (canonic1 == canonic2)
+ return;
+
+ --NumberOfSets;
+ if (Ranks[canonic1] < Ranks[canonic2]) {
+ Parents[canonic1] = canonic2;
+ Sizes[canonic2] += Sizes[canonic1];
+ } else {
+ Parents[canonic2] = canonic1;
+ Sizes[canonic1] += Sizes[canonic2];
+ Ranks[canonic2] += Ranks[canonic1] == Ranks[canonic2] ? 1 : 0;
+ }
+ }
+
+ void Expand(size_t setCount) {
+ if (setCount < Parents.size()) {
+ return;
+ }
+
+ size_t prevSize = Parents.size();
+ Parents.resize(setCount);
+ Ranks.resize(setCount);
+ Sizes.resize(setCount);
+ NumberOfSets += setCount - prevSize;
+
+ for (size_t i = prevSize; i < setCount; ++i) {
+ Parents[i] = i;
+ Ranks[i] = 0;
+ Sizes[i] = 1;
+ }
+ }
+};
diff --git a/library/cpp/disjoint_sets/ya.make b/library/cpp/disjoint_sets/ya.make
new file mode 100644
index 0000000000..2104298baa
--- /dev/null
+++ b/library/cpp/disjoint_sets/ya.make
@@ -0,0 +1,7 @@
+LIBRARY()
+
+SRCS(
+ disjoint_sets.cpp
+)
+
+END()
diff --git a/library/cpp/dwarf_backtrace/backtrace.cpp b/library/cpp/dwarf_backtrace/backtrace.cpp
new file mode 100644
index 0000000000..a955d07249
--- /dev/null
+++ b/library/cpp/dwarf_backtrace/backtrace.cpp
@@ -0,0 +1,62 @@
+#include "backtrace.h"
+
+#include <contrib/libs/backtrace/backtrace.h>
+
+#include <util/generic/yexception.h>
+#include <util/system/type_name.h>
+#include <util/system/execpath.h>
+
+namespace NDwarf {
+ namespace {
+ struct TContext {
+ TCallback& Callback;
+ int Counter = 0;
+ TMaybe<TError> Error;
+ };
+
+ void HandleLibBacktraceError(void* data, const char* msg, int errnum) {
+ auto* context = reinterpret_cast<TContext*>(data);
+ context->Error = TError{.Code = errnum, .Message=msg};
+ }
+
+ int HandleLibBacktraceFrame(void* data, uintptr_t pc, const char* filename, int lineno, const char* function) {
+ auto* context = reinterpret_cast<TContext*>(data);
+ TLineInfo lineInfo{
+ .FileName = filename != nullptr ? filename : "???",
+ .Line = lineno,
+ .Col = 0, // libbacktrace doesn't provide column numbers, so fill this field with a dummy value.
+ .FunctionName = function != nullptr ? CppDemangle(function) : "???",
+ .Address = pc,
+ .Index = context->Counter++,
+ };
+ return static_cast<int>(context->Callback(lineInfo));
+ }
+ }
+
+ TMaybe<TError> ResolveBacktrace(TArrayRef<const void* const> backtrace, TCallback callback) {
+ TContext context{.Callback = callback};
+ // Intentionally never freed (see https://a.yandex-team.ru/arc/trunk/arcadia/contrib/libs/backtrace/backtrace.h?rev=6789902#L80).
+ static auto* state = backtrace_create_state(
+ GetPersistentExecPath().c_str(),
+ 1 /* threaded */,
+ HandleLibBacktraceError,
+ &context /* data for the error callback */
+ );
+ if (nullptr == state) {
+ static const auto initError = context.Error;
+ return initError;
+ }
+ for (const void* address : backtrace) {
+ int status = backtrace_pcinfo(
+ state,
+ reinterpret_cast<uintptr_t>(address) - 1, // last byte of the call instruction
+ HandleLibBacktraceFrame,
+ HandleLibBacktraceError,
+ &context /* data for both callbacks */);
+ if (0 != status) {
+ break;
+ }
+ }
+ return context.Error;
+ }
+}
diff --git a/library/cpp/dwarf_backtrace/backtrace.h b/library/cpp/dwarf_backtrace/backtrace.h
new file mode 100644
index 0000000000..62ec36dba5
--- /dev/null
+++ b/library/cpp/dwarf_backtrace/backtrace.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include <util/generic/array_ref.h>
+#include <util/generic/maybe.h>
+#include <util/generic/string.h>
+#include <util/generic/vector.h>
+
+#include <functional>
+
+namespace NDwarf {
+ struct TLineInfo {
+ TString FileName;
+ int Line;
+ int Col;
+ TString FunctionName;
+ uintptr_t Address;
+ int Index;
+ };
+
+ struct TError {
+ int Code;
+ TString Message;
+ };
+
+ enum class EResolving {
+ Continue = 0,
+ Break = 1,
+ };
+
+ using TCallback = std::function<EResolving(const TLineInfo&)>;
+
+ // Resolves backtrace addresses and calls the callback for all line infos of inlined functions there.
+ // Stops execution if the callback returns `EResolving::Break`.
+ [[nodiscard]] TMaybe<TError> ResolveBacktrace(TArrayRef<const void* const> backtrace, TCallback callback);
+
+}
diff --git a/library/cpp/dwarf_backtrace/ya.make b/library/cpp/dwarf_backtrace/ya.make
new file mode 100644
index 0000000000..e955f9aa92
--- /dev/null
+++ b/library/cpp/dwarf_backtrace/ya.make
@@ -0,0 +1,19 @@
+LIBRARY()
+
+NO_WSHADOW()
+
+SRCS(
+ backtrace.cpp
+)
+
+PEERDIR(
+ contrib/libs/backtrace
+)
+
+END()
+
+IF (NOT OS_WINDOWS)
+ RECURSE_FOR_TESTS(
+ ut
+ )
+ENDIF()
diff --git a/library/cpp/erasure/README.md b/library/cpp/erasure/README.md
new file mode 100644
index 0000000000..7723bf7c23
--- /dev/null
+++ b/library/cpp/erasure/README.md
@@ -0,0 +1,9 @@
+# Erasure based codecs for arbitrary data
+
+C++ wrapper for LRC and Reed-Solomon erasure codecs.
+There are two backends for LRC: Jerasure(http://jerasure.org) and ISA-L(https://github.com/intel/isa-l). ISA-L is much faster - it condsiders different instrucion sets to optimize speed of encode and decode. The only limitations now are if you don't have SSE4.2 instruction set (then base variant is as slow as Jerasure) or if you run it on aarch64 architecture (however, 2.29 version will be going to support fast implementation). However, we still have to keep Jerasure because it is incompatible due to some optimization in it which affect data layout in coded blocks.
+Also see https://wiki.yandex-team.ru/yt/userdoc/erasure/, https://wiki.yandex-team.ru/ignatijjkolesnichenko/yt/erasure/.
+
+It is possible to use codecs LRC 2k-2-2 and Reed Solomon n-k for any data stream. All you need is to provide CodecTraits (see `codecs_ut.cpp` for examples). Note that ISA-L only supports WordSize equal to 8. If you use Jerasure codecs with bigger WordSize than `MaxWordSize` in `public.h`, codec is not guaranteed to be thread-safe.
+
+You can use interface in `codec.h` or use the exact codec from `lrc_isa.h`, `lrc_jerasure.h` and `reed_solomon.h` if you like.
diff --git a/library/cpp/erasure/codec.cpp b/library/cpp/erasure/codec.cpp
new file mode 100644
index 0000000000..5fbcd720b2
--- /dev/null
+++ b/library/cpp/erasure/codec.cpp
@@ -0,0 +1 @@
+#include "codec.h"
diff --git a/library/cpp/erasure/codec.h b/library/cpp/erasure/codec.h
new file mode 100644
index 0000000000..c03d25e9c8
--- /dev/null
+++ b/library/cpp/erasure/codec.h
@@ -0,0 +1,80 @@
+#pragma once
+
+#include "public.h"
+
+#include <optional>
+#include <vector>
+
+namespace NErasure {
+
+//! Describes a generic way to generate parity blocks from data blocks and
+//! to recover (repair) missing blocks.
+/*!
+ * Given N data blocks (numbered from 0 to N - 1) one can call #Encode to generate
+ * another M parity blocks (numbered from N to N + M - 1).
+ *
+ * If some of the resulting N + M blocks ever become missing one can attempt to
+ * repair the missing blocks by calling #Decode.
+ *
+ * Here N and M are fixed (codec-specific) parameters.
+ * Call #GetDataPartCount and #GetParityPartCount to figure out the
+ * the values for N and M, respectively.
+ *
+ */
+template <class TBlobType>
+struct ICodec {
+ //! Computes a sequence of parity blocks for given data blocks.
+ /*!
+ * The size of #blocks must be equal to #GetDataPartCount.
+ * The size of the returned array is equal to #GetParityPartCount.
+ */
+ virtual std::vector<TBlobType> Encode(const std::vector<TBlobType>& blocks) const = 0;
+
+ //! Decodes (repairs) missing blocks.
+ /*!
+ * #erasedIndices must contain the set of erased blocks indices.
+ * #blocks must contain known blocks (in the order specified by #GetRepairIndices).
+ * \returns The repaired blocks.
+ */
+ virtual std::vector<TBlobType> Decode(
+ const std::vector<TBlobType>& blocks,
+ const TPartIndexList& erasedIndices) const = 0;
+
+ //! Given a set of missing block indices, returns |true| if missing blocks can be repaired.
+ //! Due to performance reasons the elements of #erasedIndices must unique and sorted.
+ virtual bool CanRepair(const TPartIndexList& erasedIndices) const = 0;
+
+ //! Rapid version that works with set instead of list.
+ virtual bool CanRepair(const TPartIndexSet& erasedIndices) const = 0;
+
+ //! Given a set of missing block indices, checks if missing blocks can be repaired.
+ /*!
+ * \returns
+ * If repair is not possible, returns |std::nullopt|.
+ * Otherwise returns the indices of blocks (both data and parity) to be passed to #Decode
+ * (in this very order). Not all known blocks may be needed for repair.
+ */
+ virtual std::optional<TPartIndexList> GetRepairIndices(const TPartIndexList& erasedIndices) const = 0;
+
+ //! Returns the number of data blocks this codec can handle.
+ virtual int GetDataPartCount() const = 0;
+
+ //! Returns the number of parity blocks this codec can handle.
+ virtual int GetParityPartCount() const = 0;
+
+ //! Returns the maximum number of blocks that can always be repaired when missing.
+ virtual int GetGuaranteedRepairablePartCount() const = 0;
+
+ //! Every block passed to this codec must have size divisible by the result of #GetWordSize.
+ virtual int GetWordSize() const = 0;
+
+ // Extension methods
+
+ //! Returns the sum of #GetDataPartCount and #GetParityPartCount.
+ int GetTotalPartCount() const {
+ return GetDataPartCount() + GetParityPartCount();
+ }
+};
+
+} // namespace NErasure
+
diff --git a/library/cpp/erasure/helpers.cpp b/library/cpp/erasure/helpers.cpp
new file mode 100644
index 0000000000..74edeca52c
--- /dev/null
+++ b/library/cpp/erasure/helpers.cpp
@@ -0,0 +1,80 @@
+#include "helpers.h"
+
+#include <algorithm>
+#include <iterator>
+
+namespace NErasure {
+
+TPartIndexList MakeSegment(int begin, int end) {
+ TPartIndexList result(end - begin);
+ for (int i = begin; i < end; ++i) {
+ result[i - begin] = i;
+ }
+ return result;
+}
+
+TPartIndexList MakeSingleton(int elem) {
+ TPartIndexList result;
+ result.push_back(elem);
+ return result;
+}
+
+TPartIndexList Difference(int begin, int end, const TPartIndexList& subtrahend) {
+ size_t pos = 0;
+ TPartIndexList result;
+ for (int i = begin; i < end; ++i) {
+ while (pos < subtrahend.size() && subtrahend[pos] < i) {
+ pos += 1;
+ }
+ if (pos == subtrahend.size() || subtrahend[pos] != i) {
+ result.push_back(i);
+ }
+ }
+ return result;
+}
+
+TPartIndexList Difference(const TPartIndexList& first, const TPartIndexList& second) {
+ TPartIndexList result;
+ std::set_difference(first.begin(), first.end(), second.begin(), second.end(), std::back_inserter(result));
+ return result;
+}
+
+TPartIndexList Difference(const TPartIndexList& set, int subtrahend) {
+ return Difference(set, MakeSingleton(subtrahend));
+}
+
+TPartIndexList Intersection(const TPartIndexList& first, const TPartIndexList& second) {
+ TPartIndexList result;
+ std::set_intersection(first.begin(), first.end(), second.begin(), second.end(), std::back_inserter(result));
+ return result;
+}
+
+TPartIndexList Union(const TPartIndexList& first, const TPartIndexList& second) {
+ TPartIndexList result;
+ std::set_union(first.begin(), first.end(), second.begin(), second.end(), std::back_inserter(result));
+ return result;
+}
+
+bool Contains(const TPartIndexList& set, int elem) {
+ return std::binary_search(set.begin(), set.end(), elem);
+}
+
+TPartIndexList UniqueSortedIndices(const TPartIndexList& indices) {
+ TPartIndexList copy = indices;
+ std::sort(copy.begin(), copy.end());
+ copy.erase(std::unique(copy.begin(), copy.end()), copy.end());
+ return copy;
+}
+
+TPartIndexList ExtractRows(const TPartIndexList& matrix, int width, const TPartIndexList& rows) {
+ Y_ASSERT(matrix.size() % width == 0);
+ TPartIndexList result(width * rows.size());
+ for (size_t i = 0; i < rows.size(); ++i) {
+ auto start = matrix.begin() + rows[i] * width;
+ std::copy(start, start + width, result.begin() + i * width);
+ }
+ return result;
+}
+
+} // namespace NErasure
+
diff --git a/library/cpp/erasure/helpers.h b/library/cpp/erasure/helpers.h
new file mode 100644
index 0000000000..741186322a
--- /dev/null
+++ b/library/cpp/erasure/helpers.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include "public.h"
+
+namespace NErasure {
+
+// All vectors here are assumed to be sorted.
+
+TPartIndexList MakeSegment(int begin, int end);
+
+TPartIndexList MakeSingleton(int elem);
+
+TPartIndexList Difference(int begin, int end, const TPartIndexList& subtrahend);
+
+TPartIndexList Difference(const TPartIndexList& first, const TPartIndexList& second);
+
+TPartIndexList Difference(const TPartIndexList& first, int elem);
+
+TPartIndexList Intersection(const TPartIndexList& first, const TPartIndexList& second);
+
+TPartIndexList Union(const TPartIndexList& first, const TPartIndexList& second);
+
+bool Contains(const TPartIndexList& set, int elem);
+
+TPartIndexList UniqueSortedIndices(const TPartIndexList& indices);
+
+TPartIndexList ExtractRows(const TPartIndexList& matrix, int width, const TPartIndexList& rows);
+
+} // namespace NErasure
+
diff --git a/library/cpp/erasure/isa_erasure.cpp b/library/cpp/erasure/isa_erasure.cpp
new file mode 100644
index 0000000000..0b0199934e
--- /dev/null
+++ b/library/cpp/erasure/isa_erasure.cpp
@@ -0,0 +1 @@
+#include "isa_erasure.h"
diff --git a/library/cpp/erasure/isa_erasure.h b/library/cpp/erasure/isa_erasure.h
new file mode 100644
index 0000000000..a7df61307f
--- /dev/null
+++ b/library/cpp/erasure/isa_erasure.h
@@ -0,0 +1,170 @@
+#pragma once
+
+#include "public.h"
+
+#include "helpers.h"
+
+#include <library/cpp/yt/assert/assert.h>
+
+#include <util/generic/array_ref.h>
+#include <util/generic/ptr.h>
+#include <util/generic/singleton.h>
+
+#include <vector>
+
+extern "C" {
+ #include <contrib/libs/isa-l/include/erasure_code.h>
+}
+
+namespace NErasure {
+
+template <class TBlobType>
+static inline unsigned char* ConstCast(typename TBlobType::const_iterator blobIter) {
+ return const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(blobIter));
+}
+
+template <int DataPartCount, int ParityPartCount, class TCodecTraits, class TBlobType = typename TCodecTraits::TBlobType, class TMutableBlobType = typename TCodecTraits::TMutableBlobType>
+std::vector<TBlobType> ISAErasureEncode(
+ const std::vector<unsigned char>& encodeGFTables,
+ const std::vector<TBlobType>& dataBlocks)
+{
+ YT_VERIFY(dataBlocks.size() == DataPartCount);
+
+ size_t blockLength = dataBlocks.front().Size();
+ for (size_t i = 1; i < dataBlocks.size(); ++i) {
+ YT_VERIFY(dataBlocks[i].Size() == blockLength);
+ }
+
+ std::vector<unsigned char*> dataPointers;
+ for (const auto& block : dataBlocks) {
+ dataPointers.emplace_back(ConstCast<TBlobType>(block.Begin()));
+ }
+
+ std::vector<TMutableBlobType> parities(ParityPartCount);
+ std::vector<unsigned char*> parityPointers(ParityPartCount);
+ for (size_t i = 0; i < ParityPartCount; ++i) {
+ parities[i] = TCodecTraits::AllocateBlob(blockLength);
+ parityPointers[i] = ConstCast<TBlobType>(parities[i].Begin());
+ memset(parityPointers[i], 0, blockLength);
+ }
+
+ ec_encode_data(
+ blockLength,
+ DataPartCount,
+ ParityPartCount,
+ const_cast<unsigned char*>(encodeGFTables.data()),
+ dataPointers.data(),
+ parityPointers.data());
+
+ return std::vector<TBlobType>(parities.begin(), parities.end());
+}
+
+template <int DataPartCount, int ParityPartCount, class TCodecTraits, class TBlobType = typename TCodecTraits::TBlobType, class TMutableBlobType = typename TCodecTraits::TMutableBlobType>
+std::vector<TBlobType> ISAErasureDecode(
+ const std::vector<TBlobType>& dataBlocks,
+ const TPartIndexList& erasedIndices,
+ TConstArrayRef<TPartIndexList> groups,
+ const std::vector<unsigned char>& fullGeneratorMatrix)
+{
+ YT_VERIFY(dataBlocks.size() >= DataPartCount);
+ YT_VERIFY(erasedIndices.size() <= ParityPartCount);
+
+ size_t blockLength = dataBlocks.front().Size();
+ for (size_t i = 1; i < dataBlocks.size(); ++i) {
+ YT_VERIFY(dataBlocks[i].Size() == blockLength);
+ }
+
+ std::vector<unsigned char> partialGeneratorMatrix(DataPartCount * DataPartCount, 0);
+
+ std::vector<unsigned char*> recoveryBlocks;
+ for (size_t i = 0; i < DataPartCount; ++i) {
+ recoveryBlocks.emplace_back(ConstCast<TBlobType>(dataBlocks[i].Begin()));
+ }
+
+ // Groups check is specific for LRC.
+ std::vector<int> isGroupHealthy(2, 1);
+ for (size_t i = 0; i < 2; ++i) {
+ for (const auto& index : erasedIndices) {
+ if (!groups.empty() && Contains(groups[0], index)) {
+ isGroupHealthy[0] = 0;
+ } else if (!groups.empty() && Contains(groups[1], index)) {
+ isGroupHealthy[1] = 0;
+ }
+ }
+ }
+
+ // When a group is healthy we cannot use its local parity, thus skip it using gap.
+ size_t gap = 0;
+ size_t decodeMatrixIndex = 0;
+ size_t erasedBlockIndex = 0;
+ while (decodeMatrixIndex < DataPartCount) {
+ size_t globalIndex = decodeMatrixIndex + erasedBlockIndex + gap;
+
+ if (erasedBlockIndex < erasedIndices.size() &&
+ globalIndex == static_cast<size_t>(erasedIndices[erasedBlockIndex]))
+ {
+ ++erasedBlockIndex;
+ continue;
+ }
+
+ if (!groups.empty() && globalIndex >= DataPartCount && globalIndex < DataPartCount + 2) {
+ if (Contains(groups[0], globalIndex) && isGroupHealthy[0]) {
+ ++gap;
+ continue;
+ }
+ if (Contains(groups[1], globalIndex) && isGroupHealthy[1]) {
+ ++gap;
+ continue;
+ }
+ }
+
+ memcpy(&partialGeneratorMatrix[decodeMatrixIndex * DataPartCount], &fullGeneratorMatrix[globalIndex * DataPartCount], DataPartCount);
+ ++decodeMatrixIndex;
+ }
+
+ std::vector<unsigned char> invertedGeneratorMatrix(DataPartCount * DataPartCount, 0);
+ int res = gf_invert_matrix(partialGeneratorMatrix.data(), invertedGeneratorMatrix.data(), DataPartCount);
+ YT_VERIFY(res == 0);
+
+ std::vector<unsigned char> decodeMatrix(DataPartCount * (DataPartCount + ParityPartCount), 0);
+
+ //! Some magical code from library example.
+ for (size_t i = 0; i < erasedIndices.size(); ++i) {
+ if (erasedIndices[i] < DataPartCount) {
+ memcpy(&decodeMatrix[i * DataPartCount], &invertedGeneratorMatrix[erasedIndices[i] * DataPartCount], DataPartCount);
+ } else {
+ for (int k = 0; k < DataPartCount; ++k) {
+ int val = 0;
+ for (int j = 0; j < DataPartCount; ++j) {
+ val ^= gf_mul_erasure(invertedGeneratorMatrix[j * DataPartCount + k], fullGeneratorMatrix[DataPartCount * erasedIndices[i] + j]);
+ }
+
+ decodeMatrix[DataPartCount * i + k] = val;
+ }
+ }
+ }
+
+ std::vector<unsigned char> decodeGFTables(DataPartCount * erasedIndices.size() * 32);
+ ec_init_tables(DataPartCount, erasedIndices.size(), decodeMatrix.data(), decodeGFTables.data());
+
+ std::vector<TMutableBlobType> recoveredParts;
+ std::vector<unsigned char*> recoveredPartsPointers;
+ for (size_t i = 0; i < erasedIndices.size(); ++i) {
+ recoveredParts.emplace_back(TCodecTraits::AllocateBlob(blockLength));
+ recoveredPartsPointers.emplace_back(ConstCast<TBlobType>(recoveredParts.back().Begin()));
+ memset(recoveredPartsPointers.back(), 0, blockLength);
+ }
+
+ ec_encode_data(
+ blockLength,
+ DataPartCount,
+ erasedIndices.size(),
+ decodeGFTables.data(),
+ recoveryBlocks.data(),
+ recoveredPartsPointers.data());
+
+ return std::vector<TBlobType>(recoveredParts.begin(), recoveredParts.end());
+}
+
+} // namespace NErasure
+
diff --git a/library/cpp/erasure/lrc.cpp b/library/cpp/erasure/lrc.cpp
new file mode 100644
index 0000000000..8c6a347091
--- /dev/null
+++ b/library/cpp/erasure/lrc.cpp
@@ -0,0 +1 @@
+#include "lrc.h"
diff --git a/library/cpp/erasure/lrc.h b/library/cpp/erasure/lrc.h
new file mode 100644
index 0000000000..15185a47f4
--- /dev/null
+++ b/library/cpp/erasure/lrc.h
@@ -0,0 +1,326 @@
+#pragma once
+
+#include "helpers.h"
+
+#include <library/cpp/sse/sse.h>
+
+#include <library/cpp/yt/assert/assert.h>
+
+#include <util/generic/array_ref.h>
+
+#include <algorithm>
+#include <optional>
+
+namespace NErasure {
+
+template <class TCodecTraits, class TBlobType = typename TCodecTraits::TBlobType>
+static inline TBlobType Xor(const std::vector<TBlobType>& refs) {
+ using TBufferType = typename TCodecTraits::TBufferType;
+ size_t size = refs.front().Size();
+ TBufferType result = TCodecTraits::AllocateBuffer(size); // this also fills the buffer with zeros
+ for (const TBlobType& ref : refs) {
+ const char* data = reinterpret_cast<const char*>(ref.Begin());
+ size_t pos = 0;
+#ifdef ARCADIA_SSE
+ for (; pos + sizeof(__m128i) <= size; pos += sizeof(__m128i)) {
+ __m128i* dst = reinterpret_cast<__m128i*>(result.Begin() + pos);
+ const __m128i* src = reinterpret_cast<const __m128i*>(data + pos);
+ _mm_storeu_si128(dst, _mm_xor_si128(_mm_loadu_si128(src), _mm_loadu_si128(dst)));
+ }
+#endif
+ for (; pos < size; ++pos) {
+ *(result.Begin() + pos) ^= data[pos];
+ }
+ }
+ return TCodecTraits::FromBufferToBlob(std::move(result));
+}
+
+//! Locally Reconstructable Codes
+/*!
+ * See https://www.usenix.org/conference/usenixfederatedconferencesweek/erasure-coding-windows-azure-storage
+ * for more details.
+ */
+template <int DataPartCount, int ParityPartCount, int WordSize, class TCodecTraits>
+class TLrcCodecBase
+ : public ICodec<typename TCodecTraits::TBlobType>
+{
+ static_assert(DataPartCount % 2 == 0, "Data part count must be even.");
+ static_assert(ParityPartCount == 4, "Now we only support n-2-2 scheme for LRC codec");
+ static_assert(1 + DataPartCount / 2 < (1 << (WordSize / 2)), "Data part count should be enough small to construct proper matrix.");
+public:
+ //! Main blob for storing data.
+ using TBlobType = typename TCodecTraits::TBlobType;
+ //! Main mutable blob for decoding data.
+ using TMutableBlobType = typename TCodecTraits::TMutableBlobType;
+
+ static constexpr ui64 RequiredDataAlignment = alignof(ui64);
+
+ TLrcCodecBase() {
+ Groups_[0] = MakeSegment(0, DataPartCount / 2);
+ // Xor.
+ Groups_[0].push_back(DataPartCount);
+
+ Groups_[1] = MakeSegment(DataPartCount / 2, DataPartCount);
+ // Xor.
+ Groups_[1].push_back(DataPartCount + 1);
+
+ constexpr int totalPartCount = DataPartCount + ParityPartCount;
+ if constexpr (totalPartCount <= BitmaskOptimizationThreshold) {
+ CanRepair_.resize(1 << totalPartCount);
+ for (int mask = 0; mask < (1 << totalPartCount); ++mask) {
+ TPartIndexList erasedIndices;
+ for (size_t i = 0; i < totalPartCount; ++i) {
+ if ((mask & (1 << i)) == 0) {
+ erasedIndices.push_back(i);
+ }
+ }
+ CanRepair_[mask] = CalculateCanRepair(erasedIndices);
+ }
+ }
+ }
+
+ /*! Note that if you want to restore any internal data, blocks offsets must by WordSize * sizeof(long) aligned.
+ * Though it is possible to restore unaligned data if no more than one index in each Group is failed. See unittests for this case.
+ */
+ std::vector<TBlobType> Decode(
+ const std::vector<TBlobType>& blocks,
+ const TPartIndexList& erasedIndices) const override
+ {
+ if (erasedIndices.empty()) {
+ return std::vector<TBlobType>();
+ }
+
+ size_t blockLength = blocks.front().Size();
+ for (size_t i = 1; i < blocks.size(); ++i) {
+ YT_VERIFY(blocks[i].Size() == blockLength);
+ }
+
+ TPartIndexList indices = UniqueSortedIndices(erasedIndices);
+
+ // We can restore one block by xor.
+ if (indices.size() == 1) {
+ int index = erasedIndices.front();
+ for (size_t i = 0; i < 2; ++i) {
+ if (Contains(Groups_[i], index)) {
+ return std::vector<TBlobType>(1, Xor<TCodecTraits>(blocks));
+ }
+ }
+ }
+
+ TPartIndexList recoveryIndices = GetRepairIndices(indices).value();
+ // We can restore two blocks from different groups using xor.
+ if (indices.size() == 2 &&
+ indices.back() < DataPartCount + 2 &&
+ recoveryIndices.back() < DataPartCount + 2)
+ {
+ std::vector<TBlobType> result;
+ for (int index : indices) {
+ for (size_t groupIndex = 0; groupIndex < 2; ++groupIndex) {
+ if (!Contains(Groups_[groupIndex], index)) {
+ continue;
+ }
+
+ std::vector<TBlobType> correspondingBlocks;
+ for (int pos : Groups_[groupIndex]) {
+ for (size_t i = 0; i < blocks.size(); ++i) {
+ if (recoveryIndices[i] != pos) {
+ continue;
+ }
+ correspondingBlocks.push_back(blocks[i]);
+ }
+ }
+
+ result.push_back(Xor<TCodecTraits>(correspondingBlocks));
+ }
+ }
+ return result;
+ }
+
+ return FallbackToCodecDecode(blocks, std::move(indices));
+ }
+
+ bool CanRepair(const TPartIndexList& erasedIndices) const final {
+ constexpr int totalPartCount = DataPartCount + ParityPartCount;
+ if constexpr (totalPartCount <= BitmaskOptimizationThreshold) {
+ int mask = (1 << (totalPartCount)) - 1;
+ for (int index : erasedIndices) {
+ mask -= (1 << index);
+ }
+ return CanRepair_[mask];
+ } else {
+ return CalculateCanRepair(erasedIndices);
+ }
+ }
+
+ bool CanRepair(const TPartIndexSet& erasedIndicesMask) const final {
+ constexpr int totalPartCount = DataPartCount + ParityPartCount;
+ if constexpr (totalPartCount <= BitmaskOptimizationThreshold) {
+ TPartIndexSet mask = erasedIndicesMask;
+ return CanRepair_[mask.flip().to_ulong()];
+ } else {
+ TPartIndexList erasedIndices;
+ for (size_t i = 0; i < erasedIndicesMask.size(); ++i) {
+ if (erasedIndicesMask[i]) {
+ erasedIndices.push_back(i);
+ }
+ }
+ return CalculateCanRepair(erasedIndices);
+ }
+ }
+
+ std::optional<TPartIndexList> GetRepairIndices(const TPartIndexList& erasedIndices) const final {
+ if (erasedIndices.empty()) {
+ return TPartIndexList();
+ }
+
+ TPartIndexList indices = UniqueSortedIndices(erasedIndices);
+
+ if (indices.size() > ParityPartCount) {
+ return std::nullopt;
+ }
+
+ // One erasure from data or xor blocks.
+ if (indices.size() == 1) {
+ int index = indices.front();
+ for (size_t i = 0; i < 2; ++i) {
+ if (Contains(Groups_[i], index)) {
+ return Difference(Groups_[i], index);
+ }
+ }
+ }
+
+ // Null if we have 4 erasures in one group.
+ if (indices.size() == ParityPartCount) {
+ bool intersectsAny = true;
+ for (size_t i = 0; i < 2; ++i) {
+ if (Intersection(indices, Groups_[i]).empty()) {
+ intersectsAny = false;
+ }
+ }
+ if (!intersectsAny) {
+ return std::nullopt;
+ }
+ }
+
+ // Calculate coverage of each group.
+ int groupCoverage[2] = {};
+ for (int index : indices) {
+ for (size_t i = 0; i < 2; ++i) {
+ if (Contains(Groups_[i], index)) {
+ ++groupCoverage[i];
+ }
+ }
+ }
+
+ // Two erasures, one in each group.
+ if (indices.size() == 2 && groupCoverage[0] == 1 && groupCoverage[1] == 1) {
+ return Difference(Union(Groups_[0], Groups_[1]), indices);
+ }
+
+ // Erasures in only parity blocks.
+ if (indices.front() >= DataPartCount) {
+ return MakeSegment(0, DataPartCount);
+ }
+
+ // Remove unnecessary xor parities.
+ TPartIndexList result = Difference(0, DataPartCount + ParityPartCount, indices);
+ for (size_t i = 0; i < 2; ++i) {
+ if (groupCoverage[i] == 0 && indices.size() <= 3) {
+ result = Difference(result, DataPartCount + i);
+ }
+ }
+ return result;
+ }
+
+ int GetDataPartCount() const override {
+ return DataPartCount;
+ }
+
+ int GetParityPartCount() const override {
+ return ParityPartCount;
+ }
+
+ int GetGuaranteedRepairablePartCount() const override {
+ return ParityPartCount - 1;
+ }
+
+ int GetWordSize() const override {
+ return WordSize * sizeof(long);
+ }
+
+ virtual ~TLrcCodecBase() = default;
+
+protected:
+ // Indices of data blocks and corresponding xor (we have two xor parities).
+ TConstArrayRef<TPartIndexList> GetXorGroups() const {
+ return Groups_;
+ }
+
+ virtual std::vector<TBlobType> FallbackToCodecDecode(
+ const std::vector<TBlobType>& /* blocks */,
+ TPartIndexList /* erasedIndices */) const = 0;
+
+ template <typename T>
+ void InitializeGeneratorMatrix(T* generatorMatrix, const std::function<T(T)>& GFSquare) {
+ for (int row = 0; row < ParityPartCount; ++row) {
+ for (int column = 0; column < DataPartCount; ++column) {
+ int index = row * DataPartCount + column;
+
+ bool isFirstHalf = column < DataPartCount / 2;
+ if (row == 0) generatorMatrix[index] = isFirstHalf ? 1 : 0;
+ if (row == 1) generatorMatrix[index] = isFirstHalf ? 0 : 1;
+
+ // Let alpha_i be coefficient of first half and beta_i of the second half.
+ // Then matrix is non-singular iff:
+ // a) alpha_i, beta_j != 0
+ // b) alpha_i != beta_j
+ // c) alpha_i + alpha_k != beta_j + beta_l
+ // for any i, j, k, l.
+ if (row == 2) {
+ int shift = isFirstHalf ? 1 : (1 << (WordSize / 2));
+ int relativeColumn = isFirstHalf ? column : (column - (DataPartCount / 2));
+ generatorMatrix[index] = shift * (1 + relativeColumn);
+ }
+
+ // The last row is the square of the row before last.
+ if (row == 3) {
+ auto prev = generatorMatrix[index - DataPartCount];
+ generatorMatrix[index] = GFSquare(prev);
+ }
+ }
+ }
+ }
+
+private:
+ bool CalculateCanRepair(const TPartIndexList& erasedIndices) const {
+ TPartIndexList indices = UniqueSortedIndices(erasedIndices);
+ if (indices.size() > ParityPartCount) {
+ return false;
+ }
+
+ if (indices.size() == 1) {
+ int index = indices.front();
+ for (size_t i = 0; i < 2; ++i) {
+ if (Contains(Groups_[i], index)) {
+ return true;
+ }
+ }
+ }
+
+ // If 4 indices miss in one block we cannot recover.
+ if (indices.size() == ParityPartCount) {
+ for (size_t i = 0; i < 2; ++i) {
+ if (Intersection(indices, Groups_[i]).empty()) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ TPartIndexList Groups_[2];
+ std::vector<bool> CanRepair_;
+};
+
+} // namespace NErasure
diff --git a/library/cpp/erasure/lrc_isa.cpp b/library/cpp/erasure/lrc_isa.cpp
new file mode 100644
index 0000000000..2068f840c8
--- /dev/null
+++ b/library/cpp/erasure/lrc_isa.cpp
@@ -0,0 +1 @@
+#include "lrc_isa.h"
diff --git a/library/cpp/erasure/lrc_isa.h b/library/cpp/erasure/lrc_isa.h
new file mode 100644
index 0000000000..800dc3c5ca
--- /dev/null
+++ b/library/cpp/erasure/lrc_isa.h
@@ -0,0 +1,77 @@
+#pragma once
+
+#include "lrc.h"
+#include "helpers.h"
+
+#include "isa_erasure.h"
+
+extern "C" {
+ #include <contrib/libs/isa-l/include/erasure_code.h>
+}
+
+#include <library/cpp/sse/sse.h>
+
+#include <util/generic/array_ref.h>
+
+#include <optional>
+#include <vector>
+
+namespace NErasure {
+
+template <int DataPartCount, int ParityPartCount, int WordSize, class TCodecTraits>
+class TLrcIsa
+ : public TLrcCodecBase<DataPartCount, ParityPartCount, WordSize, TCodecTraits>
+{
+ static_assert(WordSize == 8, "ISA-l erasure codes support computations only in GF(2^8)");
+public:
+ //! Main blob for storing data.
+ using TBlobType = typename TCodecTraits::TBlobType;
+ //! Main mutable blob for decoding data.
+ using TMutableBlobType = typename TCodecTraits::TMutableBlobType;
+
+ static constexpr ui64 RequiredDataAlignment = alignof(ui64);
+
+ TLrcIsa()
+ : TLrcCodecBase<DataPartCount, ParityPartCount, WordSize, TCodecTraits>()
+ {
+ EncodeGFTables_.resize(DataPartCount * ParityPartCount * 32, 0);
+ GeneratorMatrix_.resize((DataPartCount + ParityPartCount) * DataPartCount, 0);
+
+ for (int row = 0; row < DataPartCount; ++row) {
+ GeneratorMatrix_[row * DataPartCount + row] = 1;
+ }
+ this->template InitializeGeneratorMatrix<typename decltype(GeneratorMatrix_)::value_type>(
+ &GeneratorMatrix_[DataPartCount * DataPartCount],
+ std::bind(&gf_mul_erasure, std::placeholders::_1, std::placeholders::_1));
+
+ ec_init_tables(
+ DataPartCount,
+ ParityPartCount,
+ &GeneratorMatrix_.data()[DataPartCount * DataPartCount],
+ EncodeGFTables_.data());
+ }
+
+ std::vector<TBlobType> Encode(const std::vector<TBlobType>& blocks) const override {
+ return ISAErasureEncode<DataPartCount, ParityPartCount, TCodecTraits, TBlobType, TMutableBlobType>(EncodeGFTables_, blocks);
+ }
+
+ virtual ~TLrcIsa() = default;
+
+private:
+ std::vector<TBlobType> FallbackToCodecDecode(
+ const std::vector<TBlobType>& blocks,
+ TPartIndexList erasedIndices) const override
+ {
+ return ISAErasureDecode<DataPartCount, ParityPartCount, TCodecTraits, TBlobType, TMutableBlobType>(
+ blocks,
+ std::move(erasedIndices),
+ this->GetXorGroups(),
+ GeneratorMatrix_);
+ }
+
+ std::vector<unsigned char> GeneratorMatrix_;
+ std::vector<unsigned char> EncodeGFTables_;
+};
+
+} // NErasure
+
diff --git a/library/cpp/erasure/public.cpp b/library/cpp/erasure/public.cpp
new file mode 100644
index 0000000000..4df9bcaa18
--- /dev/null
+++ b/library/cpp/erasure/public.cpp
@@ -0,0 +1 @@
+#include "public.h"
diff --git a/library/cpp/erasure/public.h b/library/cpp/erasure/public.h
new file mode 100644
index 0000000000..d5cf01297b
--- /dev/null
+++ b/library/cpp/erasure/public.h
@@ -0,0 +1,57 @@
+#pragma once
+
+#include <util/generic/buffer.h>
+#include <util/generic/yexception.h>
+#include <util/memory/blob.h>
+#include <util/string/cast.h>
+#include <util/system/src_root.h>
+
+#include <vector>
+
+#include <bitset>
+
+namespace NErasure {
+
+//! The maximum total number of blocks our erasure codec can handle.
+static constexpr int MaxTotalPartCount = 16;
+
+//! Max word size to use. `w` in jerasure notation
+static constexpr int MaxWordSize = 8;
+
+//! Max threshold to generate bitmask for CanRepair indices for LRC encoding.
+static constexpr int BitmaskOptimizationThreshold = 22;
+
+//! A vector type for holding block indexes.
+using TPartIndexList = std::vector<int>;
+
+//! Each bit corresponds to a possible block index.
+using TPartIndexSet = std::bitset<MaxTotalPartCount>;
+
+template <class TBlobType>
+struct ICodec;
+
+struct TDefaultCodecTraits {
+ using TBlobType = TBlob;
+ using TMutableBlobType = TBlob;
+ using TBufferType = TBuffer;
+
+ static inline TMutableBlobType AllocateBlob(size_t size) {
+ TBufferType buffer(size);
+ buffer.Resize(size);
+ // The buffer is cleared after this call so no use after free.
+ return TBlob::FromBuffer(buffer);
+ }
+
+ // AllocateBuffer must fill the memory with 0.
+ static inline TBufferType AllocateBuffer(size_t size) {
+ TBufferType buffer(size);
+ buffer.Fill(0, size);
+ return buffer;
+ }
+
+ static inline TBlobType FromBufferToBlob(TBufferType&& blob) {
+ return TBlobType::FromBuffer(blob);
+ }
+};
+
+} // namespace NErasure
diff --git a/library/cpp/erasure/reed_solomon.cpp b/library/cpp/erasure/reed_solomon.cpp
new file mode 100644
index 0000000000..68aa2acbfb
--- /dev/null
+++ b/library/cpp/erasure/reed_solomon.cpp
@@ -0,0 +1 @@
+#include "reed_solomon.h"
diff --git a/library/cpp/erasure/reed_solomon.h b/library/cpp/erasure/reed_solomon.h
new file mode 100644
index 0000000000..a2e268e81d
--- /dev/null
+++ b/library/cpp/erasure/reed_solomon.h
@@ -0,0 +1,60 @@
+#pragma once
+
+#include "helpers.h"
+
+#include <algorithm>
+#include <optional>
+
+namespace NErasure {
+
+template <int DataPartCount, int ParityPartCount, int WordSize, class TCodecTraits>
+class TReedSolomonBase
+ : public ICodec<typename TCodecTraits::TBlobType>
+{
+public:
+ static constexpr ui64 RequiredDataAlignment = alignof(ui64);
+
+ bool CanRepair(const TPartIndexList& erasedIndices) const final {
+ return erasedIndices.size() <= ParityPartCount;
+ }
+
+ bool CanRepair(const TPartIndexSet& erasedIndices) const final {
+ return erasedIndices.count() <= static_cast<size_t>(ParityPartCount);
+ }
+
+ std::optional<TPartIndexList> GetRepairIndices(const TPartIndexList& erasedIndices) const final {
+ if (erasedIndices.empty()) {
+ return TPartIndexList();
+ }
+
+ TPartIndexList indices = erasedIndices;
+ std::sort(indices.begin(), indices.end());
+ indices.erase(std::unique(indices.begin(), indices.end()), indices.end());
+
+ if (indices.size() > static_cast<size_t>(ParityPartCount)) {
+ return std::nullopt;
+ }
+
+ return Difference(0, DataPartCount + ParityPartCount, indices);
+ }
+
+ int GetDataPartCount() const final {
+ return DataPartCount;
+ }
+
+ int GetParityPartCount() const final {
+ return ParityPartCount;
+ }
+
+ int GetGuaranteedRepairablePartCount() const final {
+ return ParityPartCount;
+ }
+
+ int GetWordSize() const final {
+ return WordSize * sizeof(long);
+ }
+
+ virtual ~TReedSolomonBase() = default;
+};
+
+} // namespace NErasure
diff --git a/library/cpp/erasure/reed_solomon_isa.cpp b/library/cpp/erasure/reed_solomon_isa.cpp
new file mode 100644
index 0000000000..eaffde54b5
--- /dev/null
+++ b/library/cpp/erasure/reed_solomon_isa.cpp
@@ -0,0 +1 @@
+#include "reed_solomon_isa.h"
diff --git a/library/cpp/erasure/reed_solomon_isa.h b/library/cpp/erasure/reed_solomon_isa.h
new file mode 100644
index 0000000000..73a3cd630f
--- /dev/null
+++ b/library/cpp/erasure/reed_solomon_isa.h
@@ -0,0 +1,69 @@
+#pragma once
+
+#include "isa_erasure.h"
+#include "reed_solomon.h"
+
+extern "C" {
+ #include <contrib/libs/isa-l/include/erasure_code.h>
+}
+
+#include <util/generic/array_ref.h>
+
+#include <array>
+
+namespace NErasure {
+
+template <int DataPartCount, int ParityPartCount, int WordSize, class TCodecTraits>
+class TReedSolomonIsa
+ : public TReedSolomonBase<DataPartCount, ParityPartCount, WordSize, TCodecTraits>
+{
+ static_assert(WordSize == 8, "ISA-l erasure codes support computations only in GF(2^8)");
+public:
+ //! Main blob for storing data.
+ using TBlobType = typename TCodecTraits::TBlobType;
+ //! Main mutable blob for decoding data.
+ using TMutableBlobType = typename TCodecTraits::TMutableBlobType;
+
+ TReedSolomonIsa() {
+ EncodeGFTables_.resize(DataPartCount * ParityPartCount * 32, 0);
+ GeneratorMatrix_.resize((DataPartCount + ParityPartCount) * DataPartCount, 0);
+
+ gf_gen_rs_matrix(
+ GeneratorMatrix_.data(),
+ DataPartCount + ParityPartCount,
+ DataPartCount);
+
+ ec_init_tables(
+ DataPartCount,
+ ParityPartCount,
+ &GeneratorMatrix_.data()[DataPartCount * DataPartCount],
+ EncodeGFTables_.data());
+ }
+
+ virtual std::vector<TBlobType> Encode(const std::vector<TBlobType>& blocks) const override {
+ return ISAErasureEncode<DataPartCount, ParityPartCount, TCodecTraits, TBlobType, TMutableBlobType>(EncodeGFTables_, blocks);
+ }
+
+ virtual std::vector<TBlobType> Decode(
+ const std::vector<TBlobType>& blocks,
+ const TPartIndexList& erasedIndices) const override
+ {
+ if (erasedIndices.empty()) {
+ return std::vector<TBlobType>();
+ }
+
+ return ISAErasureDecode<DataPartCount, ParityPartCount, TCodecTraits, TBlobType, TMutableBlobType>(
+ blocks,
+ erasedIndices,
+ TConstArrayRef<TPartIndexList>(),
+ GeneratorMatrix_);
+ }
+
+ virtual ~TReedSolomonIsa() = default;
+
+private:
+ std::vector<unsigned char> GeneratorMatrix_;
+ std::vector<unsigned char> EncodeGFTables_;
+};
+
+} // namespace NErasure
diff --git a/library/cpp/erasure/ya.make b/library/cpp/erasure/ya.make
new file mode 100644
index 0000000000..bde816b4d8
--- /dev/null
+++ b/library/cpp/erasure/ya.make
@@ -0,0 +1,35 @@
+LIBRARY()
+
+SRCS(
+ public.cpp
+ codec.cpp
+ helpers.cpp
+
+ isa_erasure.cpp
+
+ reed_solomon.cpp
+ reed_solomon_isa.cpp
+
+ lrc.cpp
+ lrc_isa.cpp
+)
+
+PEERDIR(
+ contrib/libs/isa-l/erasure_code
+ library/cpp/sse
+ library/cpp/yt/assert
+)
+
+IF (NOT OPENSOURCE)
+ SRCS(
+ jerasure.cpp
+ reed_solomon_jerasure.cpp
+ lrc_jerasure.cpp
+ )
+
+ PEERDIR(contrib/libs/jerasure)
+ENDIF()
+
+GENERATE_ENUM_SERIALIZATION(public.h)
+
+END()
diff --git a/library/cpp/skiff/public.h b/library/cpp/skiff/public.h
new file mode 100644
index 0000000000..d67c6f26ee
--- /dev/null
+++ b/library/cpp/skiff/public.h
@@ -0,0 +1,63 @@
+#pragma once
+
+#include <vector>
+#include <memory>
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+enum class EWireType
+{
+ Nothing /* "nothing" */,
+ Int8 /* "int8" */,
+ Int16 /* "int16" */,
+ Int32 /* "int32" */,
+ Int64 /* "int64" */,
+ Int128 /* "int128" */,
+ Uint8 /* "uint8" */,
+ Uint16 /* "uint16" */,
+ Uint32 /* "uint32" */,
+ Uint64 /* "uint64" */,
+ Uint128 /* "uint128" */,
+ Double /* "double" */,
+ Boolean /* "boolean" */,
+ String32 /* "string32" */,
+ Yson32 /* "yson32" */,
+
+ Tuple /* "tuple" */,
+ Variant8 /* "variant8" */,
+ Variant16 /* "variant16" */,
+ RepeatedVariant8 /* "repeated_variant8" */,
+ RepeatedVariant16 /* "repeated_variant16" */,
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TSkiffSchema;
+using TSkiffSchemaPtr = std::shared_ptr<TSkiffSchema>;
+
+using TSkiffSchemaList = std::vector<TSkiffSchemaPtr>;
+
+class TSimpleTypeSchema;
+using TSimpleTypeSchemaPtr = std::shared_ptr<TSimpleTypeSchema>;
+
+class TSkiffValidator;
+
+class TUncheckedSkiffParser;
+class TCheckedSkiffParser;
+
+class TUncheckedSkiffWriter;
+class TCheckedSkiffWriter;
+
+#ifdef DEBUG
+using TCheckedInDebugSkiffParser = TCheckedSkiffParser;
+using TCheckedInDebugSkiffWriter = TCheckedSkiffWriter;
+#else
+using TCheckedInDebugSkiffParser = TUncheckedSkiffParser;
+using TCheckedInDebugSkiffWriter = TUncheckedSkiffWriter;
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
diff --git a/library/cpp/skiff/skiff-inl.h b/library/cpp/skiff/skiff-inl.h
new file mode 100644
index 0000000000..a3f68a9374
--- /dev/null
+++ b/library/cpp/skiff/skiff-inl.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#ifndef SKIFF_H
+#error "Direct inclusion of this file is not allowed, include skiff.h"
+// For the sake of sane code completion.
+#include "skiff.h"
+#endif
+#undef SKIFF_H
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <EWireType wireType>
+constexpr auto TUnderlyingIntegerType<wireType>::F() {
+ if constexpr (wireType == EWireType::Int8) {
+ return i8{};
+ } else if constexpr (wireType == EWireType::Int16) {
+ return i16{};
+ } else if constexpr (wireType == EWireType::Int32) {
+ return i32{};
+ } else if constexpr (wireType == EWireType::Int64) {
+ return i64{};
+ } else if constexpr (wireType == EWireType::Uint8) {
+ return ui8{};
+ } else if constexpr (wireType == EWireType::Uint16) {
+ return ui16{};
+ } else if constexpr (wireType == EWireType::Uint32) {
+ return ui32{};
+ } else if constexpr (wireType == EWireType::Uint64) {
+ return ui64{};
+ } else {
+ static_assert(wireType == EWireType::Int8, "expected integer wire type");
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
diff --git a/library/cpp/skiff/skiff.cpp b/library/cpp/skiff/skiff.cpp
new file mode 100644
index 0000000000..cbdbdfe364
--- /dev/null
+++ b/library/cpp/skiff/skiff.cpp
@@ -0,0 +1,591 @@
+#include "skiff.h"
+
+#include "skiff_validator.h"
+
+#include <util/stream/buffered.h>
+#include <util/system/byteorder.h>
+#include <util/system/unaligned_mem.h>
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool operator==(TInt128 lhs, TInt128 rhs)
+{
+ return lhs.Low == rhs.Low && lhs.High == rhs.High;
+}
+
+bool operator!=(TInt128 lhs, TInt128 rhs)
+{
+ return !(lhs == rhs);
+}
+
+bool operator==(TUint128 lhs, TUint128 rhs)
+{
+ return lhs.Low == rhs.Low && lhs.High == rhs.High;
+}
+
+bool operator!=(TUint128 lhs, TUint128 rhs)
+{
+ return !(lhs == rhs);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TUncheckedSkiffParser::TUncheckedSkiffParser(IZeroCopyInput* underlying)
+ : Underlying_(underlying)
+ , Buffer_(512 * 1024)
+{ }
+
+TUncheckedSkiffParser::TUncheckedSkiffParser(const std::shared_ptr<TSkiffSchema>& /*schema*/, IZeroCopyInput* underlying)
+ : TUncheckedSkiffParser(underlying)
+{ }
+
+i8 TUncheckedSkiffParser::ParseInt8()
+{
+ return ParseSimple<i8>();
+}
+
+i16 TUncheckedSkiffParser::ParseInt16()
+{
+ return ParseSimple<i16>();
+}
+
+i32 TUncheckedSkiffParser::ParseInt32()
+{
+ return ParseSimple<i32>();
+}
+
+i64 TUncheckedSkiffParser::ParseInt64()
+{
+ return ParseSimple<i64>();
+}
+
+ui8 TUncheckedSkiffParser::ParseUint8()
+{
+ return ParseSimple<ui8>();
+}
+
+ui16 TUncheckedSkiffParser::ParseUint16()
+{
+ return ParseSimple<ui16>();
+}
+
+ui32 TUncheckedSkiffParser::ParseUint32()
+{
+ return ParseSimple<ui32>();
+}
+
+ui64 TUncheckedSkiffParser::ParseUint64()
+{
+ return ParseSimple<ui64>();
+}
+
+TInt128 TUncheckedSkiffParser::ParseInt128()
+{
+ auto low = ParseSimple<ui64>();
+ auto high = ParseSimple<i64>();
+ return {low, high};
+}
+
+TUint128 TUncheckedSkiffParser::ParseUint128()
+{
+ auto low = ParseSimple<ui64>();
+ auto high = ParseSimple<ui64>();
+ return {low, high};
+}
+
+double TUncheckedSkiffParser::ParseDouble()
+{
+ return ParseSimple<double>();
+}
+
+bool TUncheckedSkiffParser::ParseBoolean()
+{
+ ui8 result = ParseSimple<ui8>();
+ if (result > 1) {
+ ythrow TSkiffException() << "Invalid boolean value \"" << result << "\"";
+ }
+ return result;
+}
+
+TStringBuf TUncheckedSkiffParser::ParseString32()
+{
+ ui32 len = ParseSimple<ui32>();
+ const void* data = GetData(len);
+ return TStringBuf(static_cast<const char*>(data), len);
+}
+
+TStringBuf TUncheckedSkiffParser::ParseYson32()
+{
+ return ParseString32();
+}
+
+ui8 TUncheckedSkiffParser::ParseVariant8Tag()
+{
+ return ParseSimple<ui8>();
+}
+
+ui16 TUncheckedSkiffParser::ParseVariant16Tag()
+{
+ return ParseSimple<ui16>();
+}
+
+template <typename T>
+T TUncheckedSkiffParser::ParseSimple()
+{
+ return ReadUnaligned<T>(GetData(sizeof(T)));
+}
+
+const void* TUncheckedSkiffParser::GetData(size_t size)
+{
+ if (RemainingBytes() >= size) {
+ const void* result = Position_;
+ Advance(size);
+ return result;
+ }
+
+ return GetDataViaBuffer(size);
+}
+
+const void* TUncheckedSkiffParser::GetDataViaBuffer(size_t size)
+{
+ Buffer_.Clear();
+ Buffer_.Reserve(size);
+ while (Buffer_.Size() < size) {
+ size_t toCopy = Min(size - Buffer_.Size(), RemainingBytes());
+ Buffer_.Append(Position_, toCopy);
+ Advance(toCopy);
+
+ if (RemainingBytes() == 0) {
+ RefillBuffer();
+ if (Exhausted_ && Buffer_.Size() < size) {
+ ythrow TSkiffException() << "Premature end of stream while parsing Skiff";
+ }
+ }
+ }
+ return Buffer_.Data();
+}
+
+size_t TUncheckedSkiffParser::RemainingBytes() const
+{
+ Y_ASSERT(End_ >= Position_);
+ return End_ - Position_;
+}
+
+void TUncheckedSkiffParser::Advance(size_t size)
+{
+ Y_ASSERT(size <= RemainingBytes());
+ Position_ += size;
+ ReadBytesCount_ += size;
+}
+
+void TUncheckedSkiffParser::RefillBuffer()
+{
+ size_t bufferSize = Underlying_->Next(&Position_);
+ End_ = Position_ + bufferSize;
+ if (bufferSize == 0) {
+ Exhausted_ = true;
+ }
+}
+
+bool TUncheckedSkiffParser::HasMoreData()
+{
+ if (RemainingBytes() == 0 && !Exhausted_) {
+ RefillBuffer();
+ }
+ return !(RemainingBytes() == 0 && Exhausted_);
+}
+
+void TUncheckedSkiffParser::ValidateFinished()
+{ }
+
+ui64 TUncheckedSkiffParser::GetReadBytesCount() const
+{
+ return ReadBytesCount_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TCheckedSkiffParser::TCheckedSkiffParser(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyInput* stream)
+ : Parser_(stream)
+ , Validator_(std::make_unique<TSkiffValidator>(schema))
+{ }
+
+TCheckedSkiffParser::~TCheckedSkiffParser() = default;
+
+i8 TCheckedSkiffParser::ParseInt8()
+{
+ Validator_->OnSimpleType(EWireType::Int8);
+ return Parser_.ParseInt8();
+}
+
+i16 TCheckedSkiffParser::ParseInt16()
+{
+ Validator_->OnSimpleType(EWireType::Int16);
+ return Parser_.ParseInt16();
+}
+
+i32 TCheckedSkiffParser::ParseInt32()
+{
+ Validator_->OnSimpleType(EWireType::Int32);
+ return Parser_.ParseInt32();
+}
+
+i64 TCheckedSkiffParser::ParseInt64()
+{
+ Validator_->OnSimpleType(EWireType::Int64);
+ return Parser_.ParseInt64();
+}
+
+ui8 TCheckedSkiffParser::ParseUint8()
+{
+ Validator_->OnSimpleType(EWireType::Uint8);
+ return Parser_.ParseUint8();
+}
+
+ui16 TCheckedSkiffParser::ParseUint16()
+{
+ Validator_->OnSimpleType(EWireType::Uint16);
+ return Parser_.ParseUint16();
+}
+
+ui32 TCheckedSkiffParser::ParseUint32()
+{
+ Validator_->OnSimpleType(EWireType::Uint32);
+ return Parser_.ParseUint32();
+}
+
+ui64 TCheckedSkiffParser::ParseUint64()
+{
+ Validator_->OnSimpleType(EWireType::Uint64);
+ return Parser_.ParseUint64();
+}
+
+TInt128 TCheckedSkiffParser::ParseInt128()
+{
+ Validator_->OnSimpleType(EWireType::Int128);
+ return Parser_.ParseInt128();
+}
+
+TUint128 TCheckedSkiffParser::ParseUint128()
+{
+ Validator_->OnSimpleType(EWireType::Uint128);
+ return Parser_.ParseUint128();
+}
+
+double TCheckedSkiffParser::ParseDouble()
+{
+ Validator_->OnSimpleType(EWireType::Double);
+ return Parser_.ParseDouble();
+}
+
+bool TCheckedSkiffParser::ParseBoolean()
+{
+ Validator_->OnSimpleType(EWireType::Boolean);
+ return Parser_.ParseBoolean();
+}
+
+TStringBuf TCheckedSkiffParser::ParseString32()
+{
+ Validator_->OnSimpleType(EWireType::String32);
+ return Parser_.ParseString32();
+}
+
+TStringBuf TCheckedSkiffParser::ParseYson32()
+{
+ Validator_->OnSimpleType(EWireType::Yson32);
+ return Parser_.ParseYson32();
+}
+
+ui8 TCheckedSkiffParser::ParseVariant8Tag()
+{
+ Validator_->BeforeVariant8Tag();
+ auto result = Parser_.ParseVariant8Tag();
+ Validator_->OnVariant8Tag(result);
+ return result;
+}
+
+ui16 TCheckedSkiffParser::ParseVariant16Tag()
+{
+ Validator_->BeforeVariant16Tag();
+ auto result = Parser_.ParseVariant16Tag();
+ Validator_->OnVariant16Tag(result);
+ return result;
+}
+
+bool TCheckedSkiffParser::HasMoreData()
+{
+ return Parser_.HasMoreData();
+}
+
+void TCheckedSkiffParser::ValidateFinished()
+{
+ Validator_->ValidateFinished();
+ Parser_.ValidateFinished();
+}
+
+ui64 TCheckedSkiffParser::GetReadBytesCount() const
+{
+ return Parser_.GetReadBytesCount();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TUncheckedSkiffWriter::TUncheckedSkiffWriter(IZeroCopyOutput* underlying)
+ : Underlying_(underlying)
+{ }
+
+TUncheckedSkiffWriter::TUncheckedSkiffWriter(IOutputStream* underlying)
+ : BufferedOutput_(MakeHolder<TBufferedOutput>(underlying))
+ , Underlying_(BufferedOutput_.Get())
+{ }
+
+TUncheckedSkiffWriter::TUncheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& /*schema*/, IZeroCopyOutput* underlying)
+ : TUncheckedSkiffWriter(underlying)
+{ }
+
+TUncheckedSkiffWriter::TUncheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& /*schema*/, IOutputStream* underlying)
+ : TUncheckedSkiffWriter(underlying)
+{ }
+
+TUncheckedSkiffWriter::~TUncheckedSkiffWriter()
+{
+ try {
+ Flush();
+ } catch (...) {
+ }
+}
+
+void TUncheckedSkiffWriter::WriteInt8(i8 value)
+{
+ WriteSimple<i8>(value);
+}
+
+void TUncheckedSkiffWriter::WriteInt16(i16 value)
+{
+ WriteSimple<i16>(value);
+}
+
+void TUncheckedSkiffWriter::WriteInt32(i32 value)
+{
+ WriteSimple<i32>(value);
+}
+
+void TUncheckedSkiffWriter::WriteInt64(i64 value)
+{
+ WriteSimple<i64>(value);
+}
+
+void TUncheckedSkiffWriter::WriteInt128(TInt128 value)
+{
+ WriteSimple<ui64>(value.Low);
+ WriteSimple<i64>(value.High);
+}
+
+void TUncheckedSkiffWriter::WriteUint128(TUint128 value)
+{
+ WriteSimple<ui64>(value.Low);
+ WriteSimple<ui64>(value.High);
+}
+
+void TUncheckedSkiffWriter::WriteUint8(ui8 value)
+{
+ WriteSimple<ui8>(value);
+}
+
+void TUncheckedSkiffWriter::WriteUint16(ui16 value)
+{
+ WriteSimple<ui16>(value);
+}
+
+void TUncheckedSkiffWriter::WriteUint32(ui32 value)
+{
+ WriteSimple<ui32>(value);
+}
+
+void TUncheckedSkiffWriter::WriteUint64(ui64 value)
+{
+ WriteSimple<ui64>(value);
+}
+
+void TUncheckedSkiffWriter::WriteDouble(double value)
+{
+ return WriteSimple<double>(value);
+}
+
+void TUncheckedSkiffWriter::WriteBoolean(bool value)
+{
+ return WriteSimple<ui8>(value ? 1 : 0);
+}
+
+void TUncheckedSkiffWriter::WriteString32(TStringBuf value)
+{
+ WriteSimple<ui32>(value.size());
+ Underlying_.Write(value.data(), value.size());
+}
+
+void TUncheckedSkiffWriter::WriteYson32(TStringBuf value)
+{
+ WriteSimple<ui32>(value.size());
+ Underlying_.Write(value.data(), value.size());
+}
+
+void TUncheckedSkiffWriter::WriteVariant8Tag(ui8 tag)
+{
+ WriteSimple<ui8>(tag);
+}
+
+void TUncheckedSkiffWriter::WriteVariant16Tag(ui16 tag)
+{
+ WriteSimple<ui16>(tag);
+}
+
+void TUncheckedSkiffWriter::Flush()
+{
+ Underlying_.UndoRemaining();
+ if (BufferedOutput_) {
+ BufferedOutput_->Flush();
+ }
+}
+
+template <typename T>
+Y_FORCE_INLINE void TUncheckedSkiffWriter::WriteSimple(T value)
+{
+ if constexpr (std::is_integral_v<T>) {
+ value = HostToLittle(value);
+ Underlying_.Write(&value, sizeof(T));
+ } else {
+ Underlying_.Write(&value, sizeof(T));
+ }
+}
+
+void TUncheckedSkiffWriter::Finish()
+{
+ Flush();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TCheckedSkiffWriter::TCheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyOutput* underlying)
+ : Writer_(underlying)
+ , Validator_(std::make_unique<TSkiffValidator>(schema))
+{ }
+
+TCheckedSkiffWriter::TCheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IOutputStream* underlying)
+ : Writer_(underlying)
+ , Validator_(std::make_unique<TSkiffValidator>(schema))
+{ }
+
+TCheckedSkiffWriter::~TCheckedSkiffWriter() = default;
+
+void TCheckedSkiffWriter::WriteDouble(double value)
+{
+ Validator_->OnSimpleType(EWireType::Double);
+ Writer_.WriteDouble(value);
+}
+
+void TCheckedSkiffWriter::WriteBoolean(bool value)
+{
+ Validator_->OnSimpleType(EWireType::Boolean);
+ Writer_.WriteBoolean(value);
+}
+
+void TCheckedSkiffWriter::WriteInt8(i8 value)
+{
+ Validator_->OnSimpleType(EWireType::Int8);
+ Writer_.WriteInt8(value);
+}
+
+void TCheckedSkiffWriter::WriteInt16(i16 value)
+{
+ Validator_->OnSimpleType(EWireType::Int16);
+ Writer_.WriteInt16(value);
+}
+
+void TCheckedSkiffWriter::WriteInt32(i32 value)
+{
+ Validator_->OnSimpleType(EWireType::Int32);
+ Writer_.WriteInt32(value);
+}
+
+void TCheckedSkiffWriter::WriteInt64(i64 value)
+{
+ Validator_->OnSimpleType(EWireType::Int64);
+ Writer_.WriteInt64(value);
+}
+
+void TCheckedSkiffWriter::WriteUint8(ui8 value)
+{
+ Validator_->OnSimpleType(EWireType::Uint8);
+ Writer_.WriteUint8(value);
+}
+
+void TCheckedSkiffWriter::WriteUint16(ui16 value)
+{
+ Validator_->OnSimpleType(EWireType::Uint16);
+ Writer_.WriteUint16(value);
+}
+
+void TCheckedSkiffWriter::WriteUint32(ui32 value)
+{
+ Validator_->OnSimpleType(EWireType::Uint32);
+ Writer_.WriteUint32(value);
+}
+
+void TCheckedSkiffWriter::WriteUint64(ui64 value)
+{
+ Validator_->OnSimpleType(EWireType::Uint64);
+ Writer_.WriteUint64(value);
+}
+
+void TCheckedSkiffWriter::WriteInt128(TInt128 value)
+{
+ Validator_->OnSimpleType(EWireType::Int128);
+ Writer_.WriteInt128(value);
+}
+
+void TCheckedSkiffWriter::WriteUint128(TUint128 value)
+{
+ Validator_->OnSimpleType(EWireType::Uint128);
+ Writer_.WriteUint128(value);
+}
+
+void TCheckedSkiffWriter::WriteString32(TStringBuf value)
+{
+ Validator_->OnSimpleType(EWireType::String32);
+ Writer_.WriteString32(value);
+}
+
+void TCheckedSkiffWriter::WriteYson32(TStringBuf value)
+{
+ Validator_->OnSimpleType(EWireType::Yson32);
+ Writer_.WriteYson32(value);
+}
+
+void TCheckedSkiffWriter::WriteVariant8Tag(ui8 tag)
+{
+ Validator_->OnVariant8Tag(tag);
+ Writer_.WriteVariant8Tag(tag);
+}
+
+void TCheckedSkiffWriter::WriteVariant16Tag(ui16 tag)
+{
+ Validator_->OnVariant16Tag(tag);
+ Writer_.WriteVariant16Tag(tag);
+}
+
+void TCheckedSkiffWriter::Flush()
+{
+ Writer_.Flush();
+}
+
+void TCheckedSkiffWriter::Finish()
+{
+ Validator_->ValidateFinished();
+ Writer_.Finish();
+}
+
+////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
diff --git a/library/cpp/skiff/skiff.h b/library/cpp/skiff/skiff.h
new file mode 100644
index 0000000000..183c112700
--- /dev/null
+++ b/library/cpp/skiff/skiff.h
@@ -0,0 +1,259 @@
+#pragma once
+
+#include "public.h"
+
+#include "zerocopy_output_writer.h"
+
+#include <util/generic/buffer.h>
+#include <util/generic/yexception.h>
+
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TSkiffException
+ : public yexception
+{ };
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <typename T>
+constexpr T EndOfSequenceTag()
+{
+ static_assert(std::is_integral<T>::value && std::is_unsigned<T>::value, "T must be unsigned integer");
+ return T(-1);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TInt128
+{
+ ui64 Low = 0;
+ i64 High = 0;
+};
+
+struct TUint128
+{
+ ui64 Low = 0;
+ ui64 High = 0;
+};
+
+bool operator==(TInt128 lhs, TInt128 rhs);
+bool operator!=(TInt128 lhs, TInt128 rhs);
+
+bool operator==(TUint128 lhs, TUint128 rhs);
+bool operator!=(TUint128 lhs, TUint128 rhs);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TUncheckedSkiffParser
+{
+public:
+ explicit TUncheckedSkiffParser(IZeroCopyInput* stream);
+ TUncheckedSkiffParser(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyInput* stream);
+
+ i8 ParseInt8();
+ i16 ParseInt16();
+ i32 ParseInt32();
+ i64 ParseInt64();
+
+ ui8 ParseUint8();
+ ui16 ParseUint16();
+ ui32 ParseUint32();
+ ui64 ParseUint64();
+
+ TInt128 ParseInt128();
+ TUint128 ParseUint128();
+
+ double ParseDouble();
+
+ bool ParseBoolean();
+
+ TStringBuf ParseString32();
+
+ TStringBuf ParseYson32();
+
+ ui8 ParseVariant8Tag();
+ ui16 ParseVariant16Tag();
+
+ bool HasMoreData();
+
+ void ValidateFinished();
+
+ ui64 GetReadBytesCount() const;
+
+private:
+ const void* GetData(size_t size);
+ const void* GetDataViaBuffer(size_t size);
+
+ size_t RemainingBytes() const;
+ void Advance(size_t size);
+ void RefillBuffer();
+
+ template <typename T>
+ T ParseSimple();
+
+private:
+ IZeroCopyInput* const Underlying_;
+
+ TBuffer Buffer_;
+ ui64 ReadBytesCount_ = 0;
+ char* Position_ = nullptr;
+ char* End_ = nullptr;
+ bool Exhausted_ = false;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCheckedSkiffParser
+{
+public:
+ TCheckedSkiffParser(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyInput* stream);
+ ~TCheckedSkiffParser();
+
+ i8 ParseInt8();
+ i16 ParseInt16();
+ i32 ParseInt32();
+ i64 ParseInt64();
+
+ ui8 ParseUint8();
+ ui16 ParseUint16();
+ ui32 ParseUint32();
+ ui64 ParseUint64();
+
+ TInt128 ParseInt128();
+ TUint128 ParseUint128();
+
+ double ParseDouble();
+
+ bool ParseBoolean();
+
+ TStringBuf ParseString32();
+
+ TStringBuf ParseYson32();
+
+ ui8 ParseVariant8Tag();
+ ui16 ParseVariant16Tag();
+
+ bool HasMoreData();
+
+ void ValidateFinished();
+
+ ui64 GetReadBytesCount() const;
+
+private:
+ TUncheckedSkiffParser Parser_;
+ std::unique_ptr<TSkiffValidator> Validator_;
+};
+
+////////////////////////////////////////////////////////////////////
+
+class TUncheckedSkiffWriter
+{
+public:
+ explicit TUncheckedSkiffWriter(IZeroCopyOutput* underlying);
+ explicit TUncheckedSkiffWriter(IOutputStream* underlying);
+ TUncheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyOutput* underlying);
+ TUncheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IOutputStream* underlying);
+
+ ~TUncheckedSkiffWriter();
+
+ void WriteDouble(double value);
+ void WriteBoolean(bool value);
+
+ void WriteInt8(i8 value);
+ void WriteInt16(i16 value);
+ void WriteInt32(i32 value);
+ void WriteInt64(i64 value);
+
+ void WriteUint8(ui8 value);
+ void WriteUint16(ui16 value);
+ void WriteUint32(ui32 value);
+ void WriteUint64(ui64 value);
+
+ void WriteInt128(TInt128 value);
+ void WriteUint128(TUint128 value);
+
+ void WriteString32(TStringBuf value);
+
+ void WriteYson32(TStringBuf value);
+
+ void WriteVariant8Tag(ui8 tag);
+ void WriteVariant16Tag(ui16 tag);
+
+ void Flush();
+ void Finish();
+
+private:
+
+ template <typename T>
+ void WriteSimple(T data);
+
+private:
+ THolder<TBufferedOutput> BufferedOutput_;
+ TZeroCopyOutputStreamWriter Underlying_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCheckedSkiffWriter
+{
+public:
+ TCheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IZeroCopyOutput* underlying);
+ TCheckedSkiffWriter(const std::shared_ptr<TSkiffSchema>& schema, IOutputStream* underlying);
+
+ ~TCheckedSkiffWriter();
+
+ void WriteInt8(i8 value);
+ void WriteInt16(i16 value);
+ void WriteInt32(i32 value);
+ void WriteInt64(i64 value);
+
+ void WriteUint8(ui8 value);
+ void WriteUint16(ui16 value);
+ void WriteUint32(ui32 value);
+ void WriteUint64(ui64 value);
+
+ void WriteDouble(double value);
+ void WriteBoolean(bool value);
+
+ void WriteInt128(TInt128 value);
+ void WriteUint128(TUint128 value);
+
+ void WriteString32(TStringBuf value);
+
+ void WriteYson32(TStringBuf value);
+
+ void WriteVariant8Tag(ui8 tag);
+ void WriteVariant16Tag(ui16 tag);
+
+ void Flush();
+ void Finish();
+
+private:
+ TUncheckedSkiffWriter Writer_;
+ std::unique_ptr<TSkiffValidator> Validator_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <EWireType wireType>
+class TUnderlyingIntegerType {
+private:
+ TUnderlyingIntegerType() = default;
+ static constexpr auto F();
+
+public:
+ using TValue = decltype(TUnderlyingIntegerType::F());
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
+
+#define SKIFF_H
+#include "skiff-inl.h"
+#undef SKIFF_H
diff --git a/library/cpp/skiff/skiff_schema-inl.h b/library/cpp/skiff/skiff_schema-inl.h
new file mode 100644
index 0000000000..d66325b222
--- /dev/null
+++ b/library/cpp/skiff/skiff_schema-inl.h
@@ -0,0 +1,61 @@
+#pragma once
+
+#ifndef SKIFF_SCHEMA_H
+#error "Direct inclusion of this file is not allowed, include skiff_schema.h"
+// For the sake of sane code completion.
+#include "skiff_schema.h"
+#endif
+#undef SKIFF_SCHEMA_H
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline bool IsSimpleType(EWireType type)
+{
+ switch (type) {
+ case EWireType::Int8:
+ case EWireType::Int16:
+ case EWireType::Int32:
+ case EWireType::Int64:
+ case EWireType::Int128:
+
+ case EWireType::Uint8:
+ case EWireType::Uint16:
+ case EWireType::Uint32:
+ case EWireType::Uint64:
+ case EWireType::Uint128:
+
+ case EWireType::Double:
+ case EWireType::Boolean:
+ case EWireType::String32:
+ case EWireType::Yson32:
+ case EWireType::Nothing:
+ return true;
+ case EWireType::Tuple:
+ case EWireType::Variant8:
+ case EWireType::Variant16:
+ case EWireType::RepeatedVariant8:
+ case EWireType::RepeatedVariant16:
+ return false;
+ }
+ Y_FAIL();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <EWireType WireType>
+TComplexSchema<WireType>::TComplexSchema(TSkiffSchemaList elements)
+ : TSkiffSchema(WireType)
+ , Elements_(std::move(elements))
+{ }
+
+template <EWireType WireType>
+const TSkiffSchemaList& TComplexSchema<WireType>::GetChildren() const
+{
+ return Elements_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
diff --git a/library/cpp/skiff/skiff_schema.cpp b/library/cpp/skiff/skiff_schema.cpp
new file mode 100644
index 0000000000..c762896ad0
--- /dev/null
+++ b/library/cpp/skiff/skiff_schema.cpp
@@ -0,0 +1,164 @@
+#include "skiff_schema.h"
+
+#include "skiff.h"
+
+#include <util/generic/hash.h>
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool operator==(const TSkiffSchema& lhs, const TSkiffSchema& rhs)
+{
+ if (lhs.GetWireType() != rhs.GetWireType() || lhs.GetName() != rhs.GetName()) {
+ return false;
+ }
+ const auto& lhsChildren = lhs.GetChildren();
+ const auto& rhsChildren = rhs.GetChildren();
+ return std::equal(
+ std::begin(lhsChildren),
+ std::end(lhsChildren),
+ std::begin(rhsChildren),
+ std::end(rhsChildren),
+ TSkiffSchemaPtrEqual());
+}
+
+bool operator!=(const TSkiffSchema& lhs, const TSkiffSchema& rhs)
+{
+ return !(lhs == rhs);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void PrintShortDebugString(const std::shared_ptr<const TSkiffSchema>& schema, IOutputStream* out)
+{
+ (*out) << ToString(schema->GetWireType());
+ if (!IsSimpleType(schema->GetWireType())) {
+ auto children = schema->GetChildren();
+ if (!children.empty()) {
+ (*out) << '<';
+ for (const auto& child : children) {
+ PrintShortDebugString(child, out);
+ (*out) << ';';
+ }
+ (*out) << '>';
+ }
+ }
+}
+
+TString GetShortDebugString(const std::shared_ptr<const TSkiffSchema>& schema)
+{
+ TStringStream out;
+ PrintShortDebugString(schema, &out);
+ return out.Str();
+}
+
+std::shared_ptr<TSimpleTypeSchema> CreateSimpleTypeSchema(EWireType type)
+{
+ return std::make_shared<TSimpleTypeSchema>(type);
+}
+
+static void VerifyNonemptyChildren(const TSkiffSchemaList& children, EWireType wireType)
+{
+ if (children.empty()) {
+ ythrow TSkiffException() << "\"" << ToString(wireType) << "\" must have at least one child";
+ }
+}
+
+std::shared_ptr<TTupleSchema> CreateTupleSchema(TSkiffSchemaList children)
+{
+ return std::make_shared<TTupleSchema>(std::move(children));
+}
+
+std::shared_ptr<TVariant8Schema> CreateVariant8Schema(TSkiffSchemaList children)
+{
+ VerifyNonemptyChildren(children, EWireType::Variant8);
+ return std::make_shared<TVariant8Schema>(std::move(children));
+}
+
+std::shared_ptr<TVariant16Schema> CreateVariant16Schema(TSkiffSchemaList children)
+{
+ VerifyNonemptyChildren(children, EWireType::Variant16);
+ return std::make_shared<TVariant16Schema>(std::move(children));
+}
+
+std::shared_ptr<TRepeatedVariant8Schema> CreateRepeatedVariant8Schema(TSkiffSchemaList children)
+{
+ VerifyNonemptyChildren(children, EWireType::RepeatedVariant8);
+ return std::make_shared<TRepeatedVariant8Schema>(std::move(children));
+}
+
+std::shared_ptr<TRepeatedVariant16Schema> CreateRepeatedVariant16Schema(TSkiffSchemaList children)
+{
+ VerifyNonemptyChildren(children, EWireType::RepeatedVariant16);
+ return std::make_shared<TRepeatedVariant16Schema>(std::move(children));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TSkiffSchema::TSkiffSchema(EWireType type)
+ : Type_(type)
+{ }
+
+EWireType TSkiffSchema::GetWireType() const
+{
+ return Type_;
+}
+
+std::shared_ptr<TSkiffSchema> TSkiffSchema::SetName(TString name)
+{
+ Name_ = std::move(name);
+ return shared_from_this();
+}
+
+const TString& TSkiffSchema::GetName() const
+{
+ return Name_;
+}
+
+const TSkiffSchemaList& TSkiffSchema::GetChildren() const
+{
+ static const TSkiffSchemaList children;
+ return children;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TSimpleTypeSchema::TSimpleTypeSchema(EWireType type)
+ : TSkiffSchema(type)
+{
+ Y_VERIFY(IsSimpleType(type));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+size_t TSkiffSchemaPtrHasher::operator()(const std::shared_ptr<TSkiffSchema>& schema) const
+{
+ return THash<NSkiff::TSkiffSchema>()(*schema);
+}
+
+size_t TSkiffSchemaPtrEqual::operator()(
+ const std::shared_ptr<TSkiffSchema>& lhs,
+ const std::shared_ptr<TSkiffSchema>& rhs) const
+{
+ return *lhs == *rhs;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
+
+////////////////////////////////////////////////////////////////////////////////
+
+size_t THash<NSkiff::TSkiffSchema>::operator()(const NSkiff::TSkiffSchema &schema) const
+{
+ auto hash = CombineHashes(
+ THash<TString>()(schema.GetName()),
+ static_cast<size_t>(schema.GetWireType()));
+ for (const auto& child : schema.GetChildren()) {
+ hash = CombineHashes(hash, (*this)(*child));
+ }
+ return hash;
+}
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/library/cpp/skiff/skiff_schema.h b/library/cpp/skiff/skiff_schema.h
new file mode 100644
index 0000000000..8952a84bac
--- /dev/null
+++ b/library/cpp/skiff/skiff_schema.h
@@ -0,0 +1,121 @@
+#pragma once
+
+#include "public.h"
+
+#include <util/generic/string.h>
+#include <util/string/cast.h>
+
+#include <vector>
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <EWireType WireType>
+class TComplexSchema;
+
+using TTupleSchema = TComplexSchema<EWireType::Tuple>;
+using TVariant8Schema = TComplexSchema<EWireType::Variant8>;
+using TVariant16Schema = TComplexSchema<EWireType::Variant16>;
+using TRepeatedVariant8Schema = TComplexSchema<EWireType::RepeatedVariant8>;
+using TRepeatedVariant16Schema = TComplexSchema<EWireType::RepeatedVariant16>;
+
+using TTupleSchemaPtr = std::shared_ptr<TTupleSchema>;
+using TVariant8SchemaPtr = std::shared_ptr<TVariant8Schema>;
+using TVariant16SchemaPtr = std::shared_ptr<TVariant16Schema>;
+using TRepeatedVariant8SchemaPtr = std::shared_ptr<TRepeatedVariant8Schema>;
+using TRepeatedVariant16SchemaPtr = std::shared_ptr<TRepeatedVariant16Schema>;
+
+
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TSkiffSchema
+ : public std::enable_shared_from_this<TSkiffSchema>
+{
+public:
+ virtual ~TSkiffSchema() = default;
+
+ EWireType GetWireType() const;
+ std::shared_ptr<TSkiffSchema> SetName(TString name);
+ const TString& GetName() const;
+
+ virtual const TSkiffSchemaList& GetChildren() const;
+
+protected:
+ explicit TSkiffSchema(EWireType type);
+
+private:
+ const EWireType Type_;
+ TString Name_;
+};
+
+bool operator==(const TSkiffSchema& lhs, const TSkiffSchema& rhs);
+bool operator!=(const TSkiffSchema& lhs, const TSkiffSchema& rhs);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TSimpleTypeSchema
+ : public TSkiffSchema
+{
+public:
+ explicit TSimpleTypeSchema(EWireType type);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <EWireType WireType>
+class TComplexSchema
+ : public TSkiffSchema
+{
+public:
+ explicit TComplexSchema(TSkiffSchemaList elements);
+
+ virtual const TSkiffSchemaList& GetChildren() const override;
+
+private:
+ const TSkiffSchemaList Elements_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool IsSimpleType(EWireType type);
+TString GetShortDebugString(const std::shared_ptr<const TSkiffSchema>& schema);
+void PrintShortDebugString(const std::shared_ptr<const TSkiffSchema>& schema, IOutputStream* out);
+
+std::shared_ptr<TSimpleTypeSchema> CreateSimpleTypeSchema(EWireType type);
+std::shared_ptr<TTupleSchema> CreateTupleSchema(TSkiffSchemaList children);
+std::shared_ptr<TVariant8Schema> CreateVariant8Schema(TSkiffSchemaList children);
+std::shared_ptr<TVariant16Schema> CreateVariant16Schema(TSkiffSchemaList children);
+std::shared_ptr<TRepeatedVariant8Schema> CreateRepeatedVariant8Schema(TSkiffSchemaList children);
+std::shared_ptr<TRepeatedVariant16Schema> CreateRepeatedVariant16Schema(TSkiffSchemaList children);
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TSkiffSchemaPtrHasher
+{
+ size_t operator()(const std::shared_ptr<TSkiffSchema>& schema) const;
+};
+
+struct TSkiffSchemaPtrEqual
+{
+ size_t operator()(
+ const std::shared_ptr<TSkiffSchema>& lhs,
+ const std::shared_ptr<TSkiffSchema>& rhs) const;
+};
+
+} // namespace NSkiff
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <>
+struct THash<NSkiff::TSkiffSchema>
+{
+ size_t operator()(const NSkiff::TSkiffSchema& schema) const;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+#define SKIFF_SCHEMA_H
+#include "skiff_schema-inl.h"
+#undef SKIFF_SCHEMA_H
diff --git a/library/cpp/skiff/skiff_validator.cpp b/library/cpp/skiff/skiff_validator.cpp
new file mode 100644
index 0000000000..1b1b98d5a6
--- /dev/null
+++ b/library/cpp/skiff/skiff_validator.cpp
@@ -0,0 +1,396 @@
+#include "skiff.h"
+#include "skiff_validator.h"
+
+#include <vector>
+#include <stack>
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct IValidatorNode;
+
+using TValidatorNodeList = std::vector<std::shared_ptr<IValidatorNode>>;
+using TSkiffSchemaList = std::vector<std::shared_ptr<TSkiffSchema>>;
+
+static std::shared_ptr<IValidatorNode> CreateUsageValidatorNode(const std::shared_ptr<TSkiffSchema>& skiffSchema);
+static TValidatorNodeList CreateUsageValidatorNodeList(const TSkiffSchemaList& skiffSchemaList);
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <typename T>
+inline void ThrowUnexpectedParseWrite(T wireType)
+{
+ ythrow TSkiffException() << "Unexpected parse/write of \"" << ::ToString(wireType) << "\" token";
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct IValidatorNode
+{
+ virtual ~IValidatorNode() = default;
+
+ virtual void OnBegin(TValidatorNodeStack* /*validatorNodeStack*/)
+ { }
+
+ virtual void OnChildDone(TValidatorNodeStack* /*validatorNodeStack*/)
+ {
+ Y_FAIL();
+ }
+
+ virtual void OnSimpleType(TValidatorNodeStack* /*validatorNodeStack*/, EWireType wireType)
+ {
+ ThrowUnexpectedParseWrite(wireType);
+ }
+
+ virtual void BeforeVariant8Tag()
+ {
+ ThrowUnexpectedParseWrite(EWireType::Variant8);
+ }
+
+ virtual void OnVariant8Tag(TValidatorNodeStack* /*validatorNodeStack*/, ui8 /*tag*/)
+ {
+ IValidatorNode::BeforeVariant8Tag();
+ }
+
+ virtual void BeforeVariant16Tag()
+ {
+ ThrowUnexpectedParseWrite(EWireType::Variant16);
+ }
+
+ virtual void OnVariant16Tag(TValidatorNodeStack* /*validatorNodeStack*/, ui16 /*tag*/)
+ {
+ IValidatorNode::BeforeVariant16Tag();
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TValidatorNodeStack
+{
+public:
+ explicit TValidatorNodeStack(std::shared_ptr<IValidatorNode> validator)
+ : RootValidator_(std::move(validator))
+ { }
+
+ void PushValidator(IValidatorNode* validator)
+ {
+ ValidatorStack_.push(validator);
+ validator->OnBegin(this);
+ }
+
+ void PopValidator()
+ {
+ Y_VERIFY(!ValidatorStack_.empty());
+ ValidatorStack_.pop();
+ if (!ValidatorStack_.empty()) {
+ ValidatorStack_.top()->OnChildDone(this);
+ }
+ }
+
+ void PushRootIfRequired()
+ {
+ if (ValidatorStack_.empty()) {
+ PushValidator(RootValidator_.get());
+ }
+ }
+
+ IValidatorNode* Top() const
+ {
+ Y_VERIFY(!ValidatorStack_.empty());
+ return ValidatorStack_.top();
+ }
+
+ bool IsFinished() const
+ {
+ return ValidatorStack_.empty();
+ }
+
+private:
+ const std::shared_ptr<IValidatorNode> RootValidator_;
+ std::stack<IValidatorNode*> ValidatorStack_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TNothingTypeValidator
+ : public IValidatorNode
+{
+public:
+ void OnBegin(TValidatorNodeStack* validatorNodeStack) override
+ {
+ validatorNodeStack->PopValidator();
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TSimpleTypeUsageValidator
+ : public IValidatorNode
+{
+public:
+ explicit TSimpleTypeUsageValidator(EWireType type)
+ : Type_(type)
+ { }
+
+ void OnSimpleType(TValidatorNodeStack* validatorNodeStack, EWireType type) override
+ {
+ if (type != Type_) {
+ ThrowUnexpectedParseWrite(type);
+ }
+ validatorNodeStack->PopValidator();
+ }
+
+private:
+ const EWireType Type_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <typename TTag>
+void ValidateVariantTag(TValidatorNodeStack* validatorNodeStack, TTag tag, const TValidatorNodeList& children)
+{
+ if (tag == EndOfSequenceTag<TTag>()) {
+ // Root validator is pushed into the stack before variant tag
+ // if the stack is empty.
+ validatorNodeStack->PopValidator();
+ } else if (tag >= children.size()) {
+ ythrow TSkiffException() << "Variant tag \"" << tag << "\" "
+ << "exceeds number of children \"" << children.size();
+ } else {
+ validatorNodeStack->PushValidator(children[tag].get());
+ }
+}
+
+class TVariant8TypeUsageValidator
+ : public IValidatorNode
+{
+public:
+ explicit TVariant8TypeUsageValidator(TValidatorNodeList children)
+ : Children_(std::move(children))
+ { }
+
+ void BeforeVariant8Tag() override
+ { }
+
+ void OnVariant8Tag(TValidatorNodeStack* validatorNodeStack, ui8 tag) override
+ {
+ ValidateVariantTag(validatorNodeStack, tag, Children_);
+ }
+
+ void OnChildDone(TValidatorNodeStack* validatorNodeStack) override
+ {
+ validatorNodeStack->PopValidator();
+ }
+
+private:
+ const TValidatorNodeList Children_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TVariant16TypeUsageValidator
+ : public IValidatorNode
+{
+public:
+ explicit TVariant16TypeUsageValidator(TValidatorNodeList children)
+ : Children_(std::move(children))
+ { }
+
+ void BeforeVariant16Tag() override
+ { }
+
+ void OnVariant16Tag(TValidatorNodeStack* validatorNodeStack, ui16 tag) override
+ {
+ ValidateVariantTag(validatorNodeStack, tag, Children_);
+ }
+
+ void OnChildDone(TValidatorNodeStack* validatorNodeStack) override
+ {
+ validatorNodeStack->PopValidator();
+ }
+
+private:
+ const TValidatorNodeList Children_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TRepeatedVariant8TypeUsageValidator
+ : public IValidatorNode
+{
+public:
+ explicit TRepeatedVariant8TypeUsageValidator(TValidatorNodeList children)
+ : Children_(std::move(children))
+ { }
+
+ void BeforeVariant8Tag() override
+ { }
+
+ void OnVariant8Tag(TValidatorNodeStack* validatorNodeStack, ui8 tag) override
+ {
+ ValidateVariantTag(validatorNodeStack, tag, Children_);
+ }
+
+ void OnChildDone(TValidatorNodeStack* /*validatorNodeStack*/) override
+ { }
+
+private:
+ const TValidatorNodeList Children_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TRepeatedVariant16TypeUsageValidator
+ : public IValidatorNode
+{
+public:
+ explicit TRepeatedVariant16TypeUsageValidator(TValidatorNodeList children)
+ : Children_(std::move(children))
+ { }
+
+ void BeforeVariant16Tag() override
+ { }
+
+ void OnVariant16Tag(TValidatorNodeStack* validatorNodeStack, ui16 tag) override
+ {
+ ValidateVariantTag(validatorNodeStack, tag, Children_);
+ }
+
+ void OnChildDone(TValidatorNodeStack* /*validatorNodeStack*/) override
+ { }
+
+private:
+ const TValidatorNodeList Children_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TTupleTypeUsageValidator
+ : public IValidatorNode
+{
+public:
+ explicit TTupleTypeUsageValidator(TValidatorNodeList children)
+ : Children_(std::move(children))
+ { }
+
+ void OnBegin(TValidatorNodeStack* validatorNodeStack) override
+ {
+ Position_ = 0;
+ if (!Children_.empty()) {
+ validatorNodeStack->PushValidator(Children_[0].get());
+ }
+ }
+
+ void OnChildDone(TValidatorNodeStack* validatorNodeStack) override
+ {
+ Position_++;
+ if (Position_ < Children_.size()) {
+ validatorNodeStack->PushValidator(Children_[Position_].get());
+ } else {
+ validatorNodeStack->PopValidator();
+ }
+ }
+
+private:
+ const TValidatorNodeList Children_;
+ ui32 Position_ = 0;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TSkiffValidator::TSkiffValidator(std::shared_ptr<TSkiffSchema> skiffSchema)
+ : Context_(std::make_unique<TValidatorNodeStack>(CreateUsageValidatorNode(std::move(skiffSchema))))
+{ }
+
+TSkiffValidator::~TSkiffValidator()
+{ }
+
+void TSkiffValidator::BeforeVariant8Tag()
+{
+ Context_->PushRootIfRequired();
+ Context_->Top()->BeforeVariant8Tag();
+}
+
+void TSkiffValidator::OnVariant8Tag(ui8 tag)
+{
+ Context_->PushRootIfRequired();
+ Context_->Top()->OnVariant8Tag(Context_.get(), tag);
+}
+
+void TSkiffValidator::BeforeVariant16Tag()
+{
+ Context_->PushRootIfRequired();
+ Context_->Top()->BeforeVariant16Tag();
+}
+
+void TSkiffValidator::OnVariant16Tag(ui16 tag)
+{
+ Context_->PushRootIfRequired();
+ Context_->Top()->OnVariant16Tag(Context_.get(), tag);
+}
+
+void TSkiffValidator::OnSimpleType(EWireType value)
+{
+ Context_->PushRootIfRequired();
+ Context_->Top()->OnSimpleType(Context_.get(), value);
+}
+
+void TSkiffValidator::ValidateFinished()
+{
+ if (!Context_->IsFinished()) {
+ ythrow TSkiffException() << "Parse/write is not finished";
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TValidatorNodeList CreateUsageValidatorNodeList(const TSkiffSchemaList& skiffSchemaList)
+{
+ TValidatorNodeList result;
+ result.reserve(skiffSchemaList.size());
+ for (const auto& skiffSchema : skiffSchemaList) {
+ result.push_back(CreateUsageValidatorNode(skiffSchema));
+ }
+ return result;
+}
+
+std::shared_ptr<IValidatorNode> CreateUsageValidatorNode(const std::shared_ptr<TSkiffSchema>& skiffSchema)
+{
+ switch (skiffSchema->GetWireType()) {
+ case EWireType::Int8:
+ case EWireType::Int16:
+ case EWireType::Int32:
+ case EWireType::Int64:
+ case EWireType::Int128:
+
+ case EWireType::Uint8:
+ case EWireType::Uint16:
+ case EWireType::Uint32:
+ case EWireType::Uint64:
+ case EWireType::Uint128:
+
+ case EWireType::Double:
+ case EWireType::Boolean:
+ case EWireType::String32:
+ case EWireType::Yson32:
+ return std::make_shared<TSimpleTypeUsageValidator>(skiffSchema->GetWireType());
+ case EWireType::Nothing:
+ return std::make_shared<TNothingTypeValidator>();
+ case EWireType::Tuple:
+ return std::make_shared<TTupleTypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren()));
+ case EWireType::Variant8:
+ return std::make_shared<TVariant8TypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren()));
+ case EWireType::Variant16:
+ return std::make_shared<TVariant16TypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren()));
+ case EWireType::RepeatedVariant8:
+ return std::make_shared<TRepeatedVariant8TypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren()));
+ case EWireType::RepeatedVariant16:
+ return std::make_shared<TRepeatedVariant16TypeUsageValidator>(CreateUsageValidatorNodeList(skiffSchema->GetChildren()));
+ }
+ Y_FAIL();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
diff --git a/library/cpp/skiff/skiff_validator.h b/library/cpp/skiff/skiff_validator.h
new file mode 100644
index 0000000000..522cc74db6
--- /dev/null
+++ b/library/cpp/skiff/skiff_validator.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#include "public.h"
+
+#include "skiff_schema.h"
+
+#include <util/string/cast.h>
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TValidatorNodeStack;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TSkiffValidator
+{
+public:
+ explicit TSkiffValidator(std::shared_ptr<TSkiffSchema> skiffSchema);
+ ~TSkiffValidator();
+
+ void BeforeVariant8Tag();
+ void OnVariant8Tag(ui8 tag);
+
+ void BeforeVariant16Tag();
+ void OnVariant16Tag(ui16 tag);
+
+ void OnSimpleType(EWireType value);
+
+ void ValidateFinished();
+
+private:
+ const std::unique_ptr<TValidatorNodeStack> Context_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
diff --git a/library/cpp/skiff/unittests/skiff_schema_ut.cpp b/library/cpp/skiff/unittests/skiff_schema_ut.cpp
new file mode 100644
index 0000000000..c20a560dfc
--- /dev/null
+++ b/library/cpp/skiff/unittests/skiff_schema_ut.cpp
@@ -0,0 +1,148 @@
+#include <library/cpp/skiff/skiff.h>
+#include <library/cpp/skiff/skiff_schema.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/buffer.h>
+
+using namespace NSkiff;
+
+////////////////////////////////////////////////////////////////////////////////
+
+template<>
+void Out<TSkiffSchema>(IOutputStream& s, const TSkiffSchema& schema)
+{
+ s << "TSkiffSchema:" << GetShortDebugString(schema.shared_from_this());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+Y_UNIT_TEST_SUITE(TSkiffSchemaTestSuite) {
+ Y_UNIT_TEST(TestIntEqual)
+ {
+ std::shared_ptr<TSkiffSchema> schema1 = CreateSimpleTypeSchema(EWireType::Uint64);
+ schema1->SetName("schema");
+
+ std::shared_ptr<TSkiffSchema> schema2 = CreateSimpleTypeSchema(EWireType::Uint64);
+ schema2->SetName("schema");
+
+ UNIT_ASSERT_VALUES_EQUAL(*schema1, *schema2);
+ }
+
+ Y_UNIT_TEST(TestTupleEqual)
+ {
+ std::shared_ptr<TSkiffSchema> schema1 = CreateTupleSchema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ CreateSimpleTypeSchema(EWireType::String32),
+ });
+
+ std::shared_ptr<TSkiffSchema> schema2 = CreateTupleSchema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ CreateSimpleTypeSchema(EWireType::String32),
+ });
+
+ Cerr << *schema1 << Endl;
+
+ schema1->SetName("schema");
+ UNIT_ASSERT_VALUES_UNEQUAL(*schema1, *schema2);
+
+ schema2->SetName("schema");
+ UNIT_ASSERT_VALUES_EQUAL(*schema1, *schema2);
+ }
+
+ Y_UNIT_TEST(TestHashes)
+ {
+ TSet<size_t> hashes;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Uint64);
+ schema->SetName("schema");
+ hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema));
+
+ schema = CreateSimpleTypeSchema(EWireType::Uint64);
+ hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema));
+
+ auto schema2 = CreateTupleSchema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ CreateSimpleTypeSchema(EWireType::String32),
+ });
+ schema2->SetName("s");
+ hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema2));
+
+ schema2->SetName("s0");
+ hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema2));
+
+ schema2->SetName("s");
+ hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema2));
+
+ auto schema3 = CreateRepeatedVariant16Schema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ schema2,
+ });
+ hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema3));
+
+ schema3->SetName("kek");
+ hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema3));
+
+ auto schema4 = CreateRepeatedVariant8Schema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ schema2,
+ });
+ hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema4));
+
+ schema4->SetName("kek");
+ hashes.insert(THash<NSkiff::TSkiffSchema>()(*schema4));
+
+ UNIT_ASSERT_VALUES_EQUAL(hashes.size(), 8);
+ }
+
+ Y_UNIT_TEST(TestDifferent)
+ {
+ TVector<std::shared_ptr<TSkiffSchema>> schemas;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Uint64);
+ schema->SetName("schema");
+ schemas.push_back(schema);
+ schemas.push_back(CreateSimpleTypeSchema(EWireType::Uint64));
+
+ auto schema2 = CreateTupleSchema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ CreateSimpleTypeSchema(EWireType::String32),
+ });
+ schema2->SetName("s");
+ schemas.push_back(schema2);
+
+ auto schema3 = CreateTupleSchema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ CreateSimpleTypeSchema(EWireType::String32),
+ });
+ schema3->SetName("s0");
+ schemas.push_back(schema3);
+
+ auto schema4 = CreateRepeatedVariant16Schema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ schema2,
+ });
+ schemas.push_back(schema4);
+
+ auto schema5 = CreateRepeatedVariant16Schema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ schema2,
+ });
+ schema5->SetName("kek");
+ schemas.push_back(schema5);
+
+ auto schema6 = CreateRepeatedVariant8Schema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ schema2,
+ });
+ schemas.push_back(schema6);
+
+ for (size_t i = 0; i < schemas.size(); ++i) {
+ for (size_t j = i + 1; j < schemas.size(); ++j) {
+ UNIT_ASSERT_VALUES_UNEQUAL(*schemas[i], *schemas[j]);
+ }
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/library/cpp/skiff/unittests/skiff_ut.cpp b/library/cpp/skiff/unittests/skiff_ut.cpp
new file mode 100644
index 0000000000..5e4c709611
--- /dev/null
+++ b/library/cpp/skiff/unittests/skiff_ut.cpp
@@ -0,0 +1,627 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <library/cpp/skiff/skiff.h>
+#include <library/cpp/skiff/skiff_schema.h>
+
+#include <util/stream/buffer.h>
+#include <util/string/hex.h>
+
+using namespace NSkiff;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static TString HexEncode(const TBuffer& buffer)
+{
+ auto result = HexEncode(buffer.Data(), buffer.Size());
+ result.to_lower();
+ return result;
+}
+
+Y_UNIT_TEST_SUITE(Skiff)
+{
+ Y_UNIT_TEST(TestInt8)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Int8);
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteInt8(42);
+ tokenWriter.WriteInt8(-42);
+ tokenWriter.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "2a"
+ "d6");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt8(), 42);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt8(), -42);
+ }
+
+ Y_UNIT_TEST(TestInt16)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Int16);
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteInt16(0x1234);
+ tokenWriter.WriteInt16(-0x1234);
+ tokenWriter.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "3412"
+ "cced");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt16(), 0x1234);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt16(), -0x1234);
+ }
+
+ Y_UNIT_TEST(TestInt32)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Int32);
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteInt32(0x12345678);
+ tokenWriter.WriteInt32(-0x12345678);
+ tokenWriter.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "78563412"
+ "88a9cbed");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt32(), 0x12345678);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt32(), -0x12345678);
+ }
+
+ Y_UNIT_TEST(TestInt64)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Int64);
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteInt64(-42);
+ tokenWriter.WriteInt64(100500);
+ tokenWriter.WriteInt64(-0x123456789abcdef0);
+ tokenWriter.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "d6ffffffffffffff"
+ "9488010000000000"
+ "1021436587a9cbed");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), -42);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), 100500);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), -0x123456789abcdef0);
+ }
+
+ Y_UNIT_TEST(TestUint8)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Uint8);
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteUint8(42);
+ tokenWriter.WriteUint8(200);
+ tokenWriter.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "2a"
+ "c8");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint8(), 42);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint8(), 200);
+ }
+
+ Y_UNIT_TEST(TestUint16)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Uint16);
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteUint16(0x1234);
+ tokenWriter.WriteUint16(0xfedc);
+ tokenWriter.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "3412"
+ "dcfe");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint16(), 0x1234);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint16(), 0xfedc);
+ }
+
+ Y_UNIT_TEST(TestUint32)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Uint32);
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteUint32(0x12345678);
+ tokenWriter.WriteUint32(0x87654321);
+ tokenWriter.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "78563412"
+ "21436587");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint32(), 0x12345678);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint32(), 0x87654321);
+ }
+
+
+ Y_UNIT_TEST(TestUint64)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Uint64);
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteUint64(42);
+ tokenWriter.WriteUint64(100500);
+ tokenWriter.WriteUint64(0x123456789abcdef0);
+ tokenWriter.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "2a00000000000000"
+ "9488010000000000"
+ "f0debc9a78563412");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 42);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 100500);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 0x123456789abcdef0);
+ }
+
+ Y_UNIT_TEST(TestInt128)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Int128);
+
+ const TInt128 val1 = {0x1924cd4aeb9ced82, 0x0885e83f456d6a7e};
+ const TInt128 val2 = {0xe9ba36585eccae1a, -0x7854b6f9ce448be9};
+
+ TCheckedSkiffWriter writer(schema, &bufferStream);
+ writer.WriteInt128(val1);
+ writer.WriteInt128(val2);
+ writer.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "82ed9ceb4acd2419" "7e6a6d453fe88508"
+ "1aaecc5e5836bae9" "1774bb310649ab87");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_EQUAL(parser.ParseInt128(), val1);
+ UNIT_ASSERT_EQUAL(parser.ParseInt128(), val2);
+ }
+
+ Y_UNIT_TEST(TestUint128)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Uint128);
+
+ const auto val1 = TUint128{0x1924cd4aeb9ced82, 0x0885e83f456d6a7e};
+ const auto val2 = TUint128{0xe9ba36585eccae1a, 0x8854b6f9ce448be9};
+
+ TCheckedSkiffWriter writer(schema, &bufferStream);
+ writer.WriteUint128(val1);
+ writer.WriteUint128(val2);
+ writer.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "82ed9ceb4acd2419" "7e6a6d453fe88508"
+ "1aaecc5e5836bae9" "e98b44cef9b65488");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_EQUAL(parser.ParseUint128(), val1);
+ UNIT_ASSERT_EQUAL(parser.ParseUint128(), val2);
+ }
+
+ Y_UNIT_TEST(TestBoolean)
+ {
+ auto schema = CreateSimpleTypeSchema(EWireType::Boolean);
+
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteBoolean(true);
+ tokenWriter.WriteBoolean(false);
+ tokenWriter.Finish();
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseBoolean(), true);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseBoolean(), false);
+
+ {
+ TBufferStream bufferStream;
+ bufferStream.Write('\x02');
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_EXCEPTION(parser.ParseBoolean(), std::exception);
+ }
+ }
+
+ Y_UNIT_TEST(TestVariant8)
+ {
+ auto schema = CreateVariant8Schema({
+ CreateSimpleTypeSchema(EWireType::Nothing),
+ CreateSimpleTypeSchema(EWireType::Uint64),
+ });
+
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ UNIT_ASSERT_EXCEPTION(tokenWriter.WriteUint64(42), std::exception);
+ }
+
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteVariant8Tag(0);
+ UNIT_ASSERT_EXCEPTION(tokenWriter.WriteUint64(42), std::exception);
+ }
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteVariant8Tag(1);
+ UNIT_ASSERT_EXCEPTION(tokenWriter.WriteInt64(42), std::exception);
+ }
+
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteVariant8Tag(0);
+ tokenWriter.WriteVariant8Tag(1);
+ tokenWriter.WriteUint64(42);
+ tokenWriter.Finish();
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 42);
+
+ parser.ValidateFinished();
+ }
+
+ Y_UNIT_TEST(TestTuple)
+ {
+
+ auto schema = CreateTupleSchema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ CreateSimpleTypeSchema(EWireType::String32),
+ });
+
+ {
+ TBufferStream bufferStream;
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteInt64(42);
+ tokenWriter.WriteString32("foobar");
+ tokenWriter.Finish();
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), 42);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseString32(), "foobar");
+ parser.ValidateFinished();
+ }
+ }
+
+ Y_UNIT_TEST(TestString)
+ {
+
+ auto schema = CreateSimpleTypeSchema(EWireType::String32);
+
+ {
+ TBufferStream bufferStream;
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteString32("foo");
+ tokenWriter.Finish();
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseString32(), "foo");
+
+ parser.ValidateFinished();
+ }
+
+ {
+ TBufferStream bufferStream;
+
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+ tokenWriter.WriteString32("foo");
+ tokenWriter.Finish();
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_EXCEPTION(parser.ParseInt64(), std::exception);
+ }
+ }
+
+ Y_UNIT_TEST(TestRepeatedVariant8)
+ {
+
+ auto schema = CreateRepeatedVariant8Schema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ CreateSimpleTypeSchema(EWireType::Uint64),
+ });
+
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+
+ // row 0
+ tokenWriter.WriteVariant8Tag(0);
+ tokenWriter.WriteInt64(-8);
+
+ // row 2
+ tokenWriter.WriteVariant8Tag(1);
+ tokenWriter.WriteUint64(42);
+
+ // end
+ tokenWriter.WriteVariant8Tag(EndOfSequenceTag<ui8>());
+
+ tokenWriter.Finish();
+
+ {
+ TBufferInput input(bufferStream.Buffer());
+ TCheckedSkiffParser parser(schema, &input);
+
+ // row 1
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), -8);
+
+ // row 2
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 42);
+
+ // end
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), EndOfSequenceTag<ui8>());
+
+ parser.ValidateFinished();
+ }
+
+ {
+ TBufferInput input(bufferStream.Buffer());
+ TCheckedSkiffParser parser(schema, &input);
+
+ UNIT_ASSERT_EXCEPTION(parser.ParseInt64(), std::exception);
+ }
+
+ {
+ TBufferInput input(bufferStream.Buffer());
+ TCheckedSkiffParser parser(schema, &input);
+
+ parser.ParseVariant8Tag();
+ UNIT_ASSERT_EXCEPTION(parser.ParseUint64(), std::exception);
+ }
+
+ {
+ TBufferInput input(bufferStream.Buffer());
+ TCheckedSkiffParser parser(schema, &input);
+
+ parser.ParseVariant8Tag();
+ parser.ParseInt64();
+
+ UNIT_ASSERT_EXCEPTION(parser.ValidateFinished(), std::exception);
+ }
+ }
+
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+
+ tokenWriter.WriteVariant8Tag(0);
+ UNIT_ASSERT_EXCEPTION(tokenWriter.WriteUint64(5), std::exception);
+ }
+
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+
+ tokenWriter.WriteVariant8Tag(1);
+ tokenWriter.WriteUint64(5);
+
+ UNIT_ASSERT_EXCEPTION(tokenWriter.Finish(), std::exception);
+ }
+ }
+
+ Y_UNIT_TEST(TestRepeatedVariant16)
+ {
+
+ auto schema = CreateRepeatedVariant16Schema({
+ CreateSimpleTypeSchema(EWireType::Int64),
+ CreateSimpleTypeSchema(EWireType::Uint64),
+ });
+
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+
+ // row 0
+ tokenWriter.WriteVariant16Tag(0);
+ tokenWriter.WriteInt64(-8);
+
+ // row 2
+ tokenWriter.WriteVariant16Tag(1);
+ tokenWriter.WriteUint64(42);
+
+ // end
+ tokenWriter.WriteVariant16Tag(EndOfSequenceTag<ui16>());
+
+ tokenWriter.Finish();
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+
+ // row 1
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), -8);
+
+ // row 2
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 42);
+
+ // end
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), EndOfSequenceTag<ui16>());
+
+ parser.ValidateFinished();
+ }
+
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+
+ tokenWriter.WriteVariant16Tag(0);
+ UNIT_ASSERT_EXCEPTION(tokenWriter.WriteUint64(5), std::exception);
+ }
+
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+
+ tokenWriter.WriteVariant16Tag(1);
+ tokenWriter.WriteUint64(5);
+
+ UNIT_ASSERT_EXCEPTION(tokenWriter.Finish(), std::exception);
+ }
+
+ {
+ TBufferStream bufferStream;
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+
+ // row 0
+ tokenWriter.WriteVariant16Tag(0);
+ tokenWriter.WriteInt64(-8);
+
+ // row 2
+ tokenWriter.WriteVariant16Tag(1);
+ tokenWriter.WriteUint64(42);
+
+ // end
+ tokenWriter.WriteVariant16Tag(EndOfSequenceTag<ui16>());
+
+ tokenWriter.Finish();
+
+ {
+ TBufferInput input(bufferStream.Buffer());
+ TCheckedSkiffParser parser(schema, &input);
+
+ UNIT_ASSERT_EXCEPTION(parser.ParseInt64(), std::exception);
+ }
+
+ {
+ TBufferInput input(bufferStream.Buffer());
+ TCheckedSkiffParser parser(schema, &input);
+
+ parser.ParseVariant16Tag();
+ UNIT_ASSERT_EXCEPTION(parser.ParseUint64(), std::exception);
+ }
+
+ {
+ TBufferInput input(bufferStream.Buffer());
+ TCheckedSkiffParser parser(schema, &input);
+
+ parser.ParseVariant16Tag();
+ parser.ParseInt64();
+
+ UNIT_ASSERT_EXCEPTION(parser.ValidateFinished(), std::exception);
+ }
+ }
+ }
+
+ Y_UNIT_TEST(TestStruct)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateRepeatedVariant16Schema(
+ {
+ CreateSimpleTypeSchema(EWireType::Nothing),
+ CreateTupleSchema({
+ CreateVariant8Schema({
+ CreateSimpleTypeSchema(EWireType::Nothing),
+ CreateSimpleTypeSchema(EWireType::Int64)
+ }),
+ CreateSimpleTypeSchema(EWireType::Uint64),
+ })
+ }
+ );
+
+ {
+ TCheckedSkiffWriter tokenWriter(schema, &bufferStream);
+
+ // row 0
+ tokenWriter.WriteVariant16Tag(0);
+
+ // row 1
+ tokenWriter.WriteVariant16Tag(1);
+ tokenWriter.WriteVariant8Tag(0);
+ tokenWriter.WriteUint64(1);
+
+ // row 2
+ tokenWriter.WriteVariant16Tag(1);
+ tokenWriter.WriteVariant8Tag(1);
+ tokenWriter.WriteInt64(2);
+ tokenWriter.WriteUint64(3);
+
+ // end
+ tokenWriter.WriteVariant16Tag(EndOfSequenceTag<ui16>());
+
+ tokenWriter.Finish();
+ }
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+
+ // row 0
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 0);
+
+ // row 1
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 1);
+
+ // row 2
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant8Tag(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt64(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseUint64(), 3);
+
+ // end
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseVariant16Tag(), EndOfSequenceTag<ui16>());
+
+ parser.ValidateFinished();
+ }
+
+ Y_UNIT_TEST(TestSimpleOutputStream)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Int8);
+
+ TCheckedSkiffWriter tokenWriter(schema, static_cast<IOutputStream*>(&bufferStream));
+ tokenWriter.WriteInt8(42);
+ tokenWriter.WriteInt8(-42);
+ tokenWriter.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "2a"
+ "d6");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt8(), 42);
+ UNIT_ASSERT_VALUES_EQUAL(parser.ParseInt8(), -42);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/library/cpp/skiff/unittests/ya.make b/library/cpp/skiff/unittests/ya.make
new file mode 100644
index 0000000000..d67ca8c618
--- /dev/null
+++ b/library/cpp/skiff/unittests/ya.make
@@ -0,0 +1,12 @@
+UNITTEST()
+
+SRCS(
+ skiff_ut.cpp
+ skiff_schema_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/skiff
+)
+
+END()
diff --git a/library/cpp/skiff/ya.make b/library/cpp/skiff/ya.make
new file mode 100644
index 0000000000..ff3eb55c9f
--- /dev/null
+++ b/library/cpp/skiff/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+SRCS(
+ skiff.cpp
+ skiff_schema.cpp
+ skiff_validator.cpp
+ zerocopy_output_writer.cpp
+)
+
+GENERATE_ENUM_SERIALIZATION(public.h)
+
+END()
+
+RECURSE_FOR_TESTS(
+ unittests
+)
diff --git a/library/cpp/skiff/zerocopy_output_writer-inl.h b/library/cpp/skiff/zerocopy_output_writer-inl.h
new file mode 100644
index 0000000000..6bd067c9fa
--- /dev/null
+++ b/library/cpp/skiff/zerocopy_output_writer-inl.h
@@ -0,0 +1,51 @@
+#pragma once
+#ifndef ZEROCOPY_OUTPUT_WRITER_INL_H_
+#error "Direct inclusion of this file is not allowed, include zerocopy_output_writer.h"
+// For the sake of sane code completion.
+#include "zerocopy_output_writer.h"
+#endif
+
+#include <util/system/yassert.h>
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+char* TZeroCopyOutputStreamWriter::Current() const
+{
+ return Current_;
+}
+
+ui64 TZeroCopyOutputStreamWriter::RemainingBytes() const
+{
+ return RemainingBytes_;
+}
+
+void TZeroCopyOutputStreamWriter::Advance(size_t bytes)
+{
+ Y_VERIFY(bytes <= RemainingBytes_);
+ Current_ += bytes;
+ RemainingBytes_ -= bytes;
+}
+
+void TZeroCopyOutputStreamWriter::Write(const void* buffer, size_t length)
+{
+ if (length > RemainingBytes_) {
+ UndoRemaining();
+ Output_->Write(buffer, length);
+ TotalWrittenBlockSize_ += length;
+ ObtainNextBlock();
+ } else {
+ memcpy(Current_, buffer, length);
+ Advance(length);
+ }
+}
+
+ui64 TZeroCopyOutputStreamWriter::GetTotalWrittenSize() const
+{
+ return TotalWrittenBlockSize_ - RemainingBytes_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
diff --git a/library/cpp/skiff/zerocopy_output_writer.cpp b/library/cpp/skiff/zerocopy_output_writer.cpp
new file mode 100644
index 0000000000..49492b55a4
--- /dev/null
+++ b/library/cpp/skiff/zerocopy_output_writer.cpp
@@ -0,0 +1,38 @@
+#include "zerocopy_output_writer.h"
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TZeroCopyOutputStreamWriter::TZeroCopyOutputStreamWriter(IZeroCopyOutput* output)
+ : Output_(output)
+{
+ ObtainNextBlock();
+}
+
+TZeroCopyOutputStreamWriter::~TZeroCopyOutputStreamWriter()
+{
+ if (RemainingBytes_ > 0) {
+ UndoRemaining();
+ }
+}
+
+void TZeroCopyOutputStreamWriter::ObtainNextBlock()
+{
+ if (RemainingBytes_ > 0) {
+ UndoRemaining();
+ }
+ RemainingBytes_ = Output_->Next(&Current_);
+ TotalWrittenBlockSize_ += RemainingBytes_;
+}
+
+void TZeroCopyOutputStreamWriter::UndoRemaining()
+{
+ Output_->Undo(RemainingBytes_);
+ TotalWrittenBlockSize_ -= RemainingBytes_;
+ RemainingBytes_ = 0;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
diff --git a/library/cpp/skiff/zerocopy_output_writer.h b/library/cpp/skiff/zerocopy_output_writer.h
new file mode 100644
index 0000000000..b0bccc5a63
--- /dev/null
+++ b/library/cpp/skiff/zerocopy_output_writer.h
@@ -0,0 +1,41 @@
+#pragma once
+
+#include <util/stream/zerocopy_output.h>
+
+namespace NSkiff {
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Simple wrapper around
+class TZeroCopyOutputStreamWriter
+ : private TNonCopyable
+{
+public:
+ explicit TZeroCopyOutputStreamWriter(IZeroCopyOutput* output);
+
+ ~TZeroCopyOutputStreamWriter();
+
+ Y_FORCE_INLINE char* Current() const;
+ Y_FORCE_INLINE ui64 RemainingBytes() const;
+ Y_FORCE_INLINE void Advance(size_t bytes);
+ void UndoRemaining();
+ Y_FORCE_INLINE void Write(const void* buffer, size_t length);
+ Y_FORCE_INLINE ui64 GetTotalWrittenSize() const;
+
+private:
+ void ObtainNextBlock();
+
+private:
+ IZeroCopyOutput* Output_;
+ char* Current_ = nullptr;
+ ui64 RemainingBytes_ = 0;
+ ui64 TotalWrittenBlockSize_ = 0;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NSkiff
+
+#define ZEROCOPY_OUTPUT_WRITER_INL_H_
+#include "zerocopy_output_writer-inl.h"
+#undef ZEROCOPY_OUTPUT_WRITER_INL_H_
diff --git a/library/cpp/testing/gtest/friend.h b/library/cpp/testing/gtest/friend.h
new file mode 100644
index 0000000000..551a218be0
--- /dev/null
+++ b/library/cpp/testing/gtest/friend.h
@@ -0,0 +1,5 @@
+#pragma once
+
+// Using absolute path to gtest headers in order to allow using friend.h without PEERDIRing gtest.
+#include <contrib/restricted/googletest/googletest/include/gtest/gtest_prod.h>
+
diff --git a/library/cpp/threading/blocking_queue/blocking_queue.cpp b/library/cpp/threading/blocking_queue/blocking_queue.cpp
new file mode 100644
index 0000000000..db199c80be
--- /dev/null
+++ b/library/cpp/threading/blocking_queue/blocking_queue.cpp
@@ -0,0 +1,3 @@
+#include "blocking_queue.h"
+
+// just check compilability
diff --git a/library/cpp/threading/blocking_queue/blocking_queue.h b/library/cpp/threading/blocking_queue/blocking_queue.h
new file mode 100644
index 0000000000..48d3762f68
--- /dev/null
+++ b/library/cpp/threading/blocking_queue/blocking_queue.h
@@ -0,0 +1,158 @@
+#pragma once
+
+#include <util/generic/deque.h>
+#include <util/generic/maybe.h>
+#include <util/generic/yexception.h>
+#include <util/system/condvar.h>
+#include <util/system/guard.h>
+#include <util/system/mutex.h>
+
+#include <utility>
+
+namespace NThreading {
+ ///
+ /// TBlockingQueue is a queue of elements of limited or unlimited size.
+ /// Queue provides Push and Pop operations that block if operation can't be executed
+ /// (queue is empty or maximum size is reached).
+ ///
+ /// Queue can be stopped, in that case all blocked operation will return `Nothing` / false.
+ ///
+ /// All operations are thread safe.
+ ///
+ ///
+ /// Example of usage:
+ /// TBlockingQueue<int> queue;
+ ///
+ /// ...
+ ///
+ /// // thread 1
+ /// queue.Push(42);
+ /// queue.Push(100500);
+ ///
+ /// ...
+ ///
+ /// // thread 2
+ /// while (TMaybe<int> number = queue.Pop()) {
+ /// ProcessNumber(number.GetRef());
+ /// }
+ template <class TElement>
+ class TBlockingQueue {
+ public:
+ ///
+ /// Creates blocking queue with given maxSize
+ /// if maxSize == 0 then queue is unlimited
+ TBlockingQueue(size_t maxSize)
+ : MaxSize(maxSize == 0 ? Max<size_t>() : maxSize)
+ , Stopped(false)
+ {
+ }
+
+ ///
+ /// Blocks until queue has some elements or queue is stopped or deadline is reached.
+ /// Returns `Nothing` if queue is stopped or deadline is reached.
+ /// Returns element otherwise.
+ TMaybe<TElement> Pop(TInstant deadline = TInstant::Max()) {
+ TGuard<TMutex> g(Lock);
+
+ const auto canPop = [this]() { return CanPop(); };
+ if (!CanPopCV.WaitD(Lock, deadline, canPop)) {
+ return Nothing();
+ }
+
+ if (Stopped && Queue.empty()) {
+ return Nothing();
+ }
+ TElement e = std::move(Queue.front());
+ Queue.pop_front();
+ CanPushCV.Signal();
+ return std::move(e);
+ }
+
+ TMaybe<TElement> Pop(TDuration duration) {
+ return Pop(TInstant::Now() + duration);
+ }
+
+ ///
+ /// Blocks until queue has space for new elements or queue is stopped or deadline is reached.
+ /// Returns false exception if queue is stopped and push failed or deadline is reached.
+ /// Pushes element to queue and returns true otherwise.
+ bool Push(const TElement& e, TInstant deadline = TInstant::Max()) {
+ return PushRef(e, deadline);
+ }
+
+ bool Push(TElement&& e, TInstant deadline = TInstant::Max()) {
+ return PushRef(std::move(e), deadline);
+ }
+
+ bool Push(const TElement& e, TDuration duration) {
+ return Push(e, TInstant::Now() + duration);
+ }
+
+ bool Push(TElement&& e, TDuration duration) {
+ return Push(std::move(e), TInstant::Now() + duration);
+ }
+
+ ///
+ /// Stops the queue, all blocked operations will be aborted.
+ void Stop() {
+ TGuard<TMutex> g(Lock);
+ Stopped = true;
+ CanPopCV.BroadCast();
+ CanPushCV.BroadCast();
+ }
+
+ ///
+ /// Checks whether queue is empty.
+ bool Empty() const {
+ TGuard<TMutex> g(Lock);
+ return Queue.empty();
+ }
+
+ ///
+ /// Returns size of the queue.
+ size_t Size() const {
+ TGuard<TMutex> g(Lock);
+ return Queue.size();
+ }
+
+ ///
+ /// Checks whether queue is stopped.
+ bool IsStopped() const {
+ TGuard<TMutex> g(Lock);
+ return Stopped;
+ }
+
+ private:
+ bool CanPush() const {
+ return Queue.size() < MaxSize || Stopped;
+ }
+
+ bool CanPop() const {
+ return !Queue.empty() || Stopped;
+ }
+
+ template <typename Ref>
+ bool PushRef(Ref e, TInstant deadline) {
+ TGuard<TMutex> g(Lock);
+ const auto canPush = [this]() { return CanPush(); };
+ if (!CanPushCV.WaitD(Lock, deadline, canPush)) {
+ return false;
+ }
+ if (Stopped) {
+ return false;
+ }
+ Queue.push_back(std::forward<TElement>(e));
+ CanPopCV.Signal();
+ return true;
+ }
+
+ private:
+ TMutex Lock;
+ TCondVar CanPopCV;
+ TCondVar CanPushCV;
+ TDeque<TElement> Queue;
+ size_t MaxSize;
+ bool Stopped;
+ };
+
+}
diff --git a/library/cpp/threading/blocking_queue/blocking_queue_ut.cpp b/library/cpp/threading/blocking_queue/blocking_queue_ut.cpp
new file mode 100644
index 0000000000..26e125279d
--- /dev/null
+++ b/library/cpp/threading/blocking_queue/blocking_queue_ut.cpp
@@ -0,0 +1,211 @@
+#include "blocking_queue.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/string/builder.h>
+#include <util/system/thread.h>
+
+namespace {
+ class TFunctionThread: public ISimpleThread {
+ public:
+ using TFunc = std::function<void()>;
+
+ private:
+ TFunc Func;
+
+ public:
+ TFunctionThread(const TFunc& func)
+ : Func(func)
+ {
+ }
+
+ void* ThreadProc() noexcept override {
+ Func();
+ return nullptr;
+ }
+ };
+
+}
+
+IOutputStream& operator<<(IOutputStream& o, const TMaybe<int>& val) {
+ if (val) {
+ o << "TMaybe<int>(" << val.GetRef() << ')';
+ } else {
+ o << "TMaybe<int>()";
+ }
+ return o;
+}
+
+Y_UNIT_TEST_SUITE(BlockingQueueTest) {
+ Y_UNIT_TEST(SimplePushPopTest) {
+ const size_t limit = 100;
+
+ NThreading::TBlockingQueue<int> queue(100);
+
+ for (int i = 0; i != limit; ++i) {
+ queue.Push(i);
+ }
+
+ for (int i = 0; i != limit; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), i);
+ }
+
+ UNIT_ASSERT(queue.Empty());
+ }
+
+ Y_UNIT_TEST(SimpleStopTest) {
+ const size_t limit = 100;
+
+ NThreading::TBlockingQueue<int> queue(100);
+
+ for (int i = 0; i != limit; ++i) {
+ queue.Push(i);
+ }
+ queue.Stop();
+
+ bool ok = queue.Push(100500);
+ UNIT_ASSERT_VALUES_EQUAL(ok, false);
+
+ for (int i = 0; i != limit; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), i);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), TMaybe<int>());
+ }
+
+ Y_UNIT_TEST(BigPushPop) {
+ const int limit = 100000;
+
+ NThreading::TBlockingQueue<int> queue(10);
+
+ TFunctionThread pusher([&] {
+ for (int i = 0; i != limit; ++i) {
+ if (!queue.Push(i)) {
+ break;
+ }
+ }
+ });
+
+ pusher.Start();
+
+ try {
+ for (int i = 0; i != limit; ++i) {
+ size_t size = queue.Size();
+ UNIT_ASSERT_C(size <= 10, (TStringBuilder() << "Size exceeds 10: " << size).data());
+ UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), i);
+ }
+ } catch (...) {
+ // gracefull shutdown of pusher thread if assertion fails
+ queue.Stop();
+ throw;
+ }
+
+ pusher.Join();
+ }
+
+ Y_UNIT_TEST(StopWhenMultiplePoppers) {
+ NThreading::TBlockingQueue<int> queue(10);
+ TFunctionThread popper1([&] {
+ UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), TMaybe<int>());
+ });
+ TFunctionThread popper2([&] {
+ UNIT_ASSERT_VALUES_EQUAL(queue.Pop(), TMaybe<int>());
+ });
+ popper1.Start();
+ popper2.Start();
+
+ queue.Stop();
+
+ popper1.Join();
+ popper2.Join();
+ }
+
+ Y_UNIT_TEST(StopWhenMultiplePushers) {
+ NThreading::TBlockingQueue<int> queue(1);
+ queue.Push(1);
+ TFunctionThread pusher1([&] {
+ UNIT_ASSERT_VALUES_EQUAL(queue.Push(2), false);
+ });
+ TFunctionThread pusher2([&] {
+ UNIT_ASSERT_VALUES_EQUAL(queue.Push(2), false);
+ });
+ pusher1.Start();
+ pusher2.Start();
+
+ queue.Stop();
+
+ pusher1.Join();
+ pusher2.Join();
+ }
+
+ Y_UNIT_TEST(InterruptPopByDeadline) {
+ NThreading::TBlockingQueue<int> queue1(10);
+ NThreading::TBlockingQueue<int> queue2(10);
+
+ const auto popper1DeadLine = TInstant::Now();
+ const auto popper2DeadLine = TInstant::Now() + TDuration::Seconds(2);
+
+ TFunctionThread popper1([&] {
+ UNIT_ASSERT_VALUES_EQUAL(queue1.Pop(popper1DeadLine), TMaybe<int>());
+ UNIT_ASSERT_VALUES_EQUAL(queue1.IsStopped(), false);
+ });
+
+ TFunctionThread popper2([&] {
+ UNIT_ASSERT_VALUES_EQUAL(queue2.Pop(popper2DeadLine), 2);
+ UNIT_ASSERT_VALUES_EQUAL(queue2.IsStopped(), false);
+ });
+
+ popper1.Start();
+ popper2.Start();
+
+ Sleep(TDuration::Seconds(1));
+
+ queue1.Push(1);
+ queue2.Push(2);
+
+ Sleep(TDuration::Seconds(1));
+
+ queue1.Stop();
+ queue2.Stop();
+
+ popper1.Join();
+ popper2.Join();
+ }
+
+ Y_UNIT_TEST(InterruptPushByDeadline) {
+ NThreading::TBlockingQueue<int> queue1(1);
+ NThreading::TBlockingQueue<int> queue2(1);
+
+ queue1.Push(0);
+ queue2.Push(0);
+
+ const auto pusher1DeadLine = TInstant::Now();
+ const auto pusher2DeadLine = TInstant::Now() + TDuration::Seconds(2);
+
+ TFunctionThread pusher1([&] {
+ UNIT_ASSERT_VALUES_EQUAL(queue1.Push(1, pusher1DeadLine), false);
+ UNIT_ASSERT_VALUES_EQUAL(queue1.IsStopped(), false);
+ });
+
+ TFunctionThread pusher2([&] {
+ UNIT_ASSERT_VALUES_EQUAL(queue2.Push(2, pusher2DeadLine), true);
+ UNIT_ASSERT_VALUES_EQUAL(queue2.IsStopped(), false);
+ });
+
+ pusher1.Start();
+ pusher2.Start();
+
+ Sleep(TDuration::Seconds(1));
+
+ queue1.Pop();
+ queue2.Pop();
+
+ Sleep(TDuration::Seconds(1));
+
+ queue1.Stop();
+ queue2.Stop();
+
+ pusher1.Join();
+ pusher2.Join();
+ }
+}
diff --git a/library/cpp/threading/blocking_queue/ut/ya.make b/library/cpp/threading/blocking_queue/ut/ya.make
new file mode 100644
index 0000000000..50f220d552
--- /dev/null
+++ b/library/cpp/threading/blocking_queue/ut/ya.make
@@ -0,0 +1,7 @@
+UNITTEST_FOR(library/cpp/threading/blocking_queue)
+
+SRCS(
+ blocking_queue_ut.cpp
+)
+
+END()
diff --git a/library/cpp/threading/blocking_queue/ya.make b/library/cpp/threading/blocking_queue/ya.make
new file mode 100644
index 0000000000..ce1104c1c9
--- /dev/null
+++ b/library/cpp/threading/blocking_queue/ya.make
@@ -0,0 +1,9 @@
+LIBRARY()
+
+SRCS(
+ blocking_queue.cpp
+)
+
+END()
+
+RECURSE_FOR_TESTS(ut)
diff --git a/library/cpp/threading/cron/cron.cpp b/library/cpp/threading/cron/cron.cpp
new file mode 100644
index 0000000000..e7c1c59735
--- /dev/null
+++ b/library/cpp/threading/cron/cron.cpp
@@ -0,0 +1,69 @@
+#include "cron.h"
+
+#include <library/cpp/deprecated/atomic/atomic.h>
+
+#include <util/system/thread.h>
+#include <util/system/event.h>
+
+using namespace NCron;
+
+namespace {
+ struct TPeriodicHandle: public IHandle {
+ inline TPeriodicHandle(TJob job, TDuration interval, const TString& threadName)
+ : Job(job)
+ , Interval(interval)
+ , Done(false)
+ {
+ TThread::TParams params(DoRun, this);
+ if (!threadName.empty()) {
+ params.SetName(threadName);
+ }
+ Thread = MakeHolder<TThread>(params);
+ Thread->Start();
+ }
+
+ static inline void* DoRun(void* data) noexcept {
+ ((TPeriodicHandle*)data)->Run();
+
+ return nullptr;
+ }
+
+ inline void Run() noexcept {
+ while (true) {
+ Job();
+
+ Event.WaitT(Interval);
+
+ if (AtomicGet(Done)) {
+ return;
+ }
+ }
+ }
+
+ ~TPeriodicHandle() override {
+ AtomicSet(Done, true);
+ Event.Signal();
+ Thread->Join();
+ }
+
+ TJob Job;
+ TDuration Interval;
+ TManualEvent Event;
+ TAtomic Done;
+ THolder<TThread> Thread;
+ };
+}
+
+IHandlePtr NCron::StartPeriodicJob(TJob job) {
+ return NCron::StartPeriodicJob(job, TDuration::Seconds(0), "");
+}
+
+IHandlePtr NCron::StartPeriodicJob(TJob job, TDuration interval) {
+ return NCron::StartPeriodicJob(job, interval, "");
+}
+
+IHandlePtr NCron::StartPeriodicJob(TJob job, TDuration interval, const TString& threadName) {
+ return new TPeriodicHandle(job, interval, threadName);
+}
+
+IHandle::~IHandle() = default;
diff --git a/library/cpp/threading/cron/cron.h b/library/cpp/threading/cron/cron.h
new file mode 100644
index 0000000000..77fa40c5e2
--- /dev/null
+++ b/library/cpp/threading/cron/cron.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <util/generic/ptr.h>
+#include <util/generic/function.h>
+#include <util/datetime/base.h>
+
+namespace NCron {
+ struct IHandle {
+ virtual ~IHandle();
+ };
+
+ using TJob = std::function<void()>;
+ using IHandlePtr = TAutoPtr<IHandle>;
+
+ IHandlePtr StartPeriodicJob(TJob job);
+ IHandlePtr StartPeriodicJob(TJob job, TDuration interval);
+ IHandlePtr StartPeriodicJob(TJob job, TDuration interval, const TString& threadName);
+}
diff --git a/library/cpp/threading/cron/ya.make b/library/cpp/threading/cron/ya.make
new file mode 100644
index 0000000000..ead272e837
--- /dev/null
+++ b/library/cpp/threading/cron/ya.make
@@ -0,0 +1,11 @@
+LIBRARY()
+
+SRCS(
+ cron.cpp
+)
+
+PEERDIR(
+ library/cpp/deprecated/atomic
+)
+
+END()
diff --git a/library/cpp/type_info/Readme.md b/library/cpp/type_info/Readme.md
new file mode 100644
index 0000000000..e4501c8c11
--- /dev/null
+++ b/library/cpp/type_info/Readme.md
@@ -0,0 +1,9 @@
+# TI — unified in-memory representation for Common Yandex Typesystem
+
+Common Yandex Typesystem is a type system used across data storage and processing technologies developed by Yandex. A list of systems that support it includes YT, YDB, RTMR, YQL and others.
+
+The Type Info library provides classes that represent types from Common Yandex Typesystem. It allows constructing in-memory representations of types, inspecting said representations, combining them to derive new types, as well as serializing and deserializing them.
+
+For those familiar with Protobuf, Type Info implements the concept of message descriptors for Yandex type system.
+
+Here you can find open issues: https://st.yandex-team.ru/YT/order:updated:false/filter?tags=type_info&resolution=empty() \ No newline at end of file
diff --git a/library/cpp/type_info/builder.cpp b/library/cpp/type_info/builder.cpp
new file mode 100644
index 0000000000..008e0af754
--- /dev/null
+++ b/library/cpp/type_info/builder.cpp
@@ -0,0 +1,458 @@
+#include "builder.h"
+
+#include "type_factory.h"
+
+namespace NTi {
+ TStructBuilderRaw::TStructBuilderRaw(IPoolTypeFactory& factory) noexcept
+ : Factory_(&factory)
+ {
+ }
+
+ TStructBuilderRaw::TStructBuilderRaw(IPoolTypeFactory& factory, TStructTypePtr prototype) noexcept
+ : TStructBuilderRaw(factory, prototype.Get())
+ {
+ }
+
+ TStructBuilderRaw::TStructBuilderRaw(IPoolTypeFactory& factory, const TStructType* prototype) noexcept
+ : TStructBuilderRaw(factory)
+ {
+ Members_.reserve(prototype->GetMembers().size());
+
+ if (prototype->GetFactory() == Factory_) {
+ // members are in the same factory -- can reuse them.
+ for (auto member : prototype->GetMembers()) {
+ Members_.push_back(member);
+ }
+ } else {
+ // members are in a different factory -- should copy them.
+ for (auto member : prototype->GetMembers()) {
+ AddMember(member.GetName(), member.GetTypeRaw());
+ }
+ }
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::SetName(TMaybe<TStringBuf> name) & noexcept {
+ Name_ = Factory_->AllocateStringMaybe(name);
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::SetName(TMaybe<TStringBuf> name) && noexcept {
+ return std::move(SetName(name));
+ }
+
+ bool TStructBuilderRaw::HasName() const noexcept {
+ return Name_.Defined();
+ }
+
+ TMaybe<TStringBuf> TStructBuilderRaw::GetName() const noexcept {
+ return Name_;
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::Reserve(size_t size) & noexcept {
+ Members_.reserve(size);
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::Reserve(size_t size) && noexcept {
+ return std::move(Reserve(size));
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::AddMember(TStringBuf name, TTypePtr type) & noexcept {
+ return AddMember(name, type.Get());
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::AddMember(TStringBuf name, TTypePtr type) && noexcept {
+ return std::move(AddMember(name, type));
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::AddMember(TStringBuf name, const TType* type) & noexcept {
+ Members_.emplace_back(Factory_->AllocateString(name), Factory_->Own(type));
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::AddMember(TStringBuf name, const TType* type) && noexcept {
+ return std::move(AddMember(name, type));
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::AddMemberName(TStringBuf name) & noexcept {
+ PendingMemberName_ = Factory_->AllocateString(name);
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::AddMemberName(TStringBuf name) && noexcept {
+ return std::move(AddMemberName(name));
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::DiscardMemberName() & noexcept {
+ PendingMemberName_.Clear();
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::DiscardMemberName() && noexcept {
+ return std::move(DiscardMemberName());
+ }
+
+ bool TStructBuilderRaw::HasMemberName() const noexcept {
+ return PendingMemberName_.Defined();
+ }
+
+ TMaybe<TStringBuf> TStructBuilderRaw::GetMemberName() const noexcept {
+ return PendingMemberName_;
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::AddMemberType(TTypePtr type) & noexcept {
+ return AddMemberType(type.Get());
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::AddMemberType(TTypePtr type) && noexcept {
+ return std::move(AddMemberType(type));
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::AddMemberType(const TType* type) & noexcept {
+ PendingMemberType_ = Factory_->Own(type);
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::AddMemberType(const TType* type) && noexcept {
+ return std::move(AddMemberType(type));
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::DiscardMemberType() & noexcept {
+ PendingMemberType_.Clear();
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::DiscardMemberType() && noexcept {
+ return std::move(DiscardMemberType());
+ }
+
+ bool TStructBuilderRaw::HasMemberType() const noexcept {
+ return PendingMemberType_.Defined();
+ }
+
+ TMaybe<const TType*> TStructBuilderRaw::GetMemberType() const noexcept {
+ return PendingMemberType_;
+ }
+
+ bool TStructBuilderRaw::CanAddMember() const noexcept {
+ return HasMemberName() && HasMemberType();
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::AddMember() & noexcept {
+ Y_VERIFY(CanAddMember());
+ Members_.emplace_back(*PendingMemberName_, *PendingMemberType_);
+ DiscardMember();
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::AddMember() && noexcept {
+ return std::move(AddMember());
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::DiscardMember() & noexcept {
+ DiscardMemberName();
+ DiscardMemberType();
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::DiscardMember() && noexcept {
+ return std::move(DiscardMember());
+ }
+
+ TStructType::TMembers TStructBuilderRaw::GetMembers() const noexcept {
+ return Members_;
+ }
+
+ TStructBuilderRaw& TStructBuilderRaw::Reset() & noexcept {
+ Name_ = {};
+ Members_.clear();
+ return *this;
+ }
+
+ TStructBuilderRaw TStructBuilderRaw::Reset() && noexcept {
+ return std::move(Reset());
+ }
+
+ TStructTypePtr TStructBuilderRaw::Build() {
+ return BuildRaw()->AsPtr();
+ }
+
+ const TStructType* TStructBuilderRaw::BuildRaw() {
+ return DoBuildRaw(Name_);
+ }
+
+ TVariantTypePtr TStructBuilderRaw::BuildVariant() {
+ return BuildVariantRaw()->AsPtr();
+ }
+
+ const TVariantType* TStructBuilderRaw::BuildVariantRaw() {
+ return Factory_->New<TVariantType>(Nothing(), Name_, DoBuildRaw(Nothing()));
+ }
+
+ const TStructType* TStructBuilderRaw::DoBuildRaw(TMaybe<TStringBuf> name) {
+ auto members = Factory_->NewArray<TStructType::TMember>(Members_.size(), [this](TStructType::TMember* member, size_t i) {
+ new (member) TStructType::TMember(Members_[i]);
+ });
+
+ auto sortedMembersArray = Factory_->AllocateArrayFor<size_t>(Members_.size());
+ auto sortedMembers = TArrayRef(sortedMembersArray, members.size());
+ TStructType::MakeSortedMembers(members, sortedMembers);
+
+ return Factory_->New<TStructType>(Nothing(), name, members, sortedMembers);
+ }
+
+ TTupleBuilderRaw::TTupleBuilderRaw(IPoolTypeFactory& factory) noexcept
+ : Factory_(&factory)
+ {
+ }
+
+ TTupleBuilderRaw::TTupleBuilderRaw(IPoolTypeFactory& factory, TTupleTypePtr prototype) noexcept
+ : TTupleBuilderRaw(factory, prototype.Get())
+ {
+ }
+
+ TTupleBuilderRaw::TTupleBuilderRaw(IPoolTypeFactory& factory, const TTupleType* prototype) noexcept
+ : TTupleBuilderRaw(factory)
+ {
+ Elements_.reserve(prototype->GetElements().size());
+
+ if (prototype->GetFactory() == Factory_) {
+ // elements are in the same factory -- can reuse them
+ for (auto element : prototype->GetElements()) {
+ Elements_.push_back(element);
+ }
+ } else {
+ // items are in a different factory -- should copy them
+ for (auto element : prototype->GetElements()) {
+ AddElement(element.GetTypeRaw());
+ }
+ }
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::SetName(TMaybe<TStringBuf> name) & noexcept {
+ Name_ = Factory_->AllocateStringMaybe(name);
+ return *this;
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::SetName(TMaybe<TStringBuf> name) && noexcept {
+ return std::move(SetName(name));
+ }
+
+ bool TTupleBuilderRaw::HasName() const noexcept {
+ return Name_.Defined();
+ }
+
+ TMaybe<TStringBuf> TTupleBuilderRaw::GetName() const noexcept {
+ return Name_;
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::Reserve(size_t size) & noexcept {
+ Elements_.reserve(size);
+ return *this;
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::Reserve(size_t size) && noexcept {
+ return std::move(Reserve(size));
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::AddElement(TTypePtr type) & noexcept {
+ return AddElement(type.Get());
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::AddElement(TTypePtr type) && noexcept {
+ return std::move(AddElement(type));
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::AddElement(const TType* type) & noexcept {
+ Elements_.emplace_back(Factory_->Own(type));
+ return *this;
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::AddElement(const TType* type) && noexcept {
+ return std::move(AddElement(type));
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::AddElementType(TTypePtr type) & noexcept {
+ return AddElementType(type.Get());
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::AddElementType(TTypePtr type) && noexcept {
+ return std::move(AddElementType(type));
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::AddElementType(const TType* type) & noexcept {
+ PendingElementType_ = Factory_->Own(type);
+ return *this;
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::AddElementType(const TType* type) && noexcept {
+ return std::move(AddElementType(type));
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::DiscardElementType() & noexcept {
+ PendingElementType_.Clear();
+ return *this;
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::DiscardElementType() && noexcept {
+ return std::move(DiscardElementType());
+ }
+
+ bool TTupleBuilderRaw::HasElementType() const noexcept {
+ return PendingElementType_.Defined();
+ }
+
+ TMaybe<const TType*> TTupleBuilderRaw::GetElementType() const noexcept {
+ return PendingElementType_;
+ }
+
+ bool TTupleBuilderRaw::CanAddElement() const noexcept {
+ return HasElementType();
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::AddElement() & noexcept {
+ Y_VERIFY(CanAddElement());
+ Elements_.emplace_back(*PendingElementType_);
+ DiscardElement();
+ return *this;
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::AddElement() && noexcept {
+ return std::move(AddElement());
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::DiscardElement() & noexcept {
+ DiscardElementType();
+ return *this;
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::DiscardElement() && noexcept {
+ return std::move(DiscardElement());
+ }
+
+ TTupleBuilderRaw& TTupleBuilderRaw::Reset() & noexcept {
+ Name_ = {};
+ Elements_.clear();
+ return *this;
+ }
+
+ TTupleType::TElements TTupleBuilderRaw::GetElements() const noexcept {
+ return Elements_;
+ }
+
+ TTupleBuilderRaw TTupleBuilderRaw::Reset() && noexcept {
+ return std::move(Reset());
+ }
+
+ TTupleTypePtr TTupleBuilderRaw::Build() {
+ return BuildRaw()->AsPtr();
+ }
+
+ const TTupleType* TTupleBuilderRaw::BuildRaw() {
+ return DoBuildRaw(Name_);
+ }
+
+ TVariantTypePtr TTupleBuilderRaw::BuildVariant() {
+ return BuildVariantRaw()->AsPtr();
+ }
+
+ const TVariantType* TTupleBuilderRaw::BuildVariantRaw() {
+ return Factory_->New<TVariantType>(Nothing(), Name_, DoBuildRaw(Nothing()));
+ }
+
+ const TTupleType* TTupleBuilderRaw::DoBuildRaw(TMaybe<TStringBuf> name) {
+ auto items = Factory_->NewArray<TTupleType::TElement>(Elements_.size(), [this](TTupleType::TElement* element, size_t i) {
+ new (element) TTupleType::TElement(Elements_[i]);
+ });
+
+ return Factory_->New<TTupleType>(Nothing(), name, items);
+ }
+
+ TTaggedBuilderRaw::TTaggedBuilderRaw(IPoolTypeFactory& factory) noexcept
+ : Factory_(&factory)
+ {
+ }
+
+ TTaggedBuilderRaw& TTaggedBuilderRaw::SetTag(TStringBuf tag) & noexcept {
+ Tag_ = Factory_->AllocateString(tag);
+ return *this;
+ }
+
+ TTaggedBuilderRaw TTaggedBuilderRaw::SetTag(TStringBuf tag) && noexcept {
+ return std::move(SetTag(tag));
+ }
+
+ TTaggedBuilderRaw& TTaggedBuilderRaw::DiscardTag() & noexcept {
+ Tag_.Clear();
+ return *this;
+ }
+
+ TTaggedBuilderRaw TTaggedBuilderRaw::DiscardTag() && noexcept {
+ return std::move(DiscardTag());
+ }
+
+ bool TTaggedBuilderRaw::HasTag() const noexcept {
+ return Tag_.Defined();
+ }
+
+ TMaybe<TStringBuf> TTaggedBuilderRaw::GetTag() const noexcept {
+ return Tag_;
+ }
+
+ TTaggedBuilderRaw& TTaggedBuilderRaw::SetItem(TTypePtr type) & noexcept {
+ return SetItem(type.Get());
+ }
+
+ TTaggedBuilderRaw TTaggedBuilderRaw::SetItem(TTypePtr type) && noexcept {
+ return std::move(SetItem(std::move(type)));
+ }
+
+ TTaggedBuilderRaw& TTaggedBuilderRaw::SetItem(const TType* type) & noexcept {
+ Item_ = Factory_->Own(type);
+ return *this;
+ }
+
+ TTaggedBuilderRaw TTaggedBuilderRaw::SetItem(const TType* type) && noexcept {
+ return std::move(SetItem(type));
+ }
+
+ TTaggedBuilderRaw& TTaggedBuilderRaw::DiscardItem() & noexcept {
+ Item_.Clear();
+ return *this;
+ }
+
+ TTaggedBuilderRaw TTaggedBuilderRaw::DiscardItem() && noexcept {
+ return std::move(DiscardItem());
+ }
+
+ bool TTaggedBuilderRaw::HasItem() const noexcept {
+ return Item_.Defined();
+ }
+
+ TMaybe<const TType*> TTaggedBuilderRaw::GetItem() const noexcept {
+ return Item_;
+ }
+
+ bool TTaggedBuilderRaw::CanBuild() const noexcept {
+ return HasTag() && HasItem();
+ }
+
+ TTaggedBuilderRaw& TTaggedBuilderRaw::Reset() & noexcept {
+ DiscardTag();
+ DiscardItem();
+ return *this;
+ }
+
+ TTaggedBuilderRaw TTaggedBuilderRaw::Reset() && noexcept {
+ return std::move(Reset());
+ }
+
+ TTaggedTypePtr TTaggedBuilderRaw::Build() {
+ return BuildRaw()->AsPtr();
+ }
+
+ const TTaggedType* TTaggedBuilderRaw::BuildRaw() {
+ Y_VERIFY(CanBuild());
+ return Factory_->New<TTaggedType>(Nothing(), *Item_, *Tag_);
+ }
+}
diff --git a/library/cpp/type_info/builder.h b/library/cpp/type_info/builder.h
new file mode 100644
index 0000000000..faae45ab51
--- /dev/null
+++ b/library/cpp/type_info/builder.h
@@ -0,0 +1,346 @@
+#pragma once
+
+//! @file builder.h
+//!
+//! Builders help with creating complex types piece-by-piece.
+
+#include <util/generic/vector.h>
+
+#include "type.h"
+
+#include "fwd.h"
+
+namespace NTi {
+ /// An interface for building structs using the raw interface (via memory-pool-based factory).
+ ///
+ /// This interface allows allocating structs piece-by-piece. You can feed it struct items, one-by-one, and they'll
+ /// be copied into the memory pool before the struct is created. This way, you don't have to allocate heap memory
+ /// to temporarily store item names or types.
+ ///
+ /// Note: this builder doesn't own the underlying factory.
+ class TStructBuilderRaw {
+ public:
+ /// Create a new builder with the given factory.
+ TStructBuilderRaw(IPoolTypeFactory& factory) noexcept;
+
+ /// Create a new builder with the given factory and add fields from the given struct.
+ /// Note that struct name is not copied to the builder.
+ TStructBuilderRaw(IPoolTypeFactory& factory, TStructTypePtr prototype) noexcept;
+ TStructBuilderRaw(IPoolTypeFactory& factory, const TStructType* prototype) noexcept;
+
+ /// Set a new struct name.
+ ///
+ /// Note that the name is copied to the factory right away. If you call this function twice, second call will
+ /// overwrite name from the first call, but will not remove it from the memory pool.
+ TStructBuilderRaw& SetName(TMaybe<TStringBuf> name) & noexcept;
+ TStructBuilderRaw SetName(TMaybe<TStringBuf> name) && noexcept;
+
+ /// Check if there's a struct name set.
+ bool HasName() const noexcept;
+
+ /// Get a struct name.
+ ///
+ /// Name was copied to the factory's memory pool and will live as long as the factory lives.
+ TMaybe<TStringBuf> GetName() const noexcept;
+
+ /// Reserve some place in the underlying vector that collects struct items.
+ TStructBuilderRaw& Reserve(size_t size) & noexcept;
+ TStructBuilderRaw Reserve(size_t size) && noexcept;
+
+ /// Append a new struct member.
+ TStructBuilderRaw& AddMember(TStringBuf name, TTypePtr type) & noexcept;
+ TStructBuilderRaw AddMember(TStringBuf name, TTypePtr type) && noexcept;
+ TStructBuilderRaw& AddMember(TStringBuf name, const TType* type) & noexcept;
+ TStructBuilderRaw AddMember(TStringBuf name, const TType* type) && noexcept;
+
+ /// Partial member creation interface.
+ ///
+ /// This interface allows building individual struct items piece-by-piece. You can pass member's name
+ /// and copy it to the pool without passing its type. For example:
+ ///
+ /// ```
+ /// auto builder = TStructBuilderRaw(factory);
+ /// builder.AddMemberName("name"); // add name for a new member.
+ /// builder.AddMemberType(type); // add type for a new member.
+ /// builder.AddMember(); // use added name and type to construct and append an member.
+ /// ```
+ ///
+ /// This interface is useful when you can't store member name for a long time. For example, if you building
+ /// a parser, at some point have an member name, but you don't have an member type yet. Then you can add member name
+ /// when you have it, and add member type later.
+ ///
+ /// @{
+ //-
+ /// Set name for pending member.
+ ///
+ /// Note that the name is copied to the factory right away. If you call this function twice, second call will
+ /// overwrite name from the first call, but will not remove it from the memory pool.
+ TStructBuilderRaw& AddMemberName(TStringBuf name) & noexcept;
+ TStructBuilderRaw AddMemberName(TStringBuf name) && noexcept;
+
+ /// Unset name for pending member.
+ ///
+ /// Note that member name is copied to the factory right away. If you discard it via this method,
+ /// it will not be removed from the memory pool.
+ TStructBuilderRaw& DiscardMemberName() & noexcept;
+ TStructBuilderRaw DiscardMemberName() && noexcept;
+
+ /// Check if there's a name set for pending member.
+ bool HasMemberName() const noexcept;
+
+ /// Get name for pending member.
+ ///
+ /// Name was copied to the factory's memory pool and will live as long as the factory lives.
+ TMaybe<TStringBuf> GetMemberName() const noexcept;
+
+ /// Set type for pending member.
+ ///
+ /// Note that the type is copied to the factory right away. If you call this function twice, second call will
+ /// overwrite type from the first call, but will not remove it from the memory pool.
+ TStructBuilderRaw& AddMemberType(TTypePtr type) & noexcept;
+ TStructBuilderRaw AddMemberType(TTypePtr type) && noexcept;
+ TStructBuilderRaw& AddMemberType(const TType* type) & noexcept;
+ TStructBuilderRaw AddMemberType(const TType* type) && noexcept;
+
+ /// Unset type for pending member.
+ ///
+ /// Note that member type is copied to the factory right away. If you discard it via this method,
+ /// it will not be removed from the memory pool.
+ TStructBuilderRaw& DiscardMemberType() & noexcept;
+ TStructBuilderRaw DiscardMemberType() && noexcept;
+
+ /// Check if there's a type set for pending member.
+ bool HasMemberType() const noexcept;
+
+ /// Get type for pending member.
+ TMaybe<const TType*> GetMemberType() const noexcept;
+
+ /// Check if both name and type are set for pending member.
+ bool CanAddMember() const noexcept;
+
+ /// Use data added via `AddMemberName` and `AddMemberType` to construct a new struct member
+ /// and append it to this builder. This function panics if there's no name or no type set for pending member.
+ TStructBuilderRaw& AddMember() & noexcept;
+ TStructBuilderRaw AddMember() && noexcept;
+
+ /// Discard all data added via `AddMemberName` and `AddMemberType` functions.
+ ///
+ /// Note that member name and type are copied to the factory right away. If you discard them via this method,
+ /// they will not be removed from the memory pool.
+ TStructBuilderRaw& DiscardMember() & noexcept;
+ TStructBuilderRaw DiscardMember() && noexcept;
+
+ /// @}
+
+ /// Get immutable list of items that've been added to this builder.
+ /// The returned object is invalidated when this builder dies or when `AddMember` is called.
+ TStructType::TMembers GetMembers() const noexcept;
+
+ /// Reset struct name and items.
+ TStructBuilderRaw& Reset() & noexcept;
+ TStructBuilderRaw Reset() && noexcept;
+
+ /// Create a new struct type using name and items from this builder.
+ TStructTypePtr Build();
+
+ /// Like `Build`, but returns a raw pointer.
+ const TStructType* BuildRaw();
+
+ /// Create a new variant over struct using items from this builder.
+ TVariantTypePtr BuildVariant();
+
+ /// Like `BuildVariant`, but returns a raw pointer.
+ const TVariantType* BuildVariantRaw();
+
+ private:
+ const TStructType* DoBuildRaw(TMaybe<TStringBuf> name);
+
+ private:
+ IPoolTypeFactory* Factory_;
+ TMaybe<TStringBuf> Name_;
+ TVector<TStructType::TMember> Members_;
+ TMaybe<TStringBuf> PendingMemberName_;
+ TMaybe<const TType*> PendingMemberType_;
+ };
+
+ /// An interface for building tuples using the raw interface (via memory-pool-based factory).
+ ///
+ /// Note: this builder doesn't own the underlying factory.
+ class TTupleBuilderRaw {
+ public:
+ /// Create a new builder with the given factory.
+ TTupleBuilderRaw(IPoolTypeFactory& factory) noexcept;
+
+ /// Create a new builder with the given factory and add fields from the given tuple.
+ /// Note that tuple name is not copied to the builder.
+ TTupleBuilderRaw(IPoolTypeFactory& factory, TTupleTypePtr prototype) noexcept;
+ TTupleBuilderRaw(IPoolTypeFactory& factory, const TTupleType* prototype) noexcept;
+
+ /// Set a new tuple name.
+ /// Note that the name is copied to the factory right away. If you call this function twice, second call will
+ /// overwrite name from the first call, but will not remove it from the memory pool.
+ TTupleBuilderRaw& SetName(TMaybe<TStringBuf> name) & noexcept;
+ TTupleBuilderRaw SetName(TMaybe<TStringBuf> name) && noexcept;
+
+ /// Check if there's a tuple name set.
+ bool HasName() const noexcept;
+
+ /// Get a tuple name.
+ ///
+ /// Name was copied to the factory's memory pool and will live as long as the factory lives.
+ TMaybe<TStringBuf> GetName() const noexcept;
+
+ /// Reserve some place in the underlying vector that collects tuple items.
+ TTupleBuilderRaw& Reserve(size_t size) & noexcept;
+ TTupleBuilderRaw Reserve(size_t size) && noexcept;
+
+ /// Append a new tuple item.
+ TTupleBuilderRaw& AddElement(TTypePtr type) & noexcept;
+ TTupleBuilderRaw AddElement(TTypePtr type) && noexcept;
+ TTupleBuilderRaw& AddElement(const TType* type) & noexcept;
+ TTupleBuilderRaw AddElement(const TType* type) && noexcept;
+
+ /// Partial item creation interface.
+ ///
+ /// This interface allows building individual tuple items piece-by-piece. It mirrors the same type of interface
+ /// in the struct builder.
+ ///
+ /// @{
+ //-
+ /// Set type for pending item.
+ ///
+ /// Note that the type is copied to the factory right away. If you call this function twice, second call will
+ /// overwrite type from the first call, but will not remove it from the memory pool.
+ TTupleBuilderRaw& AddElementType(TTypePtr type) & noexcept;
+ TTupleBuilderRaw AddElementType(TTypePtr type) && noexcept;
+ TTupleBuilderRaw& AddElementType(const TType* type) & noexcept;
+ TTupleBuilderRaw AddElementType(const TType* type) && noexcept;
+
+ /// Unset type for pending item.
+ ///
+ /// Note that item type is copied to the factory right away. If you discard it via this method,
+ /// it will not be removed from the memory pool.
+ TTupleBuilderRaw& DiscardElementType() & noexcept;
+ TTupleBuilderRaw DiscardElementType() && noexcept;
+
+ /// Check if there's a type set for pending item.
+ bool HasElementType() const noexcept;
+
+ /// Get type for pending item.
+ TMaybe<const TType*> GetElementType() const noexcept;
+
+ /// Check if type is set for pending item.
+ bool CanAddElement() const noexcept;
+
+ /// Use data added via `AddElementType` to construct a new tuple item and append it to this builder.
+ /// This function panics if there's no type set for pending item.
+ TTupleBuilderRaw& AddElement() & noexcept;
+ TTupleBuilderRaw AddElement() && noexcept;
+
+ /// Discard all data added via `AddElementType` function.
+ ///
+ /// Note that item type is copied to the factory right away. If you discard it via this method,
+ /// it will not be removed from the memory pool.
+ TTupleBuilderRaw& DiscardElement() & noexcept;
+ TTupleBuilderRaw DiscardElement() && noexcept;
+
+ /// @}
+
+ /// Get immutable list of items that've been added to this builder.
+ /// The returned object is invalidated when this builder dies or when `AddElement` is called.
+ TTupleType::TElements GetElements() const noexcept;
+
+ /// Reset tuple name and items.
+ TTupleBuilderRaw& Reset() & noexcept;
+ TTupleBuilderRaw Reset() && noexcept;
+
+ /// Create a new tuple type using name and items from this builder.
+ TTupleTypePtr Build();
+
+ /// Like `Build`, but returns a raw pointer.
+ const TTupleType* BuildRaw();
+
+ /// Create a new variant over tuple using items from this builder.
+ TVariantTypePtr BuildVariant();
+
+ /// Like `BuildVariant`, but returns a raw pointer.
+ const TVariantType* BuildVariantRaw();
+
+ private:
+ const TTupleType* DoBuildRaw(TMaybe<TStringBuf> name);
+
+ private:
+ IPoolTypeFactory* Factory_;
+ TMaybe<TStringBuf> Name_;
+ TVector<TTupleType::TElement> Elements_;
+ TMaybe<const TType*> PendingElementType_;
+ };
+
+ /// An interface for building tagged types using the raw interface (via memory-pool-based factory).
+ ///
+ /// Note: this builder doesn't own the underlying factory.
+ class TTaggedBuilderRaw {
+ public:
+ /// Create a new builder with the given factory.
+ TTaggedBuilderRaw(IPoolTypeFactory& factory) noexcept;
+
+ /// Set a new tag.
+ ///
+ /// Note that the tag is copied to the factory right away. If you call this function twice, second call will
+ /// overwrite tag from the first call, but will not remove it from the memory pool.
+ TTaggedBuilderRaw& SetTag(TStringBuf tag) & noexcept;
+ TTaggedBuilderRaw SetTag(TStringBuf tag) && noexcept;
+
+ /// Unset tag.
+ ///
+ /// Note that tag is copied to the factory right away. If you discard it via this method,
+ /// it will not be removed from the memory pool.
+ TTaggedBuilderRaw& DiscardTag() & noexcept;
+ TTaggedBuilderRaw DiscardTag() && noexcept;
+
+ /// Check if a tag is set.
+ bool HasTag() const noexcept;
+
+ /// Get a tag.
+ ///
+ /// The tag was copied to the factory's memory pool and will live as long as the factory lives.
+ TMaybe<TStringBuf> GetTag() const noexcept;
+
+ /// Set type that's being tagged.
+ TTaggedBuilderRaw& SetItem(TTypePtr type) & noexcept;
+ TTaggedBuilderRaw SetItem(TTypePtr type) && noexcept;
+ TTaggedBuilderRaw& SetItem(const TType* type) & noexcept;
+ TTaggedBuilderRaw SetItem(const TType* type) && noexcept;
+
+ /// Unset item type.
+ ///
+ /// Note that item type is copied to the factory right away. If you discard it via this method,
+ /// it will not be removed from the memory pool.
+ TTaggedBuilderRaw& DiscardItem() & noexcept;
+ TTaggedBuilderRaw DiscardItem() && noexcept;
+
+ /// Check if there's an item type set.
+ bool HasItem() const noexcept;
+
+ /// Get item type.
+ TMaybe<const TType*> GetItem() const noexcept;
+
+ /// Check if there's both name and item type set.
+ bool CanBuild() const noexcept;
+
+ /// Discard both tag and item.
+ TTaggedBuilderRaw& Reset() & noexcept;
+ TTaggedBuilderRaw Reset() && noexcept;
+
+ /// Create a new tagged type using name and item from this builder.
+ TTaggedTypePtr Build();
+
+ /// Like `Build`, but returns a raw pointer.
+ const TTaggedType* BuildRaw();
+
+ private:
+ IPoolTypeFactory* Factory_;
+ TMaybe<TStringBuf> Tag_;
+ TMaybe<const TType*> Item_;
+ };
+}
diff --git a/library/cpp/type_info/error.cpp b/library/cpp/type_info/error.cpp
new file mode 100644
index 0000000000..fe9ecf1d9f
--- /dev/null
+++ b/library/cpp/type_info/error.cpp
@@ -0,0 +1 @@
+#include "error.h"
diff --git a/library/cpp/type_info/error.h b/library/cpp/type_info/error.h
new file mode 100644
index 0000000000..26fd8853f9
--- /dev/null
+++ b/library/cpp/type_info/error.h
@@ -0,0 +1,33 @@
+#pragma once
+
+//! @file error.h
+//!
+//! All error classes that one can encounter when working with type info library.
+
+#include <util/generic/yexception.h>
+
+namespace NTi {
+ /// Base class for all exceptions that arise when working with Type Info library.
+ class TException: public yexception {
+ };
+
+ /// Type Info API used in an unintended way.
+ class TApiException: public TException {
+ };
+
+ /// Attempting to create an illegal type.
+ ///
+ /// For example, this exception is raised when attempting to create a struct with non-unique item names.
+ class TIllegalTypeException: public TException {
+ };
+
+ /// Type deserializer got an invalid input.
+ ///
+ /// See `TType::Serialize()` and `TType::Deserialize()` for more info on type serialization/deserialization.
+ class TDeserializationException: public TException {
+ };
+
+ /// No such item in type.
+ class TItemNotFound: public TException {
+ };
+}
diff --git a/library/cpp/type_info/fwd.h b/library/cpp/type_info/fwd.h
new file mode 100644
index 0000000000..644932f677
--- /dev/null
+++ b/library/cpp/type_info/fwd.h
@@ -0,0 +1,123 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+
+namespace NTi {
+ class ITypeFactoryInternal;
+
+ class TNamedTypeBuilderRaw;
+ class TStructBuilderRaw;
+ class TTupleBuilderRaw;
+ class TTaggedBuilderRaw;
+
+ class ITypeFactory;
+ using ITypeFactoryPtr = TIntrusivePtr<ITypeFactory>;
+
+ class IPoolTypeFactory;
+ using IPoolTypeFactoryPtr = TIntrusivePtr<IPoolTypeFactory>;
+
+ class TType;
+ using TTypePtr = TIntrusiveConstPtr<TType>;
+
+ class TVoidType;
+ using TVoidTypePtr = TIntrusiveConstPtr<TVoidType>;
+
+ class TNullType;
+ using TNullTypePtr = TIntrusiveConstPtr<TNullType>;
+
+ class TPrimitiveType;
+ using TPrimitiveTypePtr = TIntrusiveConstPtr<TPrimitiveType>;
+
+ class TBoolType;
+ using TBoolTypePtr = TIntrusiveConstPtr<TBoolType>;
+
+ class TInt8Type;
+ using TInt8TypePtr = TIntrusiveConstPtr<TInt8Type>;
+
+ class TInt16Type;
+ using TInt16TypePtr = TIntrusiveConstPtr<TInt16Type>;
+
+ class TInt32Type;
+ using TInt32TypePtr = TIntrusiveConstPtr<TInt32Type>;
+
+ class TInt64Type;
+ using TInt64TypePtr = TIntrusiveConstPtr<TInt64Type>;
+
+ class TUint8Type;
+ using TUint8TypePtr = TIntrusiveConstPtr<TUint8Type>;
+
+ class TUint16Type;
+ using TUint16TypePtr = TIntrusiveConstPtr<TUint16Type>;
+
+ class TUint32Type;
+ using TUint32TypePtr = TIntrusiveConstPtr<TUint32Type>;
+
+ class TUint64Type;
+ using TUint64TypePtr = TIntrusiveConstPtr<TUint64Type>;
+
+ class TFloatType;
+ using TFloatTypePtr = TIntrusiveConstPtr<TFloatType>;
+
+ class TDoubleType;
+ using TDoubleTypePtr = TIntrusiveConstPtr<TDoubleType>;
+
+ class TStringType;
+ using TStringTypePtr = TIntrusiveConstPtr<TStringType>;
+
+ class TUtf8Type;
+ using TUtf8TypePtr = TIntrusiveConstPtr<TUtf8Type>;
+
+ class TDateType;
+ using TDateTypePtr = TIntrusiveConstPtr<TDateType>;
+
+ class TDatetimeType;
+ using TDatetimeTypePtr = TIntrusiveConstPtr<TDatetimeType>;
+
+ class TTimestampType;
+ using TTimestampTypePtr = TIntrusiveConstPtr<TTimestampType>;
+
+ class TTzDateType;
+ using TTzDateTypePtr = TIntrusiveConstPtr<TTzDateType>;
+
+ class TTzDatetimeType;
+ using TTzDatetimeTypePtr = TIntrusiveConstPtr<TTzDatetimeType>;
+
+ class TTzTimestampType;
+ using TTzTimestampTypePtr = TIntrusiveConstPtr<TTzTimestampType>;
+
+ class TIntervalType;
+ using TIntervalTypePtr = TIntrusiveConstPtr<TIntervalType>;
+
+ class TDecimalType;
+ using TDecimalTypePtr = TIntrusiveConstPtr<TDecimalType>;
+
+ class TJsonType;
+ using TJsonTypePtr = TIntrusiveConstPtr<TJsonType>;
+
+ class TYsonType;
+ using TYsonTypePtr = TIntrusiveConstPtr<TYsonType>;
+
+ class TUuidType;
+ using TUuidTypePtr = TIntrusiveConstPtr<TUuidType>;
+
+ class TOptionalType;
+ using TOptionalTypePtr = TIntrusiveConstPtr<TOptionalType>;
+
+ class TListType;
+ using TListTypePtr = TIntrusiveConstPtr<TListType>;
+
+ class TDictType;
+ using TDictTypePtr = TIntrusiveConstPtr<TDictType>;
+
+ class TStructType;
+ using TStructTypePtr = TIntrusiveConstPtr<TStructType>;
+
+ class TTupleType;
+ using TTupleTypePtr = TIntrusiveConstPtr<TTupleType>;
+
+ class TVariantType;
+ using TVariantTypePtr = TIntrusiveConstPtr<TVariantType>;
+
+ class TTaggedType;
+ using TTaggedTypePtr = TIntrusiveConstPtr<TTaggedType>;
+}
diff --git a/library/cpp/type_info/test-data/bad-types.txt b/library/cpp/type_info/test-data/bad-types.txt
new file mode 100644
index 0000000000..c8f3120f63
--- /dev/null
+++ b/library/cpp/type_info/test-data/bad-types.txt
@@ -0,0 +1,229 @@
+#
+# The format of this file is described in README.txt
+#
+# Each test case contains 3 fields:
+# - Yson representation (that cannot be parsed to type).
+# - Expected error description.
+# - Path in yson that we expect to see in error.
+#
+# Suggested approach to writing test:
+# 1. Try to deserialize type from yson (1st field).
+# 2. Ensure that error is raised.
+# 3. (Optionally) ensure that error description matches (contains) 2nd field.
+# 4. (Optionally) ensure that error description contains path from 3rd field
+
+5 :: type must be either a string or a map :: ;;
+
+"" :: unknown type "" :: ;;
+
+"int" :: unknown type "int" :: ;;
+
+# Type names must be in lowercase.
+"Int32" :: unknown type "Int32" :: ;;
+
+{typename=int32} :: missing required key "type_name" :: ;;
+
+{type_name=5} :: "type_name" must contain a string :: ;;
+
+#
+# Decimal
+#
+
+{
+ type_name=decimal;
+ precision=3;
+} :: missing required key "scale" :: ;;
+
+{
+ type_name=decimal;
+ scale=3;
+} :: missing required key "precision" :: ;;
+
+#
+# Optional
+#
+
+{
+ type_name=optional;
+} :: missing required key "item" :: ;;
+
+{
+ item=int32;
+} :: missing required key "type_name" :: ;;
+
+#
+# List
+#
+
+{
+ type_name=list;
+} :: missing required key "item" :: ;;
+
+#
+# Struct
+#
+
+{
+ type_name=struct;
+} :: missing required key "members" :: ;;
+
+{
+ type_name=struct;
+ members=5;
+} :: "members" must contain a list :: ;;
+
+{
+ type_name=struct;
+ members=[
+ {name=foo; type=int32;};
+ 5;
+ ];
+} :: "members" must contain a list of maps :: ;;
+
+{
+ type_name=struct;
+ members=[
+ {type=int32;};
+ ];
+} :: missing required key "name" :: ;;
+
+{
+ type_name=struct;
+ members=[
+ {name=foo;};
+ ];
+} :: missing required key "type" :: ;;
+
+{
+ type_name=struct;
+ members=[
+ {name=foo; type=int32};
+ {name=bar; type=5};
+ ];
+} :: type must be either a string or a map :: /members/1/type ;;
+
+{
+ type_name=struct;
+ members=[
+ {name=4; type=int32};
+ ];
+} :: "name" must contain a string :: ;;
+
+{
+ type_name=struct;
+ elements=[
+ {name=4; type=int32};
+ ];
+} :: missing required key "members" :: ;;
+
+#
+# Tuple
+#
+
+{
+ type_name=tuple;
+} :: missing required key "elements" :: ;;
+
+{
+ type_name=tuple;
+ elements=5;
+} :: "elements" must contain a list :: ;;
+
+{
+ type_name=tuple;
+ elements=[
+ {type=int32;};
+ 5;
+ ];
+} :: "elements" must contain a list of maps :: ;;
+
+{
+ type_name=tuple;
+ elements=[
+ {};
+ ];
+} :: missing required key "type" :: /elements/0 ;;
+
+{
+ type_name=tuple;
+ elements=[
+ {type=5};
+ ];
+} :: type must be either a string or a map :: /elements/1/type ;;
+
+{
+ type_name=tuple;
+ members=[
+ {name=foo; type=int32};
+ ];
+} :: missing required key "elements" :: ;;
+
+#
+# Variant
+#
+
+# We don't specify exception message here because in C++ library that message is not good and we don't want to canonize it
+# Though fixing it is not that easy too.
+{
+ type_name=variant;
+ members=[
+ {name=foo; type=int32};
+ ];
+ elements=[
+ {type=int8};
+ ]
+} :: :: ;;
+
+{
+ type_name=variant;
+} :: missing both keys "members" and "elements" :: ;;
+
+#
+# Dict
+#
+{
+ type_name=dict;
+ key=string;
+} :: missing required key "value" :: ;;
+
+{
+ type_name=dict;
+ value=string;
+} :: missing required key "key" :: ;;
+
+{
+ type_name=dict;
+ value=string;
+} :: missing required key "key" :: ;;
+
+{
+ type_name=dict;
+ key=5;
+ value=string;
+} :: type must be either a string or a map :: /key ;;
+
+{
+ type_name=dict;
+ key=string;
+ value=5;
+} :: type must be either a string or a map :: /value ;;
+
+#
+# Tagged
+#
+
+{
+ type_name=tagged;
+ item=string;
+} :: missing required key "tag" :: ;;
+
+{
+ type_name=tagged;
+ tag=string;
+} :: missing required key "item" :: ;;
+
+{
+ type_name=tagged;
+ tag=5;
+ item=string;
+} :: "tag" must contain a string :: /tag ;;
diff --git a/library/cpp/type_info/test-data/good-types.txt b/library/cpp/type_info/test-data/good-types.txt
new file mode 100644
index 0000000000..cb082707b6
--- /dev/null
+++ b/library/cpp/type_info/test-data/good-types.txt
@@ -0,0 +1,478 @@
+#
+# The format of this file is described in README.txt
+#
+# Each test case contains 2 fields:
+# - Yson reporesentation of the type.
+# - String representation of the type.
+#
+# Suggested approach to writing test:
+# 1. Try to deserialize type from yson (1st field).
+# 2. Check that text representation of parsed type matches 2nd field.
+# 3. Serialize type to yson and deserialize it from it.
+# 4. Check that typef from 1. and 3. are equal.
+
+# Integer
+int8 :: Int8 ;;
+int16 :: Int16 ;;
+int32 :: Int32 ;;
+int64 :: Int64 ;;
+{type_name=int8} :: Int8 ;;
+{type_name=int16} :: Int16 ;;
+{type_name=int32} :: Int32 ;;
+{type_name=int64} :: Int64 ;;
+
+# Unsigned integer
+uint8 :: Uint8 ;;
+uint16 :: Uint16 ;;
+uint32 :: Uint32 ;;
+uint64 :: Uint64 ;;
+{type_name=uint8} :: Uint8 ;;
+{type_name=uint16} :: Uint16 ;;
+{type_name=uint32} :: Uint32 ;;
+{type_name=uint64} :: Uint64 ;;
+
+# Floating
+float :: Float ;;
+double :: Double ;;
+{type_name=float} :: Float ;;
+{type_name=double} :: Double ;;
+
+# Strings
+string :: String ;;
+utf8 :: Utf8 ;;
+{type_name=string} :: String ;;
+{type_name=utf8} :: Utf8 ;;
+
+# Time
+date :: Date ;;
+datetime :: Datetime ;;
+timestamp :: Timestamp ;;
+tz_date :: TzDate ;;
+tz_datetime :: TzDatetime ;;
+tz_timestamp :: TzTimestamp ;;
+interval :: Interval ;;
+{type_name=date} :: Date ;;
+{type_name=datetime} :: Datetime ;;
+{type_name=timestamp} :: Timestamp ;;
+{type_name=tz_date} :: TzDate ;;
+{type_name=tz_datetime} :: TzDatetime ;;
+{type_name=tz_timestamp} :: TzTimestamp ;;
+{type_name=interval} :: Interval ;;
+
+# Singular
+void :: Void ;;
+null :: Null ;;
+{type_name=void} :: Void ;;
+{type_name=null} :: Null ;;
+
+# UUID
+uuid :: Uuid ;;
+{type_name=uuid} :: Uuid ;;
+
+# Json / Yson
+yson :: Yson ;;
+json :: Json ;;
+{type_name=yson} :: Yson ;;
+{type_name=json} :: Json ;;
+
+{
+ type_name=string;
+ unknown_key=bar;
+} :: String ;;
+
+# Decimal
+{
+ type_name=decimal;
+ precision=3;
+ scale=2;
+} :: Decimal(3, 2) ;;
+
+{
+ type_name=decimal;
+ precision=3;
+ scale=2;
+ unknown_column=ha;
+} :: Decimal(3, 2) ;;
+
+#
+# Optional
+#
+{
+ type_name=optional;
+ item=string;
+} :: Optional<String> ;;
+
+{
+ type_name=optional;
+ item={
+ type_name=string;
+ }
+} :: Optional<String> ;;
+
+{
+ type_name=optional;
+ item={
+ type_name=optional;
+ item={
+ type_name=list;
+ item={
+ type_name=decimal;
+ precision=10;
+ scale=5;
+ }
+
+ }
+ }
+} :: Optional<Optional<List<Decimal(10, 5)>>> ;;
+
+{
+ type_name=optional;
+ item={
+ type_name=tagged;
+ item=int32;
+ tag="foo";
+ };
+} :: Optional<Tagged<Int32, 'foo'>> ;;
+
+{
+ type_name=optional;
+ item={
+ type_name=string;
+ };
+ unknown_column=ha;
+} :: Optional<String> ;;
+
+#
+# List
+#
+{
+ type_name=list;
+ item=string;
+} :: List<String> ;;
+
+{
+ type_name=list;
+ item={
+ type_name=string;
+ }
+} :: List<String> ;;
+
+{
+ type_name=list;
+ item={
+ type_name=list;
+ item={
+ type_name=optional;
+ item=string;
+ }
+ };
+} :: List<List<Optional<String>>> ;;
+
+{
+ type_name=list;
+ item={
+ type_name=tagged;
+ item=int32;
+ tag="foo";
+ };
+} :: List<Tagged<Int32, 'foo'>> ;;
+
+{
+ type_name=list;
+ item={
+ type_name=string;
+ };
+ unknown_column=ha;
+} :: List<String> ;;
+
+#
+# Dict
+#
+
+{
+ type_name=dict;
+ key=int32;
+ value=string;
+} :: Dict<Int32, String> ;;
+
+{
+ type_name=dict;
+ key={
+ type_name=optional;
+ item=string;
+ };
+ value={
+ type_name=list;
+ item={
+ type_name=int32;
+ }
+ };
+} :: Dict<Optional<String>, List<Int32>> ;;
+
+{
+ type_name=dict;
+ key={
+ type_name=tagged;
+ item=int32;
+ tag="foo";
+ };
+ value={
+ type_name=tagged;
+ item=string;
+ tag="bar";
+ };
+} :: Dict<Tagged<Int32, 'foo'>, Tagged<String, 'bar'>> ;;
+
+{
+ type_name=dict;
+ key=int32;
+ value=string;
+ unknown_column=ha;
+} :: Dict<Int32, String> ;;
+
+#
+# Struct
+#
+
+{
+ type_name=struct;
+ members=[];
+} :: Struct<> ;;
+
+{
+ type_name=struct;
+ members=[
+ {
+ name=foo;
+ type=int32;
+ };
+ {
+ name=bar;
+ type={
+ type_name=optional;
+ item=string;
+ };
+ };
+ ];
+} :: Struct<'foo': Int32, 'bar': Optional<String>> ;;
+
+{
+ type_name=struct;
+ members=[
+ {
+ name=foo;
+ type={
+ type_name=tagged;
+ item=string;
+ tag=foo;
+ };
+ };
+ ];
+} :: Struct<'foo': Tagged<String, 'foo'>> ;;
+
+{
+ type_name=struct;
+ unknown_column=ha;
+ members=[
+ {
+ unknown_column=ha;
+ name=foo;
+ type=int32;
+ };
+ {
+ name=bar;
+ type={
+ type_name=optional;
+ item=string;
+ };
+ };
+ ];
+} :: Struct<'foo': Int32, 'bar': Optional<String>> ;;
+
+#
+# Tuple
+#
+
+{
+ type_name=tuple;
+ elements=[];
+} :: Tuple<> ;;
+
+{
+ type_name=tuple;
+ elements=[
+ {
+ type=int32;
+ };
+ {
+ type={
+ type_name=optional;
+ item=string;
+ };
+ };
+ ];
+} :: Tuple<Int32, Optional<String>> ;;
+
+{
+ type_name=tuple;
+ elements=[
+ {
+ type={
+ type_name=tagged;
+ item=string;
+ tag=foo;
+ };
+ };
+ ];
+} :: Tuple<Tagged<String, 'foo'>> ;;
+
+{
+ type_name=tuple;
+ unknown_column=ha;
+ elements=[
+ {
+ type=int32;
+ unknown_column=ha;
+ };
+ {
+ type={
+ type_name=optional;
+ item=string;
+ };
+ };
+ ];
+} :: Tuple<Int32, Optional<String>> ;;
+
+#
+# Variant
+#
+
+{
+ type_name=variant;
+ elements=[
+ {
+ type=int32;
+ };
+ {
+ type={type_name=string};
+ };
+ ];
+} :: Variant<Int32, String> ;;
+
+{
+ type_name=variant;
+ members=[
+ {
+ name=foo;
+ type=int32;
+ };
+ {
+ name=bar;
+ type={
+ type_name=optional;
+ item=string;
+ };
+ };
+ ];
+} :: Variant<'foo': Int32, 'bar': Optional<String>> ;;
+
+{
+ type_name=variant;
+ elements=[
+ {
+ type={
+ type_name=tagged;
+ item=string;
+ tag=foo;
+ }
+ };
+ ];
+} :: Variant<Tagged<String, 'foo'>> ;;
+
+{
+ type_name=variant;
+ members=[
+ {
+ name=bar;
+ type={
+ type_name=tagged;
+ item=string;
+ tag=foo;
+ };
+ };
+ ];
+} :: Variant<'bar': Tagged<String, 'foo'>> ;;
+
+{
+ type_name=variant;
+ unknown_column=ha;
+ elements=[
+ {
+ type=int32;
+ unknown_column=ha;
+ };
+ {
+ type={
+ type_name=string;
+ unknown_column=ha;
+ };
+ unknown_column=ha;
+ };
+ ];
+} :: Variant<Int32, String> ;;
+
+{
+ type_name=variant;
+ unknown_column=ha;
+ members=[
+ {
+ unknown_column=ha;
+ name=foo;
+ type=int32;
+ };
+ {
+ name=bar;
+ type={
+ type_name=optional;
+ unknown_column=ha;
+ item=string;
+ };
+ };
+ ];
+} :: Variant<'foo': Int32, 'bar': Optional<String>> ;;
+
+#
+# Tagged
+#
+
+{
+ type_name=tagged;
+ item=string;
+ tag="image/png"
+} :: Tagged<String, 'image/png'> ;;
+
+{
+ type_name=tagged;
+ item={
+ type_name=optional;
+ item=string;
+ };
+ tag="image/png"
+} :: Tagged<Optional<String>, 'image/png'> ;;
+
+{
+ type_name=tagged;
+ item={
+ type_name=tagged;
+ item=string;
+ tag=foo;
+ };
+ tag=bar;
+} :: Tagged<Tagged<String, 'foo'>, 'bar'> ;;
+
+{
+ type_name=tagged;
+ item=string;
+ unknown_column=ha;
+ tag="image/png"
+} :: Tagged<String, 'image/png'> ;;
diff --git a/library/cpp/type_info/type.cpp b/library/cpp/type_info/type.cpp
new file mode 100644
index 0000000000..cee58a0a79
--- /dev/null
+++ b/library/cpp/type_info/type.cpp
@@ -0,0 +1,1662 @@
+#include "type.h"
+
+#include "type_factory.h"
+#include "type_equivalence.h"
+
+#include <util/generic/overloaded.h>
+
+#include <util/digest/murmur.h>
+#include <util/generic/hash_set.h>
+#include <util/string/escape.h>
+
+namespace {
+ inline ui64 Hash(NTi::ETypeName type) {
+ return IntHash(static_cast<ui64>(type));
+ }
+
+ inline ui64 Hash(ui64 value, ui64 seed) {
+ return MurmurHash(&value, sizeof(value), seed);
+ }
+
+ inline ui64 Hash(TStringBuf string, ui64 seed) {
+ seed = ::Hash(string.size(), seed);
+ return MurmurHash(string.data(), string.size(), seed);
+ }
+
+ inline ui64 Hash(TMaybe<TStringBuf> string, ui64 seed) {
+ if (string.Defined()) {
+ return MurmurHash(string->data(), string->size(), seed);
+ } else {
+ return seed;
+ }
+ }
+
+ TString Quote(TStringBuf s) {
+ TString result;
+ result.push_back('\'');
+ result += EscapeC(s);
+ result.push_back('\'');
+ return result;
+ }
+}
+
+namespace NTi {
+ TType::TType(TMaybe<ui64> hash, ETypeName typeName) noexcept
+ : TypeName_(typeName)
+ , HasHash_(hash.Defined())
+ , Hash_(hash.GetOrElse(0))
+ {
+ }
+
+ ui64 TType::CalculateHash() const noexcept {
+ return ::Hash(TypeName_);
+ }
+
+ TMaybe<ui64> TType::GetHashRaw() const noexcept {
+ if (HasHash_.load(std::memory_order_seq_cst)) {
+ return Hash_.load(std::memory_order_seq_cst);
+ } else {
+ return Nothing();
+ }
+ }
+
+ ui64 TType::GetHash() const {
+ if (HasHash_.load(std::memory_order_seq_cst)) {
+ return Hash_.load(std::memory_order_seq_cst);
+ } else {
+ ui64 hash = VisitRaw([](const auto* type) {
+ return type->CalculateHash();
+ });
+ Hash_.store(hash, std::memory_order_seq_cst);
+ HasHash_.store(true, std::memory_order_seq_cst);
+ return hash;
+ }
+ }
+
+ TTypePtr TType::StripTags() const noexcept {
+ return StripTagsRaw()->AsPtr();
+ }
+
+ const TType* TType::StripTagsRaw() const noexcept {
+ auto type = this;
+ while (type->IsTagged()) {
+ type = type->AsTaggedRaw()->GetItemTypeRaw();
+ }
+ return type;
+ }
+
+ TTypePtr TType::StripOptionals() const noexcept {
+ return StripOptionalsRaw()->AsPtr();
+ }
+
+ const TType* TType::StripOptionalsRaw() const noexcept {
+ auto type = this;
+ while (type->IsOptional()) {
+ type = type->AsOptionalRaw()->GetItemTypeRaw();
+ }
+ return type;
+ }
+
+ TTypePtr TType::StripTagsAndOptionals() const noexcept {
+ return StripTagsAndOptionalsRaw()->AsPtr();
+ }
+
+ const TType* TType::StripTagsAndOptionalsRaw() const noexcept {
+ auto type = this;
+ while (type->IsTagged() || type->IsOptional()) {
+ if (type->IsTagged()) {
+ type = type->AsTaggedRaw()->GetItemTypeRaw();
+ } else {
+ type = type->AsOptionalRaw()->GetItemTypeRaw();
+ }
+ }
+ return type;
+ }
+
+ const TType* TType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ return VisitRaw([&factory](const auto* type) -> const NTi::TType* {
+ return type->Clone(factory);
+ });
+ }
+
+ void TType::Drop(ITypeFactoryInternal& factory) noexcept {
+ VisitRaw([&factory](const auto* type) {
+ using T = std::remove_const_t<std::remove_pointer_t<decltype(type)>>;
+ return const_cast<T*>(type)->Drop(factory);
+ });
+ }
+
+ ITypeFactoryInternal* TType::GetFactory() const noexcept {
+ size_t manager_or_rc = FactoryOrRc_.load(std::memory_order_relaxed);
+ if (IsRc(manager_or_rc)) {
+ return NPrivate::GetDefaultHeapFactory();
+ } else if (IsFactory(manager_or_rc)) {
+ return CastToFactory(manager_or_rc);
+ } else {
+ return nullptr;
+ }
+ }
+
+ void TType::SetFactory(ITypeFactoryInternal* factory) noexcept {
+ if (factory == NPrivate::GetDefaultHeapFactory()) {
+ FactoryOrRc_.store(0b1u, std::memory_order_release);
+ } else {
+ FactoryOrRc_.store(CastFromFactory(factory), std::memory_order_release);
+ }
+ }
+
+ ITypeFactoryInternal& TType::FactoryInternal(ITypeFactory& factory) noexcept {
+ return static_cast<ITypeFactoryInternal&>(factory);
+ }
+
+ template <bool RefFactory>
+ void TType::RefImpl() noexcept {
+ size_t factoryOrRc = FactoryOrRc_.load(std::memory_order_relaxed);
+ if (Y_LIKELY(IsRc(factoryOrRc))) {
+ FactoryOrRc_.fetch_add(0b10u, std::memory_order_acq_rel);
+ } else if (Y_LIKELY(IsFactory(factoryOrRc))) {
+ auto factory = CastToFactory(factoryOrRc);
+ if (RefFactory) {
+ factory->Ref();
+ }
+ factory->RefType(this);
+ }
+ }
+
+ template void TType::RefImpl<true>() noexcept;
+ template void TType::RefImpl<false>() noexcept;
+
+ template <bool UnRefFactory>
+ void TType::UnRefImpl() noexcept {
+ size_t factoryOrRc = FactoryOrRc_.load(std::memory_order_relaxed);
+ if (Y_LIKELY(IsRc(factoryOrRc))) {
+ size_t rc = FactoryOrRc_.fetch_sub(0b10u, std::memory_order_acq_rel);
+ if (rc == 0b11u) {
+ auto factory = NPrivate::GetDefaultHeapFactory();
+ Drop(*factory);
+ factory->Delete(this);
+ }
+ } else if (Y_LIKELY(IsFactory(factoryOrRc))) {
+ auto factory = CastToFactory(factoryOrRc);
+ factory->UnRefType(this);
+ if (UnRefFactory) {
+ factory->UnRef();
+ }
+ }
+ }
+
+ template void TType::UnRefImpl<true>() noexcept;
+ template void TType::UnRefImpl<false>() noexcept;
+
+ template <bool DecRefFactory>
+ void TType::DecRefImpl() noexcept {
+ size_t factoryOrRc = FactoryOrRc_.load(std::memory_order_relaxed);
+ if (Y_LIKELY(IsRc(factoryOrRc))) {
+ size_t rc = FactoryOrRc_.fetch_sub(2, std::memory_order_acq_rel);
+ if (rc == 2) {
+ Y_FAIL("DecRef isn't supposed to drop");
+ }
+ } else if (Y_LIKELY(IsFactory(factoryOrRc))) {
+ auto factory = CastToFactory(factoryOrRc);
+ factory->DecRefType(this);
+ if (DecRefFactory) {
+ factory->DecRef();
+ }
+ }
+ }
+
+ template void TType::DecRefImpl<true>() noexcept;
+ template void TType::DecRefImpl<false>() noexcept;
+
+ long TType::RefCountImpl() const noexcept {
+ size_t factoryOrRc = FactoryOrRc_.load(std::memory_order_relaxed);
+ if (Y_LIKELY(IsRc(factoryOrRc))) {
+ return factoryOrRc >> 1u;
+ } else if (Y_LIKELY(IsFactory(factoryOrRc))) {
+ return CastToFactory(factoryOrRc)->RefCountType(this);
+ } else {
+ return 0;
+ }
+ }
+
+ template <typename T, typename TCtor>
+ const T* TType::Cached(const T* type, ITypeFactoryInternal& factory, TCtor&& ctor) {
+ const TType* result = factory.LookupCache(type);
+
+ if (result == nullptr) {
+ result = std::forward<TCtor>(ctor)();
+ factory.SaveCache(result);
+ }
+
+ Y_VERIFY(result->GetTypeName() == type->GetTypeName());
+ Y_VERIFY_DEBUG(result->GetHash() == type->GetHash());
+ return static_cast<const T*>(result);
+ }
+
+ bool operator==(const TType& lhs, const TType& rhs) {
+ Y_VERIFY(&lhs);
+ Y_VERIFY(&rhs);
+ return NEq::TStrictlyEqual().IgnoreHash(&lhs, &rhs);
+ }
+
+ bool operator!=(const TType& lhs, const TType& rhs)
+ {
+ return !(lhs == rhs);
+ }
+
+ TVoidType::TVoidType()
+ : TType({}, ETypeName::Void)
+ {
+ }
+
+ TVoidTypePtr TVoidType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TVoidType* TVoidType::InstanceRaw() {
+ static auto singleton = TVoidType();
+ return &singleton;
+ }
+
+ const TVoidType* TVoidType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TVoidType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TNullType::TNullType()
+ : TType({}, ETypeName::Null)
+ {
+ }
+
+ TNullTypePtr TNullType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TNullType* TNullType::InstanceRaw() {
+ static auto singleton = TNullType();
+ return &singleton;
+ }
+
+ const TNullType* TNullType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TNullType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TPrimitiveType::TPrimitiveType(TMaybe<ui64> hash, EPrimitiveTypeName primitiveTypeName) noexcept
+ : TType(hash, ToTypeName(primitiveTypeName))
+ {
+ }
+
+ TBoolType::TBoolType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Bool)
+ {
+ }
+
+ TBoolTypePtr TBoolType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TBoolType* TBoolType::InstanceRaw() {
+ static auto singleton = TBoolType();
+ return &singleton;
+ }
+
+ const TBoolType* TBoolType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TBoolType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TInt8Type::TInt8Type()
+ : TPrimitiveType({}, EPrimitiveTypeName::Int8)
+ {
+ }
+
+ TInt8TypePtr TInt8Type::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TInt8Type* TInt8Type::InstanceRaw() {
+ static auto singleton = TInt8Type();
+ return &singleton;
+ }
+
+ const TInt8Type* TInt8Type::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TInt8Type::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TInt16Type::TInt16Type()
+ : TPrimitiveType({}, EPrimitiveTypeName::Int16)
+ {
+ }
+
+ TInt16TypePtr TInt16Type::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TInt16Type* TInt16Type::InstanceRaw() {
+ static auto singleton = TInt16Type();
+ return &singleton;
+ }
+
+ const TInt16Type* TInt16Type::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TInt16Type::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TInt32Type::TInt32Type()
+ : TPrimitiveType({}, EPrimitiveTypeName::Int32)
+ {
+ }
+
+ TInt32TypePtr TInt32Type::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TInt32Type* TInt32Type::InstanceRaw() {
+ static auto singleton = TInt32Type();
+ return &singleton;
+ }
+
+ const TInt32Type* TInt32Type::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TInt32Type::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TInt64Type::TInt64Type()
+ : TPrimitiveType({}, EPrimitiveTypeName::Int64)
+ {
+ }
+
+ TInt64TypePtr TInt64Type::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TInt64Type* TInt64Type::InstanceRaw() {
+ static auto singleton = TInt64Type();
+ return &singleton;
+ }
+
+ const TInt64Type* TInt64Type::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TInt64Type::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TUint8Type::TUint8Type()
+ : TPrimitiveType({}, EPrimitiveTypeName::Uint8)
+ {
+ }
+
+ TUint8TypePtr TUint8Type::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TUint8Type* TUint8Type::InstanceRaw() {
+ static auto singleton = TUint8Type();
+ return &singleton;
+ }
+
+ const TUint8Type* TUint8Type::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TUint8Type::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TUint16Type::TUint16Type()
+ : TPrimitiveType({}, EPrimitiveTypeName::Uint16)
+ {
+ }
+
+ TUint16TypePtr TUint16Type::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TUint16Type* TUint16Type::InstanceRaw() {
+ static auto singleton = TUint16Type();
+ return &singleton;
+ }
+
+ const TUint16Type* TUint16Type::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TUint16Type::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TUint32Type::TUint32Type()
+ : TPrimitiveType({}, EPrimitiveTypeName::Uint32)
+ {
+ }
+
+ TUint32TypePtr TUint32Type::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TUint32Type* TUint32Type::InstanceRaw() {
+ static auto singleton = TUint32Type();
+ return &singleton;
+ }
+
+ const TUint32Type* TUint32Type::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TUint32Type::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TUint64Type::TUint64Type()
+ : TPrimitiveType({}, EPrimitiveTypeName::Uint64)
+ {
+ }
+
+ TUint64TypePtr TUint64Type::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TUint64Type* TUint64Type::InstanceRaw() {
+ static auto singleton = TUint64Type();
+ return &singleton;
+ }
+
+ const TUint64Type* TUint64Type::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TUint64Type::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TFloatType::TFloatType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Float)
+ {
+ }
+
+ TFloatTypePtr TFloatType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TFloatType* TFloatType::InstanceRaw() {
+ static auto singleton = TFloatType();
+ return &singleton;
+ }
+
+ const TFloatType* TFloatType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TFloatType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TDoubleType::TDoubleType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Double)
+ {
+ }
+
+ TDoubleTypePtr TDoubleType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TDoubleType* TDoubleType::InstanceRaw() {
+ static auto singleton = TDoubleType();
+ return &singleton;
+ }
+
+ const TDoubleType* TDoubleType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TDoubleType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TStringType::TStringType()
+ : TPrimitiveType({}, EPrimitiveTypeName::String)
+ {
+ }
+
+ TStringTypePtr TStringType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TStringType* TStringType::InstanceRaw() {
+ static auto singleton = TStringType();
+ return &singleton;
+ }
+
+ const TStringType* TStringType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TStringType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TUtf8Type::TUtf8Type()
+ : TPrimitiveType({}, EPrimitiveTypeName::Utf8)
+ {
+ }
+
+ TUtf8TypePtr TUtf8Type::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TUtf8Type* TUtf8Type::InstanceRaw() {
+ static auto singleton = TUtf8Type();
+ return &singleton;
+ }
+
+ const TUtf8Type* TUtf8Type::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TUtf8Type::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TDateType::TDateType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Date)
+ {
+ }
+
+ TDateTypePtr TDateType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TDateType* TDateType::InstanceRaw() {
+ static auto singleton = TDateType();
+ return &singleton;
+ }
+
+ const TDateType* TDateType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TDateType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TDatetimeType::TDatetimeType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Datetime)
+ {
+ }
+
+ TDatetimeTypePtr TDatetimeType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TDatetimeType* TDatetimeType::InstanceRaw() {
+ static auto singleton = TDatetimeType();
+ return &singleton;
+ }
+
+ const TDatetimeType* TDatetimeType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TDatetimeType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TTimestampType::TTimestampType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Timestamp)
+ {
+ }
+
+ TTimestampTypePtr TTimestampType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TTimestampType* TTimestampType::InstanceRaw() {
+ static auto singleton = TTimestampType();
+ return &singleton;
+ }
+
+ const TTimestampType* TTimestampType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TTimestampType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TTzDateType::TTzDateType()
+ : TPrimitiveType({}, EPrimitiveTypeName::TzDate)
+ {
+ }
+
+ TTzDateTypePtr TTzDateType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TTzDateType* TTzDateType::InstanceRaw() {
+ static auto singleton = TTzDateType();
+ return &singleton;
+ }
+
+ const TTzDateType* TTzDateType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TTzDateType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TTzDatetimeType::TTzDatetimeType()
+ : TPrimitiveType({}, EPrimitiveTypeName::TzDatetime)
+ {
+ }
+
+ TTzDatetimeTypePtr TTzDatetimeType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TTzDatetimeType* TTzDatetimeType::InstanceRaw() {
+ static auto singleton = TTzDatetimeType();
+ return &singleton;
+ }
+
+ const TTzDatetimeType* TTzDatetimeType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TTzDatetimeType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TTzTimestampType::TTzTimestampType()
+ : TPrimitiveType({}, EPrimitiveTypeName::TzTimestamp)
+ {
+ }
+
+ TTzTimestampTypePtr TTzTimestampType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TTzTimestampType* TTzTimestampType::InstanceRaw() {
+ static auto singleton = TTzTimestampType();
+ return &singleton;
+ }
+
+ const TTzTimestampType* TTzTimestampType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TTzTimestampType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TIntervalType::TIntervalType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Interval)
+ {
+ }
+
+ TIntervalTypePtr TIntervalType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TIntervalType* TIntervalType::InstanceRaw() {
+ static auto singleton = TIntervalType();
+ return &singleton;
+ }
+
+ const TIntervalType* TIntervalType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TIntervalType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TDecimalType::TDecimalType(TMaybe<ui64> hash, ui8 precision, ui8 scale) noexcept
+ : TPrimitiveType(hash, EPrimitiveTypeName::Decimal)
+ , Precision_(precision)
+ , Scale_(scale)
+ {
+ }
+
+ TDecimalTypePtr TDecimalType::Create(ITypeFactory& factory, ui8 precision, ui8 scale) {
+ return CreateRaw(factory, precision, scale)->AsPtr();
+ }
+
+ const TDecimalType* TDecimalType::CreateRaw(ITypeFactory& factory, ui8 precision, ui8 scale) {
+ Y_ENSURE_EX(
+ scale <= precision,
+ TIllegalTypeException() << "decimal scale " << (ui32)scale
+ << " should be no greater than decimal precision " << (ui32)precision);
+
+ return TDecimalType({}, precision, scale).Clone(FactoryInternal(factory));
+ }
+
+ ui64 TDecimalType::CalculateHash() const noexcept {
+ auto hash = TType::CalculateHash();
+ hash = ::Hash(Precision_, hash);
+ hash = ::Hash(Scale_, hash);
+ return hash;
+ }
+
+ const TDecimalType* TDecimalType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ return Cached(this, factory, [this, &factory]() -> const TDecimalType* {
+ auto hash = GetHashRaw();
+ auto precision = Precision_;
+ auto scale = Scale_;
+ return factory.New<TDecimalType>(hash, precision, scale);
+ });
+ }
+
+ void TDecimalType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TJsonType::TJsonType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Json)
+ {
+ }
+
+ TJsonTypePtr TJsonType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TJsonType* TJsonType::InstanceRaw() {
+ static auto singleton = TJsonType();
+ return &singleton;
+ }
+
+ const TJsonType* TJsonType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TJsonType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TYsonType::TYsonType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Yson)
+ {
+ }
+
+ TYsonTypePtr TYsonType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TYsonType* TYsonType::InstanceRaw() {
+ static auto singleton = TYsonType();
+ return &singleton;
+ }
+
+ const TYsonType* TYsonType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TYsonType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TUuidType::TUuidType()
+ : TPrimitiveType({}, EPrimitiveTypeName::Uuid)
+ {
+ }
+
+ TUuidTypePtr TUuidType::Instance() {
+ return InstanceRaw()->AsPtr();
+ }
+
+ const TUuidType* TUuidType::InstanceRaw() {
+ static auto singleton = TUuidType();
+ return &singleton;
+ }
+
+ const TUuidType* TUuidType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ Y_UNUSED(factory);
+ return InstanceRaw();
+ }
+
+ void TUuidType::Drop(ITypeFactoryInternal& factory) noexcept {
+ Y_UNUSED(factory);
+ }
+
+ TOptionalType::TOptionalType(TMaybe<ui64> hash, const TType* item) noexcept
+ : TType(hash, ETypeName::Optional)
+ , Item_(item)
+ {
+ }
+
+ TOptionalTypePtr TOptionalType::Create(ITypeFactory& factory, TTypePtr item) {
+ return CreateRaw(factory, item.Get())->AsPtr();
+ }
+
+ const TOptionalType* TOptionalType::CreateRaw(ITypeFactory& factory, const TType* item) {
+ return TOptionalType({}, item).Clone(FactoryInternal(factory));
+ }
+
+ const TOptionalType* TOptionalType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ return Cached(this, factory, [this, &factory]() -> const TOptionalType* {
+ auto hash = GetHashRaw();
+ auto item = factory.Own(Item_);
+ return factory.New<TOptionalType>(hash, item);
+ });
+ }
+
+ void TOptionalType::Drop(ITypeFactoryInternal& factory) noexcept {
+ factory.Disown(Item_);
+ }
+
+ ui64 TOptionalType::CalculateHash() const noexcept {
+ auto hash = TType::CalculateHash();
+ hash = ::Hash(Item_->GetHash(), hash);
+ return hash;
+ }
+
+ TListType::TListType(TMaybe<ui64> hash, const TType* item) noexcept
+ : TType(hash, ETypeName::List)
+ , Item_(item)
+ {
+ }
+
+ TListTypePtr TListType::Create(ITypeFactory& factory, TTypePtr item) {
+ return CreateRaw(factory, item.Get())->AsPtr();
+ }
+
+ const TListType* TListType::CreateRaw(ITypeFactory& factory, const TType* item) {
+ return TListType({}, item).Clone(FactoryInternal(factory));
+ }
+
+ const TListType* TListType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ return Cached(this, factory, [this, &factory]() -> const TListType* {
+ auto hash = GetHashRaw();
+ auto item = factory.Own(Item_);
+ return factory.New<TListType>(hash, item);
+ });
+ }
+
+ void TListType::Drop(ITypeFactoryInternal& factory) noexcept {
+ factory.Disown(Item_);
+ }
+
+ ui64 TListType::CalculateHash() const noexcept {
+ auto hash = TType::CalculateHash();
+ hash = ::Hash(Item_->GetHash(), hash);
+ return hash;
+ }
+
+ TDictType::TDictType(TMaybe<ui64> hash, const TType* key, const TType* value) noexcept
+ : TType(hash, ETypeName::Dict)
+ , Key_(key)
+ , Value_(value)
+ {
+ }
+
+ TDictTypePtr TDictType::Create(ITypeFactory& factory, TTypePtr key, TTypePtr value) {
+ return CreateRaw(factory, key.Get(), value.Get())->AsPtr();
+ }
+
+ const TDictType* TDictType::CreateRaw(ITypeFactory& factory, const TType* key, const TType* value) {
+ return TDictType({}, key, value).Clone(FactoryInternal(factory));
+ }
+
+ const TDictType* TDictType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ return Cached(this, factory, [this, &factory]() -> const TDictType* {
+ auto hash = GetHashRaw();
+ auto key = factory.Own(Key_);
+ auto value = factory.Own(Value_);
+ return factory.New<TDictType>(hash, key, value);
+ });
+ }
+
+ void TDictType::Drop(ITypeFactoryInternal& factory) noexcept {
+ factory.Disown(Key_);
+ factory.Disown(Value_);
+ }
+
+ ui64 TDictType::CalculateHash() const noexcept {
+ auto hash = TType::CalculateHash();
+ hash = ::Hash(Key_->GetHash(), hash);
+ hash = ::Hash(Value_->GetHash(), hash);
+ return hash;
+ }
+
+ TStructType::TMember::TMember(TStringBuf name, const TType* type)
+ : Name_(name)
+ , Type_(type)
+ {
+ }
+
+ ui64 TStructType::TMember::Hash() const {
+ auto hash = 0x10000;
+ hash = ::Hash(Name_, hash);
+ hash = ::Hash(Type_->GetHash(), hash);
+ return hash;
+ }
+
+ TStructType::TOwnedMember::TOwnedMember(TString name, TTypePtr type)
+ : Name_(std::move(name))
+ , Type_(std::move(type))
+ {
+ }
+
+ TStructType::TOwnedMember::operator TStructType::TMember() const& {
+ return TStructType::TMember(Name_, Type_.Get());
+ }
+
+ TStructType::TStructType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, TMembers members, TConstArrayRef<size_t> sortedItems) noexcept
+ : TType(hash, ETypeName::Struct)
+ , Name_(name)
+ , Members_(members)
+ , SortedMembers_(sortedItems)
+ {
+ }
+
+ TStructTypePtr TStructType::Create(ITypeFactory& factory, TStructType::TOwnedMembers members) {
+ return Create(factory, Nothing(), members);
+ }
+
+ TStructTypePtr TStructType::Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TStructType::TOwnedMembers members) {
+ auto rawItems = TTempArray<TMember>(members.size());
+ for (size_t i = 0; i < members.size(); ++i) {
+ new (rawItems.Data() + i) TMember(members[i]);
+ }
+ return CreateRaw(factory, name, TArrayRef(rawItems.Data(), members.size()))->AsPtr();
+ }
+
+ const TStructType* TStructType::CreateRaw(ITypeFactory& factory, TStructType::TMembers members) {
+ return CreateRaw(factory, Nothing(), members);
+ }
+
+ const TStructType* TStructType::CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, TStructType::TMembers members) {
+ auto sortedMembersArray = TTempArray<size_t>(members.size());
+ auto sortedMembers = TArrayRef(sortedMembersArray.Data(), members.size());
+ MakeSortedMembers(members, sortedMembers);
+ return TStructType({}, name, members, sortedMembers).Clone(FactoryInternal(factory));
+ }
+
+ void TStructType::MakeSortedMembers(TStructType::TMembers members, TArrayRef<size_t> sortedItems) {
+ Y_VERIFY(members.size() == sortedItems.size());
+
+ for (size_t i = 0; i < members.size(); ++i) {
+ sortedItems[i] = i;
+ }
+
+ Sort(sortedItems.begin(), sortedItems.end(), [members](size_t lhs, size_t rhs) {
+ return members[lhs].GetName() < members[rhs].GetName();
+ });
+
+ for (size_t i = 1; i < members.size(); ++i) {
+ if (members[sortedItems[i - 1]].GetName() == members[sortedItems[i]].GetName()) {
+ ythrow TIllegalTypeException() << "duplicate struct item " << Quote(members[sortedItems[i]].GetName());
+ }
+ }
+ }
+
+ const TStructType* TStructType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ return Cached(this, factory, [this, &factory]() -> const TStructType* {
+ auto hash = GetHashRaw();
+ auto name = factory.AllocateStringMaybe(Name_);
+ auto members = factory.NewArray<TMember>(Members_.size(), [this, &factory](TMember* item, size_t i) {
+ auto name = factory.AllocateString(Members_[i].GetName());
+ auto type = factory.Own(Members_[i].GetTypeRaw());
+ new (item) TMember(name, type);
+ });
+ auto sortedItems = factory.AllocateArrayFor<size_t>(SortedMembers_.size());
+ Copy(SortedMembers_.begin(), SortedMembers_.end(), sortedItems);
+ return factory.New<TStructType>(hash, name, members, TArrayRef{sortedItems, SortedMembers_.size()});
+ });
+ }
+
+ void TStructType::Drop(ITypeFactoryInternal& factory) noexcept {
+ factory.FreeStringMaybe(Name_);
+ factory.DeleteArray(Members_, [&factory](const TMember* item, size_t) {
+ factory.FreeString(item->GetName());
+ factory.Disown(item->GetTypeRaw());
+ });
+ factory.Free(const_cast<void*>(static_cast<const void*>(SortedMembers_.data())));
+ }
+
+ ui64 TStructType::CalculateHash() const noexcept {
+ auto hash = TType::CalculateHash();
+ hash = ::Hash(Name_, hash);
+ hash = ::Hash(Members_.size(), hash);
+ for (auto& item : Members_) {
+ hash = ::Hash(item.Hash(), hash);
+ }
+ return hash;
+ }
+
+ bool TStructType::HasMember(TStringBuf name) const noexcept {
+ return GetMemberIndex(name) != -1;
+ }
+
+ const TStructType::TMember& TStructType::GetMember(TStringBuf name) const {
+ auto idx = GetMemberIndex(name);
+ if (idx == -1) {
+ ythrow TItemNotFound() << "no item named " << Quote(name);
+ } else {
+ return Members_[idx];
+ }
+ }
+
+ ssize_t TStructType::GetMemberIndex(TStringBuf name) const noexcept {
+ auto it = LowerBound(SortedMembers_.begin(), SortedMembers_.end(), name, [this](size_t i, TStringBuf name) {
+ return Members_[i].GetName() < name;
+ });
+
+ if (it == SortedMembers_.end() || Members_[*it].GetName() != name) {
+ return -1;
+ } else {
+ return *it;
+ }
+ }
+
+ TTupleType::TElement::TElement(const TType* type)
+ : Type_(type)
+ {
+ }
+
+ ui64 TTupleType::TElement::Hash() const {
+ auto hash = 0x10001;
+ hash = ::Hash(Type_->GetHash(), hash);
+ return hash;
+ }
+
+ TTupleType::TOwnedElement::TOwnedElement(TTypePtr type)
+ : Type_(std::move(type))
+ {
+ }
+
+ TTupleType::TOwnedElement::operator TTupleType::TElement() const& {
+ return TTupleType::TElement(Type_.Get());
+ }
+
+ TTupleType::TTupleType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, TElements elements) noexcept
+ : TType(hash, ETypeName::Tuple)
+ , Name_(name)
+ , Elements_(elements)
+ {
+ }
+
+ TTupleTypePtr TTupleType::Create(ITypeFactory& factory, TTupleType::TOwnedElements elements) {
+ return Create(factory, Nothing(), elements);
+ }
+
+ TTupleTypePtr TTupleType::Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TTupleType::TOwnedElements elements) {
+ auto rawItems = TTempArray<TElement>(elements.size());
+ for (size_t i = 0; i < elements.size(); ++i) {
+ new (rawItems.Data() + i) TElement(elements[i]);
+ }
+ return CreateRaw(factory, name, TArrayRef(rawItems.Data(), elements.size()))->AsPtr();
+ }
+
+ const TTupleType* TTupleType::CreateRaw(ITypeFactory& factory, TTupleType::TElements elements) {
+ return CreateRaw(factory, Nothing(), elements);
+ }
+
+ const TTupleType* TTupleType::CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, TTupleType::TElements elements) {
+ return TTupleType({}, name, elements).Clone(FactoryInternal(factory));
+ }
+
+ const TTupleType* TTupleType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ return Cached(this, factory, [this, &factory]() -> const TTupleType* {
+ auto hash = GetHashRaw();
+ auto name = factory.AllocateStringMaybe(Name_);
+ auto elements = factory.NewArray<TElement>(Elements_.size(), [this, &factory](TElement* item, size_t i) {
+ auto type = factory.Own(Elements_[i].GetTypeRaw());
+ new (item) TElement(type);
+ });
+ return factory.New<TTupleType>(hash, name, elements);
+ });
+ }
+
+ void TTupleType::Drop(ITypeFactoryInternal& factory) noexcept {
+ factory.FreeStringMaybe(Name_);
+ factory.DeleteArray(Elements_, [&factory](const TElement* item, size_t) {
+ factory.Disown(item->GetTypeRaw());
+ });
+ }
+
+ ui64 TTupleType::CalculateHash() const noexcept {
+ auto hash = TType::CalculateHash();
+ hash = ::Hash(Name_, hash);
+ hash = ::Hash(Elements_.size(), hash);
+ for (auto& item : Elements_) {
+ hash = ::Hash(item.Hash(), hash);
+ }
+ return hash;
+ }
+
+ TVariantType::TVariantType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, const TType* inner) noexcept
+ : TType(hash, ETypeName::Variant)
+ , Name_(name)
+ , Underlying_(inner)
+ {
+ }
+
+ TVariantTypePtr TVariantType::Create(ITypeFactory& factory, TTypePtr inner) {
+ return Create(factory, Nothing(), std::move(inner));
+ }
+
+ TVariantTypePtr TVariantType::Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TTypePtr inner) {
+ return CreateRaw(factory, name, inner.Get())->AsPtr();
+ }
+
+ const TVariantType* TVariantType::CreateRaw(ITypeFactory& factory, const TType* inner) {
+ return CreateRaw(factory, Nothing(), inner);
+ }
+
+ const TVariantType* TVariantType::CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, const TType* inner) {
+ inner->VisitRaw(TOverloaded{
+ [&](const TStructType* s) {
+ Y_ENSURE_EX(
+ !s->GetMembers().empty(),
+ TIllegalTypeException() << "variant should contain at least one alternative");
+ },
+ [&](const TTupleType* t) {
+ Y_ENSURE_EX(
+ !t->GetElements().empty(),
+ TIllegalTypeException() << "variant should contain at least one alternative");
+ },
+ [](const TType* t) {
+ ythrow TIllegalTypeException() << "variants can only contain structs and tuples, got "
+ << t->GetTypeName() << " instead";
+ }});
+
+ return TVariantType({}, name, inner).Clone(FactoryInternal(factory));
+ }
+
+ const TVariantType* TVariantType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ return Cached(this, factory, [this, &factory]() -> const TVariantType* {
+ auto hash = GetHashRaw();
+ auto name = factory.AllocateStringMaybe(Name_);
+ auto inner = factory.Own(Underlying_);
+ return factory.New<TVariantType>(hash, name, inner);
+ });
+ }
+
+ void TVariantType::Drop(ITypeFactoryInternal& factory) noexcept {
+ factory.FreeStringMaybe(Name_);
+ factory.Disown(Underlying_);
+ }
+
+ ui64 TVariantType::CalculateHash() const noexcept {
+ auto hash = TType::CalculateHash();
+ hash = ::Hash(Name_, hash);
+ hash = ::Hash(Underlying_->GetHash(), hash);
+ return hash;
+ }
+
+ TTaggedType::TTaggedType(TMaybe<ui64> hash, const TType* item, TStringBuf tag) noexcept
+ : TType(hash, ETypeName::Tagged)
+ , Item_(item)
+ , Tag_(tag)
+ {
+ }
+
+ TTaggedTypePtr TTaggedType::Create(ITypeFactory& factory, TTypePtr type, TStringBuf tag) {
+ return CreateRaw(factory, type.Get(), tag)->AsPtr();
+ }
+
+ const TTaggedType* TTaggedType::CreateRaw(ITypeFactory& factory, const TType* type, TStringBuf tag) {
+ return TTaggedType({}, type, tag).Clone(FactoryInternal(factory));
+ }
+
+ const TTaggedType* TTaggedType::Clone(ITypeFactoryInternal& factory) const noexcept {
+ return Cached(this, factory, [this, &factory]() -> const TTaggedType* {
+ auto hash = GetHashRaw();
+ auto item = factory.Own(Item_);
+ auto tag = factory.AllocateString(Tag_);
+ return factory.New<TTaggedType>(hash, item, tag);
+ });
+ }
+
+ void TTaggedType::Drop(ITypeFactoryInternal& factory) noexcept {
+ factory.FreeString(Tag_);
+ factory.Disown(Item_);
+ }
+
+ ui64 TTaggedType::CalculateHash() const noexcept {
+ auto hash = TType::CalculateHash();
+ hash = ::Hash(Tag_, hash);
+ hash = ::Hash(Item_->GetHash(), hash);
+ return hash;
+ }
+
+ TVoidTypePtr Void() {
+ return NPrivate::GetDefaultHeapFactory()->Void();
+ }
+
+ TNullTypePtr Null() {
+ return NPrivate::GetDefaultHeapFactory()->Null();
+ }
+
+ TBoolTypePtr Bool() {
+ return NPrivate::GetDefaultHeapFactory()->Bool();
+ }
+
+ TInt8TypePtr Int8() {
+ return NPrivate::GetDefaultHeapFactory()->Int8();
+ }
+
+ TInt16TypePtr Int16() {
+ return NPrivate::GetDefaultHeapFactory()->Int16();
+ }
+
+ TInt32TypePtr Int32() {
+ return NPrivate::GetDefaultHeapFactory()->Int32();
+ }
+
+ TInt64TypePtr Int64() {
+ return NPrivate::GetDefaultHeapFactory()->Int64();
+ }
+
+ TUint8TypePtr Uint8() {
+ return NPrivate::GetDefaultHeapFactory()->Uint8();
+ }
+
+ TUint16TypePtr Uint16() {
+ return NPrivate::GetDefaultHeapFactory()->Uint16();
+ }
+
+ TUint32TypePtr Uint32() {
+ return NPrivate::GetDefaultHeapFactory()->Uint32();
+ }
+
+ TUint64TypePtr Uint64() {
+ return NPrivate::GetDefaultHeapFactory()->Uint64();
+ }
+
+ TFloatTypePtr Float() {
+ return NPrivate::GetDefaultHeapFactory()->Float();
+ }
+
+ TDoubleTypePtr Double() {
+ return NPrivate::GetDefaultHeapFactory()->Double();
+ }
+
+ TStringTypePtr String() {
+ return NPrivate::GetDefaultHeapFactory()->String();
+ }
+
+ TUtf8TypePtr Utf8() {
+ return NPrivate::GetDefaultHeapFactory()->Utf8();
+ }
+
+ TDateTypePtr Date() {
+ return NPrivate::GetDefaultHeapFactory()->Date();
+ }
+
+ TDatetimeTypePtr Datetime() {
+ return NPrivate::GetDefaultHeapFactory()->Datetime();
+ }
+
+ TTimestampTypePtr Timestamp() {
+ return NPrivate::GetDefaultHeapFactory()->Timestamp();
+ }
+
+ TTzDateTypePtr TzDate() {
+ return NPrivate::GetDefaultHeapFactory()->TzDate();
+ }
+
+ TTzDatetimeTypePtr TzDatetime() {
+ return NPrivate::GetDefaultHeapFactory()->TzDatetime();
+ }
+
+ TTzTimestampTypePtr TzTimestamp() {
+ return NPrivate::GetDefaultHeapFactory()->TzTimestamp();
+ }
+
+ TIntervalTypePtr Interval() {
+ return NPrivate::GetDefaultHeapFactory()->Interval();
+ }
+
+ TDecimalTypePtr Decimal(ui8 precision, ui8 scale) {
+ return NPrivate::GetDefaultHeapFactory()->Decimal(precision, scale);
+ }
+
+ TJsonTypePtr Json() {
+ return NPrivate::GetDefaultHeapFactory()->Json();
+ }
+
+ TYsonTypePtr Yson() {
+ return NPrivate::GetDefaultHeapFactory()->Yson();
+ }
+
+ TUuidTypePtr Uuid() {
+ return NPrivate::GetDefaultHeapFactory()->Uuid();
+ }
+
+ TOptionalTypePtr Optional(TTypePtr item) {
+ return NPrivate::GetDefaultHeapFactory()->Optional(std::move(item));
+ }
+
+ TListTypePtr List(TTypePtr item) {
+ return NPrivate::GetDefaultHeapFactory()->List(std::move(item));
+ }
+
+ TDictTypePtr Dict(TTypePtr key, TTypePtr value) {
+ return NPrivate::GetDefaultHeapFactory()->Dict(std::move(key), std::move(value));
+ }
+
+ TStructTypePtr Struct(TStructType::TOwnedMembers members) {
+ return NPrivate::GetDefaultHeapFactory()->Struct(members);
+ }
+
+ TStructTypePtr Struct(TMaybe<TStringBuf> name, TStructType::TOwnedMembers members) {
+ return NPrivate::GetDefaultHeapFactory()->Struct(name, members);
+ }
+
+ TTupleTypePtr Tuple(TTupleType::TOwnedElements elements) {
+ return NPrivate::GetDefaultHeapFactory()->Tuple(elements);
+ }
+
+ TTupleTypePtr Tuple(TMaybe<TStringBuf> name, TTupleType::TOwnedElements elements) {
+ return NPrivate::GetDefaultHeapFactory()->Tuple(name, elements);
+ }
+
+ TVariantTypePtr Variant(TTypePtr underlying) {
+ return NPrivate::GetDefaultHeapFactory()->Variant(std::move(underlying));
+ }
+
+ TVariantTypePtr Variant(TMaybe<TStringBuf> name, TTypePtr underlying) {
+ return NPrivate::GetDefaultHeapFactory()->Variant(name, std::move(underlying));
+ }
+
+ TTaggedTypePtr Tagged(TTypePtr type, TStringBuf tag) {
+ return NPrivate::GetDefaultHeapFactory()->Tagged(std::move(type), tag);
+ }
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TType, o, v) {
+ v.VisitRaw([&o](const auto* v) { o << *v; });
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TPrimitiveType, o, v) {
+ v.VisitPrimitiveRaw([&o](const auto* v) { o << *v; });
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TVoidType, o, v) {
+ Y_UNUSED(v);
+ o << "Void";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TNullType, o, v) {
+ Y_UNUSED(v);
+ o << "Null";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TBoolType, o, v) {
+ Y_UNUSED(v);
+ o << "Bool";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TInt8Type, o, v) {
+ Y_UNUSED(v);
+ o << "Int8";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TInt16Type, o, v) {
+ Y_UNUSED(v);
+ o << "Int16";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TInt32Type, o, v) {
+ Y_UNUSED(v);
+ o << "Int32";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TInt64Type, o, v) {
+ Y_UNUSED(v);
+ o << "Int64";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TUint8Type, o, v) {
+ Y_UNUSED(v);
+ o << "Uint8";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TUint16Type, o, v) {
+ Y_UNUSED(v);
+ o << "Uint16";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TUint32Type, o, v) {
+ Y_UNUSED(v);
+ o << "Uint32";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TUint64Type, o, v) {
+ Y_UNUSED(v);
+ o << "Uint64";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TFloatType, o, v) {
+ Y_UNUSED(v);
+ o << "Float";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TDoubleType, o, v) {
+ Y_UNUSED(v);
+ o << "Double";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TStringType, o, v) {
+ Y_UNUSED(v);
+ o << "String";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TUtf8Type, o, v) {
+ Y_UNUSED(v);
+ o << "Utf8";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TDateType, o, v) {
+ Y_UNUSED(v);
+ o << "Date";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TDatetimeType, o, v) {
+ Y_UNUSED(v);
+ o << "Datetime";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TTimestampType, o, v) {
+ Y_UNUSED(v);
+ o << "Timestamp";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TTzDateType, o, v) {
+ Y_UNUSED(v);
+ o << "TzDate";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TTzDatetimeType, o, v) {
+ Y_UNUSED(v);
+ o << "TzDatetime";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TTzTimestampType, o, v) {
+ Y_UNUSED(v);
+ o << "TzTimestamp";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TIntervalType, o, v) {
+ Y_UNUSED(v);
+ o << "Interval";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TDecimalType, o, v) {
+ o << "Decimal(" << (i32)v.GetPrecision() << ", " << (i32)v.GetScale() << ')';
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TJsonType, o, v) {
+ Y_UNUSED(v);
+ o << "Json";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TYsonType, o, v) {
+ Y_UNUSED(v);
+ o << "Yson";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TUuidType, o, v) {
+ Y_UNUSED(v);
+ o << "Uuid";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TOptionalType, o, v) {
+ o << "Optional<" << *v.GetItemTypeRaw() << ">";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TListType, o, v) {
+ o << "List<" << *v.GetItemTypeRaw() << ">";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TDictType, o, v) {
+ o << "Dict<" << *v.GetKeyTypeRaw() << ", " << *v.GetValueTypeRaw() << ">";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TStructType, o, v) {
+ o << "Struct";
+
+ if (v.GetName().Defined()) {
+ o << "[" << Quote(*v.GetName()) << "]";
+ }
+
+ o << "<";
+ const char* sep = "";
+ for (auto& item : v.GetMembers()) {
+ o << sep << Quote(item.GetName()) << ": " << *item.GetTypeRaw();
+ sep = ", ";
+ }
+ o << ">";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TTupleType, o, v) {
+ o << "Tuple";
+
+ if (v.GetName().Defined()) {
+ o << "[" << Quote(*v.GetName()) << "]";
+ }
+
+ o << "<";
+ const char* sep = "";
+ for (auto& item : v.GetElements()) {
+ o << sep << *item.GetTypeRaw();
+ sep = ", ";
+ }
+ o << ">";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TVariantType, o, v) {
+ o << "Variant";
+
+ if (v.GetName().Defined()) {
+ o << "[" << Quote(*v.GetName()) << "]";
+ }
+
+ o << "<";
+ v.VisitUnderlyingRaw(
+ TOverloaded{
+ [&o](const NTi::TStructType* s) {
+ const char* sep = "";
+ for (auto& item : s->GetMembers()) {
+ o << sep << Quote(item.GetName()) << ": " << *item.GetTypeRaw();
+ sep = ", ";
+ }
+ },
+ [&o](const NTi::TTupleType* t) {
+ const char* sep = "";
+ for (auto& item : t->GetElements()) {
+ o << sep << *item.GetTypeRaw();
+ sep = ", ";
+ }
+ }});
+ o << ">";
+}
+
+Y_DECLARE_OUT_SPEC(, NTi::TTaggedType, o, v) {
+ o << "Tagged<" << *v.GetItemTypeRaw() << ", " << Quote(v.GetTag()) << ">";
+}
+
+static_assert(std::is_trivially_destructible_v<NTi::TVoidType>);
+static_assert(std::is_trivially_destructible_v<NTi::TNullType>);
+static_assert(std::is_trivially_destructible_v<NTi::TPrimitiveType>);
+static_assert(std::is_trivially_destructible_v<NTi::TBoolType>);
+static_assert(std::is_trivially_destructible_v<NTi::TInt8Type>);
+static_assert(std::is_trivially_destructible_v<NTi::TInt16Type>);
+static_assert(std::is_trivially_destructible_v<NTi::TInt32Type>);
+static_assert(std::is_trivially_destructible_v<NTi::TInt64Type>);
+static_assert(std::is_trivially_destructible_v<NTi::TUint8Type>);
+static_assert(std::is_trivially_destructible_v<NTi::TUint16Type>);
+static_assert(std::is_trivially_destructible_v<NTi::TUint32Type>);
+static_assert(std::is_trivially_destructible_v<NTi::TUint64Type>);
+static_assert(std::is_trivially_destructible_v<NTi::TFloatType>);
+static_assert(std::is_trivially_destructible_v<NTi::TDoubleType>);
+static_assert(std::is_trivially_destructible_v<NTi::TStringType>);
+static_assert(std::is_trivially_destructible_v<NTi::TUtf8Type>);
+static_assert(std::is_trivially_destructible_v<NTi::TDateType>);
+static_assert(std::is_trivially_destructible_v<NTi::TDatetimeType>);
+static_assert(std::is_trivially_destructible_v<NTi::TTimestampType>);
+static_assert(std::is_trivially_destructible_v<NTi::TTzDateType>);
+static_assert(std::is_trivially_destructible_v<NTi::TTzDatetimeType>);
+static_assert(std::is_trivially_destructible_v<NTi::TTzTimestampType>);
+static_assert(std::is_trivially_destructible_v<NTi::TIntervalType>);
+static_assert(std::is_trivially_destructible_v<NTi::TDecimalType>);
+static_assert(std::is_trivially_destructible_v<NTi::TJsonType>);
+static_assert(std::is_trivially_destructible_v<NTi::TYsonType>);
+static_assert(std::is_trivially_destructible_v<NTi::TUuidType>);
+static_assert(std::is_trivially_destructible_v<NTi::TOptionalType>);
+static_assert(std::is_trivially_destructible_v<NTi::TListType>);
+static_assert(std::is_trivially_destructible_v<NTi::TDictType>);
+static_assert(std::is_trivially_destructible_v<NTi::TStructType>);
+static_assert(std::is_trivially_destructible_v<NTi::TStructType::TMember>);
+static_assert(std::is_trivially_destructible_v<NTi::TTupleType>);
+static_assert(std::is_trivially_destructible_v<NTi::TTupleType::TElement>);
+static_assert(std::is_trivially_destructible_v<NTi::TVariantType>);
+static_assert(std::is_trivially_destructible_v<NTi::TTaggedType>);
diff --git a/library/cpp/type_info/type.h b/library/cpp/type_info/type.h
new file mode 100644
index 0000000000..1577f1cee3
--- /dev/null
+++ b/library/cpp/type_info/type.h
@@ -0,0 +1,2421 @@
+#pragma once
+
+//! @file type.h
+//!
+//! Hierarchy of classes that represent types.
+#include "fwd.h"
+
+#include "error.h"
+#include "type_list.h"
+
+#include <atomic>
+#include <util/generic/array_ref.h>
+#include <util/generic/maybe.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/string.h>
+
+namespace NTi {
+ /// Represents a single type.
+ ///
+ /// Create instances of types using type factory (see `NTi::ITypeFactory`).
+ ///
+ /// Introspect them using associated methods and functions that work with `ETypeName`.
+ ///
+ /// Pattern-match them using the `Visit` method.
+ ///
+ /// Serialize and deserialize them using functions from `NTi::NIo`.
+ class TType {
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+ template <typename T>
+ friend class ::TDefaultIntrusivePtrOps;
+
+ public:
+ TTypePtr AsPtr() const noexcept {
+ return const_cast<TType*>(this);
+ }
+
+ protected:
+ explicit TType(TMaybe<ui64> hash, ETypeName typeName) noexcept;
+
+ protected:
+ /// Calculate hash for this type. This function is lazily called by `GetHash`.
+ ///
+ /// Note: this function is not marked as `virtual` because we use our own dispatch via `Visit`.
+ ui64 CalculateHash() const noexcept;
+
+ public:
+ /// Get hash of this type. Hashes follow the 'strict equivalence' relation (see `type_equivalence.h`).
+ ui64 GetHash() const;
+
+ /// Get hash of this type. If hash is not calculated, returns nothing.
+ TMaybe<ui64> GetHashRaw() const noexcept;
+
+ /// Get name of this type as a `NTi::ETypeName` enumerator.
+ ETypeName GetTypeName() const noexcept {
+ return TypeName_;
+ }
+
+ /// @name Simple type downcast functions
+ ///
+ /// Check if type is of the given subclass and convert between subclasses.
+ /// Conversions panic if downcasting into an incompatible type.
+ ///
+ /// @{
+ inline bool IsVoid() const noexcept;
+ inline TVoidTypePtr AsVoid() const noexcept;
+ inline const TVoidType* AsVoidRaw() const noexcept;
+
+ inline bool IsNull() const noexcept;
+ inline TNullTypePtr AsNull() const noexcept;
+ inline const TNullType* AsNullRaw() const noexcept;
+
+ inline bool IsPrimitive() const noexcept;
+ inline TPrimitiveTypePtr AsPrimitive() const noexcept;
+ inline const TPrimitiveType* AsPrimitiveRaw() const noexcept;
+
+ inline bool IsBool() const noexcept;
+ inline TBoolTypePtr AsBool() const noexcept;
+ inline const TBoolType* AsBoolRaw() const noexcept;
+
+ inline bool IsInt8() const noexcept;
+ inline TInt8TypePtr AsInt8() const noexcept;
+ inline const TInt8Type* AsInt8Raw() const noexcept;
+
+ inline bool IsInt16() const noexcept;
+ inline TInt16TypePtr AsInt16() const noexcept;
+ inline const TInt16Type* AsInt16Raw() const noexcept;
+
+ inline bool IsInt32() const noexcept;
+ inline TInt32TypePtr AsInt32() const noexcept;
+ inline const TInt32Type* AsInt32Raw() const noexcept;
+
+ inline bool IsInt64() const noexcept;
+ inline TInt64TypePtr AsInt64() const noexcept;
+ inline const TInt64Type* AsInt64Raw() const noexcept;
+
+ inline bool IsUint8() const noexcept;
+ inline TUint8TypePtr AsUint8() const noexcept;
+ inline const TUint8Type* AsUint8Raw() const noexcept;
+
+ inline bool IsUint16() const noexcept;
+ inline TUint16TypePtr AsUint16() const noexcept;
+ inline const TUint16Type* AsUint16Raw() const noexcept;
+
+ inline bool IsUint32() const noexcept;
+ inline TUint32TypePtr AsUint32() const noexcept;
+ inline const TUint32Type* AsUint32Raw() const noexcept;
+
+ inline bool IsUint64() const noexcept;
+ inline TUint64TypePtr AsUint64() const noexcept;
+ inline const TUint64Type* AsUint64Raw() const noexcept;
+
+ inline bool IsFloat() const noexcept;
+ inline TFloatTypePtr AsFloat() const noexcept;
+ inline const TFloatType* AsFloatRaw() const noexcept;
+
+ inline bool IsDouble() const noexcept;
+ inline TDoubleTypePtr AsDouble() const noexcept;
+ inline const TDoubleType* AsDoubleRaw() const noexcept;
+
+ inline bool IsString() const noexcept;
+ inline TStringTypePtr AsString() const noexcept;
+ inline const TStringType* AsStringRaw() const noexcept;
+
+ inline bool IsUtf8() const noexcept;
+ inline TUtf8TypePtr AsUtf8() const noexcept;
+ inline const TUtf8Type* AsUtf8Raw() const noexcept;
+
+ inline bool IsDate() const noexcept;
+ inline TDateTypePtr AsDate() const noexcept;
+ inline const TDateType* AsDateRaw() const noexcept;
+
+ inline bool IsDatetime() const noexcept;
+ inline TDatetimeTypePtr AsDatetime() const noexcept;
+ inline const TDatetimeType* AsDatetimeRaw() const noexcept;
+
+ inline bool IsTimestamp() const noexcept;
+ inline TTimestampTypePtr AsTimestamp() const noexcept;
+ inline const TTimestampType* AsTimestampRaw() const noexcept;
+
+ inline bool IsTzDate() const noexcept;
+ inline TTzDateTypePtr AsTzDate() const noexcept;
+ inline const TTzDateType* AsTzDateRaw() const noexcept;
+
+ inline bool IsTzDatetime() const noexcept;
+ inline TTzDatetimeTypePtr AsTzDatetime() const noexcept;
+ inline const TTzDatetimeType* AsTzDatetimeRaw() const noexcept;
+
+ inline bool IsTzTimestamp() const noexcept;
+ inline TTzTimestampTypePtr AsTzTimestamp() const noexcept;
+ inline const TTzTimestampType* AsTzTimestampRaw() const noexcept;
+
+ inline bool IsInterval() const noexcept;
+ inline TIntervalTypePtr AsInterval() const noexcept;
+ inline const TIntervalType* AsIntervalRaw() const noexcept;
+
+ inline bool IsDecimal() const noexcept;
+ inline TDecimalTypePtr AsDecimal() const noexcept;
+ inline const TDecimalType* AsDecimalRaw() const noexcept;
+
+ inline bool IsJson() const noexcept;
+ inline TJsonTypePtr AsJson() const noexcept;
+ inline const TJsonType* AsJsonRaw() const noexcept;
+
+ inline bool IsYson() const noexcept;
+ inline TYsonTypePtr AsYson() const noexcept;
+ inline const TYsonType* AsYsonRaw() const noexcept;
+
+ inline bool IsUuid() const noexcept;
+ inline TUuidTypePtr AsUuid() const noexcept;
+ inline const TUuidType* AsUuidRaw() const noexcept;
+
+ inline bool IsOptional() const noexcept;
+ inline TOptionalTypePtr AsOptional() const noexcept;
+ inline const TOptionalType* AsOptionalRaw() const noexcept;
+
+ inline bool IsList() const noexcept;
+ inline TListTypePtr AsList() const noexcept;
+ inline const TListType* AsListRaw() const noexcept;
+
+ inline bool IsDict() const noexcept;
+ inline TDictTypePtr AsDict() const noexcept;
+ inline const TDictType* AsDictRaw() const noexcept;
+
+ inline bool IsStruct() const noexcept;
+ inline TStructTypePtr AsStruct() const noexcept;
+ inline const TStructType* AsStructRaw() const noexcept;
+
+ inline bool IsTuple() const noexcept;
+ inline TTupleTypePtr AsTuple() const noexcept;
+ inline const TTupleType* AsTupleRaw() const noexcept;
+
+ inline bool IsVariant() const noexcept;
+ inline TVariantTypePtr AsVariant() const noexcept;
+ inline const TVariantType* AsVariantRaw() const noexcept;
+
+ inline bool IsTagged() const noexcept;
+ inline TTaggedTypePtr AsTagged() const noexcept;
+ inline const TTaggedType* AsTaggedRaw() const noexcept;
+
+ /// @}
+
+ /// Recursively descends to tagged types and returns first non-tagged type.
+ TTypePtr StripTags() const noexcept;
+
+ /// Like `StripTags`, but returns a raw pointer.
+ const TType* StripTagsRaw() const noexcept;
+
+ /// Recursively descends to optional types and returns first non-optional type.
+ TTypePtr StripOptionals() const noexcept;
+
+ /// Like `StripOptionals`, but returns a raw pointer.
+ const TType* StripOptionalsRaw() const noexcept;
+
+ /// Recursively descends to tagged and optional types and returns first non-tagged non-optional type.
+ TTypePtr StripTagsAndOptionals() const noexcept;
+
+ /// Like `StripTagsAndOptionals`, but returns a raw pointer.
+ const TType* StripTagsAndOptionalsRaw() const noexcept;
+
+ /// Cast this base class down to the most-derived class and pass it to the `visitor`.
+ ///
+ /// This function is used as a safer alternative to manually downcasting types via `IsType`/`AsType` calls.
+ /// It works like `std::visit` for types, except that it doesn't produce as much code bloat as `std::visit`
+ /// does, and can be optimized better. It casts an instance of `NTi::TType` down to the most-derived class,
+ /// and passes an intrusive pointer to that concrete type to the `visitor` functor. That is, `visitor` should
+ /// be a callable which can handle `NTi::TVoidTypePtr`, `NTi::TOptionalTypePtr`, etc.
+ ///
+ /// This function returns whatever the `visitor` returns.
+ ///
+ ///
+ /// # Example: visitor
+ ///
+ /// A simple visitor that returns name for a type would look like this:
+ ///
+ /// ```
+ /// struct TGetNameVisitor {
+ /// TString operator()(TVoidTypePtr) {
+ /// return "Void";
+ /// }
+ ///
+ /// TString operator()(TStringTypePtr) {
+ /// return "String";
+ /// }
+ ///
+ /// // ...
+ ///
+ /// TString operator()(TStructTypePtr type) {
+ /// return TString(type->GetName().GetOrElse("Struct"));;
+ /// }
+ ///
+ /// // ...
+ /// }
+ /// ```
+ ///
+ /// Now, we can use this visitor as following:
+ ///
+ /// ```
+ /// TString typeName = type->Visit(TGetNameVisitor());
+ /// ```
+ ///
+ ///
+ /// # Example: overloaded struct
+ ///
+ /// Writing a separate struct each time one needs a visitor is tedious. Thanks to C++17 magic, we may avoid it.
+ /// Using lambdas and `TOverloaded` from `library/cpp/overloaded` allows replacing separate struct with
+ /// a bunch of lambdas:
+ ///
+ /// ```
+ /// TString typeName = type->Visit(TOverloaded{
+ /// [](TVoidTypePtr) -> TString {
+ /// return "Void";
+ /// },
+ /// [](TStringTypePtr) -> TString {
+ /// return "String";
+ /// },
+ ///
+ /// // ...
+ ///
+ /// });
+ /// ```
+ ///
+ ///
+ /// # Example: handling all primitives at once
+ ///
+ /// Since all primitives derive from `TPrimitiveType`, they can be handled all at once,
+ /// by accepting `TPrimitiveTypePtr`:
+ ///
+ /// ```
+ /// TString typeName = type->Visit(TOverloaded{
+ /// // All primitive types are handled by this lambda.
+ /// [](TPrimitiveTypePtr) -> TString {
+ /// return "Primitive";
+ /// },
+ ///
+ /// // Special handler for string type. Strings are handled by this lambda
+ /// // because of how C++ prioritizes overloads.
+ /// [](TStringTypePtr) -> TString {
+ /// return "String";
+ /// },
+ ///
+ /// // ...
+ ///
+ /// });
+ /// ```
+ template <typename V>
+ inline decltype(auto) Visit(V&& visitor) const;
+
+ /// Like `Visit`, but passes const raw pointers to the visitor.
+ template <typename V>
+ inline decltype(auto) VisitRaw(V&& visitor) const;
+
+ /// @}
+
+ protected:
+ /// @name Internal interface for adoption semantics support
+ ///
+ /// Do not call these functions manually!
+ ///
+ /// See `type_factory.h`'s section on implementation details for more info.
+ ///
+ /// @{
+ //-
+ /// Create a new instance of the class using this instance as a prototype.
+ ///
+ /// This is a [virtual copy constructor]. Typical implementation does the following:
+ ///
+ /// 1. for nested types, if any, it calls the factory's `Own` function. The `Own` acquires internal
+ /// ownership over the nested types, thus guaranteeing that they'll outlive the object that've
+ /// owned them. Depending on the particular factory implementation, `Own` may either recursively deepcopy
+ /// the whole nested type, increment some reference counter, or do nothing;
+ /// 2. for other resources owned by this type (i.e. arrays, strings, etc.), it copies them into the given
+ /// factory by calling factory's `New` and `Allocate` functions;
+ /// 3. finally, it creates a new instance of the type by invoking its constructor via the factory's
+ /// `New` function.
+ ///
+ /// Note: there are no guarantees on the value stored in `FactoryOrRc_` during invocation of this function.
+ /// Specifically, creating types on the stack (`FactoryOrRc_` is `0` in this case) and moving them
+ /// into a factory is a valid technique used extensively throughout this library.
+ ///
+ /// See `type_factory.h`'s section on implementation details for more info.
+ ///
+ /// Note: this function is not marked as `virtual` because we use our own dispatch via `Visit`.
+ ///
+ /// [virtual move constructor]: https://isocpp.org/wiki/faq/virtual-functions#virtual-ctors
+ const TType* Clone(ITypeFactoryInternal& factory) const noexcept;
+
+ /// Release internal resources that were allocated in `Clone`.
+ ///
+ /// This function is the opposite of `Clone`. It releases all memory that was allocated within `Clone`,
+ /// and disowns nested types.
+ ///
+ /// This function is called by factories that perform active memory management, such as the default
+ /// heap factory. Typical implementation does the following:
+ ///
+ /// 1. for each `Clone`'s call to `Own` it calls `Disown`. The `Disown` releases internal ownership
+ /// over the nested types, thus allowing factory to free their memory. Depending
+ /// on the particular factory implementation, `Disown` may either decrement some reference counter,
+ /// call `Drop` and free the underlying memory, or do nothing;
+ /// 2. for each `Clone`'s call to `New` and `Allocate`, it calls `Delete` and `Free`;
+ /// 3. it should *not* call `Delete(this)` to mirror `Clone`'s final call to `New` (see the third bullet
+ /// in the `Clone`'s documentation). It is the factory's job to release memory under the type that's
+ /// being dropped.
+ ///
+ /// Note: there are no guarantees on whether this method will be called or not. For example, the default
+ /// heap factory will call it when some type's reference counter reaches zero. The default memory pool
+ /// factory will not call it.
+ ///
+ /// Note: this function is not marked as `virtual` because we use our own dispatch via `Visit`.
+ ///
+ /// See `type_factory.h`'s section on implementation details for more info.
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+
+ /// Get factory that manages this instance.
+ ///
+ /// If this instance is refcounted, returns the default heap factory. If it is unmanaged, returns `nullptr`.
+ /// Otherwise, returns pointer to the instance's factory.
+ ///
+ /// Remember that factories are not thread safe, thus using factory from this method may not be safe.
+ ITypeFactoryInternal* GetFactory() const noexcept;
+
+ /// Mark this instance as managed by the given factory.
+ void SetFactory(ITypeFactoryInternal* factory) noexcept;
+
+ /// Get factory's internal interface.
+ static ITypeFactoryInternal& FactoryInternal(ITypeFactory& factory) noexcept;
+
+ /// @}
+
+ protected:
+ /// @name Internal interface for reference counting
+ ///
+ /// See `type_factory.h`'s section on implementation details for more info.
+ ///
+ /// @{
+ //-
+ /// Increase reference count of this type.
+ void RefSelf() noexcept {
+ RefImpl</* RefFactory = */ false>();
+ }
+
+ /// Increase reference count of this type and its factory.
+ void Ref() noexcept {
+ RefImpl</* RefFactory = */ true>();
+ }
+
+ /// Decrease reference count of this type.
+ void UnRefSelf() noexcept {
+ UnRefImpl</* UnRefFactory = */ false>();
+ }
+
+ /// Decrease reference count of this type and its factory.
+ void UnRef() noexcept {
+ UnRefImpl</* UnRefFactory = */ true>();
+ }
+
+ /// Decrease reference count of this type. Panic if any of it reaches zero.
+ void DecRefSelf() noexcept {
+ DecRefImpl</* DecRefFactory = */ false>();
+ }
+
+ /// Decrease reference count of type and its factory. Panic if any of it reaches zero.
+ void DecRef() noexcept {
+ DecRefImpl</* DecRefFactory = */ true>();
+ }
+
+ /// Get current reference count for this type.
+ long RefCount() const noexcept {
+ return RefCountImpl();
+ }
+
+ /// @}
+
+ private:
+ template <bool RefFactory>
+ void RefImpl() noexcept;
+
+ template <bool UnRefFactory>
+ void UnRefImpl() noexcept;
+
+ template <bool DecRefFactory>
+ void DecRefImpl() noexcept;
+
+ long RefCountImpl() const noexcept;
+
+ protected:
+ /// Helper for implementing `Clone` with caching.
+ template <typename T, typename TCtor>
+ static const T* Cached(const T* type, ITypeFactoryInternal& factory, TCtor&& ctor);
+
+ private:
+ /// Pointer to the type factory that've created this instance.
+ /// If this instance is static, this variable contains zero.
+ /// If this instance was created by the default heap factory, this variable is used as a reference counter.
+ std::atomic<size_t> FactoryOrRc_ = 0;
+
+ /// Name of this type. Can be used to check before downcast.
+ ETypeName TypeName_;
+
+ /// Hash is calculated lazily.
+ mutable std::atomic<bool> HasHash_;
+ mutable std::atomic<ui64> Hash_;
+
+ private:
+ static bool IsRc(size_t factoryOrRc) noexcept {
+ return factoryOrRc & 1u;
+ }
+ static bool IsFactory(size_t factoryOrRc) noexcept {
+ return factoryOrRc != 0 && !IsRc(factoryOrRc);
+ }
+ static size_t CastFromFactory(ITypeFactoryInternal* factory) noexcept {
+ return reinterpret_cast<size_t>(factory);
+ }
+ static ITypeFactoryInternal* CastToFactory(size_t factoryOrRc) noexcept {
+ return reinterpret_cast<ITypeFactoryInternal*>(factoryOrRc);
+ }
+ };
+
+ static_assert(sizeof(TType) == 24);
+
+ bool operator==(const TTypePtr& lhs, const TTypePtr& rhs) = delete;
+ bool operator!=(const TTypePtr& lhs, const TTypePtr& rhs) = delete;
+
+ /// @brief Check for strict equivalence of the types.
+ ///
+ /// @see NTi::NEq::TStrictlyEqual
+ ///
+ /// @{
+ bool operator==(const TType& lhs, const TType& rhs);
+ bool operator!=(const TType& lhs, const TType& rhs);
+ /// @}
+
+ /// A singular type. This type has only one value. When serialized, it takes no space because it carries no data.
+ ///
+ /// Historically, YQL's `Void` is what's known as unit type (see https://en.wikipedia.org/wiki/Unit_type),
+ /// i.e. a type with only one possible value. This is similar to Python's `NoneType` or Rust's `()`.
+ class TVoidType final: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TVoidTypePtr AsPtr() const noexcept {
+ return const_cast<TVoidType*>(this);
+ }
+
+ private:
+ explicit TVoidType();
+
+ public:
+ static TVoidTypePtr Instance();
+ static const TVoidType* InstanceRaw();
+
+ protected:
+ const TVoidType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// Create new `Void` type using the default heap factory.
+ TVoidTypePtr Void();
+
+ /// A singular type, value of an empty optional.
+ ///
+ /// This type is used by YQL for `NULL` literal. Use `TVoidType` unless you have reasons to use this one.
+ class TNullType final: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TNullTypePtr AsPtr() const noexcept {
+ return const_cast<TNullType*>(this);
+ }
+
+ private:
+ explicit TNullType();
+
+ public:
+ static TNullTypePtr Instance();
+ static const TNullType* InstanceRaw();
+
+ protected:
+ const TNullType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// Base class for all primitive types.
+ class TPrimitiveType: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TPrimitiveTypePtr AsPtr() const noexcept {
+ return const_cast<TPrimitiveType*>(this);
+ }
+
+ protected:
+ explicit TPrimitiveType(TMaybe<ui64> hash, EPrimitiveTypeName primitiveTypeName) noexcept;
+
+ public:
+ /// Get name of this primitive type as a `NTi::EPrimitiveTypeName` enumerator.
+ EPrimitiveTypeName GetPrimitiveTypeName() const noexcept {
+ return ToPrimitiveTypeName(GetTypeName());
+ }
+
+ /// Cast this scalar class down to the most-derived type and pass it to the `visitor`.
+ ///
+ /// This function works like `NTi::TType::Visit`, but only handles primitive types.
+ ///
+ ///
+ /// # Example:
+ ///
+ /// ```
+ /// auto name = scalar->VisitPrimitive(TOverloaded{
+ /// [](TBoolTypePtr t) { return "Bool" },
+ /// [](TStringTypePtr t) { return "String" },
+ /// // ...
+ /// });
+ /// ```
+ template <typename V>
+ inline decltype(auto) VisitPrimitive(V&& visitor) const;
+
+ /// Like `VisitPrimitive`, but passes raw pointers to the visitor.
+ template <typename V>
+ inline decltype(auto) VisitPrimitiveRaw(V&& visitor) const;
+ };
+
+ /// A logical type capable of holding one of the two values: true or false.
+ class TBoolType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TBoolTypePtr AsPtr() const noexcept {
+ return const_cast<TBoolType*>(this);
+ }
+
+ private:
+ explicit TBoolType();
+
+ public:
+ static TBoolTypePtr Instance();
+ static const TBoolType* InstanceRaw();
+
+ protected:
+ const TBoolType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A signed integer, one byte.
+ class TInt8Type final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TInt8TypePtr AsPtr() const noexcept {
+ return const_cast<TInt8Type*>(this);
+ }
+
+ private:
+ explicit TInt8Type();
+
+ public:
+ static TInt8TypePtr Instance();
+ static const TInt8Type* InstanceRaw();
+
+ protected:
+ const TInt8Type* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A signed integer, two bytes.
+ class TInt16Type final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TInt16TypePtr AsPtr() const noexcept {
+ return const_cast<TInt16Type*>(this);
+ }
+
+ private:
+ explicit TInt16Type();
+
+ public:
+ static TInt16TypePtr Instance();
+ static const TInt16Type* InstanceRaw();
+
+ protected:
+ const TInt16Type* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A signed integer, four bytes.
+ class TInt32Type final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TInt32TypePtr AsPtr() const noexcept {
+ return const_cast<TInt32Type*>(this);
+ }
+
+ private:
+ explicit TInt32Type();
+
+ public:
+ static TInt32TypePtr Instance();
+ static const TInt32Type* InstanceRaw();
+
+ protected:
+ const TInt32Type* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A signed integer, eight bytes.
+ class TInt64Type final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TInt64TypePtr AsPtr() const noexcept {
+ return const_cast<TInt64Type*>(this);
+ }
+
+ private:
+ explicit TInt64Type();
+
+ public:
+ static TInt64TypePtr Instance();
+ static const TInt64Type* InstanceRaw();
+
+ protected:
+ const TInt64Type* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// An unsigned integer, one byte.
+ class TUint8Type final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TUint8TypePtr AsPtr() const noexcept {
+ return const_cast<TUint8Type*>(this);
+ }
+
+ private:
+ explicit TUint8Type();
+
+ public:
+ static TUint8TypePtr Instance();
+ static const TUint8Type* InstanceRaw();
+
+ protected:
+ const TUint8Type* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// An unsigned integer, two bytes.
+ class TUint16Type final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TUint16TypePtr AsPtr() const noexcept {
+ return const_cast<TUint16Type*>(this);
+ }
+
+ private:
+ explicit TUint16Type();
+
+ public:
+ static TUint16TypePtr Instance();
+ static const TUint16Type* InstanceRaw();
+
+ protected:
+ const TUint16Type* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// An unsigned integer, four bytes.
+ class TUint32Type final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TUint32TypePtr AsPtr() const noexcept {
+ return const_cast<TUint32Type*>(this);
+ }
+
+ private:
+ explicit TUint32Type();
+
+ public:
+ static TUint32TypePtr Instance();
+ static const TUint32Type* InstanceRaw();
+
+ protected:
+ const TUint32Type* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// An unsigned integer, eight bytes.
+ class TUint64Type final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TUint64TypePtr AsPtr() const noexcept {
+ return const_cast<TUint64Type*>(this);
+ }
+
+ private:
+ explicit TUint64Type();
+
+ public:
+ static TUint64TypePtr Instance();
+ static const TUint64Type* InstanceRaw();
+
+ protected:
+ const TUint64Type* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A floating point number, four bytes.
+ class TFloatType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TFloatTypePtr AsPtr() const noexcept {
+ return const_cast<TFloatType*>(this);
+ }
+
+ private:
+ explicit TFloatType();
+
+ public:
+ static TFloatTypePtr Instance();
+ static const TFloatType* InstanceRaw();
+
+ protected:
+ const TFloatType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A floating point number, eight bytes.
+ class TDoubleType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TDoubleTypePtr AsPtr() const noexcept {
+ return const_cast<TDoubleType*>(this);
+ }
+
+ private:
+ explicit TDoubleType();
+
+ public:
+ static TDoubleTypePtr Instance();
+ static const TDoubleType* InstanceRaw();
+
+ protected:
+ const TDoubleType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A binary blob.
+ ///
+ /// This type can be used for any binary data. For text, consider using type `Utf8`.
+ class TStringType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TStringTypePtr AsPtr() const noexcept {
+ return const_cast<TStringType*>(this);
+ }
+
+ private:
+ explicit TStringType();
+
+ public:
+ static TStringTypePtr Instance();
+ static const TStringType* InstanceRaw();
+
+ protected:
+ const TStringType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A utf-8 encoded text.
+ class TUtf8Type final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TUtf8TypePtr AsPtr() const noexcept {
+ return const_cast<TUtf8Type*>(this);
+ }
+
+ private:
+ explicit TUtf8Type();
+
+ public:
+ static TUtf8TypePtr Instance();
+ static const TUtf8Type* InstanceRaw();
+
+ protected:
+ const TUtf8Type* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// An absolute point in time in range `[1970-01-01, 2106-01-01)`, precision up to days.
+ class TDateType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TDateTypePtr AsPtr() const noexcept {
+ return const_cast<TDateType*>(this);
+ }
+
+ private:
+ explicit TDateType();
+
+ public:
+ static TDateTypePtr Instance();
+ static const TDateType* InstanceRaw();
+
+ protected:
+ const TDateType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// An absolute point in time in range `[1970-01-01, 2106-01-01)`, precision up to seconds.
+ class TDatetimeType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TDatetimeTypePtr AsPtr() const noexcept {
+ return const_cast<TDatetimeType*>(this);
+ }
+
+ private:
+ explicit TDatetimeType();
+
+ public:
+ static TDatetimeTypePtr Instance();
+ static const TDatetimeType* InstanceRaw();
+
+ protected:
+ const TDatetimeType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// An absolute point in time in range `[1970-01-01, 2106-01-01)`, precision up to microseconds.
+ class TTimestampType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TTimestampTypePtr AsPtr() const noexcept {
+ return const_cast<TTimestampType*>(this);
+ }
+
+ private:
+ explicit TTimestampType();
+
+ public:
+ static TTimestampTypePtr Instance();
+ static const TTimestampType* InstanceRaw();
+
+ protected:
+ const TTimestampType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// `TDateType` with additional timezone mark.
+ class TTzDateType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TTzDateTypePtr AsPtr() const noexcept {
+ return const_cast<TTzDateType*>(this);
+ }
+
+ private:
+ explicit TTzDateType();
+
+ public:
+ static TTzDateTypePtr Instance();
+ static const TTzDateType* InstanceRaw();
+
+ protected:
+ const TTzDateType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// `TDatetimeType` with additional timezone mark.
+ class TTzDatetimeType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TTzDatetimeTypePtr AsPtr() const noexcept {
+ return const_cast<TTzDatetimeType*>(this);
+ }
+
+ private:
+ explicit TTzDatetimeType();
+
+ public:
+ static TTzDatetimeTypePtr Instance();
+ static const TTzDatetimeType* InstanceRaw();
+
+ protected:
+ const TTzDatetimeType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// `TTimestampType` with additional timezone mark.
+ class TTzTimestampType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TTzTimestampTypePtr AsPtr() const noexcept {
+ return const_cast<TTzTimestampType*>(this);
+ }
+
+ private:
+ explicit TTzTimestampType();
+
+ public:
+ static TTzTimestampTypePtr Instance();
+ static const TTzTimestampType* InstanceRaw();
+
+ protected:
+ const TTzTimestampType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// Signed delta between two timestamps.
+ class TIntervalType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TIntervalTypePtr AsPtr() const noexcept {
+ return const_cast<TIntervalType*>(this);
+ }
+
+ private:
+ explicit TIntervalType();
+
+ public:
+ static TIntervalTypePtr Instance();
+ static const TIntervalType* InstanceRaw();
+
+ protected:
+ const TIntervalType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A 128-bit number with controlled exponent/significand size.
+ ///
+ /// Decimal is a type for extra precise calculations. Internally, it is represented by a float-like 128-bit number.
+ ///
+ /// Two parameters control number of decimal digits in the decimal value. `Precision` is the total number
+ /// of decimal digits. `Scale` is the number of digits after the decimal point.
+ class TDecimalType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TDecimalTypePtr AsPtr() const noexcept {
+ return const_cast<TDecimalType*>(this);
+ }
+
+ private:
+ explicit TDecimalType(TMaybe<ui64> hash, ui8 precision, ui8 scale) noexcept;
+ static TDecimalTypePtr Create(ITypeFactory& factory, ui8 precision, ui8 scale);
+ static const TDecimalType* CreateRaw(ITypeFactory& factory, ui8 precision, ui8 scale);
+
+ protected:
+ ui64 CalculateHash() const noexcept;
+
+ public:
+ /// Get total number of decimal digits.
+ ui8 GetPrecision() const noexcept {
+ return Precision_;
+ }
+
+ /// Get number of decimal digits after the decimal point.
+ ui8 GetScale() const noexcept {
+ return Scale_;
+ }
+
+ protected:
+ const TDecimalType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+
+ private:
+ ui8 Precision_;
+ ui8 Scale_;
+ };
+
+ /// A string with valid JSON.
+ class TJsonType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TJsonTypePtr AsPtr() const noexcept {
+ return const_cast<TJsonType*>(this);
+ }
+
+ private:
+ explicit TJsonType();
+
+ public:
+ static TJsonTypePtr Instance();
+ static const TJsonType* InstanceRaw();
+
+ protected:
+ const TJsonType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A string with valid YSON.
+ class TYsonType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TYsonTypePtr AsPtr() const noexcept {
+ return const_cast<TYsonType*>(this);
+ }
+
+ private:
+ explicit TYsonType();
+
+ public:
+ static TYsonTypePtr Instance();
+ static const TYsonType* InstanceRaw();
+
+ protected:
+ const TYsonType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// A string with valid UUID.
+ class TUuidType final: public TPrimitiveType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TUuidTypePtr AsPtr() const noexcept {
+ return const_cast<TUuidType*>(this);
+ }
+
+ private:
+ explicit TUuidType();
+
+ public:
+ static TUuidTypePtr Instance();
+ static const TUuidType* InstanceRaw();
+
+ protected:
+ const TUuidType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+ };
+
+ /// Object which can store a value or a singular `NULL` value.
+ ///
+ /// This type is used to encode a value or its absence.
+ class TOptionalType final: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TOptionalTypePtr AsPtr() const noexcept {
+ return const_cast<TOptionalType*>(this);
+ }
+
+ private:
+ explicit TOptionalType(TMaybe<ui64> hash, const TType* item) noexcept;
+ static TOptionalTypePtr Create(ITypeFactory& factory, TTypePtr item);
+ static const TOptionalType* CreateRaw(ITypeFactory& factory, const TType* item);
+
+ protected:
+ ui64 CalculateHash() const noexcept;
+
+ public:
+ /// Get underlying type.
+ TTypePtr GetItemType() const noexcept {
+ return GetItemTypeRaw()->AsPtr();
+ }
+
+ /// Like `GetMemberType`, but returns a raw pointer.
+ const TType* GetItemTypeRaw() const noexcept {
+ return Item_;
+ }
+
+ protected:
+ const TOptionalType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+
+ private:
+ const TType* Item_;
+ };
+
+ /// A variable-size collection of homogeneous values.
+ class TListType final: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TListTypePtr AsPtr() const noexcept {
+ return const_cast<TListType*>(this);
+ }
+
+ private:
+ explicit TListType(TMaybe<ui64> hash, const TType* item) noexcept;
+ static TListTypePtr Create(ITypeFactory& factory, TTypePtr item);
+ static const TListType* CreateRaw(ITypeFactory& factory, const TType* item);
+
+ protected:
+ ui64 CalculateHash() const noexcept;
+
+ public:
+ /// Get underlying type.
+ TTypePtr GetItemType() const noexcept {
+ return GetItemTypeRaw()->AsPtr();
+ }
+
+ /// Like `GetMemberType`, but returns a raw pointer.
+ const TType* GetItemTypeRaw() const noexcept {
+ return Item_;
+ }
+
+ protected:
+ const TListType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+
+ private:
+ const TType* Item_;
+ };
+
+ /// An associative key-value container.
+ ///
+ /// Values of this type are usually represented as hashmaps.
+ class TDictType final: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TDictTypePtr AsPtr() const noexcept {
+ return const_cast<TDictType*>(this);
+ }
+
+ private:
+ explicit TDictType(TMaybe<ui64> hash, const TType* key, const TType* value) noexcept;
+ static TDictTypePtr Create(ITypeFactory& factory, TTypePtr key, TTypePtr value);
+ static const TDictType* CreateRaw(ITypeFactory& factory, const TType* key, const TType* value);
+
+ protected:
+ ui64 CalculateHash() const noexcept;
+
+ public:
+ /// Get the key type.
+ TTypePtr GetKeyType() const noexcept {
+ return GetKeyTypeRaw()->AsPtr();
+ }
+
+ /// Like `GetKeyType`, but returns a raw pointer.
+ const TType* GetKeyTypeRaw() const noexcept {
+ return Key_;
+ }
+
+ /// Get the value type.
+ TTypePtr GetValueType() const noexcept {
+ return GetValueTypeRaw()->AsPtr();
+ }
+
+ /// Like `GetValueType`, but returns a raw pointer.
+ const TType* GetValueTypeRaw() const noexcept {
+ return Value_;
+ }
+
+ protected:
+ const TDictType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+
+ private:
+ const TType* Key_;
+ const TType* Value_;
+ };
+
+ /// A fixed-size collection of named heterogeneous values.
+ ///
+ /// Structs represent multiple values grouped into a single entity. Values stored in a struct are called items.
+ /// Each item have an associated type and a name.
+ ///
+ /// Struct type is represented by an array of item types and their names.
+ ///
+ /// Even though struct elements are accessed by their names, we use vector to represent struct type because order
+ /// of struct items matters. If affects memory layout of a struct, how struct is serialized and deserialized.
+ /// Items order is vital for struct versioning. New fields should always be added to the end of the struct.
+ /// This way older parsers can read values serialized by newer writers: they'll simply read known head of a struct
+ /// and skip tail that contains unknown fields.
+ ///
+ ///
+ /// # Struct names
+ ///
+ /// Each struct defined by YDL must have an associated name. This name is used to refer struct in code, to generate
+ /// code representing this struct in other programming languages, and to report errors. The struct's name is saved
+ /// in this field.
+ ///
+ /// Note that, even though YDL requires struct names, name field is optional because other systems might
+ /// use anonymous structs (or maybe they dont't use struct names at all).
+ ///
+ /// Note also that type aliases (especially `newtype`) use tags to name types, so primitives don't have name field.
+ class TStructType final: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+ friend class TStructBuilderRaw;
+
+ public:
+ TStructTypePtr AsPtr() const noexcept {
+ return const_cast<TStructType*>(this);
+ }
+
+ public:
+ /// A single struct element.
+ class TMember {
+ public:
+ TMember(TStringBuf name, const TType* type);
+
+ public:
+ /// Get name of this item.
+ TStringBuf GetName() const {
+ return Name_;
+ }
+
+ /// Get type of this item.
+ TTypePtr GetType() const {
+ return GetTypeRaw()->AsPtr();
+ }
+
+ /// Like `GetType`, but returns a raw pointer.
+ const TType* GetTypeRaw() const {
+ return Type_;
+ }
+
+ /// Calculate this item's hash. Hashes follow the 'strict equivalence' (see type_equivalence.h).
+ ui64 Hash() const;
+
+ private:
+ TStringBuf Name_;
+ const TType* Type_;
+ };
+
+ using TMembers = TConstArrayRef<TMember>;
+
+ /// Like `TMember`, but owns its contents. Used in non-raw constructors to guarantee data validity.
+ class TOwnedMember {
+ public:
+ TOwnedMember(TString name, TTypePtr type);
+
+ public:
+ operator TMember() const&;
+
+ private:
+ TString Name_;
+ TTypePtr Type_;
+ };
+
+ using TOwnedMembers = TConstArrayRef<TOwnedMember>;
+
+ private:
+ explicit TStructType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, TMembers members, TConstArrayRef<size_t> sortedMembers) noexcept;
+ static TStructTypePtr Create(ITypeFactory& factory, TOwnedMembers members);
+ static TStructTypePtr Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TOwnedMembers members);
+ static const TStructType* CreateRaw(ITypeFactory& factory, TMembers members);
+ static const TStructType* CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, TMembers members);
+ static void MakeSortedMembers(TMembers members, TArrayRef<size_t> sortedItems);
+
+ protected:
+ ui64 CalculateHash() const noexcept;
+
+ public:
+ /// Get name of this struct.
+ TMaybe<TStringBuf> GetName() const noexcept {
+ return Name_;
+ }
+
+ /// Get description of structure members.
+ TMembers GetMembers() const noexcept {
+ return Members_;
+ }
+
+ /// Check if there is an item with the given name in this struct.
+ bool HasMember(TStringBuf name) const noexcept;
+
+ /// Lookup struct item by name. Throw an error if there is no such item.
+ const TMember& GetMember(TStringBuf name) const;
+
+ /// Lookup struct item by name, return its index or `-1`, if item was not found.
+ /// This function works in `O(log(n))` time, where `n` is number of struct members.
+ ssize_t GetMemberIndex(TStringBuf name) const noexcept;
+
+ protected:
+ const TStructType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+
+ private:
+ TMaybe<TStringBuf> Name_;
+ TMembers Members_;
+ TConstArrayRef<size_t> SortedMembers_;
+ };
+
+ /// A fixed-size collection of named heterogeneous values.
+ ///
+ /// Tuples, like structs, represent multiple values, also called items, grouped into a single entity. Unlike
+ /// structs, though, tuple items are unnamed. Instead of names, they are accessed by their indexes.
+ ///
+ /// For a particular tuple type, number of items, their order and types are fixed, they should be known before
+ /// creating instances of tuples and can't change over time.
+ ///
+ ///
+ /// # Tuple names
+ ///
+ /// YDL requires each tuple definition to have a name (see `TStructType`). The name might not be mandatory
+ /// in other systems, so name field is optional.
+ class TTupleType: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+ friend class TTupleBuilderRaw;
+
+ public:
+ TTupleTypePtr AsPtr() const noexcept {
+ return const_cast<TTupleType*>(this);
+ }
+
+ public:
+ /// A single tuple element.
+ class TElement {
+ public:
+ TElement(const TType* type);
+
+ public:
+ /// Get type of this item.
+ TTypePtr GetType() const {
+ return GetTypeRaw()->AsPtr();
+ }
+
+ /// Like `GetType`, but returns a raw pointer.
+ const TType* GetTypeRaw() const {
+ return Type_;
+ }
+
+ /// Calculate this item's hash. Hashes follow the 'strict equivalence' (see type_equivalence.h).
+ ui64 Hash() const;
+
+ private:
+ const TType* Type_;
+ };
+
+ using TElements = TConstArrayRef<TElement>;
+
+ /// Like `TElement`, but owns its contents. Used in non-raw constructors to guarantee data validity.
+ class TOwnedElement {
+ public:
+ TOwnedElement(TTypePtr type);
+
+ public:
+ operator TElement() const&;
+
+ private:
+ TTypePtr Type_;
+ };
+
+ using TOwnedElements = TConstArrayRef<TOwnedElement>;
+
+ private:
+ explicit TTupleType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, TElements items) noexcept;
+ static TTupleTypePtr Create(ITypeFactory& factory, TOwnedElements items);
+ static TTupleTypePtr Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TOwnedElements items);
+ static const TTupleType* CreateRaw(ITypeFactory& factory, TElements items);
+ static const TTupleType* CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, TElements items);
+
+ protected:
+ ui64 CalculateHash() const noexcept;
+
+ public:
+ /// Get name of this type.
+ TMaybe<TStringBuf> GetName() const noexcept {
+ return Name_;
+ }
+
+ /// Get description of tuple items.
+ TElements GetElements() const noexcept {
+ return Elements_;
+ }
+
+ protected:
+ const TTupleType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+
+ private:
+ TMaybe<TStringBuf> Name_;
+ TElements Elements_;
+ };
+
+ /// A tagged union with named or unnamed alternatives (a.k.a. variant over struct or variant over tuple).
+ ///
+ /// Variants are used to store values of different types.
+ ///
+ /// For example, a variant over struct which holds an ip address could look like
+ /// `Ip = Variant<v4: IpV4, v6: IpV6>`, where `IpV4` and `IpV6` are some other types. Now, a value of type `Ip`
+ /// could store either a value of type `IpV4` or a value of type `IpV6`.
+ ///
+ /// Even though item types can be the same, each item represents a distinct state of a variant. For example,
+ /// a variant for a user identifier can look like `Uid = Variant<yuid: String, ip: String>`. This variant can
+ /// contain either user's yandexuid of user's ip. Despite both items are of the same type `String`, `Uid` which
+ /// contains a `yuid` and `Uid` which contains an `ip` are never equal.
+ /// That is, `Uid.yuid("000000") != Uid.ip("000000")`.
+ ///
+ /// Exactly like with structs or tuples, order of variant items matter. Indexes of variant items are used
+ /// instead of names when variant is serialized.
+ ///
+ ///
+ /// # Variant names
+ ///
+ /// YDL requires each variant definition to have a name (see `TStructType`). The name might not be mandatory
+ /// in other systems, so name field is optional.
+ class TVariantType final: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+ friend class TStructBuilderRaw;
+ friend class TTupleBuilderRaw;
+
+ public:
+ TVariantTypePtr AsPtr() const noexcept {
+ return const_cast<TVariantType*>(this);
+ }
+
+ private:
+ explicit TVariantType(TMaybe<ui64> hash, TMaybe<TStringBuf> name, const TType* inner) noexcept;
+ static TVariantTypePtr Create(ITypeFactory& factory, TTypePtr inner);
+ static TVariantTypePtr Create(ITypeFactory& factory, TMaybe<TStringBuf> name, TTypePtr inner);
+ static const TVariantType* CreateRaw(ITypeFactory& factory, const TType* inner);
+ static const TVariantType* CreateRaw(ITypeFactory& factory, TMaybe<TStringBuf> name, const TType* inner);
+
+ protected:
+ ui64 CalculateHash() const noexcept;
+
+ public:
+ /// Get name of this variant.
+ TMaybe<TStringBuf> GetName() const noexcept {
+ return Name_;
+ }
+
+ /// Get vector of variant items.
+ TTypePtr GetUnderlyingType() const noexcept {
+ return Underlying_->AsPtr();
+ }
+
+ /// Like `GetUnderlyingType`, but returns a raw pointer.
+ const TType* GetUnderlyingTypeRaw() const noexcept {
+ return Underlying_;
+ }
+
+ /// Check if this variant's inner type is a struct.
+ bool IsVariantOverStruct() const noexcept {
+ return GetUnderlyingTypeRaw()->GetTypeName() == ETypeName::Struct;
+ }
+
+ /// Check if this variant's inner type is a tuple.
+ bool IsVariantOverTuple() const noexcept {
+ return GetUnderlyingTypeRaw()->GetTypeName() == ETypeName::Tuple;
+ }
+
+ /// Visit inner type of this variant. This function works like `Visit`, but only casts inner type
+ /// to struct or tuple, so you don't need to handle other types in a visitor.
+ template <typename V>
+ inline decltype(auto) VisitUnderlying(V&& visitor) const;
+
+ /// Like `VisitUnderlying`, but passes const raw pointers to the visitor.
+ template <typename V>
+ inline decltype(auto) VisitUnderlyingRaw(V&& visitor) const;
+
+ protected:
+ const TVariantType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+
+ private:
+ TMaybe<TStringBuf> Name_;
+ const TType* Underlying_;
+ };
+
+ /// Named or tagged or user-defined type.
+ ///
+ /// Tags are used to create new types from existing ones by assigning them a tag, i.e. a name. They wrap other types
+ /// adding them additional semantics.
+ ///
+ /// On physical level, tags do not change types. Both `Tagged<Int32, 'GeoId'>` and `Int32` have exactly the same
+ /// representation when serialized.
+ ///
+ /// On logical level, tags change semantics of a type. This can affect how types are displayed, how they
+ /// are checked and converted, etc.
+ class TTaggedType final: public TType {
+ friend class TType;
+ friend class ITypeFactoryInternal;
+ friend class ITypeFactory;
+ friend class IPoolTypeFactory;
+
+ public:
+ TTaggedTypePtr AsPtr() const noexcept {
+ return const_cast<TTaggedType*>(this);
+ }
+
+ private:
+ explicit TTaggedType(TMaybe<ui64> hash, const TType* type, TStringBuf tag) noexcept;
+ static TTaggedTypePtr Create(ITypeFactory& factory, TTypePtr type, TStringBuf tag);
+ static const TTaggedType* CreateRaw(ITypeFactory& factory, const TType* type, TStringBuf tag);
+
+ protected:
+ ui64 CalculateHash() const noexcept;
+
+ public:
+ /// Get tag content, i.e. name of the new type.
+ TStringBuf GetTag() const noexcept {
+ return Tag_;
+ }
+
+ /// Get the wrapped type.
+ TTypePtr GetItemType() const noexcept {
+ return GetItemTypeRaw()->AsPtr();
+ }
+
+ /// Like `GetMemberType`, but returns a raw pointer.
+ const TType* GetItemTypeRaw() const noexcept {
+ return Item_;
+ }
+
+ protected:
+ const TTaggedType* Clone(ITypeFactoryInternal& factory) const noexcept;
+ void Drop(ITypeFactoryInternal& factory) noexcept;
+
+ private:
+ const TType* Item_;
+ TStringBuf Tag_;
+ };
+
+#ifdef __JETBRAINS_IDE__
+#pragma clang diagnostic push
+#pragma ide diagnostic ignored "cppcoreguidelines-pro-type-static-cast-downcast"
+#endif
+
+ bool TType::IsPrimitive() const noexcept {
+ return ::NTi::IsPrimitive(TypeName_);
+ }
+
+ TPrimitiveTypePtr TType::AsPrimitive() const noexcept {
+ return AsPrimitiveRaw()->AsPtr();
+ }
+
+ const TPrimitiveType* TType::AsPrimitiveRaw() const noexcept {
+ Y_VERIFY(IsPrimitive());
+ return static_cast<const TPrimitiveType*>(this);
+ }
+
+ bool TType::IsVoid() const noexcept {
+ return TypeName_ == ETypeName::Void;
+ }
+
+ TVoidTypePtr TType::AsVoid() const noexcept {
+ return AsVoidRaw()->AsPtr();
+ }
+
+ const TVoidType* TType::AsVoidRaw() const noexcept {
+ Y_VERIFY(IsVoid());
+ return static_cast<const TVoidType*>(this);
+ }
+
+ bool TType::IsNull() const noexcept {
+ return TypeName_ == ETypeName::Null;
+ }
+
+ TNullTypePtr TType::AsNull() const noexcept {
+ return AsNullRaw()->AsPtr();
+ }
+
+ const TNullType* TType::AsNullRaw() const noexcept {
+ Y_VERIFY(IsNull());
+ return static_cast<const TNullType*>(this);
+ }
+
+ bool TType::IsBool() const noexcept {
+ return TypeName_ == ETypeName::Bool;
+ }
+
+ TBoolTypePtr TType::AsBool() const noexcept {
+ return AsBoolRaw()->AsPtr();
+ }
+
+ const TBoolType* TType::AsBoolRaw() const noexcept {
+ Y_VERIFY(IsBool());
+ return static_cast<const TBoolType*>(this);
+ }
+
+ bool TType::IsInt8() const noexcept {
+ return TypeName_ == ETypeName::Int8;
+ }
+
+ TInt8TypePtr TType::AsInt8() const noexcept {
+ return AsInt8Raw()->AsPtr();
+ }
+
+ const TInt8Type* TType::AsInt8Raw() const noexcept {
+ Y_VERIFY(IsInt8());
+ return static_cast<const TInt8Type*>(this);
+ }
+
+ bool TType::IsInt16() const noexcept {
+ return TypeName_ == ETypeName::Int16;
+ }
+
+ TInt16TypePtr TType::AsInt16() const noexcept {
+ return AsInt16Raw()->AsPtr();
+ }
+
+ const TInt16Type* TType::AsInt16Raw() const noexcept {
+ Y_VERIFY(IsInt16());
+ return static_cast<const TInt16Type*>(this);
+ }
+
+ bool TType::IsInt32() const noexcept {
+ return TypeName_ == ETypeName::Int32;
+ }
+
+ TInt32TypePtr TType::AsInt32() const noexcept {
+ return AsInt32Raw()->AsPtr();
+ }
+
+ const TInt32Type* TType::AsInt32Raw() const noexcept {
+ Y_VERIFY(IsInt32());
+ return static_cast<const TInt32Type*>(this);
+ }
+
+ bool TType::IsInt64() const noexcept {
+ return TypeName_ == ETypeName::Int64;
+ }
+
+ TInt64TypePtr TType::AsInt64() const noexcept {
+ return AsInt64Raw()->AsPtr();
+ }
+
+ const TInt64Type* TType::AsInt64Raw() const noexcept {
+ Y_VERIFY(IsInt64());
+ return static_cast<const TInt64Type*>(this);
+ }
+
+ bool TType::IsUint8() const noexcept {
+ return TypeName_ == ETypeName::Uint8;
+ }
+
+ TUint8TypePtr TType::AsUint8() const noexcept {
+ return AsUint8Raw()->AsPtr();
+ }
+
+ const TUint8Type* TType::AsUint8Raw() const noexcept {
+ Y_VERIFY(IsUint8());
+ return static_cast<const TUint8Type*>(this);
+ }
+
+ bool TType::IsUint16() const noexcept {
+ return TypeName_ == ETypeName::Uint16;
+ }
+
+ TUint16TypePtr TType::AsUint16() const noexcept {
+ return AsUint16Raw()->AsPtr();
+ }
+
+ const TUint16Type* TType::AsUint16Raw() const noexcept {
+ Y_VERIFY(IsUint16());
+ return static_cast<const TUint16Type*>(this);
+ }
+
+ bool TType::IsUint32() const noexcept {
+ return TypeName_ == ETypeName::Uint32;
+ }
+
+ TUint32TypePtr TType::AsUint32() const noexcept {
+ return AsUint32Raw()->AsPtr();
+ }
+
+ const TUint32Type* TType::AsUint32Raw() const noexcept {
+ Y_VERIFY(IsUint32());
+ return static_cast<const TUint32Type*>(this);
+ }
+
+ bool TType::IsUint64() const noexcept {
+ return TypeName_ == ETypeName::Uint64;
+ }
+
+ TUint64TypePtr TType::AsUint64() const noexcept {
+ return AsUint64Raw()->AsPtr();
+ }
+
+ const TUint64Type* TType::AsUint64Raw() const noexcept {
+ Y_VERIFY(IsUint64());
+ return static_cast<const TUint64Type*>(this);
+ }
+
+ bool TType::IsFloat() const noexcept {
+ return TypeName_ == ETypeName::Float;
+ }
+
+ TFloatTypePtr TType::AsFloat() const noexcept {
+ return AsFloatRaw()->AsPtr();
+ }
+
+ const TFloatType* TType::AsFloatRaw() const noexcept {
+ Y_VERIFY(IsFloat());
+ return static_cast<const TFloatType*>(this);
+ }
+
+ bool TType::IsDouble() const noexcept {
+ return TypeName_ == ETypeName::Double;
+ }
+
+ TDoubleTypePtr TType::AsDouble() const noexcept {
+ return AsDoubleRaw()->AsPtr();
+ }
+
+ const TDoubleType* TType::AsDoubleRaw() const noexcept {
+ Y_VERIFY(IsDouble());
+ return static_cast<const TDoubleType*>(this);
+ }
+
+ bool TType::IsString() const noexcept {
+ return TypeName_ == ETypeName::String;
+ }
+
+ TStringTypePtr TType::AsString() const noexcept {
+ return AsStringRaw()->AsPtr();
+ }
+
+ const TStringType* TType::AsStringRaw() const noexcept {
+ Y_VERIFY(IsString());
+ return static_cast<const TStringType*>(this);
+ }
+
+ bool TType::IsUtf8() const noexcept {
+ return TypeName_ == ETypeName::Utf8;
+ }
+
+ TUtf8TypePtr TType::AsUtf8() const noexcept {
+ return AsUtf8Raw()->AsPtr();
+ }
+
+ const TUtf8Type* TType::AsUtf8Raw() const noexcept {
+ Y_VERIFY(IsUtf8());
+ return static_cast<const TUtf8Type*>(this);
+ }
+
+ bool TType::IsDate() const noexcept {
+ return TypeName_ == ETypeName::Date;
+ }
+
+ TDateTypePtr TType::AsDate() const noexcept {
+ return AsDateRaw()->AsPtr();
+ }
+
+ const TDateType* TType::AsDateRaw() const noexcept {
+ Y_VERIFY(IsDate());
+ return static_cast<const TDateType*>(this);
+ }
+
+ bool TType::IsDatetime() const noexcept {
+ return TypeName_ == ETypeName::Datetime;
+ }
+
+ TDatetimeTypePtr TType::AsDatetime() const noexcept {
+ return AsDatetimeRaw()->AsPtr();
+ }
+
+ const TDatetimeType* TType::AsDatetimeRaw() const noexcept {
+ Y_VERIFY(IsDatetime());
+ return static_cast<const TDatetimeType*>(this);
+ }
+
+ bool TType::IsTimestamp() const noexcept {
+ return TypeName_ == ETypeName::Timestamp;
+ }
+
+ TTimestampTypePtr TType::AsTimestamp() const noexcept {
+ return AsTimestampRaw()->AsPtr();
+ }
+
+ const TTimestampType* TType::AsTimestampRaw() const noexcept {
+ Y_VERIFY(IsTimestamp());
+ return static_cast<const TTimestampType*>(this);
+ }
+
+ bool TType::IsTzDate() const noexcept {
+ return TypeName_ == ETypeName::TzDate;
+ }
+
+ TTzDateTypePtr TType::AsTzDate() const noexcept {
+ return AsTzDateRaw()->AsPtr();
+ }
+
+ const TTzDateType* TType::AsTzDateRaw() const noexcept {
+ Y_VERIFY(IsTzDate());
+ return static_cast<const TTzDateType*>(this);
+ }
+
+ bool TType::IsTzDatetime() const noexcept {
+ return TypeName_ == ETypeName::TzDatetime;
+ }
+
+ TTzDatetimeTypePtr TType::AsTzDatetime() const noexcept {
+ return AsTzDatetimeRaw()->AsPtr();
+ }
+
+ const TTzDatetimeType* TType::AsTzDatetimeRaw() const noexcept {
+ Y_VERIFY(IsTzDatetime());
+ return static_cast<const TTzDatetimeType*>(this);
+ }
+
+ bool TType::IsTzTimestamp() const noexcept {
+ return TypeName_ == ETypeName::TzTimestamp;
+ }
+
+ TTzTimestampTypePtr TType::AsTzTimestamp() const noexcept {
+ return AsTzTimestampRaw()->AsPtr();
+ }
+
+ const TTzTimestampType* TType::AsTzTimestampRaw() const noexcept {
+ Y_VERIFY(IsTzTimestamp());
+ return static_cast<const TTzTimestampType*>(this);
+ }
+
+ bool TType::IsInterval() const noexcept {
+ return TypeName_ == ETypeName::Interval;
+ }
+
+ TIntervalTypePtr TType::AsInterval() const noexcept {
+ return AsIntervalRaw()->AsPtr();
+ }
+
+ const TIntervalType* TType::AsIntervalRaw() const noexcept {
+ Y_VERIFY(IsInterval());
+ return static_cast<const TIntervalType*>(this);
+ }
+
+ bool TType::IsDecimal() const noexcept {
+ return TypeName_ == ETypeName::Decimal;
+ }
+
+ TDecimalTypePtr TType::AsDecimal() const noexcept {
+ return AsDecimalRaw()->AsPtr();
+ }
+
+ const TDecimalType* TType::AsDecimalRaw() const noexcept {
+ Y_VERIFY(IsDecimal());
+ return static_cast<const TDecimalType*>(this);
+ }
+
+ bool TType::IsJson() const noexcept {
+ return TypeName_ == ETypeName::Json;
+ }
+
+ TJsonTypePtr TType::AsJson() const noexcept {
+ return AsJsonRaw()->AsPtr();
+ }
+
+ const TJsonType* TType::AsJsonRaw() const noexcept {
+ Y_VERIFY(IsJson());
+ return static_cast<const TJsonType*>(this);
+ }
+
+ bool TType::IsYson() const noexcept {
+ return TypeName_ == ETypeName::Yson;
+ }
+
+ TYsonTypePtr TType::AsYson() const noexcept {
+ return AsYsonRaw()->AsPtr();
+ }
+
+ const TYsonType* TType::AsYsonRaw() const noexcept {
+ Y_VERIFY(IsYson());
+ return static_cast<const TYsonType*>(this);
+ }
+
+ bool TType::IsUuid() const noexcept {
+ return TypeName_ == ETypeName::Uuid;
+ }
+
+ TUuidTypePtr TType::AsUuid() const noexcept {
+ return AsUuidRaw()->AsPtr();
+ }
+
+ const TUuidType* TType::AsUuidRaw() const noexcept {
+ Y_VERIFY(IsUuid());
+ return static_cast<const TUuidType*>(this);
+ }
+
+ bool TType::IsOptional() const noexcept {
+ return TypeName_ == ETypeName::Optional;
+ }
+
+ TOptionalTypePtr TType::AsOptional() const noexcept {
+ return AsOptionalRaw()->AsPtr();
+ }
+
+ const TOptionalType* TType::AsOptionalRaw() const noexcept {
+ Y_VERIFY(IsOptional());
+ return static_cast<const TOptionalType*>(this);
+ }
+
+ bool TType::IsList() const noexcept {
+ return TypeName_ == ETypeName::List;
+ }
+
+ TListTypePtr TType::AsList() const noexcept {
+ return AsListRaw()->AsPtr();
+ }
+
+ const TListType* TType::AsListRaw() const noexcept {
+ Y_VERIFY(IsList());
+ return static_cast<const TListType*>(this);
+ }
+
+ bool TType::IsDict() const noexcept {
+ return TypeName_ == ETypeName::Dict;
+ }
+
+ TDictTypePtr TType::AsDict() const noexcept {
+ return AsDictRaw()->AsPtr();
+ }
+
+ const TDictType* TType::AsDictRaw() const noexcept {
+ Y_VERIFY(IsDict());
+ return static_cast<const TDictType*>(this);
+ }
+
+ bool TType::IsStruct() const noexcept {
+ return TypeName_ == ETypeName::Struct;
+ }
+
+ TStructTypePtr TType::AsStruct() const noexcept {
+ return AsStructRaw()->AsPtr();
+ }
+
+ const TStructType* TType::AsStructRaw() const noexcept {
+ Y_VERIFY(IsStruct());
+ return static_cast<const TStructType*>(this);
+ }
+
+ bool TType::IsTuple() const noexcept {
+ return TypeName_ == ETypeName::Tuple;
+ }
+
+ TTupleTypePtr TType::AsTuple() const noexcept {
+ return AsTupleRaw()->AsPtr();
+ }
+
+ const TTupleType* TType::AsTupleRaw() const noexcept {
+ Y_VERIFY(IsTuple());
+ return static_cast<const TTupleType*>(this);
+ }
+
+ bool TType::IsVariant() const noexcept {
+ return TypeName_ == ETypeName::Variant;
+ }
+
+ TVariantTypePtr TType::AsVariant() const noexcept {
+ return AsVariantRaw()->AsPtr();
+ }
+
+ const TVariantType* TType::AsVariantRaw() const noexcept {
+ Y_VERIFY(IsVariant());
+ return static_cast<const TVariantType*>(this);
+ }
+
+ bool TType::IsTagged() const noexcept {
+ return TypeName_ == ETypeName::Tagged;
+ }
+
+ TTaggedTypePtr TType::AsTagged() const noexcept {
+ return AsTaggedRaw()->AsPtr();
+ }
+
+ const TTaggedType* TType::AsTaggedRaw() const noexcept {
+ Y_VERIFY(IsTagged());
+ return static_cast<const TTaggedType*>(this);
+ }
+
+#ifdef __JETBRAINS_IDE__
+#pragma clang diagnostic pop
+#endif
+
+ template <typename V>
+ decltype(auto) TType::Visit(V&& visitor) const {
+ switch (TypeName_) {
+ case ETypeName::Bool:
+ return std::forward<V>(visitor)(this->AsBool());
+ case ETypeName::Int8:
+ return std::forward<V>(visitor)(this->AsInt8());
+ case ETypeName::Int16:
+ return std::forward<V>(visitor)(this->AsInt16());
+ case ETypeName::Int32:
+ return std::forward<V>(visitor)(this->AsInt32());
+ case ETypeName::Int64:
+ return std::forward<V>(visitor)(this->AsInt64());
+ case ETypeName::Uint8:
+ return std::forward<V>(visitor)(this->AsUint8());
+ case ETypeName::Uint16:
+ return std::forward<V>(visitor)(this->AsUint16());
+ case ETypeName::Uint32:
+ return std::forward<V>(visitor)(this->AsUint32());
+ case ETypeName::Uint64:
+ return std::forward<V>(visitor)(this->AsUint64());
+ case ETypeName::Float:
+ return std::forward<V>(visitor)(this->AsFloat());
+ case ETypeName::Double:
+ return std::forward<V>(visitor)(this->AsDouble());
+ case ETypeName::String:
+ return std::forward<V>(visitor)(this->AsString());
+ case ETypeName::Utf8:
+ return std::forward<V>(visitor)(this->AsUtf8());
+ case ETypeName::Date:
+ return std::forward<V>(visitor)(this->AsDate());
+ case ETypeName::Datetime:
+ return std::forward<V>(visitor)(this->AsDatetime());
+ case ETypeName::Timestamp:
+ return std::forward<V>(visitor)(this->AsTimestamp());
+ case ETypeName::TzDate:
+ return std::forward<V>(visitor)(this->AsTzDate());
+ case ETypeName::TzDatetime:
+ return std::forward<V>(visitor)(this->AsTzDatetime());
+ case ETypeName::TzTimestamp:
+ return std::forward<V>(visitor)(this->AsTzTimestamp());
+ case ETypeName::Interval:
+ return std::forward<V>(visitor)(this->AsInterval());
+ case ETypeName::Decimal:
+ return std::forward<V>(visitor)(this->AsDecimal());
+ case ETypeName::Json:
+ return std::forward<V>(visitor)(this->AsJson());
+ case ETypeName::Yson:
+ return std::forward<V>(visitor)(this->AsYson());
+ case ETypeName::Uuid:
+ return std::forward<V>(visitor)(this->AsUuid());
+ case ETypeName::Void:
+ return std::forward<V>(visitor)(this->AsVoid());
+ case ETypeName::Null:
+ return std::forward<V>(visitor)(this->AsNull());
+ case ETypeName::Optional:
+ return std::forward<V>(visitor)(this->AsOptional());
+ case ETypeName::List:
+ return std::forward<V>(visitor)(this->AsList());
+ case ETypeName::Dict:
+ return std::forward<V>(visitor)(this->AsDict());
+ case ETypeName::Struct:
+ return std::forward<V>(visitor)(this->AsStruct());
+ case ETypeName::Tuple:
+ return std::forward<V>(visitor)(this->AsTuple());
+ case ETypeName::Variant:
+ return std::forward<V>(visitor)(this->AsVariant());
+ case ETypeName::Tagged:
+ return std::forward<V>(visitor)(this->AsTagged());
+ }
+
+ Y_UNREACHABLE();
+ }
+
+ template <typename V>
+ decltype(auto) TType::VisitRaw(V&& visitor) const {
+ switch (TypeName_) {
+ case ETypeName::Bool:
+ return std::forward<V>(visitor)(this->AsBoolRaw());
+ case ETypeName::Int8:
+ return std::forward<V>(visitor)(this->AsInt8Raw());
+ case ETypeName::Int16:
+ return std::forward<V>(visitor)(this->AsInt16Raw());
+ case ETypeName::Int32:
+ return std::forward<V>(visitor)(this->AsInt32Raw());
+ case ETypeName::Int64:
+ return std::forward<V>(visitor)(this->AsInt64Raw());
+ case ETypeName::Uint8:
+ return std::forward<V>(visitor)(this->AsUint8Raw());
+ case ETypeName::Uint16:
+ return std::forward<V>(visitor)(this->AsUint16Raw());
+ case ETypeName::Uint32:
+ return std::forward<V>(visitor)(this->AsUint32Raw());
+ case ETypeName::Uint64:
+ return std::forward<V>(visitor)(this->AsUint64Raw());
+ case ETypeName::Float:
+ return std::forward<V>(visitor)(this->AsFloatRaw());
+ case ETypeName::Double:
+ return std::forward<V>(visitor)(this->AsDoubleRaw());
+ case ETypeName::String:
+ return std::forward<V>(visitor)(this->AsStringRaw());
+ case ETypeName::Utf8:
+ return std::forward<V>(visitor)(this->AsUtf8Raw());
+ case ETypeName::Date:
+ return std::forward<V>(visitor)(this->AsDateRaw());
+ case ETypeName::Datetime:
+ return std::forward<V>(visitor)(this->AsDatetimeRaw());
+ case ETypeName::Timestamp:
+ return std::forward<V>(visitor)(this->AsTimestampRaw());
+ case ETypeName::TzDate:
+ return std::forward<V>(visitor)(this->AsTzDateRaw());
+ case ETypeName::TzDatetime:
+ return std::forward<V>(visitor)(this->AsTzDatetimeRaw());
+ case ETypeName::TzTimestamp:
+ return std::forward<V>(visitor)(this->AsTzTimestampRaw());
+ case ETypeName::Interval:
+ return std::forward<V>(visitor)(this->AsIntervalRaw());
+ case ETypeName::Decimal:
+ return std::forward<V>(visitor)(this->AsDecimalRaw());
+ case ETypeName::Json:
+ return std::forward<V>(visitor)(this->AsJsonRaw());
+ case ETypeName::Yson:
+ return std::forward<V>(visitor)(this->AsYsonRaw());
+ case ETypeName::Uuid:
+ return std::forward<V>(visitor)(this->AsUuidRaw());
+ case ETypeName::Void:
+ return std::forward<V>(visitor)(this->AsVoidRaw());
+ case ETypeName::Null:
+ return std::forward<V>(visitor)(this->AsNullRaw());
+ case ETypeName::Optional:
+ return std::forward<V>(visitor)(this->AsOptionalRaw());
+ case ETypeName::List:
+ return std::forward<V>(visitor)(this->AsListRaw());
+ case ETypeName::Dict:
+ return std::forward<V>(visitor)(this->AsDictRaw());
+ case ETypeName::Struct:
+ return std::forward<V>(visitor)(this->AsStructRaw());
+ case ETypeName::Tuple:
+ return std::forward<V>(visitor)(this->AsTupleRaw());
+ case ETypeName::Variant:
+ return std::forward<V>(visitor)(this->AsVariantRaw());
+ case ETypeName::Tagged:
+ return std::forward<V>(visitor)(this->AsTaggedRaw());
+ }
+
+ Y_UNREACHABLE();
+ }
+
+ template <typename V>
+ decltype(auto) TPrimitiveType::VisitPrimitive(V&& visitor) const {
+ switch (GetPrimitiveTypeName()) {
+ case EPrimitiveTypeName::Bool:
+ return std::forward<V>(visitor)(this->AsBool());
+ case EPrimitiveTypeName::Int8:
+ return std::forward<V>(visitor)(this->AsInt8());
+ case EPrimitiveTypeName::Int16:
+ return std::forward<V>(visitor)(this->AsInt16());
+ case EPrimitiveTypeName::Int32:
+ return std::forward<V>(visitor)(this->AsInt32());
+ case EPrimitiveTypeName::Int64:
+ return std::forward<V>(visitor)(this->AsInt64());
+ case EPrimitiveTypeName::Uint8:
+ return std::forward<V>(visitor)(this->AsUint8());
+ case EPrimitiveTypeName::Uint16:
+ return std::forward<V>(visitor)(this->AsUint16());
+ case EPrimitiveTypeName::Uint32:
+ return std::forward<V>(visitor)(this->AsUint32());
+ case EPrimitiveTypeName::Uint64:
+ return std::forward<V>(visitor)(this->AsUint64());
+ case EPrimitiveTypeName::Float:
+ return std::forward<V>(visitor)(this->AsFloat());
+ case EPrimitiveTypeName::Double:
+ return std::forward<V>(visitor)(this->AsDouble());
+ case EPrimitiveTypeName::String:
+ return std::forward<V>(visitor)(this->AsString());
+ case EPrimitiveTypeName::Utf8:
+ return std::forward<V>(visitor)(this->AsUtf8());
+ case EPrimitiveTypeName::Date:
+ return std::forward<V>(visitor)(this->AsDate());
+ case EPrimitiveTypeName::Datetime:
+ return std::forward<V>(visitor)(this->AsDatetime());
+ case EPrimitiveTypeName::Timestamp:
+ return std::forward<V>(visitor)(this->AsTimestamp());
+ case EPrimitiveTypeName::TzDate:
+ return std::forward<V>(visitor)(this->AsTzDate());
+ case EPrimitiveTypeName::TzDatetime:
+ return std::forward<V>(visitor)(this->AsTzDatetime());
+ case EPrimitiveTypeName::TzTimestamp:
+ return std::forward<V>(visitor)(this->AsTzTimestamp());
+ case EPrimitiveTypeName::Interval:
+ return std::forward<V>(visitor)(this->AsInterval());
+ case EPrimitiveTypeName::Decimal:
+ return std::forward<V>(visitor)(this->AsDecimal());
+ case EPrimitiveTypeName::Json:
+ return std::forward<V>(visitor)(this->AsJson());
+ case EPrimitiveTypeName::Yson:
+ return std::forward<V>(visitor)(this->AsYson());
+ case EPrimitiveTypeName::Uuid:
+ return std::forward<V>(visitor)(this->AsUuid());
+ }
+
+ Y_UNREACHABLE();
+ }
+
+ template <typename V>
+ decltype(auto) TPrimitiveType::VisitPrimitiveRaw(V&& visitor) const {
+ switch (GetPrimitiveTypeName()) {
+ case EPrimitiveTypeName::Bool:
+ return std::forward<V>(visitor)(this->AsBoolRaw());
+ case EPrimitiveTypeName::Int8:
+ return std::forward<V>(visitor)(this->AsInt8Raw());
+ case EPrimitiveTypeName::Int16:
+ return std::forward<V>(visitor)(this->AsInt16Raw());
+ case EPrimitiveTypeName::Int32:
+ return std::forward<V>(visitor)(this->AsInt32Raw());
+ case EPrimitiveTypeName::Int64:
+ return std::forward<V>(visitor)(this->AsInt64Raw());
+ case EPrimitiveTypeName::Uint8:
+ return std::forward<V>(visitor)(this->AsUint8Raw());
+ case EPrimitiveTypeName::Uint16:
+ return std::forward<V>(visitor)(this->AsUint16Raw());
+ case EPrimitiveTypeName::Uint32:
+ return std::forward<V>(visitor)(this->AsUint32Raw());
+ case EPrimitiveTypeName::Uint64:
+ return std::forward<V>(visitor)(this->AsUint64Raw());
+ case EPrimitiveTypeName::Float:
+ return std::forward<V>(visitor)(this->AsFloatRaw());
+ case EPrimitiveTypeName::Double:
+ return std::forward<V>(visitor)(this->AsDoubleRaw());
+ case EPrimitiveTypeName::String:
+ return std::forward<V>(visitor)(this->AsStringRaw());
+ case EPrimitiveTypeName::Utf8:
+ return std::forward<V>(visitor)(this->AsUtf8Raw());
+ case EPrimitiveTypeName::Date:
+ return std::forward<V>(visitor)(this->AsDateRaw());
+ case EPrimitiveTypeName::Datetime:
+ return std::forward<V>(visitor)(this->AsDatetimeRaw());
+ case EPrimitiveTypeName::Timestamp:
+ return std::forward<V>(visitor)(this->AsTimestampRaw());
+ case EPrimitiveTypeName::TzDate:
+ return std::forward<V>(visitor)(this->AsTzDateRaw());
+ case EPrimitiveTypeName::TzDatetime:
+ return std::forward<V>(visitor)(this->AsTzDatetimeRaw());
+ case EPrimitiveTypeName::TzTimestamp:
+ return std::forward<V>(visitor)(this->AsTzTimestampRaw());
+ case EPrimitiveTypeName::Interval:
+ return std::forward<V>(visitor)(this->AsIntervalRaw());
+ case EPrimitiveTypeName::Decimal:
+ return std::forward<V>(visitor)(this->AsDecimalRaw());
+ case EPrimitiveTypeName::Json:
+ return std::forward<V>(visitor)(this->AsJsonRaw());
+ case EPrimitiveTypeName::Yson:
+ return std::forward<V>(visitor)(this->AsYsonRaw());
+ case EPrimitiveTypeName::Uuid:
+ return std::forward<V>(visitor)(this->AsUuidRaw());
+ }
+
+ Y_UNREACHABLE();
+ }
+
+ template <typename V>
+ decltype(auto) TVariantType::VisitUnderlying(V&& visitor) const {
+ switch (GetUnderlyingTypeRaw()->GetTypeName()) {
+ case ETypeName::Struct:
+ return std::forward<V>(visitor)(this->GetUnderlyingTypeRaw()->AsStruct());
+ case ETypeName::Tuple:
+ return std::forward<V>(visitor)(this->GetUnderlyingTypeRaw()->AsTuple());
+ default:
+ Y_UNREACHABLE();
+ }
+ }
+
+ template <typename V>
+ decltype(auto) TVariantType::VisitUnderlyingRaw(V&& visitor) const {
+ switch (GetUnderlyingTypeRaw()->GetTypeName()) {
+ case ETypeName::Struct:
+ return std::forward<V>(visitor)(this->GetUnderlyingTypeRaw()->AsStructRaw());
+ case ETypeName::Tuple:
+ return std::forward<V>(visitor)(this->GetUnderlyingTypeRaw()->AsTupleRaw());
+ default:
+ Y_UNREACHABLE();
+ }
+ }
+} // namespace NTi
diff --git a/library/cpp/type_info/type_complexity.cpp b/library/cpp/type_info/type_complexity.cpp
new file mode 100644
index 0000000000..073c24b909
--- /dev/null
+++ b/library/cpp/type_info/type_complexity.cpp
@@ -0,0 +1,78 @@
+#include "type_complexity.h"
+
+#include "type.h"
+
+
+namespace NTi {
+
+int ComputeTypeComplexity(const TTypePtr& type)
+{
+ return ComputeTypeComplexity(type.Get());
+}
+
+int ComputeTypeComplexity(const TType* type)
+{
+ switch (type->GetTypeName()) {
+ case ETypeName::Bool:
+ case ETypeName::Int8:
+ case ETypeName::Int16:
+ case ETypeName::Int32:
+ case ETypeName::Int64:
+ case ETypeName::Uint8:
+ case ETypeName::Uint16:
+ case ETypeName::Uint32:
+ case ETypeName::Uint64:
+ case ETypeName::Float:
+ case ETypeName::Double:
+ case ETypeName::String:
+ case ETypeName::Utf8:
+ case ETypeName::Date:
+ case ETypeName::Datetime:
+ case ETypeName::Timestamp:
+ case ETypeName::TzDate:
+ case ETypeName::TzDatetime:
+ case ETypeName::TzTimestamp:
+ case ETypeName::Interval:
+ case ETypeName::Decimal:
+ case ETypeName::Json:
+ case ETypeName::Yson:
+ case ETypeName::Uuid:
+ case ETypeName::Void:
+ case ETypeName::Null:
+ return 1;
+
+ case ETypeName::Optional:
+ return 1 + ComputeTypeComplexity(type->AsOptionalRaw()->GetItemTypeRaw());
+
+ case ETypeName::List:
+ return 1 + ComputeTypeComplexity(type->AsListRaw()->GetItemTypeRaw());
+
+ case ETypeName::Dict:
+ return 1 + ComputeTypeComplexity(type->AsDictRaw()->GetKeyTypeRaw())
+ + ComputeTypeComplexity(type->AsDictRaw()->GetValueTypeRaw());
+ case ETypeName::Struct: {
+ int result = 1;
+ for (const auto& member : type->AsStructRaw()->GetMembers()) {
+ result += ComputeTypeComplexity(member.GetTypeRaw());
+ }
+ return result;
+ }
+ case ETypeName::Tuple: {
+ int result = 1;
+ for (const auto& element : type->AsTupleRaw()->GetElements()) {
+ result += ComputeTypeComplexity(element.GetTypeRaw());
+ }
+ return result;
+ }
+ case ETypeName::Variant: {
+ return ComputeTypeComplexity(type->AsVariantRaw()->GetUnderlyingTypeRaw());
+ }
+ case ETypeName::Tagged: {
+ return 1 + ComputeTypeComplexity(type->AsTaggedRaw()->GetItemType());
+ }
+ }
+ Y_FAIL("internal error: unreachable code");
+}
+
+} // namespace NTi
+
diff --git a/library/cpp/type_info/type_complexity.h b/library/cpp/type_info/type_complexity.h
new file mode 100644
index 0000000000..fcf367eb81
--- /dev/null
+++ b/library/cpp/type_info/type_complexity.h
@@ -0,0 +1,18 @@
+#pragma once
+#include "fwd.h"
+
+namespace NTi {
+
+ ///
+ /// Compute type complexity.
+ /// Roughly speaking type complexity is a number of nodes in the schema tree.
+ /// Examples:
+ /// - Type complexity of simple or singular type (i.e. Int64, String, Null, Decimal) is 1.
+ /// - Type complexity of `Optional<Int64>` is 2
+ /// - Type complexity of `Struct<a:Int64,b:Optional<String>` is 4 (1 for Int64, 2 for Optional<String> and 1 for struct)
+ ///
+ /// Systems might impose restrictions on the type complexity.
+ int ComputeTypeComplexity(const TTypePtr& type);
+ int ComputeTypeComplexity(const TType* type);
+
+} // namespace NTi
diff --git a/library/cpp/type_info/type_constructors.h b/library/cpp/type_info/type_constructors.h
new file mode 100644
index 0000000000..439d5a1887
--- /dev/null
+++ b/library/cpp/type_info/type_constructors.h
@@ -0,0 +1,110 @@
+#pragma once
+
+//! @file type_constructors.h
+
+#include "type.h"
+
+namespace NTi {
+ /// Create new `Null` type using the default heap factory.
+ TNullTypePtr Null();
+
+ /// Create new `Bool` type using the default heap factory.
+ TBoolTypePtr Bool();
+
+ /// Create new `Int8` type using the default heap factory.
+ TInt8TypePtr Int8();
+
+ /// Create new `Int16` type using the default heap factory.
+ TInt16TypePtr Int16();
+
+ /// Create new `Int32` type using the default heap factory.
+ TInt32TypePtr Int32();
+
+ /// Create new `Int64` type using the default heap factory.
+ TInt64TypePtr Int64();
+
+ /// Create new `Uint8` type using the default heap factory.
+ TUint8TypePtr Uint8();
+
+ /// Create new `Uint16` type using the default heap factory.
+ TUint16TypePtr Uint16();
+
+ /// Create new `Uint32` type using the default heap factory.
+ TUint32TypePtr Uint32();
+
+ /// Create new `Uint64` type using the default heap factory.
+ TUint64TypePtr Uint64();
+
+ /// Create new `Float` type using the default heap factory.
+ TFloatTypePtr Float();
+
+ /// Create new `Double` type using the default heap factory.
+ TDoubleTypePtr Double();
+
+ /// Create new `String` type using the default heap factory.
+ TStringTypePtr String();
+
+ /// Create new `Utf8` type using the default heap factory.
+ TUtf8TypePtr Utf8();
+
+ /// Create new `Date` type using the default heap factory.
+ TDateTypePtr Date();
+
+ /// Create new `Datetime` type using the default heap factory.
+ TDatetimeTypePtr Datetime();
+
+ /// Create new `Timestamp` type using the default heap factory.
+ TTimestampTypePtr Timestamp();
+
+ /// Create new `TzDate` type using the default heap factory.
+ TTzDateTypePtr TzDate();
+
+ /// Create new `TzDatetime` type using the default heap factory.
+ TTzDatetimeTypePtr TzDatetime();
+
+ /// Create new `TzTimestamp` type using the default heap factory.
+ TTzTimestampTypePtr TzTimestamp();
+
+ /// Create new `Interval` type using the default heap factory.
+ TIntervalTypePtr Interval();
+
+ /// Create new `Decimal` type using the default heap factory.
+ TDecimalTypePtr Decimal(ui8 precision, ui8 scale);
+
+ /// Create new `Json` type using the default heap factory.
+ TJsonTypePtr Json();
+
+ /// Create new `Yson` type using the default heap factory.
+ TYsonTypePtr Yson();
+
+ /// Create new `Uuid` type using the default heap factory.
+ TUuidTypePtr Uuid();
+
+ /// Create new `Optional` type using the default heap factory.
+ TOptionalTypePtr Optional(TTypePtr item);
+
+ /// Create new `List` type using the default heap factory.
+ TListTypePtr List(TTypePtr item);
+
+ /// Create new `Dict` type using the default heap factory.
+ TDictTypePtr Dict(TTypePtr key, TTypePtr value);
+
+ /// Create new `Struct` type using the default heap factory.
+ TStructTypePtr Struct(TStructType::TOwnedMembers items);
+ /// Create new `Struct` type using the default heap factory.
+ TStructTypePtr Struct(TMaybe<TStringBuf> name, TStructType::TOwnedMembers items);
+
+ /// Create new `Tuple` type using the default heap factory.
+ TTupleTypePtr Tuple(TTupleType::TOwnedElements items);
+ /// Create new `Tuple` type using the default heap factory.
+ TTupleTypePtr Tuple(TMaybe<TStringBuf> name, TTupleType::TOwnedElements items);
+
+ /// Create new `Variant` type using the default heap factory.
+ TVariantTypePtr Variant(TTypePtr underlying);
+ /// Create new `Variant` type using the default heap factory.
+ TVariantTypePtr Variant(TMaybe<TStringBuf> name, TTypePtr underlying);
+
+ /// Create new `Tagged` type using the default heap factory.
+ TTaggedTypePtr Tagged(TTypePtr type, TStringBuf tag);
+
+} // namespace NTi
diff --git a/library/cpp/type_info/type_equivalence.cpp b/library/cpp/type_info/type_equivalence.cpp
new file mode 100644
index 0000000000..9f22f0308c
--- /dev/null
+++ b/library/cpp/type_info/type_equivalence.cpp
@@ -0,0 +1,286 @@
+#include "type_equivalence.h"
+
+#include <util/generic/overloaded.h>
+
+#include "type.h"
+
+namespace NTi::NEq {
+ namespace {
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TType* lhs, const TType* rhs);
+
+ // template <bool IgnoreHash>
+ // bool StrictlyEqual(const TDocumentation& lhs, const TDocumentation& rhs) {
+ // ...
+ // }
+
+ // template <bool IgnoreHash>
+ // bool StrictlyEqual(const TAnnotations& lhs, const TAnnotations& rhs) {
+ // ...
+ // }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TVoidType&, const TVoidType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TNullType&, const TNullType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TBoolType&, const TBoolType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TInt8Type&, const TInt8Type&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TInt16Type&, const TInt16Type&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TInt32Type&, const TInt32Type&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TInt64Type&, const TInt64Type&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TUint8Type&, const TUint8Type&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TUint16Type&, const TUint16Type&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TUint32Type&, const TUint32Type&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TUint64Type&, const TUint64Type&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TFloatType&, const TFloatType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TDoubleType&, const TDoubleType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TStringType&, const TStringType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TUtf8Type&, const TUtf8Type&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TDateType&, const TDateType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TDatetimeType&, const TDatetimeType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TTimestampType&, const TTimestampType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TTzDateType&, const TTzDateType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TTzDatetimeType&, const TTzDatetimeType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TTzTimestampType&, const TTzTimestampType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TIntervalType&, const TIntervalType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TDecimalType& lhs, const TDecimalType& rhs) {
+ return lhs.GetPrecision() == rhs.GetPrecision() && lhs.GetScale() == rhs.GetScale();
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TJsonType&, const TJsonType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TYsonType&, const TYsonType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TUuidType&, const TUuidType&) {
+ return true;
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TOptionalType& lhs, const TOptionalType& rhs) {
+ return StrictlyEqual<IgnoreHash>(lhs.GetItemTypeRaw(), rhs.GetItemTypeRaw());
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TListType& lhs, const TListType& rhs) {
+ return StrictlyEqual<IgnoreHash>(lhs.GetItemTypeRaw(), rhs.GetItemTypeRaw());
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TDictType& lhs, const TDictType& rhs) {
+ return StrictlyEqual<IgnoreHash>(lhs.GetKeyTypeRaw(), rhs.GetKeyTypeRaw()) &&
+ StrictlyEqual<IgnoreHash>(lhs.GetValueTypeRaw(), rhs.GetValueTypeRaw());
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TStructType& lhs, const TStructType& rhs) {
+ if (lhs.GetName() != rhs.GetName()) {
+ return false;
+ }
+
+ return std::equal(
+ lhs.GetMembers().begin(), lhs.GetMembers().end(),
+ rhs.GetMembers().begin(), rhs.GetMembers().end(),
+ [](const TStructType::TMember& lhs, const TStructType::TMember& rhs) -> bool {
+ return lhs.GetName() == rhs.GetName() &&
+ StrictlyEqual<IgnoreHash>(lhs.GetTypeRaw(), rhs.GetTypeRaw());
+ // && StrictlyEqual(lhs.GetAnnotations(), rhs.GetAnnotations())
+ // && StrictlyEqual(lhs.GetDocumentation(), rhs.GetDocumentation());
+ });
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TTupleType& lhs, const TTupleType& rhs) {
+ if (lhs.GetName() != rhs.GetName()) {
+ return false;
+ }
+
+ return std::equal(
+ lhs.GetElements().begin(), lhs.GetElements().end(),
+ rhs.GetElements().begin(), rhs.GetElements().end(),
+ [](const TTupleType::TElement& lhs, const TTupleType::TElement& rhs) -> bool {
+ return StrictlyEqual<IgnoreHash>(lhs.GetTypeRaw(), rhs.GetTypeRaw());
+ // && StrictlyEqual(lhs.GetAnnotations(), rhs.GetAnnotations())
+ // && StrictlyEqual(lhs.GetDocumentation(), rhs.GetDocumentation());
+ });
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TVariantType& lhs, const TVariantType& rhs) {
+ if (lhs.GetName() != rhs.GetName()) {
+ return false;
+ }
+
+ return StrictlyEqual<IgnoreHash>(lhs.GetUnderlyingTypeRaw(), rhs.GetUnderlyingTypeRaw());
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TTaggedType& lhs, const TTaggedType& rhs) {
+ return lhs.GetTag() == rhs.GetTag() &&
+ StrictlyEqual<IgnoreHash>(lhs.GetItemTypeRaw(), rhs.GetItemTypeRaw());
+ }
+
+ template <bool IgnoreHash>
+ bool StrictlyEqual(const TType* lhs, const TType* rhs) {
+ if (lhs == rhs) {
+ return true;
+ }
+
+ if (lhs == nullptr || rhs == nullptr) {
+ return false;
+ }
+
+ if (!IgnoreHash && lhs->GetHash() != rhs->GetHash()) {
+ return false;
+ }
+
+ if (lhs->GetTypeName() != rhs->GetTypeName()) {
+ return false;
+ }
+
+ // FIXME: update `TStrictlyEqual`'s docs to explicitly state that documentation and annotations
+ // must be the same for types to compare strictly equal.
+
+ // if (!StrictlyEqual(lhs->GetAnnotations(), rhs->GetAnnotations())) {
+ // return false;
+ // }
+
+ // if (!StrictlyEqual(lhs->GetDocumentation(), rhs->GetDocumentation())) {
+ // return false;
+ // }
+
+ return lhs->VisitRaw([&rhs](const auto* lhs) {
+ using TSameType = decltype(lhs);
+ return rhs->VisitRaw(TOverloaded{
+ [lhs](TSameType rhs) {
+ return StrictlyEqual<IgnoreHash>(*lhs, *rhs);
+ },
+ [](const auto* rhs) {
+ static_assert(!std::is_same_v<decltype(lhs), decltype(rhs)>);
+ return false;
+ }});
+ });
+ }
+ }
+
+ bool TStrictlyEqual::operator()(const TType* lhs, const TType* rhs) const {
+ return StrictlyEqual<false>(lhs, rhs);
+ }
+
+ bool TStrictlyEqual::operator()(TTypePtr lhs, TTypePtr rhs) const {
+ return operator()(lhs.Get(), rhs.Get());
+ }
+
+ bool TStrictlyEqual::IgnoreHash(const TType* lhs, const TType* rhs) const {
+ return StrictlyEqual<true>(lhs, rhs);
+ }
+
+ bool TStrictlyEqual::IgnoreHash(TTypePtr lhs, TTypePtr rhs) const {
+ return IgnoreHash(lhs.Get(), rhs.Get());
+ }
+
+ ui64 TStrictlyEqualHash::operator()(const TType* type) const {
+ if (type == nullptr) {
+ return 0;
+ }
+
+ return type->GetHash();
+ }
+
+ ui64 TStrictlyEqualHash::operator()(TTypePtr type) const {
+ return operator()(type.Get());
+ }
+}
diff --git a/library/cpp/type_info/type_equivalence.h b/library/cpp/type_info/type_equivalence.h
new file mode 100644
index 0000000000..70351dc689
--- /dev/null
+++ b/library/cpp/type_info/type_equivalence.h
@@ -0,0 +1,36 @@
+#pragma once
+
+//! @file type_equivalence.h
+//!
+//! Relations between types.
+//!
+//! Type info declares multiple ways to compare types. There's strict, nominal, structural and other equivalences,
+//! as well as subtyping. See corresponding functors for more info.
+//!
+//! At the moment, only strict equivalence is implemented because others are not yet standartized.
+
+#include <type_traits>
+
+#include "fwd.h"
+
+#include <util/system/types.h>
+
+namespace NTi::NEq {
+ /// Strict equivalence is the strongest form of type equivalence. If two types are strictly equal,
+ /// they're literally the same type for all intents and purposes. This includes struct, tuple, variant and enum
+ /// names, order of their items, names of their items, names of tagged types, and so on.
+ struct TStrictlyEqual {
+ bool operator()(const TType* lhs, const TType* rhs) const;
+ bool operator()(TTypePtr lhs, TTypePtr rhs) const;
+
+ /// Compare types without calculating and comparing their hashes first.
+ bool IgnoreHash(const TType* lhs, const TType* rhs) const;
+ bool IgnoreHash(TTypePtr lhs, TTypePtr rhs) const;
+ };
+
+ /// Hash that follows the strict equality rules (see `TStrictlyEqual`).
+ struct TStrictlyEqualHash {
+ ui64 operator()(const TType* type) const;
+ ui64 operator()(TTypePtr type) const;
+ };
+}
diff --git a/library/cpp/type_info/type_factory.cpp b/library/cpp/type_info/type_factory.cpp
new file mode 100644
index 0000000000..9d60307938
--- /dev/null
+++ b/library/cpp/type_info/type_factory.cpp
@@ -0,0 +1,495 @@
+#include "type_factory.h"
+
+#include "type.h"
+#include "type_equivalence.h"
+
+#include <util/memory/pool.h>
+#include <util/generic/hash_set.h>
+
+#include <cstdlib>
+
+namespace NTi {
+ TVoidTypePtr ITypeFactory::Void() {
+ return TVoidType::Instance();
+ }
+
+ const TVoidType* IPoolTypeFactory::VoidRaw() {
+ return TVoidType::InstanceRaw();
+ }
+
+ TNullTypePtr ITypeFactory::Null() {
+ return TNullType::Instance();
+ }
+
+ const TNullType* IPoolTypeFactory::NullRaw() {
+ return TNullType::InstanceRaw();
+ }
+
+ TBoolTypePtr ITypeFactory::Bool() {
+ return TBoolType::Instance();
+ }
+
+ const TBoolType* IPoolTypeFactory::BoolRaw() {
+ return TBoolType::InstanceRaw();
+ }
+
+ TInt8TypePtr ITypeFactory::Int8() {
+ return TInt8Type::Instance();
+ }
+
+ const TInt8Type* IPoolTypeFactory::Int8Raw() {
+ return TInt8Type::InstanceRaw();
+ }
+
+ TInt16TypePtr ITypeFactory::Int16() {
+ return TInt16Type::Instance();
+ }
+
+ const TInt16Type* IPoolTypeFactory::Int16Raw() {
+ return TInt16Type::InstanceRaw();
+ }
+
+ TInt32TypePtr ITypeFactory::Int32() {
+ return TInt32Type::Instance();
+ }
+
+ const TInt32Type* IPoolTypeFactory::Int32Raw() {
+ return TInt32Type::InstanceRaw();
+ }
+
+ TInt64TypePtr ITypeFactory::Int64() {
+ return TInt64Type::Instance();
+ }
+
+ const TInt64Type* IPoolTypeFactory::Int64Raw() {
+ return TInt64Type::InstanceRaw();
+ }
+
+ TUint8TypePtr ITypeFactory::Uint8() {
+ return TUint8Type::Instance();
+ }
+
+ const TUint8Type* IPoolTypeFactory::Uint8Raw() {
+ return TUint8Type::InstanceRaw();
+ }
+
+ TUint16TypePtr ITypeFactory::Uint16() {
+ return TUint16Type::Instance();
+ }
+
+ const TUint16Type* IPoolTypeFactory::Uint16Raw() {
+ return TUint16Type::InstanceRaw();
+ }
+
+ TUint32TypePtr ITypeFactory::Uint32() {
+ return TUint32Type::Instance();
+ }
+
+ const TUint32Type* IPoolTypeFactory::Uint32Raw() {
+ return TUint32Type::InstanceRaw();
+ }
+
+ TUint64TypePtr ITypeFactory::Uint64() {
+ return TUint64Type::Instance();
+ }
+
+ const TUint64Type* IPoolTypeFactory::Uint64Raw() {
+ return TUint64Type::InstanceRaw();
+ }
+
+ TFloatTypePtr ITypeFactory::Float() {
+ return TFloatType::Instance();
+ }
+
+ const TFloatType* IPoolTypeFactory::FloatRaw() {
+ return TFloatType::InstanceRaw();
+ }
+
+ TDoubleTypePtr ITypeFactory::Double() {
+ return TDoubleType::Instance();
+ }
+
+ const TDoubleType* IPoolTypeFactory::DoubleRaw() {
+ return TDoubleType::InstanceRaw();
+ }
+
+ TStringTypePtr ITypeFactory::String() {
+ return TStringType::Instance();
+ }
+
+ const TStringType* IPoolTypeFactory::StringRaw() {
+ return TStringType::InstanceRaw();
+ }
+
+ TUtf8TypePtr ITypeFactory::Utf8() {
+ return TUtf8Type::Instance();
+ }
+
+ const TUtf8Type* IPoolTypeFactory::Utf8Raw() {
+ return TUtf8Type::InstanceRaw();
+ }
+
+ TDateTypePtr ITypeFactory::Date() {
+ return TDateType::Instance();
+ }
+
+ const TDateType* IPoolTypeFactory::DateRaw() {
+ return TDateType::InstanceRaw();
+ }
+
+ TDatetimeTypePtr ITypeFactory::Datetime() {
+ return TDatetimeType::Instance();
+ }
+
+ const TDatetimeType* IPoolTypeFactory::DatetimeRaw() {
+ return TDatetimeType::InstanceRaw();
+ }
+
+ TTimestampTypePtr ITypeFactory::Timestamp() {
+ return TTimestampType::Instance();
+ }
+
+ const TTimestampType* IPoolTypeFactory::TimestampRaw() {
+ return TTimestampType::InstanceRaw();
+ }
+
+ TTzDateTypePtr ITypeFactory::TzDate() {
+ return TTzDateType::Instance();
+ }
+
+ const TTzDateType* IPoolTypeFactory::TzDateRaw() {
+ return TTzDateType::InstanceRaw();
+ }
+
+ TTzDatetimeTypePtr ITypeFactory::TzDatetime() {
+ return TTzDatetimeType::Instance();
+ }
+
+ const TTzDatetimeType* IPoolTypeFactory::TzDatetimeRaw() {
+ return TTzDatetimeType::InstanceRaw();
+ }
+
+ TTzTimestampTypePtr ITypeFactory::TzTimestamp() {
+ return TTzTimestampType::Instance();
+ }
+
+ const TTzTimestampType* IPoolTypeFactory::TzTimestampRaw() {
+ return TTzTimestampType::InstanceRaw();
+ }
+
+ TIntervalTypePtr ITypeFactory::Interval() {
+ return TIntervalType::Instance();
+ }
+
+ const TIntervalType* IPoolTypeFactory::IntervalRaw() {
+ return TIntervalType::InstanceRaw();
+ }
+
+ TDecimalTypePtr ITypeFactory::Decimal(ui8 precision, ui8 scale) {
+ return TDecimalType::Create(*this, precision, scale);
+ }
+
+ const TDecimalType* IPoolTypeFactory::DecimalRaw(ui8 precision, ui8 scale) {
+ return TDecimalType::CreateRaw(*this, precision, scale);
+ }
+
+ TJsonTypePtr ITypeFactory::Json() {
+ return TJsonType::Instance();
+ }
+
+ const TJsonType* IPoolTypeFactory::JsonRaw() {
+ return TJsonType::InstanceRaw();
+ }
+
+ TYsonTypePtr ITypeFactory::Yson() {
+ return TYsonType::Instance();
+ }
+
+ const TYsonType* IPoolTypeFactory::YsonRaw() {
+ return TYsonType::InstanceRaw();
+ }
+
+ TUuidTypePtr ITypeFactory::Uuid() {
+ return TUuidType::Instance();
+ }
+
+ const TUuidType* IPoolTypeFactory::UuidRaw() {
+ return TUuidType::InstanceRaw();
+ }
+
+ TOptionalTypePtr ITypeFactory::Optional(TTypePtr item) {
+ return TOptionalType::Create(*this, std::move(item));
+ }
+
+ const TOptionalType* IPoolTypeFactory::OptionalRaw(const TType* item) {
+ return TOptionalType::CreateRaw(*this, item);
+ }
+
+ TListTypePtr ITypeFactory::List(TTypePtr item) {
+ return TListType::Create(*this, std::move(item));
+ }
+
+ const TListType* IPoolTypeFactory::ListRaw(const TType* item) {
+ return TListType::CreateRaw(*this, item);
+ }
+
+ TDictTypePtr ITypeFactory::Dict(TTypePtr key, TTypePtr value) {
+ return TDictType::Create(*this, std::move(key), std::move(value));
+ }
+
+ const TDictType* IPoolTypeFactory::DictRaw(const TType* key, const TType* value) {
+ return TDictType::CreateRaw(*this, key, value);
+ }
+
+ TStructTypePtr ITypeFactory::Struct(TStructType::TOwnedMembers items) {
+ return TStructType::Create(*this, items);
+ }
+
+ TStructTypePtr ITypeFactory::Struct(TMaybe<TStringBuf> name, TStructType::TOwnedMembers items) {
+ return TStructType::Create(*this, name, items);
+ }
+
+ const TStructType* IPoolTypeFactory::StructRaw(TStructType::TMembers items) {
+ return TStructType::CreateRaw(*this, items);
+ }
+
+ const TStructType* IPoolTypeFactory::StructRaw(TMaybe<TStringBuf> name, TStructType::TMembers items) {
+ return TStructType::CreateRaw(*this, name, items);
+ }
+
+ TTupleTypePtr ITypeFactory::Tuple(TTupleType::TOwnedElements items) {
+ return TTupleType::Create(*this, items);
+ }
+
+ TTupleTypePtr ITypeFactory::Tuple(TMaybe<TStringBuf> name, TTupleType::TOwnedElements items) {
+ return TTupleType::Create(*this, name, items);
+ }
+
+ const TTupleType* IPoolTypeFactory::TupleRaw(TTupleType::TElements items) {
+ return TTupleType::CreateRaw(*this, items);
+ }
+
+ const TTupleType* IPoolTypeFactory::TupleRaw(TMaybe<TStringBuf> name, TTupleType::TElements items) {
+ return TTupleType::CreateRaw(*this, name, items);
+ }
+
+ TVariantTypePtr ITypeFactory::Variant(TTypePtr inner) {
+ return TVariantType::Create(*this, std::move(inner));
+ }
+
+ TVariantTypePtr ITypeFactory::Variant(TMaybe<TStringBuf> name, TTypePtr inner) {
+ return TVariantType::Create(*this, name, std::move(inner));
+ }
+
+ const TVariantType* IPoolTypeFactory::VariantRaw(const TType* inner) {
+ return TVariantType::CreateRaw(*this, inner);
+ }
+
+ const TVariantType* IPoolTypeFactory::VariantRaw(TMaybe<TStringBuf> name, const TType* inner) {
+ return TVariantType::CreateRaw(*this, name, inner);
+ }
+
+ TTaggedTypePtr ITypeFactory::Tagged(TTypePtr type, TStringBuf tag) {
+ return TTaggedType::Create(*this, std::move(type), tag);
+ }
+
+ const TTaggedType* IPoolTypeFactory::TaggedRaw(const TType* type, TStringBuf tag) {
+ return TTaggedType::CreateRaw(*this, type, tag);
+ }
+
+ namespace {
+ class TPoolFactory: public NTi::IPoolTypeFactory {
+ public:
+ TPoolFactory(size_t initial, TMemoryPool::IGrowPolicy* grow, IAllocator* alloc, TMemoryPool::TOptions options)
+ : Pool_(initial, grow, alloc, options)
+ {
+ }
+
+ public:
+ void* Allocate(size_t size, size_t align) noexcept override {
+ return Pool_.Allocate(size, align);
+ }
+
+ void Free(void* data) noexcept override {
+ Y_UNUSED(data);
+ }
+
+ protected:
+ const NTi::TType* LookupCache(const NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ return nullptr;
+ }
+
+ void SaveCache(const NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ }
+
+ void Ref() noexcept override {
+ Counter_.Inc();
+ }
+
+ void UnRef() noexcept override {
+ if (Counter_.Dec() == 0) {
+ delete this;
+ }
+ }
+
+ void DecRef() noexcept override {
+ if (Counter_.Dec() == 0) {
+ Y_FAIL("DecRef is not supposed to drop");
+ }
+ }
+
+ long RefCount() const noexcept override {
+ return Counter_.Val();
+ }
+
+ void RefType(NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ }
+
+ void UnRefType(NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ }
+
+ void DecRefType(NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ }
+
+ long RefCountType(const NTi::TType* type) const noexcept override {
+ Y_UNUSED(type);
+ return RefCount();
+ }
+
+ public:
+ size_t Available() const noexcept override {
+ return Pool_.Available();
+ }
+
+ size_t MemoryAllocated() const noexcept override {
+ return Pool_.MemoryAllocated();
+ }
+
+ size_t MemoryWaste() const noexcept override {
+ return Pool_.MemoryWaste();
+ }
+
+ private:
+ TAtomicCounter Counter_;
+ TMemoryPool Pool_;
+ };
+
+ class TPoolFactoryDedup: public TPoolFactory {
+ public:
+ using TPoolFactory::TPoolFactory;
+
+ public:
+ const NTi::TType* LookupCache(const NTi::TType* type) noexcept override {
+ if (auto it = Cache_.find(type); it != Cache_.end()) {
+ return *it;
+ } else {
+ return nullptr;
+ }
+ }
+
+ void SaveCache(const NTi::TType* type) noexcept override {
+ Cache_.insert(type);
+ }
+
+ protected:
+ TStringBuf AllocateString(TStringBuf str) noexcept override {
+ if (str.empty()) {
+ return TStringBuf(); // `str` could still point somewhere whereas empty strbuf points to NULL
+ }
+
+ if (auto it = StringCache_.find(str); it != StringCache_.end()) {
+ return *it;
+ } else {
+ return *StringCache_.insert(it, ITypeFactoryInternal::AllocateString(str));
+ }
+ }
+
+ private:
+ THashSet<const NTi::TType*, NTi::NEq::TStrictlyEqualHash, NTi::NEq::TStrictlyEqual> Cache_;
+ THashSet<TStringBuf> StringCache_;
+ };
+
+ class THeapFactory: public NTi::ITypeFactory {
+ public:
+ void* Allocate(size_t size, size_t align) noexcept override {
+ Y_UNUSED(align);
+ return malloc(size);
+ }
+
+ void Free(void* data) noexcept override {
+ free(data);
+ }
+
+ protected:
+ const NTi::TType* LookupCache(const NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ return nullptr;
+ }
+
+ void SaveCache(const NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ }
+
+ void Ref() noexcept override {
+ // nothing
+ }
+
+ void UnRef() noexcept override {
+ // nothing
+ }
+
+ void DecRef() noexcept override {
+ // nothing
+ }
+
+ long RefCount() const noexcept override {
+ return 0;
+ }
+
+ void RefType(NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ Y_FAIL("not supposed to be called");
+ }
+
+ void UnRefType(NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ Y_FAIL("not supposed to be called");
+ }
+
+ void DecRefType(NTi::TType* type) noexcept override {
+ Y_UNUSED(type);
+ Y_FAIL("not supposed to be called");
+ }
+
+ long RefCountType(const NTi::TType* type) const noexcept override {
+ Y_UNUSED(type);
+ Y_FAIL("not supposed to be called");
+ }
+ };
+
+ THeapFactory HEAP_FACTORY;
+ }
+
+ IPoolTypeFactoryPtr PoolFactory(bool deduplicate, size_t initial, TMemoryPool::IGrowPolicy* grow, IAllocator* alloc, TMemoryPool::TOptions options) {
+ if (deduplicate) {
+ return new TPoolFactoryDedup(initial, grow, alloc, options);
+ } else {
+ return new TPoolFactory(initial, grow, alloc, options);
+ }
+ }
+
+ namespace NPrivate {
+ ITypeFactory* GetDefaultHeapFactory() {
+ return &HEAP_FACTORY;
+ }
+ }
+
+ ITypeFactoryPtr HeapFactory() {
+ return NPrivate::GetDefaultHeapFactory();
+ }
+}
diff --git a/library/cpp/type_info/type_factory.h b/library/cpp/type_info/type_factory.h
new file mode 100644
index 0000000000..df47c7082b
--- /dev/null
+++ b/library/cpp/type_info/type_factory.h
@@ -0,0 +1,906 @@
+#pragma once
+
+//! @file type_factory.h
+//!
+//! Type factory creates type instances and manages their lifetimes and destruction.
+//!
+//! Type info supports multiple ways of allocating type instances and managing their lifetimes:
+//!
+//! - [heap-based allocation] is the standard memory allocation method for C++. It offers great flexibility
+//! as it can allocate memory regions of almost arbitrary size and alignment, deallocate them, re-use them
+//! for new allocations or return the to the operating system. This flexibility comes with a price, though.
+//! Heap allocators usually require synchronisation (i.e. a mutex or a spinlock), and some memory overhead to track
+//! allocated regions. Also, they provide no guarantees on data locality and CPU cache friendliness whatsoever.
+//!
+//! When using a heap-based factory, each type instance have a separate reference counter. `TTypePtr`s will
+//! increment and decrement this counter. Whenever it reaches zero, they will destroy the type instance and free
+//! its memory. This is the standard reference counting technique, just like in `TRefCounted`.
+//!
+//! - [memory pool] is a data structure for faster memory allocation. It pre-allocates large chunks of memory and uses
+//! them to allocate smaller objects. This way, we don't have to `malloc` memory for each new object.
+//! Also, we can drop all objects at once by discarding all memory chunks.
+//!
+//! When using memory-pool-based factory, types don't have individual reference counters. Instead, they store
+//! a pointer to the factory that've created them. Whenever such type is referenced or unreferenced, the factory's
+//! own reference counter is incremented or decremented. Factory dies and releases all pool's memory when the last
+//! reference to a type in its pool dies.
+//!
+//! [heap-based allocation]: https://en.wikipedia.org/wiki/Memory_management#Dynamic_memory_allocation
+//! [Reference counting]: https://en.wikipedia.org/wiki/Reference_counting
+//! [memory pool]: https://en.wikipedia.org/wiki/Region-based_memory_management
+//!
+//! The rule of thumb is: if you have an intrusive pointer to a type, you can be sure it points to an alive object;
+//! if you have a raw pointer to a type, it's your responsibility to ensure its validity.
+//!
+//! Whenever you have a valid raw pointer to a type, you can promote it to an intrusive pointer by calling `AsPtr`.
+//! This is always safe because there's no way to create a type instance on the stack or in the static memory
+//! of a program.
+//!
+//!
+//! # Implementation details
+//!
+//! We're building a hierarchy of classes that work equally well with both heap-based allocation model
+//! and memory-pool-based allocation model. In order to understand how we do it, we should first understand how
+//! object ownership works in both models.
+//!
+//!
+//! ## Ownership schema
+//!
+//! Depending on the chosen factory implementation, ownership schema may vary.
+//!
+//! When using the heap-based factory, each type has its own reference counter. User own types, and types own
+//! their nested types. For `Optional<String>`, user owns the `Optional` type, and `Optional` type
+//! owns the `String` type:
+//!
+//! ```text
+//! Legend: A ───> B -- A owns B | A ─ ─> B -- A points to B
+//!
+//! User code: Objects:
+//! ┌─────────────┐ ┌────────────────┐
+//! │ IFactoryPtr │───────>│ Factory │
+//! └─────────────┘ └────────────────┘
+//! ┌─────────────┐ ┌────────────────┐
+//! │ TTypePtr │───────>│ Type: Optional │──┐
+//! └─────────────┘ └────────────────┘ │
+//! ┌──────────────────────┘
+//! │ ┌────────────────┐
+//! └─>│ Type: String │
+//! └────────────────┘
+//! ```
+//!
+//! When using a pool-based factory, all allocated types are owned by pool. So, in the `Optional<String>` example,
+//! user owns factory, factory owns the `Optional` type and the `String` type. All references between types
+//! are just borrows:
+//!
+//! ```text
+//! Legend: A ───> B -- A owns B | A ─ ─> B -- A points to B
+//!
+//! User code: Objects:
+//! ┌─────────────┐ ┌────────────────┐
+//! │ IFactoryPtr │─┌─────>│ Factory │──┐
+//! └─────────────┘ │ └────────────────┘ │
+//! ┌─────────────┐ │ ┌────────────────┐ │
+//! │ TTypePtr │─┘─ ─ ─>│ Type: Optional │<─┤
+//! └─────────────┘ └───────┬────────┘ │
+//! ┌ ─ ─ ─ ─ ─ ─┘ │
+//! ╎ ┌────────────────┐ │
+//! └─ ─>│ Type: String │<─┘
+//! └────────────────┘
+//! ```
+//!
+//! Notice how `TTypePtr` points to `Optional`, but doesn't own it. Instead, it owns the factory which,
+//! transitively, owns `Optional`. This way we can be sure that the factory will not be destroyed before
+//! all `IFactoryPtr`s and `TTypePtr`s die, thus guaranteeing that all `TTypePtr` always stay valid.
+//!
+//! With that in mind, we can derive two types of ownership here.
+//!
+//! Whenever `TTypePtr` owns some type, we call it 'external ownership' (because it's code that is external
+//! to this library owns something). In this situation, ref and unref procedures must increment type's own reference
+//! counter, if there is one, and also increment factory's reference counter. `NTi::TType::Ref` and `NTi::TType::Unref`
+//! do this by calling `NTi::ITypeFactoryInternal::Ref`, `NTi::ITypeFactoryInternal::RefType`,
+//! `NTi::ITypeFactoryInternal::UnRef`, `NTi::ITypeFactoryInternal::UnRefType`.
+//!
+//! Whenever one type owns another type, we call it 'internal ownership'. In this situation, ref and unref procedures
+//! must only increment and decrement type's own reference counter. They should *not* increment or decrement
+//! factory's reference counter. `NTi::TType::RefSelf` and `NTi::TType::UnrefSelf` do this by calling
+//! `NTi::ITypeFactoryInternal::RefType` and `NTi::ITypeFactoryInternal::UnRefType`, and not calling
+//! `NTi::ITypeFactoryInternal::Ref` and `NTi::ITypeFactoryInternal::UnRef`.
+//!
+//!
+//! ## Adoption semantics
+//!
+//! Let's see now what should happen when we create some complex type using more that one factory.
+//!
+//! Suppose we're creating a container type, such as `Optional`, using factory `a`. If its nested type is
+//! managed by the same factory, we have no issues at all:
+//!
+//! ```
+//! auto a = NTi::PoolFactory();
+//! auto string = a->String();
+//! auto optional = a->Optional(string);
+//! ```
+//!
+//! But what if the nested type is managed by some other factory `b`? Consider:
+//!
+//! ```
+//! auto b = NTi::PoolFactory();
+//! auto a = NTi::PoolFactory();
+//! auto string = b->String();
+//! auto optional = a->Optional(string);
+//! ```
+//!
+//! Well, we're in trouble. It's not enough for `optional` to just acquire internal ownership over `string`.
+//! Actually, `optional` has to own both `string` and its factory, `b`. Otherwise, if `b` dies, `string` will
+//! die with it, leaving `optional` with a dangling pointer.
+//!
+//! However, we can't have a type owning its factory. It will create a loop. The only safe solution is to copy
+//! the nested type from one factory to another. If we have some type managed by factory `a`, all nested types
+//! must also be managed by the same factory `a`.
+//!
+//! So, whenever we create a type, we must ensure that all its nested types are managed by the same factory.
+//! If they're not, we must deep-copy them to the current factory. This is called the 'adoption semantics'. That is,
+//! we adopt nested types into the current factory.
+//!
+//! In fact, adoption semantics is a bit more complicated that this. Some types are not managed by any factory.
+//! They have static storage duration and therefore need no management. So, if we see that there's no factory
+//! associated with some type, we don't perform deep-copy. Instead, we just return it as is.
+//!
+//!
+//! ## Overview
+//!
+//! So, now we're ready to put this puzzle together.
+//!
+//! When we create a new type instance, we do the following:
+//!
+//! 1: we adopt all nested types and acquire internal ownership over them. This is done by the `Own` function;
+//! 2: we allocate some memory for the type. This is done by `New`, `Allocate` and other functions;
+//! 3: we initialize all this memory;
+//! 4: finally, we acquire external ownership over the newly created type and return it to the user.
+//!
+//! Steps 1 through 3 are performed by the `NTi::TType::Clone` function. Step 4 is performed by the factory.
+//!
+//! When we adopt some type, we check if it's managed by any factory other that the current one. If needed,
+//! we deep-copy it by the `NTi::TType::Clone` function. That is, `Clone` copies type to a new factory and adopts
+//! nested types, adoption calls nested type's `Clone`, causing recursive deep-copying.
+//!
+//! When we release external or internal ownership over some type, factory might decide to destroy it (if, for
+//! example, this type's reference count reached zero). In this case, we need to release internal ownership
+//! over the nested types, and release all memory that was allocated in step 3.
+//! This is done by `NTi::TType::Drop` function.
+
+#include <util/generic/maybe.h>
+#include <util/generic/ptr.h>
+#include <util/generic/strbuf.h>
+#include <util/memory/pool.h>
+
+#include "type.h"
+
+#include "fwd.h"
+
+namespace NTi {
+ namespace NPrivate {
+ /// If `T` is derived from `NTi::TType` (ignoring cv-qualification), provides the member constant `value`
+ /// equal to `true`. Otherwise `value` is `false`.
+ template <typename T>
+ struct TIsType: public std::is_base_of<TType, T> {
+ };
+
+ /// Helper template variable for `TIsType`.
+ template <typename T>
+ inline constexpr const bool IsTypeV = TIsType<T>::value;
+
+ /// Statically assert that `T` is a managed object.
+ template <typename T>
+ void AssertIsType() {
+ static_assert(IsTypeV<T>, "object should be derived from 'NTi::TType'");
+ }
+ }
+
+ /// Internal interface for type factory.
+ ///
+ /// This interface is accessible from `TType` implementation, but not from outside.
+ class ITypeFactoryInternal {
+ friend class TType;
+ template <typename T>
+ friend class ::TDefaultIntrusivePtrOps;
+
+ public:
+ virtual ~ITypeFactoryInternal() = default;
+
+ public:
+ /// @name Object creation and lifetime management interface
+ ///
+ /// @{
+ //-
+ /// Create a new object of type `T` by allocating memory for it and constructing it with arguments `args`.
+ ///
+ /// If `T` is derived from `TType`, it will be linked to this factory via the `NTi::TType::SetFactory` method.
+ ///
+ /// This method does not acquire any ownership over the created object, not internal nor external. It just
+ /// allocates some memory and calls a constructor and that's it.
+ template <typename T, typename... Args>
+ inline T* New(Args&&... args) {
+ // Note: it's important to understand difference between `New` and `Create` - read the docs!
+
+ static_assert(std::is_trivially_destructible_v<T>, "can only create trivially destructible types");
+
+ auto value = new (AllocateFor<T>()) T(std::forward<Args>(args)...);
+
+ if constexpr (NPrivate::IsTypeV<T>) {
+ value->SetFactory(this);
+ }
+
+ return value;
+ }
+
+ /// Delete an object that was created via the `New` function.
+ ///
+ /// This function essentially just calls `Free`. Note that `New` can only create trivially destructible types,
+ /// thus `Delete` does not neet to call object's destructor.
+ template <typename T>
+ inline void Delete(T* obj) noexcept {
+ Free(obj);
+ }
+
+ /// Create a new array of `count` objects of type `T`.
+ ///
+ /// This function will allocate an array of `count` objects of type `T`. Then, for each object in the array,
+ /// it will call the `ctor(T* memory, size_t index)`, passing a pointer to the uninitialized object
+ /// and its index in the array. `ctor` should call placement new on the passed pointer.
+ ///
+ /// If `T` is derived from `TType`, it will be linked to this factory via the `NTi::TType::SetFactory` method.
+ ///
+ /// This method does not acquire any ownership over the created array elements, not internal nor external.
+ /// It just allocates some memory and calls a constructor and that's it.
+ ///
+ ///
+ /// # Examples
+ ///
+ /// Copy vector to factory:
+ ///
+ /// ```
+ /// TVector<T> source = ...;
+ /// TArrayRef<T> copied = NMem::NewArray<T>(factory, source.size(), [&source](T* ptr, size_t i) {
+ /// new (ptr) T(source[i]);
+ /// });
+ /// ```
+ template <typename T, typename TCtor>
+ inline TArrayRef<T> NewArray(size_t count, TCtor&& ctor) {
+ // Note: it's important to understand difference between `New` and `Create` - read the docs!
+
+ static_assert(std::is_trivially_destructible_v<T>, "can only create trivially destructible types");
+
+ auto items = AllocateArrayFor<T>(count);
+
+ for (size_t i = 0; i < count; ++i) {
+ auto value = items + i;
+ ctor(value, i);
+ if constexpr (NPrivate::IsTypeV<T>) {
+ value->SetFactory(this);
+ }
+ }
+
+ return TArrayRef(items, count);
+ }
+
+ /// Delete an object that was created via the `NewArray` function.
+ ///
+ /// This function runs `dtor` on each element of the array and then just calls `Free`. Note that `NewArray`
+ /// can only create arrays for trivially destructible types, thus `DeleteArray`
+ /// does not call object destructors.
+ template <typename T, typename TDtor>
+ inline void DeleteArray(TArrayRef<T> obj, TDtor&& dtor) noexcept {
+ for (size_t i = 0; i < obj.size(); ++i) {
+ auto value = obj.data() + i;
+ dtor(value, i);
+ }
+
+ Free(const_cast<void*>(static_cast<const void*>(obj.data())));
+ }
+
+ /// Adopt `type` into this factory.
+ ///
+ /// This function works like `AdoptRaw`, but it wraps its result to an intrusive pointer, thus acquiring
+ /// external ownership over it.
+ ///
+ /// It is used by external code to copy types between factories.
+ template <typename T>
+ inline TIntrusiveConstPtr<T> Adopt(TIntrusiveConstPtr<T> type) noexcept {
+ NPrivate::AssertIsType<T>();
+
+ return TIntrusiveConstPtr<T>(const_cast<T*>(AdoptRaw<T>(type.Get())));
+ }
+
+ /// Adopt `type` into this factory.
+ ///
+ /// This function is used to transfer types between factories. It has the following semantics:
+ ///
+ /// - if `type` is managed by this factory, this function returns its argument unchanged;
+ /// - if `type` is not managed by any factory, this function also returns its argument unchanged;
+ /// - if `type` is managed by another factory, this function deep-copies `type` into this factory and returns
+ /// a pointer to the new deep-copied value.
+ ///
+ /// This function uses `NTi::TType::Clone` to deep-copy types. It does not acquire any ownership over
+ /// the returned object, not internal nor external. If you need to adopt and acquire external ownership
+ /// (i.e. you're returning type to a user), use `Adopt`. If you need to adopt and acquire internal ownership
+ /// (i.e. you're writing `NTi::TType::Clone` implementation), use `Own`.
+ template <typename T>
+ inline const T* AdoptRaw(const T* type) noexcept {
+ NPrivate::AssertIsType<T>();
+
+ ITypeFactoryInternal* typeFactory = type->GetFactory();
+ if (typeFactory == nullptr || typeFactory == this) {
+ return type;
+ } else {
+ return type->Clone(*this);
+ }
+ }
+
+ /// Adopt some type and acquire internal ownership over it.
+ ///
+ /// This function works like `AdoptRaw`, but acquires internal ownership over the returned type.
+ ///
+ /// It is used by `NTi::TType::Clone` implementations to acquire ownership over the nested types.
+ template <typename T>
+ inline const T* Own(const T* type) noexcept {
+ NPrivate::AssertIsType<T>();
+
+ type = AdoptRaw(type);
+ const_cast<T*>(type)->RefSelf();
+ return type;
+ }
+
+ /// Release internal ownership over some type.
+ ///
+ /// This function is used by `NTi::TType::Drop` to release internal ownership over nested types.
+ template <typename T>
+ inline void Disown(const T* type) noexcept {
+ NPrivate::AssertIsType<T>();
+
+ const_cast<T*>(type)->UnRefSelf();
+ }
+
+ /// @}
+
+ public:
+ /// @name Memory management interface
+ ///
+ /// Functions for dealing with raw unmanaged memory.
+ ///
+ /// @{
+ //-
+ /// Allocate a new chunk of memory of size `size` aligned on `align`.
+ ///
+ /// The behaviour is undefined if `size` is zero, `align` is zero, is not a power of two,
+ /// or greater than `PLATFORM_DATA_ALIGN` (i.e., over-aligned).
+ ///
+ /// This function panics if memory can't be allocated.
+ ///
+ /// Returned memory may or may not be initialized.
+ virtual void* Allocate(size_t size, size_t align) noexcept = 0;
+
+ /// Allocate a chunk of memory suitable for storing an object of type `T`.
+ ///
+ /// This function panics if memory can't be allocated.
+ ///
+ /// Returned memory may or may not be initialized.
+ template <typename T>
+ T* AllocateFor() noexcept {
+ static_assert(alignof(T) <= PLATFORM_DATA_ALIGN, "over-aligned types are not supported");
+ return static_cast<T*>(Allocate(sizeof(T), alignof(T)));
+ }
+
+ /// Allocate a new chunk of memory suitable for storing `count` of objects of size `size` aligned on `align`.
+ ///
+ /// The behaviour is undefined if `count` is zero, `size` is zero, `align` is zero, is not a power of two,
+ /// or greater than `PLATFORM_DATA_ALIGN` (i.e., over-aligned).
+ ///
+ /// This function panics if memory can't be allocated.
+ ///
+ /// Returned memory may or may not be initialized.
+ void* AllocateArray(size_t count, size_t size, size_t align) noexcept {
+ return Allocate(size * count, align);
+ }
+
+ /// Allocate a chunk of memory suitable for storing an array of `count` objects of type `T`.
+ ///
+ /// This function panics if memory can't be allocated.
+ ///
+ /// Returned memory may or may not be initialized.
+ template <typename T>
+ T* AllocateArrayFor(size_t count) noexcept {
+ static_assert(alignof(T) <= PLATFORM_DATA_ALIGN, "over-aligned types are not supported");
+ return static_cast<T*>(AllocateArray(count, sizeof(T), alignof(T)));
+ }
+
+ /// Reclaim a chunk of memory memory that was allocated via one of the above `Allocate*` functions.
+ ///
+ /// This function is not suitable for freeing memory allocated via `AllocateString` or `AllocateStringMaybe`.
+ ///
+ /// Behaviour of this function varies between implementations. Specifically, in the memory-pool-based factory,
+ /// this function does nothing, while in heap-based factory, it calls `free`.
+ ///
+ /// Passing a null pointer here is ok and does nothing.
+ ///
+ /// The behaviour is undefined if the given pointer is not null and it wasn't obtained from a call
+ /// to the `Allocate` function or it was obtained from a call to the `Allocate` function
+ /// of a different factory.
+ virtual void Free(void* data) noexcept = 0;
+
+ /// Allocate memory for string `str` and copy `str` to the allocated memory.
+ ///
+ /// Note: the returned string is not guaranteed to be null-terminated.
+ ///
+ /// Note: some factories may cache results of this function to avoid repeated allocations. Read documentation
+ /// for specific factory for more info.
+ virtual TStringBuf AllocateString(TStringBuf str) noexcept {
+ return TStringBuf(MemCopy(AllocateArrayFor<char>(str.size()), str.data(), str.size()), str.size());
+ }
+
+ /// Reclaim a chunk of memory memory that was allocated via the `AllocateString` function.
+ virtual void FreeString(TStringBuf str) noexcept {
+ Free(const_cast<char*>(str.Data()));
+ }
+
+ /// Like `AllocateString`, but works with `TMaybe<TStringBuf>`.
+ ///
+ /// If the given `str` contains a value, pass it to `AllocateString`.
+ /// If it contains no value, return an empty `TMaybe`.
+ TMaybe<TStringBuf> AllocateStringMaybe(TMaybe<TStringBuf> str) noexcept {
+ if (str.Defined()) {
+ return AllocateString(str.GetRef());
+ } else {
+ return Nothing();
+ }
+ }
+
+ /// Reclaim a chunk of memory memory that was allocated via the `AllocateStringMaybe` function.
+ void FreeStringMaybe(TMaybe<TStringBuf> str) noexcept {
+ if (str.Defined()) {
+ FreeString(str.GetRef());
+ }
+ }
+
+ /// @}
+
+ public:
+ /// @name Type caching and deduplication interface
+ ///
+ /// @{
+ //-
+ /// Lookup the given type in the factory cache. Return cached version of the type or `nullptr`.
+ virtual const TType* LookupCache(const TType* type) noexcept = 0;
+
+ /// Save the given type to the factory cache. Type must be allocated via this factory.
+ virtual void SaveCache(const TType* type) noexcept = 0;
+
+ /// @}
+
+ public:
+ /// @name Factory reference counting interface
+ ///
+ /// @{
+ //-
+ /// Increase reference count of this factory.
+ virtual void Ref() noexcept = 0;
+
+ /// Decrease reference count of this factory.
+ /// If reference count reaches zero, drop this factory.
+ virtual void UnRef() noexcept = 0;
+
+ /// Decrease reference count of this factory.
+ /// If reference count reaches zero, panic.
+ virtual void DecRef() noexcept = 0;
+
+ /// Get current reference count of this factory.
+ virtual long RefCount() const noexcept = 0;
+
+ /// @}
+
+ public:
+ /// @name Type instance reference counting interface
+ ///
+ /// @{
+ //-
+ /// Increase reference count of some object managed by this factory.
+ virtual void RefType(TType*) noexcept = 0;
+
+ /// Decrease reference count of some object managed by this factory.
+ /// If reference count reaches zero, call `TType::Drop` and free object's memory, if needed.
+ virtual void UnRefType(TType*) noexcept = 0;
+
+ /// Decrease reference count of some object managed by this factory.
+ /// If reference count reaches zero, panic.
+ virtual void DecRefType(TType*) noexcept = 0;
+
+ /// Get current reference count of some object managed by this factory.
+ virtual long RefCountType(const TType*) const noexcept = 0;
+
+ /// @}
+ };
+
+ /// Base interface for type factory. There are functions to create type instances, and also a function to
+ /// adopt a type from one factory to another. See the file-level documentation for more info.
+ class ITypeFactory: protected ITypeFactoryInternal {
+ friend class TType;
+ template <typename T>
+ friend class ::TDefaultIntrusivePtrOps;
+
+ public:
+ /// Adopt type into this factory.
+ ///
+ /// This function is used to transfer types between factories. It has the following semantics:
+ ///
+ /// - if `type` is managed by this factory, this function returns its argument unchanged;
+ /// - if `type` is not managed by any factory, this function also returns its argument unchanged;
+ /// - if `type` is managed by another factory, this function deep-copies `type` into this factory and returns
+ /// a pointer to the new deep-copied value.
+ ///
+ /// This function should be used on API borders, whenever you receive ownership over some type that wasn't
+ /// created by you. For example, you may ensure that the type you got is allocated in heap, or you may want to
+ /// copy some type into a memory pool that you have control over.
+ template <typename T>
+ inline TIntrusiveConstPtr<T> Adopt(TIntrusiveConstPtr<T> value) noexcept {
+ return ITypeFactoryInternal::Adopt<T>(std::move(value));
+ }
+
+ public:
+ /// Create a new instance of a type using this factory.
+ /// @{
+ //-
+ /// Create a new `Void` type. See `NTi::TVoidType` for more info.
+ TVoidTypePtr Void();
+
+ /// Create a new `Null` type. See `NTi::TNullType` for more info.
+ TNullTypePtr Null();
+
+ /// Create a new `Bool` type. See `NTi::TBoolType` for more info.
+ TBoolTypePtr Bool();
+
+ /// Create a new `Int8` type. See `NTi::TInt8Type` for more info.
+ TInt8TypePtr Int8();
+
+ /// Create a new `Int16` type. See `NTi::TInt16Type` for more info.
+ TInt16TypePtr Int16();
+
+ /// Create a new `Int32` type. See `NTi::TInt32Type` for more info.
+ TInt32TypePtr Int32();
+
+ /// Create a new `Int64` type. See `NTi::TInt64Type` for more info.
+ TInt64TypePtr Int64();
+
+ /// Create a new `Uint8` type. See `NTi::TUint8Type` for more info.
+ TUint8TypePtr Uint8();
+
+ /// Create a new `Uint16` type. See `NTi::TUint16Type` for more info.
+ TUint16TypePtr Uint16();
+
+ /// Create a new `Uint32` type. See `NTi::TUint32Type` for more info.
+ TUint32TypePtr Uint32();
+
+ /// Create a new `Uint64` type. See `NTi::TUint64Type` for more info.
+ TUint64TypePtr Uint64();
+
+ /// Create a new `Float` type. See `NTi::TFloatType` for more info.
+ TFloatTypePtr Float();
+
+ /// Create a new `Double` type. See `NTi::TDoubleType` for more info.
+ TDoubleTypePtr Double();
+
+ /// Create a new `String` type. See `NTi::TStringType` for more info.
+ TStringTypePtr String();
+
+ /// Create a new `Utf8` type. See `NTi::TUtf8Type` for more info.
+ TUtf8TypePtr Utf8();
+
+ /// Create a new `Date` type. See `NTi::TDateType` for more info.
+ TDateTypePtr Date();
+
+ /// Create a new `Datetime` type. See `NTi::TDatetimeType` for more info.
+ TDatetimeTypePtr Datetime();
+
+ /// Create a new `Timestamp` type. See `NTi::TTimestampType` for more info.
+ TTimestampTypePtr Timestamp();
+
+ /// Create a new `TzDate` type. See `NTi::TTzDateType` for more info.
+ TTzDateTypePtr TzDate();
+
+ /// Create a new `TzDatetime` type. See `NTi::TTzDatetimeType` for more info.
+ TTzDatetimeTypePtr TzDatetime();
+
+ /// Create a new `TzTimestamp` type. See `NTi::TTzTimestampType` for more info.
+ TTzTimestampTypePtr TzTimestamp();
+
+ /// Create a new `Interval` type. See `NTi::TIntervalType` for more info.
+ TIntervalTypePtr Interval();
+
+ /// Create a new `Decimal` type. See `NTi::TDecimalType` for more info.
+ TDecimalTypePtr Decimal(ui8 precision, ui8 scale);
+
+ /// Create a new `Json` type. See `NTi::TJsonType` for more info.
+ TJsonTypePtr Json();
+
+ /// Create a new `Yson` type. See `NTi::TYsonType` for more info.
+ TYsonTypePtr Yson();
+
+ /// Create a new `Uuid` type. See `NTi::TUuidType` for more info.
+ TUuidTypePtr Uuid();
+
+ /// Create a new `Optional` type. See `NTi::TOptionalType` for more info.
+ /// If `item` is managed by some other factory, it will be deep-copied into this factory.
+ TOptionalTypePtr Optional(TTypePtr item);
+
+ /// Create a new `List` type. See `NTi::TListType` for more info.
+ /// If `item` is managed by some other factory, it will be deep-copied into this factory.
+ TListTypePtr List(TTypePtr item);
+
+ /// Create a new `Dict` type. See `NTi::TDictType` for more info.
+ /// If `key` or `value` are managed by some other factory, they will be deep-copied into this factory.
+ TDictTypePtr Dict(TTypePtr key, TTypePtr value);
+
+ /// Create a new `Struct` type. See `NTi::TStructType` for more info.
+ /// If item types are managed by some other factory, they will be deep-copied into this factory.
+ TStructTypePtr Struct(TStructType::TOwnedMembers items);
+ TStructTypePtr Struct(TMaybe<TStringBuf> name, TStructType::TOwnedMembers items);
+
+ /// Create a new `Tuple` type. See `NTi::TTupleType` for more info.
+ /// If item types are managed by some other factory, they will be deep-copied into this factory.
+ TTupleTypePtr Tuple(TTupleType::TOwnedElements items);
+ TTupleTypePtr Tuple(TMaybe<TStringBuf> name, TTupleType::TOwnedElements items);
+
+ /// Create a new `Variant` type. See `NTi::TVariantType` for more info.
+ /// If `inner` is managed by some other factory, it will be deep-copied into this factory.
+ TVariantTypePtr Variant(TTypePtr inner);
+ TVariantTypePtr Variant(TMaybe<TStringBuf> name, TTypePtr inner);
+
+ /// Create a new `Tagged` type. See `NTi::TTaggedType` for more info.
+ /// If `type` is managed by some other factory, it will be deep-copied into this factory.
+ TTaggedTypePtr Tagged(TTypePtr type, TStringBuf tag);
+
+ /// @}
+ };
+
+ /// Type factory over a memory pool.
+ ///
+ /// This interface provides some additional info related to the underlying memory pool state.
+ ///
+ /// Also, since all types that're created via this factory will live exactly as long as this factory lives,
+ /// one doesn't have to actually refcount them. Thus, it is safe to work with raw pointers instead of intrusive
+ /// pointers when using this kind of factory. So we provide the 'raw' interface that creates type instances
+ /// as usual, but doesn't increment any reference counters, thus saving some atomic operations. It still
+ /// adopts nested types into this factory, though.
+ class IPoolTypeFactory: public ITypeFactory {
+ friend class TNamedTypeBuilderRaw;
+ friend class TStructBuilderRaw;
+ friend class TTupleBuilderRaw;
+ friend class TTaggedBuilderRaw;
+
+ public:
+ /// Memory available in the current chunk.
+ ///
+ /// Allocating object of size greater than this number will cause allocation of a new chunk. When this happens,
+ /// available memory in the current chunk is lost, i.e. it'll never be used again.
+ virtual size_t Available() const noexcept = 0;
+
+ /// Number of bytes that're currently used by some object (i.e. they were allocated).
+ ///
+ /// Total memory consumed by arena is `MemoryAllocated() + MemoryWaste()`.
+ virtual size_t MemoryAllocated() const noexcept = 0;
+
+ /// Number of bytes that're not used by anyone.
+ ///
+ /// Total memory consumed by arena is `MemoryAllocated() + MemoryWaste()`.
+ ///
+ /// Memory that's lost and will not be reused is `MemoryWaste() - Available()`.
+ virtual size_t MemoryWaste() const noexcept = 0;
+
+ public:
+ /// Adopt type into this factory.
+ ///
+ /// This works like `ITypeFactory::Adopt`, but returns a raw pointer.
+ template <typename T>
+ inline const T* AdoptRaw(const T* value) noexcept {
+ return ITypeFactoryInternal::AdoptRaw<T>(value);
+ }
+
+ public:
+ /// Raw versions of type constructors. See the class-level documentation for more info.
+ ///
+ /// @{
+ //-
+ /// Create a new `Void` type. See `NTi::TVoidType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TVoidType* VoidRaw();
+
+ /// Create a new `Null` type. See `NTi::TNullType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TNullType* NullRaw();
+
+ /// Create a new `Bool` type. See `NTi::TBoolType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TBoolType* BoolRaw();
+
+ /// Create a new `Int8` type. See `NTi::TInt8Type` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TInt8Type* Int8Raw();
+
+ /// Create a new `Int16` type. See `NTi::TInt16Type` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TInt16Type* Int16Raw();
+
+ /// Create a new `Int32` type. See `NTi::TInt32Type` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TInt32Type* Int32Raw();
+
+ /// Create a new `Int64` type. See `NTi::TInt64Type` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TInt64Type* Int64Raw();
+
+ /// Create a new `Uint8` type. See `NTi::TUint8Type` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TUint8Type* Uint8Raw();
+
+ /// Create a new `Uint16` type. See `NTi::TUint16Type` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TUint16Type* Uint16Raw();
+
+ /// Create a new `Uint32` type. See `NTi::TUint32Type` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TUint32Type* Uint32Raw();
+
+ /// Create a new `Uint64` type. See `NTi::TUint64Type` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TUint64Type* Uint64Raw();
+
+ /// Create a new `Float` type. See `NTi::TFloatType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TFloatType* FloatRaw();
+
+ /// Create a new `Double` type. See `NTi::TDoubleType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TDoubleType* DoubleRaw();
+
+ /// Create a new `String` type. See `NTi::TStringType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TStringType* StringRaw();
+
+ /// Create a new `Utf8` type. See `NTi::TUtf8Type` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TUtf8Type* Utf8Raw();
+
+ /// Create a new `Date` type. See `NTi::TDateType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TDateType* DateRaw();
+
+ /// Create a new `Datetime` type. See `NTi::TDatetimeType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TDatetimeType* DatetimeRaw();
+
+ /// Create a new `Timestamp` type. See `NTi::TTimestampType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TTimestampType* TimestampRaw();
+
+ /// Create a new `TzDate` type. See `NTi::TTzDateType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TTzDateType* TzDateRaw();
+
+ /// Create a new `TzDatetime` type. See `NTi::TTzDatetimeType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TTzDatetimeType* TzDatetimeRaw();
+
+ /// Create a new `TzTimestamp` type. See `NTi::TTzTimestampType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TTzTimestampType* TzTimestampRaw();
+
+ /// Create a new `Interval` type. See `NTi::TIntervalType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TIntervalType* IntervalRaw();
+
+ /// Create a new `Decimal` type. See `NTi::TDecimalType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TDecimalType* DecimalRaw(ui8 precision, ui8 scale);
+
+ /// Create a new `Json` type. See `NTi::TJsonType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TJsonType* JsonRaw();
+
+ /// Create a new `Yson` type. See `NTi::TYsonType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TYsonType* YsonRaw();
+
+ /// Create a new `Uuid` type. See `NTi::TUuidType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TUuidType* UuidRaw();
+
+ /// Create a new `Optional` type. See `NTi::TOptionalType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TOptionalType* OptionalRaw(const TType* item);
+
+ /// Create a new `List` type. See `NTi::TListType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TListType* ListRaw(const TType* item);
+
+ /// Create a new `Dict` type. See `NTi::TDictType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TDictType* DictRaw(const TType* key, const TType* value);
+
+ /// Create a new `Struct` type. See `NTi::TStructType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TStructType* StructRaw(TStructType::TMembers items);
+ const TStructType* StructRaw(TMaybe<TStringBuf> name, TStructType::TMembers items);
+
+ /// Create a new `Tuple` type. See `NTi::TTupleType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TTupleType* TupleRaw(TTupleType::TElements items);
+ const TTupleType* TupleRaw(TMaybe<TStringBuf> name, TTupleType::TElements items);
+
+ /// Create a new `Variant` type. See `NTi::TVariantType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TVariantType* VariantRaw(const TType* inner);
+ const TVariantType* VariantRaw(TMaybe<TStringBuf> name, const TType* inner);
+
+ /// Create a new `Tagged` type. See `NTi::TTaggedType` for more info.
+ /// The returned object will live for as long as this factory lives.
+ const TTaggedType* TaggedRaw(const TType* type, TStringBuf tag);
+
+ /// @}
+ };
+
+ namespace NPrivate {
+ ITypeFactory* GetDefaultHeapFactory();
+ }
+
+ /// Create a heap-based factory.
+ ///
+ /// This factory uses heap to allocate type instances, and reference counting to track individual type lifetimes.
+ ///
+ /// Choose this factory if you need safety and flexibility.
+ ITypeFactoryPtr HeapFactory();
+
+ /// Create a memory-pool-based factory.
+ ///
+ /// This factory uses a memory pool to allocate type instances. All allocated memory will be released when
+ /// the factory dies.
+ ///
+ /// Choose this factory if you need speed and memory efficiency. You have to understand how memory pool works.
+ /// We advise testing your code with msan when using this factory.
+ ///
+ /// If in doubt, use a heap-based factory instead.
+ ///
+ /// @param deduplicate
+ /// if enabled, factory will cache all types and avoid allocations when creating a new type that is
+ /// strictly-equal (see `type_equivalence.h`) to some previously created type.
+ ///
+ /// For example:
+ ///
+ /// ```
+ /// auto a = factory.Optional(factory.String());
+ /// auto b = factory.Optional(factory.String());
+ /// Y_ASSERT(a == b);
+ /// ```
+ ///
+ /// If `deduplicate` is disabled, this assert will fail. If, however, `deduplicate` is enabled,
+ /// the assert will not fail because `a` and `b` will point to the same type instance.
+ ///
+ /// This behaviour slows down creation process, but can save some memory when working with complex types.
+ ///
+ /// @param initial
+ /// number of bytes in the first page of the memory pool.
+ /// By default, we pre-allocate `64` bytes.
+ ///
+ /// @param grow
+ /// policy for calculating size for a next pool page.
+ /// By default, next page is twice as big as the previous one.
+ ///
+ /// @param alloc
+ /// underlying allocator that'll be used to allocate pool pages.
+ ///
+ /// @param options
+ /// other memory pool options.
+ IPoolTypeFactoryPtr PoolFactory(
+ bool deduplicate = true,
+ size_t initial = 64,
+ TMemoryPool::IGrowPolicy* grow = TMemoryPool::TExpGrow::Instance(),
+ IAllocator* alloc = TDefaultAllocator::Instance(),
+ TMemoryPool::TOptions options = {});
+}
diff --git a/library/cpp/type_info/type_info.cpp b/library/cpp/type_info/type_info.cpp
new file mode 100644
index 0000000000..8c14b158d0
--- /dev/null
+++ b/library/cpp/type_info/type_info.cpp
@@ -0,0 +1 @@
+#include "type_info.h"
diff --git a/library/cpp/type_info/type_info.h b/library/cpp/type_info/type_info.h
new file mode 100644
index 0000000000..5c6e249c11
--- /dev/null
+++ b/library/cpp/type_info/type_info.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include "builder.h"
+#include "error.h"
+#include "type.h"
+#include "type_constructors.h"
+#include "type_equivalence.h"
+#include "type_factory.h"
+#include "type_io.h"
+#include "type_list.h"
diff --git a/library/cpp/type_info/type_io.cpp b/library/cpp/type_info/type_io.cpp
new file mode 100644
index 0000000000..90b98c8baa
--- /dev/null
+++ b/library/cpp/type_info/type_io.cpp
@@ -0,0 +1,1186 @@
+#include "type_io.h"
+
+#include "builder.h"
+#include "type_constructors.h"
+#include "type_factory.h"
+
+#include <util/generic/overloaded.h>
+
+#include <library/cpp/yson_pull/read_ops.h>
+
+#include <util/stream/mem.h>
+#include <util/string/cast.h>
+#include <util/generic/vector.h>
+#include <util/generic/scope.h>
+
+namespace NTi::NIo {
+ namespace {
+ class TYsonDeserializer: private TNonCopyable {
+ public:
+ TYsonDeserializer(IPoolTypeFactory* factory, NYsonPull::TReader* reader)
+ : Factory_(factory)
+ , Reader_(reader)
+ {
+ }
+
+ public:
+ const TType* ReadType() {
+ if (++Depth_ > 100) {
+ ythrow TDeserializationException() << "types are nested too deep";
+ }
+
+ Y_DEFER {
+ Depth_--;
+ };
+
+ auto event = Reader_->NextEvent();
+
+ if (event.Type() == NYsonPull::EEventType::BeginStream) {
+ event = Reader_->NextEvent();
+ }
+
+ if (event.Type() == NYsonPull::EEventType::EndStream) {
+ if (Depth_ == 1) {
+ return nullptr;
+ } else {
+ ythrow TDeserializationException() << "unexpected end of stream";
+ }
+ }
+
+ if (event.Type() == NYsonPull::EEventType::Scalar && event.AsScalar().Type() == NYsonPull::EScalarType::String) {
+ return ReadTypeFromData(TypeNameStringToEnum(event.AsString()), std::monostate{});
+ } else if (event.Type() == NYsonPull::EEventType::BeginMap) {
+ return ReadTypeFromMap();
+ } else {
+ ythrow TDeserializationException() << "type must be either a string or a map";
+ }
+ }
+
+ private:
+ struct TDictData {
+ const TType *Key, *Value;
+ };
+ struct TDecimalData {
+ ui8 Precision, Scale;
+ };
+ using TTypeData = std::variant<
+ std::monostate,
+ TDictData,
+ TDecimalData,
+ TStructBuilderRaw,
+ TTupleBuilderRaw,
+ TTaggedBuilderRaw>;
+
+ const TType* ReadTypeFromMap() {
+ TMaybe<ETypeName> typeName;
+ TTypeData data;
+
+ while (true) {
+ auto event = Reader_->NextEvent();
+ if (event.Type() == NYsonPull::EEventType::Key) {
+ auto mapKey = event.AsString();
+
+ if (mapKey == "type_name") {
+ if (typeName.Defined()) {
+ ythrow TDeserializationException() << R"(duplicate key R"(type_name"))";
+ } else {
+ typeName = TypeNameStringToEnum(ReadString(R"("type_name")"));
+ }
+ } else if (mapKey == "item") {
+ if (std::holds_alternative<std::monostate>(data)) {
+ data = TTaggedBuilderRaw(*Factory_);
+ }
+ if (std::holds_alternative<TTaggedBuilderRaw>(data)) {
+ auto& builder = std::get<TTaggedBuilderRaw>(data);
+ if (!builder.HasItem()) {
+ builder.SetItem(ReadType());
+ } else {
+ ythrow TDeserializationException() << R"(duplicate key "item")";
+ }
+ } else {
+ ythrow TDeserializationException() << R"(unexpected key "item")";
+ }
+ } else if (mapKey == "key") {
+ if (std::holds_alternative<std::monostate>(data)) {
+ data = TDictData{nullptr, nullptr};
+ }
+ if (std::holds_alternative<TDictData>(data)) {
+ auto& dictData = std::get<TDictData>(data);
+ if (dictData.Key == nullptr) {
+ dictData.Key = ReadType();
+ } else {
+ ythrow TDeserializationException() << R"(duplicate key "key")";
+ }
+ } else {
+ ythrow TDeserializationException() << R"(unexpected key "key")";
+ }
+ } else if (mapKey == "value") {
+ if (std::holds_alternative<std::monostate>(data)) {
+ data = TDictData{nullptr, nullptr};
+ }
+ if (std::holds_alternative<TDictData>(data)) {
+ auto& dictData = std::get<TDictData>(data);
+ if (dictData.Value == nullptr) {
+ dictData.Value = ReadType();
+ } else {
+ ythrow TDeserializationException() << R"(duplicate key "value")";
+ }
+ } else {
+ ythrow TDeserializationException() << R"(unexpected key "value")";
+ }
+ } else if (mapKey == "members") {
+ if (std::holds_alternative<std::monostate>(data)) {
+ data = TStructBuilderRaw(*Factory_);
+ }
+ if (std::holds_alternative<TStructBuilderRaw>(data)) {
+ ReadMembers(std::get<TStructBuilderRaw>(data));
+ } else {
+ ythrow TDeserializationException() << R"(unexpected key "members")";
+ }
+ } else if (mapKey == "elements") {
+ if (std::holds_alternative<std::monostate>(data)) {
+ data = TTupleBuilderRaw(*Factory_);
+ }
+ if (std::holds_alternative<TTupleBuilderRaw>(data)) {
+ ReadElements(std::get<TTupleBuilderRaw>(data));
+ } else {
+ ythrow TDeserializationException() << R"(unexpected key "elements")";
+ }
+ } else if (mapKey == "tag") {
+ if (std::holds_alternative<std::monostate>(data)) {
+ data = TTaggedBuilderRaw(*Factory_);
+ }
+ if (std::holds_alternative<TTaggedBuilderRaw>(data)) {
+ auto& builder = std::get<TTaggedBuilderRaw>(data);
+ if (!builder.HasTag()) {
+ builder.SetTag(ReadString(R"("tag")"));
+ } else {
+ ythrow TDeserializationException() << R"(duplicate key "tag")";
+ }
+ } else {
+ ythrow TDeserializationException() << R"(unexpected key "tag")";
+ }
+ } else if (mapKey == "precision") {
+ if (std::holds_alternative<std::monostate>(data)) {
+ data = TDecimalData{ReadSmallInt(R"("precision")"), 0};
+ } else if (std::holds_alternative<TDecimalData>(data)) {
+ auto& decimalData = std::get<TDecimalData>(data);
+ if (decimalData.Precision == 0) {
+ decimalData.Precision = ReadSmallInt(R"("precision")");
+ } else {
+ ythrow TDeserializationException() << R"(duplicate key "precision")";
+ }
+ } else {
+ ythrow TDeserializationException() << R"(unexpected key "precision")";
+ }
+ } else if (mapKey == "scale") {
+ if (std::holds_alternative<std::monostate>(data)) {
+ data = TDecimalData{0, ReadSmallInt(R"("scale")")};
+ } else if (std::holds_alternative<TDecimalData>(data)) {
+ auto& decimalData = std::get<TDecimalData>(data);
+ if (decimalData.Scale == 0) {
+ decimalData.Scale = ReadSmallInt(R"("scale")");
+ } else {
+ ythrow TDeserializationException() << R"(duplicate key "scale")";
+ }
+ } else {
+ ythrow TDeserializationException() << R"(unexpected key "scale")";
+ }
+ } else {
+ NYsonPull::NReadOps::SkipValue(*Reader_);
+ }
+ } else if (event.Type() == NYsonPull::EEventType::EndMap) {
+ if (!typeName.Defined()) {
+ ythrow TDeserializationException() << R"(missing required key "type_name")";
+ }
+
+ return ReadTypeFromData(*typeName, std::move(data));
+ } else {
+ ythrow TDeserializationException() << "unexpected event " << event.Type();
+ }
+ }
+ }
+
+ const TType* ReadTypeFromData(ETypeName typeName, TTypeData data) {
+ const TType* type;
+
+ switch (typeName) {
+ case ETypeName::Bool:
+ type = TBoolType::InstanceRaw();
+ break;
+ case ETypeName::Int8:
+ type = TInt8Type::InstanceRaw();
+ break;
+ case ETypeName::Int16:
+ type = TInt16Type::InstanceRaw();
+ break;
+ case ETypeName::Int32:
+ type = TInt32Type::InstanceRaw();
+ break;
+ case ETypeName::Int64:
+ type = TInt64Type::InstanceRaw();
+ break;
+ case ETypeName::Uint8:
+ type = TUint8Type::InstanceRaw();
+ break;
+ case ETypeName::Uint16:
+ type = TUint16Type::InstanceRaw();
+ break;
+ case ETypeName::Uint32:
+ type = TUint32Type::InstanceRaw();
+ break;
+ case ETypeName::Uint64:
+ type = TUint64Type::InstanceRaw();
+ break;
+ case ETypeName::Float:
+ type = TFloatType::InstanceRaw();
+ break;
+ case ETypeName::Double:
+ type = TDoubleType::InstanceRaw();
+ break;
+ case ETypeName::String:
+ type = TStringType::InstanceRaw();
+ break;
+ case ETypeName::Utf8:
+ type = TUtf8Type::InstanceRaw();
+ break;
+ case ETypeName::Date:
+ type = TDateType::InstanceRaw();
+ break;
+ case ETypeName::Datetime:
+ type = TDatetimeType::InstanceRaw();
+ break;
+ case ETypeName::Timestamp:
+ type = TTimestampType::InstanceRaw();
+ break;
+ case ETypeName::TzDate:
+ type = TTzDateType::InstanceRaw();
+ break;
+ case ETypeName::TzDatetime:
+ type = TTzDatetimeType::InstanceRaw();
+ break;
+ case ETypeName::TzTimestamp:
+ type = TTzTimestampType::InstanceRaw();
+ break;
+ case ETypeName::Interval:
+ type = TIntervalType::InstanceRaw();
+ break;
+ case ETypeName::Decimal: {
+ if (!std::holds_alternative<TDecimalData>(data)) {
+ ythrow TDeserializationException() << R"(missing required keys "precision" and "scale" for type Decimal)";
+ }
+
+ auto& decimalData = std::get<TDecimalData>(data);
+
+ if (decimalData.Precision == 0) {
+ ythrow TDeserializationException() << R"(missing required key "precision" for type Decimal)";
+ }
+
+ if (decimalData.Scale == 0) {
+ ythrow TDeserializationException() << R"(missing required key "scale" for type Decimal)";
+ }
+
+ return Factory_->DecimalRaw(decimalData.Precision, decimalData.Scale);
+ }
+ case ETypeName::Json:
+ type = TJsonType::InstanceRaw();
+ break;
+ case ETypeName::Yson:
+ type = TYsonType::InstanceRaw();
+ break;
+ case ETypeName::Uuid:
+ type = TUuidType::InstanceRaw();
+ break;
+ case ETypeName::Void:
+ type = TVoidType::InstanceRaw();
+ break;
+ case ETypeName::Null:
+ type = TNullType::InstanceRaw();
+ break;
+ case ETypeName::Optional: {
+ if (!std::holds_alternative<TTaggedBuilderRaw>(data)) {
+ ythrow TDeserializationException() << R"(missing required key "item" for type Optional)";
+ }
+
+ auto& builder = std::get<TTaggedBuilderRaw>(data);
+
+ if (!builder.HasItem()) {
+ ythrow TDeserializationException() << R"(missing required key "item" for type Optional)";
+ }
+
+ return Factory_->OptionalRaw(*builder.GetItem());
+ }
+ case ETypeName::List: {
+ if (!std::holds_alternative<TTaggedBuilderRaw>(data)) {
+ ythrow TDeserializationException() << R"(missing required key "item" for type List)";
+ }
+
+ auto& builder = std::get<TTaggedBuilderRaw>(data);
+
+ if (!builder.HasItem()) {
+ ythrow TDeserializationException() << R"(missing required key "item" for type List)";
+ }
+
+ return Factory_->ListRaw(*builder.GetItem());
+ }
+ case ETypeName::Dict: {
+ if (!std::holds_alternative<TDictData>(data)) {
+ ythrow TDeserializationException() << R"(missing required keys "key" and "value" for type Dict)";
+ }
+
+ auto& dictData = std::get<TDictData>(data);
+
+ if (dictData.Key == nullptr) {
+ ythrow TDeserializationException() << R"(missing required key "key" for type Dict)";
+ }
+
+ if (dictData.Value == nullptr) {
+ ythrow TDeserializationException() << R"(missing required key "value" for type Dict)";
+ }
+
+ return Factory_->DictRaw(dictData.Key, dictData.Value);
+ }
+ case ETypeName::Struct: {
+ if (!std::holds_alternative<TStructBuilderRaw>(data)) {
+ ythrow TDeserializationException() << R"(missing required key "members" for type Struct)";
+ }
+
+ return std::get<TStructBuilderRaw>(data).BuildRaw();
+ }
+ case ETypeName::Tuple: {
+ if (!std::holds_alternative<TTupleBuilderRaw>(data)) {
+ ythrow TDeserializationException() << R"(missing required key "elements" for type Tuple)";
+ }
+
+ return std::get<TTupleBuilderRaw>(data).BuildRaw();
+ }
+ case ETypeName::Variant: {
+ if (std::holds_alternative<TStructBuilderRaw>(data)) {
+ return std::get<TStructBuilderRaw>(data).BuildVariantRaw();
+ } else if (std::holds_alternative<TTupleBuilderRaw>(data)) {
+ return std::get<TTupleBuilderRaw>(data).BuildVariantRaw();
+ } else {
+ ythrow TDeserializationException() << R"(missing both keys "members" and "elements" for type Variant)";
+ }
+ }
+ case ETypeName::Tagged: {
+ if (!std::holds_alternative<TTaggedBuilderRaw>(data)) {
+ ythrow TDeserializationException() << R"(missing required keys "tag" and "item" for type Tagged)";
+ }
+
+ auto& builder = std::get<TTaggedBuilderRaw>(data);
+
+ if (!builder.HasItem()) {
+ ythrow TDeserializationException() << R"(missing required key "item" for type Tagged)";
+ }
+
+ if (!builder.HasTag()) {
+ ythrow TDeserializationException() << R"(missing required key "tag" for type Tagged)";
+ }
+
+ return builder.BuildRaw();
+ }
+ }
+
+ if (!std::holds_alternative<std::monostate>(data)) {
+ ythrow TDeserializationException() << "unexpected key for type " << typeName;
+ }
+
+ return type;
+ }
+
+ TStringBuf ReadString(TStringBuf what) {
+ auto event = Reader_->NextEvent();
+
+ if (event.Type() != NYsonPull::EEventType::Scalar || event.AsScalar().Type() != NYsonPull::EScalarType::String) {
+ ythrow TDeserializationException() << what << " must contain a string";
+ }
+
+ return event.AsString();
+ }
+
+ ui8 ReadSmallInt(TStringBuf what) {
+ auto event = Reader_->NextEvent();
+
+ if (event.Type() != NYsonPull::EEventType::Scalar || event.AsScalar().Type() != NYsonPull::EScalarType::Int64) {
+ ythrow TDeserializationException() << what << " must contain a signed integer";
+ }
+
+ auto result = event.AsScalar().AsInt64();
+
+ if (result <= 0) {
+ ythrow TDeserializationException() << what << " must be greater than zero";
+ }
+
+ if (result > Max<ui8>()) {
+ ythrow TDeserializationException() << what << " is too big";
+ }
+
+ return static_cast<ui8>(result);
+ }
+
+ void ReadMembers(TStructBuilderRaw& builder) {
+ if (Reader_->NextEvent().Type() != NYsonPull::EEventType::BeginList) {
+ ythrow TDeserializationException() << R"("members" must contain a list)";
+ }
+
+ while (true) {
+ auto event = Reader_->NextEvent();
+
+ if (event.Type() == NYsonPull::EEventType::BeginMap) {
+ while (true) {
+ auto event = Reader_->NextEvent();
+
+ if (event.Type() == NYsonPull::EEventType::Key) {
+ auto mapKey = event.AsString();
+ if (mapKey == "name") {
+ if (builder.HasMemberName()) {
+ ythrow TDeserializationException() << R"(duplicate key "name")";
+ }
+
+ builder.AddMemberName(ReadString(R"("name")"));
+ } else if (mapKey == "type") {
+ if (builder.HasMemberType()) {
+ ythrow TDeserializationException() << R"(duplicate key "type")";
+ }
+
+ builder.AddMemberType(ReadType());
+ } else {
+ NYsonPull::NReadOps::SkipValue(*Reader_);
+ }
+ } else if (event.Type() == NYsonPull::EEventType::EndMap) {
+ if (!builder.HasMemberName()) {
+ ythrow TDeserializationException() << R"(missing required key "name")";
+ }
+ if (!builder.HasMemberType()) {
+ ythrow TDeserializationException() << R"(missing required key "type")";
+ }
+
+ builder.AddMember();
+ break;
+ } else {
+ ythrow TDeserializationException() << "unexpected event " << event.Type();
+ }
+ }
+ } else if (event.Type() == NYsonPull::EEventType::EndList) {
+ break;
+ } else {
+ ythrow TDeserializationException() << R"("members" must contain a list of maps)";
+ }
+ }
+ }
+
+ void ReadElements(TTupleBuilderRaw& builder) {
+ if (Reader_->NextEvent().Type() != NYsonPull::EEventType::BeginList) {
+ ythrow TDeserializationException() << R"("elements" must contain a list)";
+ }
+
+ while (true) {
+ auto event = Reader_->NextEvent();
+
+ if (event.Type() == NYsonPull::EEventType::BeginMap) {
+ while (true) {
+ auto event = Reader_->NextEvent();
+
+ if (event.Type() == NYsonPull::EEventType::Key) {
+ auto mapKey = event.AsString();
+ if (mapKey == "type") {
+ if (builder.HasElementType()) {
+ ythrow TDeserializationException() << R"(duplicate key "type")";
+ }
+
+ builder.AddElementType(ReadType());
+ } else {
+ NYsonPull::NReadOps::SkipValue(*Reader_);
+ }
+ } else if (event.Type() == NYsonPull::EEventType::EndMap) {
+ if (!builder.HasElementType()) {
+ ythrow TDeserializationException() << R"(missing required key "type")";
+ }
+
+ builder.AddElement();
+ break;
+ } else {
+ ythrow TDeserializationException() << "unexpected event " << event.Type();
+ }
+ }
+ } else if (event.Type() == NYsonPull::EEventType::EndList) {
+ break;
+ } else {
+ ythrow TDeserializationException() << R"("elements" must contain a list of maps)";
+ }
+ }
+ }
+
+ static ETypeName TypeNameStringToEnum(TStringBuf name) {
+ static const THashMap<TStringBuf, ETypeName> dispatch = {
+ {"void", ETypeName::Void},
+ {"null", ETypeName::Null},
+ {"bool", ETypeName::Bool},
+ {"int8", ETypeName::Int8},
+ {"int16", ETypeName::Int16},
+ {"int32", ETypeName::Int32},
+ {"int64", ETypeName::Int64},
+ {"uint8", ETypeName::Uint8},
+ {"uint16", ETypeName::Uint16},
+ {"uint32", ETypeName::Uint32},
+ {"uint64", ETypeName::Uint64},
+ {"float", ETypeName::Float},
+ {"double", ETypeName::Double},
+ {"string", ETypeName::String},
+ {"utf8", ETypeName::Utf8},
+ {"date", ETypeName::Date},
+ {"datetime", ETypeName::Datetime},
+ {"timestamp", ETypeName::Timestamp},
+ {"tz_date", ETypeName::TzDate},
+ {"tz_datetime", ETypeName::TzDatetime},
+ {"tz_timestamp", ETypeName::TzTimestamp},
+ {"interval", ETypeName::Interval},
+ {"json", ETypeName::Json},
+ {"yson", ETypeName::Yson},
+ {"uuid", ETypeName::Uuid},
+ {"decimal", ETypeName::Decimal},
+ {"optional", ETypeName::Optional},
+ {"list", ETypeName::List},
+ {"dict", ETypeName::Dict},
+ {"struct", ETypeName::Struct},
+ {"tuple", ETypeName::Tuple},
+ {"variant", ETypeName::Variant},
+ {"tagged", ETypeName::Tagged},
+ };
+
+ if (auto it = dispatch.find(name); it != dispatch.end()) {
+ return it->second;
+ } else {
+ ythrow TDeserializationException() << "unknown type " << TString{name}.Quote();
+ }
+ }
+
+ private:
+ IPoolTypeFactory* Factory_;
+ NYsonPull::TReader* Reader_;
+ size_t Depth_ = 0;
+ };
+ }
+
+ TTypePtr DeserializeYson(ITypeFactory& factory, NYsonPull::TReader& reader, bool deduplicate) {
+ auto pool = PoolFactory(deduplicate, 2048);
+ auto type = DeserializeYsonRaw(*pool, reader);
+ return factory.Adopt(type->AsPtr());
+ }
+
+ TTypePtr DeserializeYson(ITypeFactory& factory, TStringBuf data, bool deduplicate) {
+ auto reader = NYsonPull::TReader(NYsonPull::NInput::FromMemory(data), NYsonPull::EStreamType::Node);
+ return DeserializeYson(factory, reader, deduplicate);
+ }
+
+ TTypePtr DeserializeYson(ITypeFactory& factory, IInputStream& input, bool deduplicate) {
+ auto reader = NYsonPull::TReader(NYsonPull::NInput::FromInputStream(&input), NYsonPull::EStreamType::Node);
+ return DeserializeYson(factory, reader, deduplicate);
+ }
+
+ const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, NYsonPull::TReader& reader) {
+ if (reader.LastEvent().Type() != NYsonPull::EEventType::BeginStream) {
+ ythrow TDeserializationException() << "stream contains extraneous data";
+ }
+
+ auto type = DeserializeYsonMultipleRaw(factory, reader);
+
+ if (type == nullptr) {
+ ythrow TDeserializationException() << "unexpected end of stream";
+ }
+
+ if (reader.NextEvent().Type() != NYsonPull::EEventType::EndStream) {
+ ythrow TDeserializationException() << "stream contains extraneous data";
+ }
+
+ return type;
+ }
+
+ const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, TStringBuf data) {
+ auto reader = NYsonPull::TReader(NYsonPull::NInput::FromMemory(data), NYsonPull::EStreamType::Node);
+ return DeserializeYsonRaw(factory, reader);
+ }
+
+ const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, IInputStream& input) {
+ auto reader = NYsonPull::TReader(NYsonPull::NInput::FromInputStream(&input), NYsonPull::EStreamType::Node);
+ return DeserializeYsonRaw(factory, reader);
+ }
+
+ const TType* DeserializeYsonMultipleRaw(IPoolTypeFactory& factory, NYsonPull::TReader& reader) {
+ return TYsonDeserializer(&factory, &reader).ReadType();
+ }
+
+ TString SerializeYson(const TType* type, bool humanReadable, bool includeTags) {
+ auto result = TString();
+ auto writer = humanReadable
+ ? NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node)
+ : NYsonPull::MakeBinaryWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node);
+ SerializeYson(type, writer.GetConsumer(), includeTags);
+ return result;
+ }
+
+ void SerializeYson(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags) {
+ consumer.OnBeginStream();
+ SerializeYsonMultiple(type, consumer, includeTags);
+ consumer.OnEndStream();
+ }
+
+ void SerializeYson(const TType* type, IOutputStream& stream, bool humanReadable, bool includeTags) {
+ auto writer = humanReadable
+ ? NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromOutputStream(&stream), NYsonPull::EStreamType::Node)
+ : NYsonPull::MakeBinaryWriter(NYsonPull::NOutput::FromOutputStream(&stream), NYsonPull::EStreamType::Node);
+ SerializeYson(type, writer.GetConsumer(), includeTags);
+ }
+
+ void SerializeYsonMultiple(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags) {
+ type->VisitRaw(TOverloaded{
+ [&consumer](const TVoidType*) {
+ consumer.OnScalarString("void");
+ },
+ [&consumer](const TNullType*) {
+ consumer.OnScalarString("null");
+ },
+ [&consumer](const TBoolType*) {
+ consumer.OnScalarString("bool");
+ },
+ [&consumer](const TInt8Type*) {
+ consumer.OnScalarString("int8");
+ },
+ [&consumer](const TInt16Type*) {
+ consumer.OnScalarString("int16");
+ },
+ [&consumer](const TInt32Type*) {
+ consumer.OnScalarString("int32");
+ },
+ [&consumer](const TInt64Type*) {
+ consumer.OnScalarString("int64");
+ },
+ [&consumer](const TUint8Type*) {
+ consumer.OnScalarString("uint8");
+ },
+ [&consumer](const TUint16Type*) {
+ consumer.OnScalarString("uint16");
+ },
+ [&consumer](const TUint32Type*) {
+ consumer.OnScalarString("uint32");
+ },
+ [&consumer](const TUint64Type*) {
+ consumer.OnScalarString("uint64");
+ },
+ [&consumer](const TFloatType*) {
+ consumer.OnScalarString("float");
+ },
+ [&consumer](const TDoubleType*) {
+ consumer.OnScalarString("double");
+ },
+ [&consumer](const TStringType*) {
+ consumer.OnScalarString("string");
+ },
+ [&consumer](const TUtf8Type*) {
+ consumer.OnScalarString("utf8");
+ },
+ [&consumer](const TDateType*) {
+ consumer.OnScalarString("date");
+ },
+ [&consumer](const TDatetimeType*) {
+ consumer.OnScalarString("datetime");
+ },
+ [&consumer](const TTimestampType*) {
+ consumer.OnScalarString("timestamp");
+ },
+ [&consumer](const TTzDateType*) {
+ consumer.OnScalarString("tz_date");
+ },
+ [&consumer](const TTzDatetimeType*) {
+ consumer.OnScalarString("tz_datetime");
+ },
+ [&consumer](const TTzTimestampType*) {
+ consumer.OnScalarString("tz_timestamp");
+ },
+ [&consumer](const TIntervalType*) {
+ consumer.OnScalarString("interval");
+ },
+ [&consumer](const TJsonType*) {
+ consumer.OnScalarString("json");
+ },
+ [&consumer](const TYsonType*) {
+ consumer.OnScalarString("yson");
+ },
+ [&consumer](const TUuidType*) {
+ consumer.OnScalarString("uuid");
+ },
+ [&consumer](const TDecimalType* t) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type_name");
+ consumer.OnScalarString("decimal");
+
+ consumer.OnKey("precision");
+ consumer.OnScalarInt64(t->GetPrecision());
+
+ consumer.OnKey("scale");
+ consumer.OnScalarInt64(t->GetScale());
+
+ consumer.OnEndMap();
+ },
+ [&consumer, includeTags](const TOptionalType* t) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type_name");
+ consumer.OnScalarString("optional");
+
+ consumer.OnKey("item");
+ SerializeYsonMultiple(t->GetItemTypeRaw(), consumer, includeTags);
+
+ consumer.OnEndMap();
+ },
+ [&consumer, includeTags](const TListType* t) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type_name");
+ consumer.OnScalarString("list");
+
+ consumer.OnKey("item");
+ SerializeYsonMultiple(t->GetItemTypeRaw(), consumer, includeTags);
+
+ consumer.OnEndMap();
+ },
+ [&consumer, includeTags](const TDictType* t) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type_name");
+ consumer.OnScalarString("dict");
+
+ consumer.OnKey("key");
+ SerializeYsonMultiple(t->GetKeyTypeRaw(), consumer, includeTags);
+
+ consumer.OnKey("value");
+ SerializeYsonMultiple(t->GetValueTypeRaw(), consumer, includeTags);
+
+ consumer.OnEndMap();
+ },
+ [&consumer, includeTags](const TStructType* t) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type_name");
+ consumer.OnScalarString("struct");
+
+ consumer.OnKey("members");
+ consumer.OnBeginList();
+ for (auto& item : t->GetMembers()) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("name");
+ consumer.OnScalarString(item.GetName());
+
+ consumer.OnKey("type");
+ SerializeYsonMultiple(item.GetTypeRaw(), consumer, includeTags);
+
+ consumer.OnEndMap();
+ }
+ consumer.OnEndList();
+
+ consumer.OnEndMap();
+ },
+ [&consumer, includeTags](const TTupleType* t) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type_name");
+ consumer.OnScalarString("tuple");
+
+ consumer.OnKey("elements");
+ consumer.OnBeginList();
+ for (auto& item : t->GetElements()) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type");
+ SerializeYsonMultiple(item.GetTypeRaw(), consumer, includeTags);
+
+ consumer.OnEndMap();
+ }
+ consumer.OnEndList();
+
+ consumer.OnEndMap();
+ },
+ [&consumer, includeTags](const TVariantType* t) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type_name");
+ consumer.OnScalarString("variant");
+
+ t->VisitUnderlyingRaw(
+ TOverloaded{
+ [&consumer, includeTags](const TStructType* t) {
+ // Warning: we loose struct's name here.
+ // See https://ml.yandex-team.ru/thread/data-com-dev/171136785840079161/
+
+ consumer.OnKey("members");
+ consumer.OnBeginList();
+ for (auto& item : t->GetMembers()) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("name");
+ consumer.OnScalarString(item.GetName());
+
+ consumer.OnKey("type");
+ SerializeYsonMultiple(item.GetTypeRaw(), consumer, includeTags);
+
+ consumer.OnEndMap();
+ }
+ consumer.OnEndList();
+ },
+ [&consumer, includeTags](const TTupleType* t) {
+ // Warning: we loose tuple's name here.
+ // See https://ml.yandex-team.ru/thread/data-com-dev/171136785840079161/
+
+ consumer.OnKey("elements");
+ consumer.OnBeginList();
+ for (auto& item : t->GetElements()) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type");
+ SerializeYsonMultiple(item.GetTypeRaw(), consumer, includeTags);
+
+ consumer.OnEndMap();
+ }
+ consumer.OnEndList();
+ }});
+
+ consumer.OnEndMap();
+ },
+ [&consumer, includeTags](const TTaggedType* t) {
+ if (includeTags) {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("type_name");
+ consumer.OnScalarString("tagged");
+
+ consumer.OnKey("tag");
+ consumer.OnScalarString(t->GetTag());
+
+ consumer.OnKey("item");
+ SerializeYsonMultiple(t->GetItemTypeRaw(), consumer, includeTags);
+
+ consumer.OnEndMap();
+ } else {
+ SerializeYsonMultiple(t->GetItemTypeRaw(), consumer, includeTags);
+ }
+ },
+ });
+ }
+
+ namespace {
+ void WriteVoidType(NYsonPull::IConsumer& consumer) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("VoidType");
+ consumer.OnEndList();
+ }
+
+ void WriteNullType(NYsonPull::IConsumer& consumer) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("NullType");
+ consumer.OnEndList();
+ }
+
+ void WriteDataType(NYsonPull::IConsumer& consumer, EPrimitiveTypeName name) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("DataType");
+ consumer.OnScalarString(ToString(name));
+ consumer.OnEndList();
+ }
+ }
+
+ void AsYqlType(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags) {
+ type->VisitRaw(TOverloaded{
+ [&consumer](const TVoidType*) {
+ WriteVoidType(consumer);
+ },
+ [&consumer](const TNullType*) {
+ WriteNullType(consumer);
+ },
+ [&consumer](const TBoolType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Bool);
+ },
+ [&consumer](const TInt8Type*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Int8);
+ },
+ [&consumer](const TInt16Type*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Int16);
+ },
+ [&consumer](const TInt32Type*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Int32);
+ },
+ [&consumer](const TInt64Type*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Int64);
+ },
+ [&consumer](const TUint8Type*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Uint8);
+ },
+ [&consumer](const TUint16Type*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Uint16);
+ },
+ [&consumer](const TUint32Type*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Uint32);
+ },
+ [&consumer](const TUint64Type*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Uint64);
+ },
+ [&consumer](const TFloatType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Float);
+ },
+ [&consumer](const TDoubleType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Double);
+ },
+ [&consumer](const TStringType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::String);
+ },
+ [&consumer](const TUtf8Type*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Utf8);
+ },
+ [&consumer](const TDateType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Date);
+ },
+ [&consumer](const TDatetimeType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Datetime);
+ },
+ [&consumer](const TTimestampType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Timestamp);
+ },
+ [&consumer](const TTzDateType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::TzDate);
+ },
+ [&consumer](const TTzDatetimeType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::TzDatetime);
+ },
+ [&consumer](const TTzTimestampType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::TzTimestamp);
+ },
+ [&consumer](const TIntervalType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Interval);
+ },
+ [&consumer](const TJsonType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Json);
+ },
+ [&consumer](const TYsonType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Yson);
+ },
+ [&consumer](const TUuidType*) {
+ WriteDataType(consumer, EPrimitiveTypeName::Uuid);
+ },
+ [&consumer](const TDecimalType* t) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("DataType");
+ consumer.OnScalarString("Decimal");
+ consumer.OnScalarString(ToString(t->GetPrecision()));
+ consumer.OnScalarString(ToString(t->GetScale()));
+ consumer.OnEndList();
+ },
+ [&consumer, includeTags](const TOptionalType* t) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("OptionalType");
+ AsYqlType(t->GetItemTypeRaw(), consumer, includeTags);
+ consumer.OnEndList();
+ },
+ [&consumer, includeTags](const TListType* t) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("ListType");
+ AsYqlType(t->GetItemTypeRaw(), consumer, includeTags);
+ consumer.OnEndList();
+ },
+ [&consumer, includeTags](const TDictType* t) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("DictType");
+ AsYqlType(t->GetKeyTypeRaw(), consumer, includeTags);
+ AsYqlType(t->GetValueTypeRaw(), consumer, includeTags);
+ consumer.OnEndList();
+ },
+ [&consumer, includeTags](const TStructType* t) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("StructType");
+ {
+ consumer.OnBeginList();
+ for (auto& item : t->GetMembers()) {
+ consumer.OnBeginList();
+ consumer.OnScalarString(item.GetName());
+ AsYqlType(item.GetTypeRaw(), consumer, includeTags);
+ consumer.OnEndList();
+ }
+ consumer.OnEndList();
+ }
+ consumer.OnEndList();
+ },
+ [&consumer, includeTags](const TTupleType* t) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("TupleType");
+ {
+ consumer.OnBeginList();
+ for (auto& item : t->GetElements()) {
+ AsYqlType(item.GetTypeRaw(), consumer, includeTags);
+ }
+ consumer.OnEndList();
+ }
+
+ consumer.OnEndList();
+ },
+ [&consumer, includeTags](const TVariantType* t) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("VariantType");
+ AsYqlType(t->GetUnderlyingTypeRaw(), consumer, includeTags);
+ consumer.OnEndList();
+ },
+ [&consumer, includeTags](const TTaggedType* t) {
+ if (includeTags) {
+ consumer.OnBeginList();
+ consumer.OnScalarString("TaggedType");
+ consumer.OnScalarString(t->GetTag());
+ AsYqlType(t->GetItemTypeRaw(), consumer, includeTags);
+ consumer.OnEndList();
+ } else {
+ AsYqlType(t->GetItemTypeRaw(), consumer, includeTags);
+ }
+ },
+ });
+ }
+
+ TString AsYqlType(const NTi::TType* type, bool includeTags) {
+ auto result = TString();
+ auto writer = NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node);
+ writer.BeginStream();
+ AsYqlType(type, writer.GetConsumer(), includeTags);
+ writer.EndStream();
+ return result;
+ }
+
+ void AsYqlRowSpec(const TType* maybeTagged, NYsonPull::IConsumer& consumer, bool includeTags) {
+ auto* type = maybeTagged->StripTagsRaw();
+
+ if (!type->IsStruct()) {
+ ythrow TApiException() << "AsYqlRowSpec expected a struct type but got " << type->GetTypeName();
+ }
+
+ consumer.OnBeginMap();
+ consumer.OnKey("StrictSchema");
+ consumer.OnScalarBoolean(true);
+ consumer.OnKey("Type");
+ AsYqlType(type, consumer, includeTags);
+ consumer.OnEndMap();
+ }
+
+ TString AsYqlRowSpec(const NTi::TType* type, bool includeTags) {
+ auto result = TString();
+ auto writer = NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node);
+ writer.BeginStream();
+ AsYqlRowSpec(type, writer.GetConsumer(), includeTags);
+ writer.EndStream();
+ return result;
+ }
+
+ void AsYtSchema(const TType* maybeTagged, NYsonPull::IConsumer& consumer, bool failOnEmptyStruct) {
+ auto* type = maybeTagged->StripTagsRaw();
+
+ if (!type->IsStruct()) {
+ ythrow TApiException() << "AsYtSchema expected a struct type but got " << type->GetTypeName();
+ }
+
+ auto* structType = type->AsStructRaw();
+
+ if (structType->GetMembers().empty()) {
+ if (failOnEmptyStruct) {
+ ythrow TApiException() << "AsYtSchema expected a non-empty struct";
+ }
+
+ AsYtSchema(Struct({{"_yql_fake_column", Optional(Bool())}}).Get(), consumer);
+ return;
+ }
+
+ consumer.OnBeginAttributes();
+
+ consumer.OnKey("strict");
+ consumer.OnScalarBoolean(true);
+
+ consumer.OnKey("unique_keys");
+ consumer.OnScalarBoolean(false);
+
+ consumer.OnEndAttributes();
+
+ consumer.OnBeginList();
+ for (auto& item : structType->GetMembers()) {
+ auto* itemType = item.GetTypeRaw()->StripTagsRaw();
+
+ bool required = true;
+
+ if (itemType->IsOptional()) {
+ // toplevel optionals make non-required columns
+ itemType = itemType->AsOptionalRaw()->GetItemTypeRaw();
+ required = false;
+ }
+
+ TStringBuf typeString = itemType->VisitRaw(TOverloaded{
+ [](const TVoidType*) -> TStringBuf { return "any"; },
+ [](const TNullType*) -> TStringBuf { return "any"; },
+ [](const TBoolType*) -> TStringBuf { return "boolean"; },
+ [](const TInt8Type*) -> TStringBuf { return "int8"; },
+ [](const TInt16Type*) -> TStringBuf { return "int16"; },
+ [](const TInt32Type*) -> TStringBuf { return "int32"; },
+ [](const TInt64Type*) -> TStringBuf { return "int64"; },
+ [](const TUint8Type*) -> TStringBuf { return "uint8"; },
+ [](const TUint16Type*) -> TStringBuf { return "uint16"; },
+ [](const TUint32Type*) -> TStringBuf { return "uint32"; },
+ [](const TUint64Type*) -> TStringBuf { return "uint64"; },
+ [](const TFloatType*) -> TStringBuf { return "double"; },
+ [](const TDoubleType*) -> TStringBuf { return "double"; },
+ [](const TStringType*) -> TStringBuf { return "string"; },
+ [](const TUtf8Type*) -> TStringBuf { return "utf8"; },
+ [](const TDateType*) -> TStringBuf { return "uint16"; },
+ [](const TDatetimeType*) -> TStringBuf { return "uint32"; },
+ [](const TTimestampType*) -> TStringBuf { return "uint64"; },
+ [](const TTzDateType*) -> TStringBuf { return "string"; },
+ [](const TTzDatetimeType*) -> TStringBuf { return "string"; },
+ [](const TTzTimestampType*) -> TStringBuf { return "string"; },
+ [](const TIntervalType*) -> TStringBuf { return "int64"; },
+ [](const TJsonType*) -> TStringBuf { return "string"; },
+ [](const TYsonType*) -> TStringBuf { return "any"; },
+ [](const TUuidType*) -> TStringBuf { return "string"; },
+ [](const TDecimalType*) -> TStringBuf { return "string"; },
+ [](const TOptionalType*) -> TStringBuf { return "any"; },
+ [](const TListType*) -> TStringBuf { return "any"; },
+ [](const TDictType*) -> TStringBuf { return "any"; },
+ [](const TStructType*) -> TStringBuf { return "any"; },
+ [](const TTupleType*) -> TStringBuf { return "any"; },
+ [](const TVariantType*) -> TStringBuf { return "any"; },
+ [](const TTaggedType*) -> TStringBuf { return "any"; },
+ });
+
+ if (typeString == "any") {
+ // columns of type `any` cannot be required
+ required = false;
+ }
+
+ {
+ consumer.OnBeginMap();
+
+ consumer.OnKey("name");
+ consumer.OnScalarString(item.GetName());
+
+ consumer.OnKey("required");
+ consumer.OnScalarBoolean(required);
+
+ consumer.OnKey("type");
+ consumer.OnScalarString(typeString);
+
+ consumer.OnEndMap();
+ }
+ }
+ consumer.OnEndList();
+ }
+
+ TString AsYtSchema(const NTi::TType* type, bool failOnEmptyStruct) {
+ auto result = TString();
+ auto writer = NYsonPull::MakePrettyTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::Node);
+ writer.BeginStream();
+ AsYtSchema(type, writer.GetConsumer(), failOnEmptyStruct);
+ writer.EndStream();
+ return result;
+ }
+}
diff --git a/library/cpp/type_info/type_io.h b/library/cpp/type_info/type_io.h
new file mode 100644
index 0000000000..e400414cff
--- /dev/null
+++ b/library/cpp/type_info/type_io.h
@@ -0,0 +1,115 @@
+#pragma once
+
+//! @file type_io.h
+//!
+//! Utilities for serializing and deserializing type instances.
+
+#include "type.h"
+
+#include <library/cpp/yson_pull/yson.h>
+
+namespace NTi::NIo {
+ /// Load type from a serialized representation.
+ ///
+ /// Serialization uses YSON (either binary or text). Contents are described in the [docs page].
+ ///
+ /// Throws `TTypeDeserializationException` if input is not valid.
+ ///
+ /// [docs page]: https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/type_info/docs/types_serialization.md
+ ///
+ /// @param factory factory that will be used to allocate new type. Technically, type will be deserialized into
+ /// a temporary pool factory and then adopted into a given one.
+ /// @param reader yson pull reader that'll be used to read types.
+ /// @param deduplicate use deduplication while creating new types. See `NTi::PoolFactory` function for more info.
+ /// @{
+ TTypePtr DeserializeYson(ITypeFactory& factory, NYsonPull::TReader& reader, bool deduplicate = true);
+ TTypePtr DeserializeYson(ITypeFactory& factory, TStringBuf data, bool deduplicate = true);
+ TTypePtr DeserializeYson(ITypeFactory& factory, IInputStream& input, bool deduplicate = true);
+ /// @}
+
+ /// Like `Deserialize`, but returns a raw pointer.
+ /// @{
+ const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, NYsonPull::TReader& reader);
+ const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, TStringBuf data);
+ const TType* DeserializeYsonRaw(IPoolTypeFactory& factory, IInputStream& input);
+ /// @}
+
+ /// Like `Deserialize`, but allows deserializing multiple types from the same reader.
+ ///
+ /// This function takes a reader created with `NYsonPull::EStreamType::ListFragment` mode. It reads a type,
+ /// but doesn't fails if there is no `BeginStream` event. If the reader is empty, it returns nullptr.
+ ///
+ /// This function mirrors `SerializeMultiple`. Call it multiple times on the same reader to read multiple types.
+ const TType* DeserializeYsonMultipleRaw(IPoolTypeFactory& factory, NYsonPull::TReader& reader);
+
+ /// Serialize this type info.
+ ///
+ /// Serialization uses YSON (either binary or text). Contents are described in the [RFC].
+ ///
+ /// [RFC]: https://a.yandex-team.ru/arc/trunk/arcadia/logfeller/mvp/docs/types_serialization.md
+ ///
+ /// @param humanReadable use pretty textual format instead of a binary one.
+ /// @param includeTags when disabled, tagged types will be removed from the result, only tagged type contents
+ /// will be dumped. For example, `Tagged<'Url', String>` will be rendered as just `String`.
+ /// This is useful if you only care about physical layout of a type and don't want to export
+ /// any semantical meaning.
+ /// @{
+ void SerializeYson(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags = true);
+ void SerializeYson(const TType* type, IOutputStream& stream, bool humanReadable = false, bool includeTags = true);
+ TString SerializeYson(const TType* type, bool humanReadable = false, bool includeTags = true);
+ /// @}
+
+ /// Like `Serialize`, but allows serializing multiple types into the same consumer.
+ ///
+ /// This function takes a consumer created with `NYsonPull::EStreamType::ListFragment` mode. It writes type,
+ /// but doesn't emit the `BeginStream` and `EndStream` commands.
+ ///
+ /// Call this function multiple times on the same consumer to write multiple types. Note that you must emit
+ /// the `BeginStream` and `EndStream` commands to the consumer yourself.
+ void SerializeYsonMultiple(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags = true);
+
+ /// Convert type to the lisp-like representation used in YQL row specs.
+ ///
+ /// TODO: move this code to yql/
+ ///
+ /// @param includeTags when disabled, tagged types will be removed from the result, only tagged type contents
+ /// will be dumped. For example, `Tagged<'Url', String>` will be rendered as just `String`.
+ /// This is useful if you only care about physical layout of a type and don't want to export
+ /// any semantic meaning.
+ /// @{
+ void AsYqlType(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags = true);
+ TString AsYqlType(const TType* type, bool includeTags = true);
+ /// @}
+
+ /// Generate a strict YQL row spec. Toplevel tags will be ignored.
+ ///
+ /// Throws `TApiException` if the type is not a (possibly tagged) struct.
+ ///
+ /// TODO: move this code to yql/
+ ///
+ /// @param type type that'll be converted to YQL row spec.
+ /// @param consumer yson pull consumer. Attention: `OnBeginStream` and `OnEndStream` should be emitted manually
+ /// before and after calling this function.
+ /// @param includeTags same as in `TType::AsYqlType`.
+ /// @{
+ void AsYqlRowSpec(const TType* type, NYsonPull::IConsumer& consumer, bool includeTags = true);
+ TString AsYqlRowSpec(const TType* type, bool includeTags = true);
+ /// @}
+
+ /// Generate a strict YT schema (only types are exported, no index/sorting information).
+ ///
+ /// Throws `TApiException` if the type is not a (possibly tagged) struct.
+ ///
+ /// The schema is generated according to the translation rules of YQL types, i.e. container types translate
+ /// to `Any`.
+ ///
+ /// TODO: move this code to mapreduce/yt/
+ ///
+ /// @param failOnEmptyStruct if true, will throw `TApiException` if called on a struct with no fields;
+ /// if false, will emit a strict YT schema with a single column `'_yql_fake_column'`
+ /// of type `Optional<Bool>` (this is how YQL handles empty tables).
+ /// @{
+ void AsYtSchema(const TType* type, NYsonPull::IConsumer& consumer, bool failOnEmptyStruct = true);
+ TString AsYtSchema(const TType* type, bool failOnEmptyStruct = true);
+ /// @}
+}
diff --git a/library/cpp/type_info/type_list.cpp b/library/cpp/type_info/type_list.cpp
new file mode 100644
index 0000000000..b741b2858b
--- /dev/null
+++ b/library/cpp/type_info/type_list.cpp
@@ -0,0 +1 @@
+#include "type_list.h"
diff --git a/library/cpp/type_info/type_list.h b/library/cpp/type_info/type_list.h
new file mode 100644
index 0000000000..c87c51918d
--- /dev/null
+++ b/library/cpp/type_info/type_list.h
@@ -0,0 +1,183 @@
+#pragma once
+
+//! @file type_list.h
+//!
+//! Enum with all type names that are included in the Common Type System.
+//!
+//!
+//! # Primitive and non-primitive types
+//!
+//! Some systems only work with primitive types, so we've split the enum in two: the first contains primitive types,
+//! and the second contains all types, including the primitive ones. This way systems that are only interested
+//! in primitives can stop handling containers and make all their switch-cases exhaustive.
+//!
+//! Consequently, the class hierarchy follows the same division: there is `NTi::TPrimitiveType`,
+//! from which all primitives are derived.
+//!
+//!
+//! # Enumerator values
+//!
+//! Enumerator values are implementation detail and should not be relied upon. In particular, use `NTi::ToTypeName`
+//! and `NTi::ToPrimitiveTypeName` to safely cast between `NTi::EPrimitiveTypeName` and `NTi::ETypeName`. Also, don't
+//! use enumerator values for serialization and deserialization — convert enumerator values to strings
+//! it you need persistence.
+
+#include <util/system/types.h>
+#include <util/generic/variant.h>
+
+namespace NTi {
+ /// Enum with names of all primitive types.
+ ///
+ /// See the file-level documentation.
+ enum class EPrimitiveTypeName : i32 {
+ Bool,
+
+ Int8,
+ Int16,
+ Int32,
+ Int64,
+ Uint8,
+ Uint16,
+ Uint32,
+ Uint64,
+
+ Float,
+ Double,
+
+ String,
+ Utf8,
+
+ Date,
+ Datetime,
+ Timestamp,
+ TzDate,
+ TzDatetime,
+ TzTimestamp,
+ Interval,
+
+ Decimal,
+ Json,
+ Yson,
+ Uuid,
+ };
+
+ /// Enum with names of all types, including primitives.
+ ///
+ /// See the file-level documentation.
+ enum class ETypeName : i32 {
+ //
+ // # Primitive types
+
+ Bool,
+
+ Int8,
+ Int16,
+ Int32,
+ Int64,
+ Uint8,
+ Uint16,
+ Uint32,
+ Uint64,
+
+ Float,
+ Double,
+
+ String,
+ Utf8,
+
+ Date,
+ Datetime,
+ Timestamp,
+ TzDate,
+ TzDatetime,
+ TzTimestamp,
+ Interval,
+
+ Decimal,
+ Json,
+ Yson,
+ Uuid,
+
+ FIRST_PRIMITIVE = Bool,
+ LAST_PRIMITIVE = Uuid,
+
+ //
+ // # Singular types
+
+ Void,
+ Null,
+
+ FIRST_SINGULAR = Void,
+ LAST_SINGULAR = Null,
+
+ //
+ // # Containers
+
+ Optional,
+ List,
+ Dict,
+ Struct,
+ Tuple,
+ Variant,
+ Tagged,
+
+ FIRST_CONTAINER = Optional,
+ LAST_CONTAINER = Tagged,
+ };
+
+ /// Return true if the given type is a primitive one.
+ ///
+ /// Primitive type is a type that have no type parameters and is not a singular one.
+ inline constexpr bool IsPrimitive(ETypeName typeName) {
+ return ETypeName::FIRST_PRIMITIVE <= typeName && typeName <= ETypeName::LAST_PRIMITIVE;
+ }
+
+ /// Return true if the given type is one of singular types.
+ ///
+ /// Singular type is a type that has only one instance and therefore carries no information,
+ /// i.e. occupy zero-length memory buffer.
+ inline constexpr bool IsSingular(ETypeName typeName) {
+ return ETypeName::FIRST_SINGULAR <= typeName && typeName <= ETypeName::LAST_SINGULAR;
+ }
+
+ /// Return true if the given type is one of containers.
+ ///
+ /// Container type is a type that has type parameters.
+ inline constexpr bool IsContainer(ETypeName typeName) {
+ return ETypeName::FIRST_CONTAINER <= typeName && typeName <= ETypeName::LAST_CONTAINER;
+ }
+
+ /// Return true if the given type has any type parameters.
+ inline constexpr bool HasTypeParameters(ETypeName typeName) {
+ return IsContainer(typeName);
+ }
+
+ /// Return true if the given type has any non-type parameters.
+ inline constexpr bool HasNonTypeParameters(ETypeName typeName) {
+ return typeName == ETypeName::Decimal;
+ }
+
+ /// Return true if the given type has any type or non-type parameters.
+ inline constexpr bool HasParameters(ETypeName typeName) {
+ return HasTypeParameters(typeName) || HasNonTypeParameters(typeName);
+ }
+
+ /// Safely cast `NTi::EPrimitiveTypeName` to `NTi::ETypeName`.
+ ///
+ /// Enumerator values should not relied upon, therefore users should not cast `NTi::EPrimitiveTypeName`
+ /// to `NTi::ETypeName` using `static_cast`.
+ inline constexpr ETypeName ToTypeName(EPrimitiveTypeName primitiveTypeName) {
+ // Note: there's a test in ut/type_list.cpp that checks this is a safe conversion
+ return static_cast<ETypeName>(primitiveTypeName);
+ }
+
+ /// Cast `NTi::ETypeName` to `NTi::EPrimitiveTypeName`, panic if the given type is not a primitive one.
+ ///
+ /// Enumerator values should not relied upon, therefore users should not cast `NTi::ETypeName`
+ /// to `NTi::EPrimitiveTypeName` using `static_cast`.
+ inline constexpr EPrimitiveTypeName ToPrimitiveTypeName(ETypeName typeName) {
+ Y_VERIFY(IsPrimitive(typeName));
+ // Note: there's a test in ut/type_list.cpp that checks this is a safe conversion
+ return static_cast<EPrimitiveTypeName>(typeName);
+ }
+}
diff --git a/library/cpp/type_info/ut/builder.cpp b/library/cpp/type_info/ut/builder.cpp
new file mode 100644
index 0000000000..43b7bb8c9b
--- /dev/null
+++ b/library/cpp/type_info/ut/builder.cpp
@@ -0,0 +1,125 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include "utils.h"
+
+class Builder: public NTesting::TTest {
+public:
+ void SetUp() override {
+ F = NTi::PoolFactory(false);
+ }
+
+ void TearDown() override {
+ F.Reset();
+ }
+
+ NTi::IPoolTypeFactoryPtr F;
+};
+
+TEST_F(Builder, TaggedBuilder) {
+ auto builder = NTi::TTaggedBuilderRaw(*F);
+
+ ASSERT_FALSE(builder.CanBuild());
+ ASSERT_FALSE(builder.HasTag());
+ ASSERT_FALSE(builder.HasItem());
+
+ UNIT_ASSERT_EQUAL(builder.GetTag(), Nothing());
+ UNIT_ASSERT_EQUAL(builder.GetItem(), Nothing());
+
+ {
+ auto tag = TString("Url");
+ builder.SetTag(tag);
+ }
+
+ ASSERT_FALSE(builder.CanBuild());
+ ASSERT_TRUE(builder.HasTag());
+ ASSERT_FALSE(builder.HasItem());
+
+ UNIT_ASSERT_EQUAL(builder.GetTag(), MakeMaybe<TStringBuf>("Url"));
+ UNIT_ASSERT_EQUAL(builder.GetItem(), Nothing());
+
+ {
+ auto ty = NTi::String();
+ builder.SetItem(ty);
+ }
+
+ ASSERT_TRUE(builder.CanBuild());
+ ASSERT_TRUE(builder.HasTag());
+ ASSERT_TRUE(builder.HasItem());
+
+ UNIT_ASSERT_EQUAL(builder.GetTag(), MakeMaybe<TStringBuf>("Url"));
+ UNIT_ASSERT_EQUAL(builder.GetItem(), MakeMaybe<const NTi::TType*>(NTi::TStringType::InstanceRaw()));
+
+ {
+ auto tagged = builder.Build();
+ ASSERT_STRICT_EQ(tagged, NTi::Tagged(NTi::String(), "Url"));
+ }
+
+ ASSERT_TRUE(builder.CanBuild());
+
+ builder.Reset();
+
+ ASSERT_FALSE(builder.CanBuild());
+ ASSERT_FALSE(builder.HasTag());
+ ASSERT_FALSE(builder.HasItem());
+
+ UNIT_ASSERT_EQUAL(builder.GetTag(), Nothing());
+ UNIT_ASSERT_EQUAL(builder.GetItem(), Nothing());
+
+ builder.SetTag("T");
+ builder.SetItem(NTi::String());
+
+ ASSERT_TRUE(builder.CanBuild());
+ ASSERT_TRUE(builder.HasTag());
+ ASSERT_TRUE(builder.HasItem());
+
+ UNIT_ASSERT_EQUAL(builder.GetTag(), MakeMaybe<TStringBuf>("T"));
+ UNIT_ASSERT_EQUAL(builder.GetItem(), MakeMaybe<const NTi::TType*>(NTi::TStringType::InstanceRaw()));
+
+ builder.DiscardItem();
+
+ ASSERT_FALSE(builder.CanBuild());
+ ASSERT_TRUE(builder.HasTag());
+ ASSERT_FALSE(builder.HasItem());
+
+ UNIT_ASSERT_EQUAL(builder.GetTag(), MakeMaybe<TStringBuf>("T"));
+ UNIT_ASSERT_EQUAL(builder.GetItem(), Nothing());
+
+ builder.DiscardTag();
+
+ builder.SetTag("T");
+ builder.SetItem(NTi::String());
+
+ ASSERT_TRUE(builder.CanBuild());
+ ASSERT_TRUE(builder.HasTag());
+ ASSERT_TRUE(builder.HasItem());
+}
+
+TEST_F(Builder, TaggedBuilderChaining) {
+ auto builder = NTi::TTaggedBuilderRaw(*F)
+ .SetTag("Uuu")
+ .SetItem(NTi::Optional(NTi::String()));
+
+ ASSERT_STRICT_EQ(builder.Build(), NTi::Tagged(NTi::Optional(NTi::String()), "Uuu"));
+
+ builder = std::move(builder)
+ .DiscardTag()
+ .SetTag("Urls");
+
+ ASSERT_STRICT_EQ(builder.Build(), NTi::Tagged(NTi::Optional(NTi::String()), "Urls"));
+
+ builder = std::move(builder)
+ .DiscardItem()
+ .SetItem(F->ListRaw(F->StringRaw()));
+
+ ASSERT_STRICT_EQ(builder.Build(), NTi::Tagged(NTi::List(NTi::String()), "Urls"));
+
+ auto type = std::move(builder)
+ .Reset()
+ .SetTag("Url")
+ .SetItem(F->StringRaw())
+ .Build();
+
+ ASSERT_STRICT_EQ(type, NTi::Tagged(NTi::String(), "Url"));
+}
diff --git a/library/cpp/type_info/ut/test_data.cpp b/library/cpp/type_info/ut/test_data.cpp
new file mode 100644
index 0000000000..36944e7bc4
--- /dev/null
+++ b/library/cpp/type_info/ut/test_data.cpp
@@ -0,0 +1,91 @@
+#include <library/cpp/testing/unittest/gtest.h>
+#include <library/cpp/resource/resource.h>
+
+#include <library/cpp/type_info/type_info.h>
+#include <library/cpp/type_info/type_io.h>
+
+#include <util/string/strip.h>
+#include <util/string/split.h>
+
+using namespace NTi;
+
+std::vector<std::vector<TString>> ParseData(TStringBuf data, int expectedFieldsCount) {
+ TString noComments;
+ {
+ TMemoryInput in(data);
+ TString line;
+ while (in.ReadLine(line)) {
+ if (StripString(line).StartsWith('#')) {
+ continue;
+ }
+ noComments += line;
+ }
+ }
+
+ std::vector<std::vector<TString>> result;
+ for (TStringBuf record : StringSplitter(noComments).SplitByString(";;")) {
+ record = StripString(record);
+ if (record.Empty()) {
+ continue;
+ }
+ std::vector<TString> fields;
+ for (TStringBuf field : StringSplitter(record).SplitByString("::")) {
+ fields.emplace_back(StripString(field));
+ }
+ if (static_cast<int>(fields.size()) != expectedFieldsCount) {
+ ythrow yexception() << "Unexpected field count expected: " << expectedFieldsCount << " actual: " << fields.size();
+ }
+ result.push_back(fields);
+ }
+ return result;
+}
+
+TEST(TestData, GoodTypes) {
+ auto records = ParseData(NResource::Find("/good"), 2);
+
+ for (const auto& record : records) {
+ const auto& typeYson = record.at(0);
+ const auto& typeText = record.at(1);
+ TString context = TStringBuilder()
+ << "text: " << typeText << Endl
+ << "yson: " << typeYson << Endl;
+ auto wrapError = [&] (const std::exception& ex) {
+ return yexception() << "Unexpected error: " << ex.what() << '\n' << context;
+ };
+
+ TTypePtr type;
+ try {
+ type = NIo::DeserializeYson(*HeapFactory(), typeYson);
+ } catch (const std::exception& ex) {
+ ythrow wrapError(ex);
+ }
+ UNIT_ASSERT_VALUES_EQUAL_C(ToString(*type), typeText, context);
+
+ TTypePtr type2;
+ try {
+ auto yson2 = NIo::SerializeYson(type.Get(), true);
+ type2 = NIo::DeserializeYson(*HeapFactory(), yson2);
+ } catch (const std::exception& ex) {
+ ythrow wrapError(ex);
+ }
+ UNIT_ASSERT_VALUES_EQUAL_C(*type, *type2, context);
+ }
+}
+
+TEST(TestData, BadTypes) {
+ auto records = ParseData(NResource::Find("/bad"), 3);
+
+ for (const auto& record : records) {
+ const auto& typeYson = record.at(0);
+ const auto& exceptionMessage = record.at(1);
+
+ TString context = TStringBuilder()
+ << "exception: " << exceptionMessage << Endl
+ << "yson: " << typeYson << Endl;
+ UNIT_ASSERT_EXCEPTION_CONTAINS_C(
+ NIo::DeserializeYson(*HeapFactory(), typeYson),
+ yexception,
+ exceptionMessage,
+ context);
+ }
+} \ No newline at end of file
diff --git a/library/cpp/type_info/ut/type_basics.cpp b/library/cpp/type_info/ut/type_basics.cpp
new file mode 100644
index 0000000000..83996d63d3
--- /dev/null
+++ b/library/cpp/type_info/ut/type_basics.cpp
@@ -0,0 +1,381 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include "utils.h"
+
+TEST_TF(TypeBasics, Void) {
+ auto t = f.Void();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Void);
+ ASSERT_TRUE(t->IsVoid());
+}
+
+TEST_TF(TypeBasics, Bool) {
+ auto t = f.Bool();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Bool);
+ ASSERT_TRUE(t->IsBool());
+}
+
+TEST_TF(TypeBasics, Int8) {
+ auto t = f.Int8();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int8);
+ ASSERT_TRUE(t->IsInt8());
+}
+
+TEST_TF(TypeBasics, Int16) {
+ auto t = f.Int16();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int16);
+ ASSERT_TRUE(t->IsInt16());
+}
+
+TEST_TF(TypeBasics, Int32) {
+ auto t = f.Int32();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int32);
+ ASSERT_TRUE(t->IsInt32());
+}
+
+TEST_TF(TypeBasics, Int64) {
+ auto t = f.Int64();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int64);
+ ASSERT_TRUE(t->IsInt64());
+}
+
+TEST_TF(TypeBasics, Uint8) {
+ auto t = f.Uint8();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint8);
+ ASSERT_TRUE(t->IsUint8());
+}
+
+TEST_TF(TypeBasics, Uint16) {
+ auto t = f.Uint16();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint16);
+ ASSERT_TRUE(t->IsUint16());
+}
+
+TEST_TF(TypeBasics, Uint32) {
+ auto t = f.Uint32();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint32);
+ ASSERT_TRUE(t->IsUint32());
+}
+
+TEST_TF(TypeBasics, Uint64) {
+ auto t = f.Uint64();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint64);
+ ASSERT_TRUE(t->IsUint64());
+}
+
+TEST_TF(TypeBasics, Float) {
+ auto t = f.Float();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Float);
+ ASSERT_TRUE(t->IsFloat());
+}
+
+TEST_TF(TypeBasics, Double) {
+ auto t = f.Double();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Double);
+ ASSERT_TRUE(t->IsDouble());
+}
+
+TEST_TF(TypeBasics, String) {
+ auto t = f.String();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::String);
+ ASSERT_TRUE(t->IsString());
+}
+
+TEST_TF(TypeBasics, Utf8) {
+ auto t = f.Utf8();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Utf8);
+ ASSERT_TRUE(t->IsUtf8());
+}
+
+TEST_TF(TypeBasics, Date) {
+ auto t = f.Date();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Date);
+ ASSERT_TRUE(t->IsDate());
+}
+
+TEST_TF(TypeBasics, Datetime) {
+ auto t = f.Datetime();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Datetime);
+ ASSERT_TRUE(t->IsDatetime());
+}
+
+TEST_TF(TypeBasics, Timestamp) {
+ auto t = f.Timestamp();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Timestamp);
+ ASSERT_TRUE(t->IsTimestamp());
+}
+
+TEST_TF(TypeBasics, TzDate) {
+ auto t = f.TzDate();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzDate);
+ ASSERT_TRUE(t->IsTzDate());
+}
+
+TEST_TF(TypeBasics, TzDatetime) {
+ auto t = f.TzDatetime();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzDatetime);
+ ASSERT_TRUE(t->IsTzDatetime());
+}
+
+TEST_TF(TypeBasics, TzTimestamp) {
+ auto t = f.TzTimestamp();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzTimestamp);
+ ASSERT_TRUE(t->IsTzTimestamp());
+}
+
+TEST_TF(TypeBasics, Interval) {
+ auto t = f.Interval();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Interval);
+ ASSERT_TRUE(t->IsInterval());
+}
+
+TEST_TF(TypeBasics, Decimal) {
+ auto t = f.Decimal(20, 10);
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Decimal);
+ ASSERT_TRUE(t->IsDecimal());
+ ASSERT_EQ(t->GetPrecision(), 20);
+ ASSERT_EQ(t->GetScale(), 10);
+}
+
+TEST_TF(TypeBasics, Json) {
+ auto t = f.Json();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Json);
+ ASSERT_TRUE(t->IsJson());
+}
+
+TEST_TF(TypeBasics, Yson) {
+ auto t = f.Yson();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Yson);
+ ASSERT_TRUE(t->IsYson());
+}
+
+TEST_TF(TypeBasics, Uuid) {
+ auto t = f.Uuid();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uuid);
+ ASSERT_TRUE(t->IsUuid());
+}
+
+TEST_TF(TypeBasics, Optional) {
+ auto t = f.Optional(f.Void());
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Optional);
+ ASSERT_TRUE(t->IsOptional());
+ ASSERT_TRUE(t->GetItemType()->IsVoid());
+ ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw());
+}
+
+TEST_TF(TypeBasics, List) {
+ auto t = f.List(f.Void());
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::List);
+ ASSERT_TRUE(t->IsList());
+ ASSERT_TRUE(t->GetItemType()->IsVoid());
+ ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw());
+}
+
+TEST_TF(TypeBasics, Dict) {
+ auto t = f.Dict(f.Void(), f.String());
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Dict);
+ ASSERT_TRUE(t->IsDict());
+ ASSERT_TRUE(t->GetKeyType()->IsVoid());
+ ASSERT_EQ(t->GetKeyType().Get(), t->GetKeyTypeRaw());
+ ASSERT_TRUE(t->GetValueType()->IsString());
+ ASSERT_EQ(t->GetValueType().Get(), t->GetValueTypeRaw());
+}
+
+TEST_TF(TypeBasics, EmptyStruct) {
+ auto t = f.Struct({});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct);
+ ASSERT_TRUE(t->IsStruct());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_EQ(t->GetMembers().size(), 0);
+}
+
+TEST_TF(TypeBasics, NamedEmptyStruct) {
+ auto t = f.Struct("S", {});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct);
+ ASSERT_TRUE(t->IsStruct());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "S");
+ ASSERT_EQ(t->GetMembers().size(), 0);
+}
+
+TEST_TF(TypeBasics, Struct) {
+ auto t = f.Struct({{"a", f.Int64()}, {"b", f.Int8()}});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct);
+ ASSERT_TRUE(t->IsStruct());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_EQ(t->GetMembers().size(), 2);
+ ASSERT_EQ(t->GetMembers()[0].GetName(), "a");
+ ASSERT_TRUE(t->GetMembers()[0].GetTypeRaw()->IsInt64());
+ ASSERT_EQ(t->GetMembers()[0].GetType().Get(), t->GetMembers()[0].GetTypeRaw());
+ ASSERT_EQ(t->GetMembers()[1].GetName(), "b");
+ ASSERT_TRUE(t->GetMembers()[1].GetTypeRaw()->IsInt8());
+ ASSERT_EQ(t->GetMembers()[1].GetType().Get(), t->GetMembers()[1].GetTypeRaw());
+}
+
+TEST_TF(TypeBasics, NamedStruct) {
+ auto t = f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct);
+ ASSERT_TRUE(t->IsStruct());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "S");
+ ASSERT_EQ(t->GetMembers().size(), 2);
+ ASSERT_EQ(t->GetMembers()[0].GetName(), "a");
+ ASSERT_TRUE(t->GetMembers()[0].GetTypeRaw()->IsInt64());
+ ASSERT_EQ(t->GetMembers()[0].GetType().Get(), t->GetMembers()[0].GetTypeRaw());
+ ASSERT_EQ(t->GetMembers()[1].GetName(), "b");
+ ASSERT_TRUE(t->GetMembers()[1].GetTypeRaw()->IsInt8());
+ ASSERT_EQ(t->GetMembers()[1].GetType().Get(), t->GetMembers()[1].GetTypeRaw());
+}
+
+TEST_TF(TypeBasics, EmptyTuple) {
+ auto t = f.Tuple({});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple);
+ ASSERT_TRUE(t->IsTuple());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_EQ(t->GetElements().size(), 0);
+}
+
+TEST_TF(TypeBasics, NamedEmptyTuple) {
+ auto t = f.Tuple("T", {});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple);
+ ASSERT_TRUE(t->IsTuple());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "T");
+ ASSERT_EQ(t->GetElements().size(), 0);
+}
+
+TEST_TF(TypeBasics, Tuple) {
+ auto t = f.Tuple({{f.Int64()}, {f.Int8()}});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple);
+ ASSERT_TRUE(t->IsTuple());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_EQ(t->GetElements().size(), 2);
+ ASSERT_TRUE(t->GetElements()[0].GetTypeRaw()->IsInt64());
+ ASSERT_EQ(t->GetElements()[0].GetType().Get(), t->GetElements()[0].GetTypeRaw());
+ ASSERT_TRUE(t->GetElements()[1].GetTypeRaw()->IsInt8());
+ ASSERT_EQ(t->GetElements()[1].GetType().Get(), t->GetElements()[1].GetTypeRaw());
+}
+
+TEST_TF(TypeBasics, NamedTuple) {
+ auto t = f.Tuple("T", {{f.Int64()}, {f.Int8()}});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple);
+ ASSERT_TRUE(t->IsTuple());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "T");
+ ASSERT_EQ(t->GetElements().size(), 2);
+ ASSERT_TRUE(t->GetElements()[0].GetTypeRaw()->IsInt64());
+ ASSERT_EQ(t->GetElements()[0].GetType().Get(), t->GetElements()[0].GetTypeRaw());
+ ASSERT_TRUE(t->GetElements()[1].GetTypeRaw()->IsInt8());
+ ASSERT_EQ(t->GetElements()[1].GetType().Get(), t->GetElements()[1].GetTypeRaw());
+}
+
+TEST_TF(TypeBasics, VariantOverStruct) {
+ auto t = f.Variant(f.Struct("Inner", {{"x", f.Void()}}));
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant);
+ ASSERT_TRUE(t->IsVariant());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_TRUE(t->IsVariantOverStruct());
+ ASSERT_FALSE(t->IsVariantOverTuple());
+ ASSERT_EQ(t->GetUnderlyingType()->AsStruct()->GetName().GetRef(), "Inner");
+ ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw());
+}
+
+TEST_TF(TypeBasics, NamedVariantOverStruct) {
+ auto t = f.Variant("V", f.Struct("Inner", {{"x", f.Void()}}));
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant);
+ ASSERT_TRUE(t->IsVariant());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "V");
+ ASSERT_TRUE(t->IsVariantOverStruct());
+ ASSERT_FALSE(t->IsVariantOverTuple());
+ ASSERT_EQ(t->GetUnderlyingType()->AsStruct()->GetName().GetRef(), "Inner");
+ ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw());
+}
+
+TEST_TF(TypeBasics, VariantOverTuple) {
+ auto t = f.Variant(f.Tuple("Inner", {{f.Void()}}));
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant);
+ ASSERT_TRUE(t->IsVariant());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_FALSE(t->IsVariantOverStruct());
+ ASSERT_TRUE(t->IsVariantOverTuple());
+ ASSERT_EQ(t->GetUnderlyingType()->AsTuple()->GetName().GetRef(), "Inner");
+ ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw());
+}
+
+TEST_TF(TypeBasics, NamedVariantOverTuple) {
+ auto t = f.Variant("V", f.Tuple("Inner", {{f.Void()}}));
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant);
+ ASSERT_TRUE(t->IsVariant());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "V");
+ ASSERT_FALSE(t->IsVariantOverStruct());
+ ASSERT_TRUE(t->IsVariantOverTuple());
+ ASSERT_EQ(t->GetUnderlyingType()->AsTuple()->GetName().GetRef(), "Inner");
+ ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw());
+}
+
+TEST_TF(TypeBasics, Tagged) {
+ auto t = f.Tagged(f.Void(), "T");
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tagged);
+ ASSERT_TRUE(t->IsTagged());
+ ASSERT_EQ(t->GetTag(), "T");
+ ASSERT_TRUE(t->GetItemType()->IsVoid());
+ ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw());
+}
+
+TEST_TF(TypeBasics, EmptyStructLookupByName) {
+ auto s = f.Struct({});
+
+ ASSERT_FALSE(s->HasMember("item"));
+ ASSERT_EQ(s->GetMemberIndex("item"), -1);
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ [=]() {
+ s->GetMember("item");
+ }(),
+ NTi::TItemNotFound, "no item named 'item'");
+}
+
+TEST_TF(TypeBasics, StructLookupByName) {
+ auto s = f.Struct({
+ {"a", f.Void()},
+ {"field2", f.String()},
+ {"field1", f.Utf8()},
+ {"", f.Null()},
+ });
+
+ ASSERT_TRUE(s->HasMember("a"));
+ ASSERT_EQ(s->GetMemberIndex("a"), 0);
+
+ ASSERT_TRUE(s->HasMember("field2"));
+ ASSERT_EQ(s->GetMemberIndex("field2"), 1);
+
+ ASSERT_TRUE(s->HasMember("field1"));
+ ASSERT_EQ(s->GetMemberIndex("field1"), 2);
+
+ ASSERT_TRUE(s->HasMember(""));
+ ASSERT_EQ(s->GetMemberIndex(""), 3);
+
+ ASSERT_FALSE(s->HasMember("b"));
+ ASSERT_EQ(s->GetMemberIndex("b"), -1);
+
+ ASSERT_EQ(s->GetMember("a").GetName(), "a");
+ ASSERT_TRUE(s->GetMember("a").GetTypeRaw()->IsVoid());
+
+ ASSERT_EQ(s->GetMember("field2").GetName(), "field2");
+ ASSERT_TRUE(s->GetMember("field2").GetTypeRaw()->IsString());
+
+ ASSERT_EQ(s->GetMember("field1").GetName(), "field1");
+ ASSERT_TRUE(s->GetMember("field1").GetTypeRaw()->IsUtf8());
+
+ ASSERT_EQ(s->GetMember("").GetName(), "");
+ ASSERT_TRUE(s->GetMember("").GetTypeRaw()->IsNull());
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ [=]() {
+ s->GetMember("b");
+ }(),
+ NTi::TItemNotFound, "no item named 'b'");
+}
diff --git a/library/cpp/type_info/ut/type_complexity_ut.cpp b/library/cpp/type_info/ut/type_complexity_ut.cpp
new file mode 100644
index 0000000000..3b4f6e5372
--- /dev/null
+++ b/library/cpp/type_info/ut/type_complexity_ut.cpp
@@ -0,0 +1,33 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_complexity.h>
+#include <library/cpp/type_info/type_constructors.h>
+
+using namespace NTi;
+
+TEST(TypeComplexity, Test)
+{
+ EXPECT_EQ(ComputeTypeComplexity(Int64()), 1);
+ EXPECT_EQ(ComputeTypeComplexity(String()), 1);
+ EXPECT_EQ(ComputeTypeComplexity(Null()), 1);
+ EXPECT_EQ(ComputeTypeComplexity(Decimal(4, 2)), 1);
+
+ EXPECT_EQ(ComputeTypeComplexity(Optional(Utf8())), 2);
+ EXPECT_EQ(ComputeTypeComplexity(List(Json())), 2);
+ EXPECT_EQ(ComputeTypeComplexity(Tagged(String(), "jpeg")), 2);
+ EXPECT_EQ(ComputeTypeComplexity(Dict(String(), Optional(Int64()))), 4);
+ EXPECT_EQ(ComputeTypeComplexity(Struct({
+ {"a", String()},
+ {"b", List(Optional(Int64()))},
+ })), 5);
+ EXPECT_EQ(ComputeTypeComplexity(Tuple({{Float()}, {Float()}})), 3);
+
+ EXPECT_EQ(ComputeTypeComplexity(Variant(Struct({
+ {"a", String()},
+ {"b", Int64()},
+ }))), 3);
+ EXPECT_EQ(ComputeTypeComplexity(Tuple({
+ {String()},
+ {Int64()},
+ })), 3);
+} \ No newline at end of file
diff --git a/library/cpp/type_info/ut/type_constraints.cpp b/library/cpp/type_info/ut/type_constraints.cpp
new file mode 100644
index 0000000000..0f56bd89b0
--- /dev/null
+++ b/library/cpp/type_info/ut/type_constraints.cpp
@@ -0,0 +1,53 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include "utils.h"
+
+TEST_TF(TypeConstraints, DecimalScale) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ f.Decimal(20, 21);
+ }(),
+ NTi::TIllegalTypeException, "decimal scale 21 should be no greater than decimal precision 20");
+}
+
+TEST_TF(TypeConstraints, StructDuplicateItem) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ f.Struct({{"a", f.Void()}, {"a", f.String()}});
+ }(),
+ NTi::TIllegalTypeException, "duplicate struct item 'a'");
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ f.Struct({{"a", f.Void()}, {"b", f.Bool()}, {"a", f.String()}});
+ }(),
+ NTi::TIllegalTypeException, "duplicate struct item 'a'");
+}
+
+TEST_TF(TypeConstraints, StructEmpty) {
+ f.Struct({}); // empty structs are ok, this should not fail
+}
+
+TEST_TF(TypeConstraints, TupleEmpty) {
+ f.Tuple({}); // empty tuples are ok, this should not fail
+}
+
+TEST_TF(TypeConstraints, VariantStructEmpty) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ f.Variant(f.Struct({}));
+ }(),
+ NTi::TIllegalTypeException, "variant should contain at least one alternative");
+}
+
+TEST_TF(TypeConstraints, VariantTupleEmpty) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ f.Variant(f.Tuple({}));
+ }(),
+ NTi::TIllegalTypeException, "variant should contain at least one alternative");
+}
+
+TEST_TF(TypeConstraints, VariantWrongInnerType) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ f.Variant(f.String());
+ }(),
+ NTi::TIllegalTypeException, "variants can only contain structs and tuples, got String instead");
+}
diff --git a/library/cpp/type_info/ut/type_deserialize.cpp b/library/cpp/type_info/ut/type_deserialize.cpp
new file mode 100644
index 0000000000..9e93a26bee
--- /dev/null
+++ b/library/cpp/type_info/ut/type_deserialize.cpp
@@ -0,0 +1,528 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include <library/cpp/yson_pull/yson.h>
+
+#include "utils.h"
+
+TEST(TypeDeserialize, Void) {
+ ASSERT_DESERIALIZED_EQ(NTi::Void(), R"(void)");
+ ASSERT_DESERIALIZED_EQ(NTi::Void(), R"({type_name=void})");
+}
+
+TEST(TypeDeserialize, Null) {
+ ASSERT_DESERIALIZED_EQ(NTi::Null(), R"(null)");
+ ASSERT_DESERIALIZED_EQ(NTi::Null(), R"({type_name=null})");
+}
+
+TEST(TypeDeserialize, Bool) {
+ ASSERT_DESERIALIZED_EQ(NTi::Bool(), R"(bool)");
+ ASSERT_DESERIALIZED_EQ(NTi::Bool(), R"({type_name=bool})");
+}
+
+TEST(TypeDeserialize, Int8) {
+ ASSERT_DESERIALIZED_EQ(NTi::Int8(), R"(int8)");
+ ASSERT_DESERIALIZED_EQ(NTi::Int8(), R"({type_name=int8})");
+}
+
+TEST(TypeDeserialize, Int16) {
+ ASSERT_DESERIALIZED_EQ(NTi::Int16(), R"(int16)");
+ ASSERT_DESERIALIZED_EQ(NTi::Int16(), R"({type_name=int16})");
+}
+
+TEST(TypeDeserialize, Int32) {
+ ASSERT_DESERIALIZED_EQ(NTi::Int32(), R"(int32)");
+ ASSERT_DESERIALIZED_EQ(NTi::Int32(), R"({type_name=int32})");
+}
+
+TEST(TypeDeserialize, Int64) {
+ ASSERT_DESERIALIZED_EQ(NTi::Int64(), R"(int64)");
+ ASSERT_DESERIALIZED_EQ(NTi::Int64(), R"({type_name=int64})");
+}
+
+TEST(TypeDeserialize, Uint8) {
+ ASSERT_DESERIALIZED_EQ(NTi::Uint8(), R"(uint8)");
+ ASSERT_DESERIALIZED_EQ(NTi::Uint8(), R"({type_name=uint8})");
+}
+
+TEST(TypeDeserialize, Uint16) {
+ ASSERT_DESERIALIZED_EQ(NTi::Uint16(), R"(uint16)");
+ ASSERT_DESERIALIZED_EQ(NTi::Uint16(), R"({type_name=uint16})");
+}
+
+TEST(TypeDeserialize, Uint32) {
+ ASSERT_DESERIALIZED_EQ(NTi::Uint32(), R"(uint32)");
+ ASSERT_DESERIALIZED_EQ(NTi::Uint32(), R"({type_name=uint32})");
+}
+
+TEST(TypeDeserialize, Uint64) {
+ ASSERT_DESERIALIZED_EQ(NTi::Uint64(), R"(uint64)");
+ ASSERT_DESERIALIZED_EQ(NTi::Uint64(), R"({type_name=uint64})");
+}
+
+TEST(TypeDeserialize, Float) {
+ ASSERT_DESERIALIZED_EQ(NTi::Float(), R"(float)");
+ ASSERT_DESERIALIZED_EQ(NTi::Float(), R"({type_name=float})");
+}
+
+TEST(TypeDeserialize, Double) {
+ ASSERT_DESERIALIZED_EQ(NTi::Double(), R"(double)");
+ ASSERT_DESERIALIZED_EQ(NTi::Double(), R"({type_name=double})");
+}
+
+TEST(TypeDeserialize, String) {
+ ASSERT_DESERIALIZED_EQ(NTi::String(), R"(string)");
+ ASSERT_DESERIALIZED_EQ(NTi::String(), R"({type_name=string})");
+}
+
+TEST(TypeDeserialize, Utf8) {
+ ASSERT_DESERIALIZED_EQ(NTi::Utf8(), R"(utf8)");
+ ASSERT_DESERIALIZED_EQ(NTi::Utf8(), R"({type_name=utf8})");
+}
+
+TEST(TypeDeserialize, Date) {
+ ASSERT_DESERIALIZED_EQ(NTi::Date(), R"(date)");
+ ASSERT_DESERIALIZED_EQ(NTi::Date(), R"({type_name=date})");
+}
+
+TEST(TypeDeserialize, Datetime) {
+ ASSERT_DESERIALIZED_EQ(NTi::Datetime(), R"(datetime)");
+ ASSERT_DESERIALIZED_EQ(NTi::Datetime(), R"({type_name=datetime})");
+}
+
+TEST(TypeDeserialize, Timestamp) {
+ ASSERT_DESERIALIZED_EQ(NTi::Timestamp(), R"(timestamp)");
+ ASSERT_DESERIALIZED_EQ(NTi::Timestamp(), R"({type_name=timestamp})");
+}
+
+TEST(TypeDeserialize, TzDate) {
+ ASSERT_DESERIALIZED_EQ(NTi::TzDate(), R"(tz_date)");
+ ASSERT_DESERIALIZED_EQ(NTi::TzDate(), R"({type_name=tz_date})");
+}
+
+TEST(TypeDeserialize, TzDatetime) {
+ ASSERT_DESERIALIZED_EQ(NTi::TzDatetime(), R"(tz_datetime)");
+ ASSERT_DESERIALIZED_EQ(NTi::TzDatetime(), R"({type_name=tz_datetime})");
+}
+
+TEST(TypeDeserialize, TzTimestamp) {
+ ASSERT_DESERIALIZED_EQ(NTi::TzTimestamp(), R"(tz_timestamp)");
+ ASSERT_DESERIALIZED_EQ(NTi::TzTimestamp(), R"({type_name=tz_timestamp})");
+}
+
+TEST(TypeDeserialize, Interval) {
+ ASSERT_DESERIALIZED_EQ(NTi::Interval(), R"(interval)");
+ ASSERT_DESERIALIZED_EQ(NTi::Interval(), R"({type_name=interval})");
+}
+
+TEST(TypeDeserialize, Decimal) {
+ ASSERT_DESERIALIZED_EQ(NTi::Decimal(20, 10), R"({type_name=decimal; precision=20; scale=10})");
+ ASSERT_DESERIALIZED_EQ(NTi::Decimal(20, 10), R"({scale=10; type_name=decimal; precision=20})");
+ ASSERT_DESERIALIZED_EQ(NTi::Decimal(10, 10), R"({type_name=decimal; precision=10; scale=10})");
+}
+
+TEST(TypeDeserialize, DecimalMissingTypeParameters) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({precision=20; scale=10})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=decimal; scale=10})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "precision")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=decimal; precision=20})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "scale")");
+}
+
+TEST(TypeDeserialize, Json) {
+ ASSERT_DESERIALIZED_EQ(NTi::Json(), R"(json)");
+ ASSERT_DESERIALIZED_EQ(NTi::Json(), R"({type_name=json})");
+}
+
+TEST(TypeDeserialize, Yson) {
+ ASSERT_DESERIALIZED_EQ(NTi::Yson(), R"(yson)");
+ ASSERT_DESERIALIZED_EQ(NTi::Yson(), R"({type_name=yson})");
+}
+
+TEST(TypeDeserialize, Uuid) {
+ ASSERT_DESERIALIZED_EQ(NTi::Uuid(), R"(uuid)");
+ ASSERT_DESERIALIZED_EQ(NTi::Uuid(), R"({type_name=uuid})");
+}
+
+TEST(TypeDeserialize, Optional) {
+ ASSERT_DESERIALIZED_EQ(NTi::Optional(NTi::Void()), R"({type_name=optional; item=void})");
+ ASSERT_DESERIALIZED_EQ(NTi::Optional(NTi::String()), R"({type_name=optional; item=string})");
+ ASSERT_DESERIALIZED_EQ(NTi::Optional(NTi::String()), R"({item=string; type_name=optional})");
+}
+
+TEST(TypeDeserialize, OptionalMissingTypeParameters) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({item=string})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=optional})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "item")");
+}
+
+TEST(TypeDeserialize, List) {
+ ASSERT_DESERIALIZED_EQ(NTi::List(NTi::Void()), R"({type_name=list; item=void})");
+ ASSERT_DESERIALIZED_EQ(NTi::List(NTi::String()), R"({type_name=list; item=string})");
+ ASSERT_DESERIALIZED_EQ(NTi::List(NTi::String()), R"({item=string; type_name=list})");
+}
+
+TEST(TypeDeserialize, ListMissingTypeParameters) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({item=string})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=list})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "item")");
+}
+
+TEST(TypeDeserialize, Dict) {
+ ASSERT_DESERIALIZED_EQ(NTi::Dict(NTi::Void(), NTi::Void()), R"({type_name=dict; key=void; value=void})");
+ ASSERT_DESERIALIZED_EQ(NTi::Dict(NTi::Int32(), NTi::String()), R"({type_name=dict; key=int32; value=string})");
+ ASSERT_DESERIALIZED_EQ(NTi::Dict(NTi::Int32(), NTi::String()), R"({key=int32; value=string; type_name=dict})");
+ ASSERT_DESERIALIZED_EQ(NTi::Dict(NTi::Int32(), NTi::String()), R"({value=string; key=int32; type_name=dict})");
+}
+
+TEST(TypeDeserialize, DictMissingTypeParameters) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({key=string; value=string})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=dict})");
+ }(),
+ NTi::TDeserializationException, R"(missing required keys "key" and "value")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=dict; value=string})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "key")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=dict; key=string})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "value")");
+}
+
+TEST(TypeDeserialize, StructEmpty) {
+ ASSERT_DESERIALIZED_EQ(
+ NTi::Struct({}),
+ R"({type_name=struct; members=[]})");
+ ASSERT_DESERIALIZED_EQ(
+ NTi::Struct({}),
+ R"({members=[]; type_name=struct})");
+}
+
+TEST(TypeDeserialize, Struct) {
+ auto ty = NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}});
+
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=struct; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]; type_name=struct})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=struct; members=[{type=string; name=ItemB}; {name=ItemA; type={item=int64; type_name=list}}]})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=struct; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]; name=#})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=struct; name=#; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})");
+}
+
+TEST(TypeDeserialize, StructMissingTypeParameters) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({members=[]})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=struct})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "members")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=struct; name=S})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "members")");
+}
+
+TEST(TypeDeserialize, TupleEmpty) {
+ ASSERT_DESERIALIZED_EQ(
+ NTi::Tuple({}),
+ R"({type_name=tuple; elements=[]})");
+
+ ASSERT_DESERIALIZED_EQ(
+ NTi::Tuple({}),
+ R"({elements=[]; type_name=tuple})");
+}
+
+TEST(TypeDeserialize, Tuple) {
+ auto ty = NTi::Tuple({{NTi::String()}, {NTi::List(NTi::Int64())}});
+
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=tuple; elements=[{type=string}; {type={type_name=list; item=int64}}]})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({elements=[{type=string}; {type={item=int64; type_name=list}}]; type_name=tuple})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=tuple; name=#; elements=[{type=string}; {type={type_name=list; item=int64}}]})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=tuple; elements=[{type=string}; {type={type_name=list; item=int64}}]; name=#})");
+}
+
+TEST(TypeDeserialize, TupleMissingTypeParameters) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({elements=[]})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=tuple})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "elements")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=tuple; name=T})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "elements")");
+}
+
+TEST(TypeDeserialize, VariantStruct) {
+ auto ty = NTi::Variant(NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}}));
+
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=variant; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]; type_name=variant})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=variant; name=#; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=variant; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]; name=#})");
+}
+
+TEST(TypeDeserialize, VariantTuple) {
+ auto ty = NTi::Variant(NTi::Tuple({{NTi::String()}, {NTi::List(NTi::Int64())}}));
+
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=variant; elements=[{type=string}; {type={type_name=list; item=int64}}]})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({elements=[{type=string}; {type={type_name=list; item=int64}}]; type_name=variant})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=variant; name=#; elements=[{type=string}; {type={type_name=list; item=int64}}]})");
+ ASSERT_DESERIALIZED_EQ(
+ ty,
+ R"({type_name=variant; elements=[{type=string}; {type={type_name=list; item=int64}}]; name=#})");
+}
+
+TEST(TypeDeserialize, VariantMissingTypeParameters) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({elements=[]})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({name=X})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=variant})");
+ }(),
+ NTi::TDeserializationException, R"(missing both keys "members" and "elements")");
+}
+
+TEST(TypeDeserialize, Tagged) {
+ ASSERT_DESERIALIZED_EQ(
+ NTi::Tagged(NTi::String(), "Url"),
+ R"({type_name=tagged; tag=Url; item=string})");
+ ASSERT_DESERIALIZED_EQ(
+ NTi::Tagged(NTi::String(), "Url"),
+ R"({type_name=tagged; item=string; tag=Url})");
+ ASSERT_DESERIALIZED_EQ(
+ NTi::Tagged(NTi::String(), "Url"),
+ R"({item=string; tag=Url; type_name=tagged})");
+}
+
+TEST(TypeDeserialize, TaggedMissingTypeParameters) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({tag=T; item=string})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({tag=T})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=tagged; tag=T})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "item")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({type_name=tagged; item=string})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "tag")");
+}
+
+TEST(TypeDeserialize, ComplexTypeAsString) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(decimal)");
+ }(),
+ NTi::TDeserializationException, R"(missing required keys "precision" and "scale")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(optional)");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "item")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(list)");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "item")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(dict)");
+ }(),
+ NTi::TDeserializationException, R"(missing required keys "key" and "value")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(struct)");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "members")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(tuple)");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "elements")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(variant)");
+ }(),
+ NTi::TDeserializationException, R"(missing both keys "members" and "elements")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"(tagged)");
+ }(),
+ NTi::TDeserializationException, R"(missing required keys "tag" and "item")");
+}
+
+TEST(TypeDeserialize, MissingTypeName) {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({item=string})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({tag=Url})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ []() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), R"({key=string; value=int32})");
+ }(),
+ NTi::TDeserializationException, R"(missing required key "type_name")");
+}
+
+TEST(TypeDeserialize, UnknownKeys) {
+ auto tupleType = NTi::Tuple({{NTi::String()}, {NTi::List(NTi::Int64())}});
+
+ ASSERT_DESERIALIZED_EQ(
+ tupleType,
+ R"({type_name=tuple; unknown_key=<>0; elements=[{type=string}; {type={type_name=list; item=int64}}]})");
+
+ ASSERT_DESERIALIZED_EQ(
+ tupleType,
+ R"({type_name=tuple; elements=[{unknown_key={foo=<>0}; type=string}; {type={type_name=list; item=int64}}]})");
+
+ auto structType = NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}});
+ ASSERT_DESERIALIZED_EQ(
+ structType,
+ R"({type_name=struct; unknown_key=[]; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})");
+ ASSERT_DESERIALIZED_EQ(
+ structType,
+ R"({type_name=struct; members=[{name=ItemB; unknown_key=foo; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})");
+
+ auto utf8Type = NTi::Utf8();
+ ASSERT_DESERIALIZED_EQ(
+ utf8Type,
+ R"({type_name=utf8; unknown_key=[];})");
+}
+
+
+TEST(TypeDeserialize, DeepType) {
+ auto ty = TStringBuilder();
+ for (size_t i = 0; i < 100; ++i)
+ ty << "{type_name=optional; item=";
+ ty << "string";
+ for (size_t i = 0; i < 100; ++i)
+ ty << "}";
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&ty]() {
+ NTi::NIo::DeserializeYson(*NTi::HeapFactory(), ty);
+ }(),
+ NTi::TDeserializationException, R"(too deep)");
+}
+
+TEST(TypeDeserialize, MultipleTypes) {
+ auto ty = "{type_name=optional; item=string}; {type_name=list; item=utf8}";
+
+ auto reader = NYsonPull::TReader(NYsonPull::NInput::FromMemory(ty), NYsonPull::EStreamType::ListFragment);
+ auto factory = NTi::PoolFactory();
+
+ {
+ auto ty1 = NTi::Optional(NTi::String());
+ ASSERT_STRICT_EQ(NTi::NIo::DeserializeYsonMultipleRaw(*factory, reader), ty1.Get());
+ }
+
+ {
+ auto ty2 = NTi::List(NTi::Utf8());
+ ASSERT_STRICT_EQ(NTi::NIo::DeserializeYsonMultipleRaw(*factory, reader), ty2.Get());
+ }
+
+ ASSERT_EQ(NTi::NIo::DeserializeYsonMultipleRaw(*factory, reader), nullptr);
+}
diff --git a/library/cpp/type_info/ut/type_equivalence.cpp b/library/cpp/type_info/ut/type_equivalence.cpp
new file mode 100644
index 0000000000..ca697620ad
--- /dev/null
+++ b/library/cpp/type_info/ut/type_equivalence.cpp
@@ -0,0 +1,394 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include "utils.h"
+
+TEST_TF(TypeEquivalence, StrictEqSelf) {
+ ASSERT_STRICT_EQ(
+ f.Void(),
+ f.Void());
+ ASSERT_STRICT_EQ(
+ f.Bool(),
+ f.Bool());
+ ASSERT_STRICT_EQ(
+ f.Int8(),
+ f.Int8());
+ ASSERT_STRICT_EQ(
+ f.Int16(),
+ f.Int16());
+ ASSERT_STRICT_EQ(
+ f.Int32(),
+ f.Int32());
+ ASSERT_STRICT_EQ(
+ f.Int64(),
+ f.Int64());
+ ASSERT_STRICT_EQ(
+ f.Uint8(),
+ f.Uint8());
+ ASSERT_STRICT_EQ(
+ f.Uint16(),
+ f.Uint16());
+ ASSERT_STRICT_EQ(
+ f.Uint32(),
+ f.Uint32());
+ ASSERT_STRICT_EQ(
+ f.Uint64(),
+ f.Uint64());
+ ASSERT_STRICT_EQ(
+ f.Float(),
+ f.Float());
+ ASSERT_STRICT_EQ(
+ f.Double(),
+ f.Double());
+ ASSERT_STRICT_EQ(
+ f.String(),
+ f.String());
+ ASSERT_STRICT_EQ(
+ f.Utf8(),
+ f.Utf8());
+ ASSERT_STRICT_EQ(
+ f.Date(),
+ f.Date());
+ ASSERT_STRICT_EQ(
+ f.Datetime(),
+ f.Datetime());
+ ASSERT_STRICT_EQ(
+ f.Timestamp(),
+ f.Timestamp());
+ ASSERT_STRICT_EQ(
+ f.TzDate(),
+ f.TzDate());
+ ASSERT_STRICT_EQ(
+ f.TzDatetime(),
+ f.TzDatetime());
+ ASSERT_STRICT_EQ(
+ f.TzTimestamp(),
+ f.TzTimestamp());
+ ASSERT_STRICT_EQ(
+ f.Interval(),
+ f.Interval());
+ ASSERT_STRICT_EQ(
+ f.Decimal(20, 10),
+ f.Decimal(20, 10));
+ ASSERT_STRICT_EQ(
+ f.Json(),
+ f.Json());
+ ASSERT_STRICT_EQ(
+ f.Yson(),
+ f.Yson());
+ ASSERT_STRICT_EQ(
+ f.Uuid(),
+ f.Uuid());
+ ASSERT_STRICT_EQ(
+ f.Optional(f.Void()),
+ f.Optional(f.Void()));
+ ASSERT_STRICT_EQ(
+ f.List(f.Void()),
+ f.List(f.Void()));
+ ASSERT_STRICT_EQ(
+ f.Dict(f.Void(), f.String()),
+ f.Dict(f.Void(), f.String()));
+ ASSERT_STRICT_EQ(
+ f.Struct({}),
+ f.Struct({}));
+ ASSERT_STRICT_EQ(
+ f.Struct("S", {}),
+ f.Struct("S", {}));
+ ASSERT_STRICT_EQ(
+ f.Struct({{"a", f.Int64()}, {"b", f.Int8()}}),
+ f.Struct({{"a", f.Int64()}, {"b", f.Int8()}}));
+ ASSERT_STRICT_EQ(
+ f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}}),
+ f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}}));
+ ASSERT_STRICT_EQ(
+ f.Tuple({}),
+ f.Tuple({}));
+ ASSERT_STRICT_EQ(
+ f.Tuple("T", {}),
+ f.Tuple("T", {}));
+ ASSERT_STRICT_EQ(
+ f.Tuple({{f.Int64()}, {f.Int8()}}),
+ f.Tuple({{f.Int64()}, {f.Int8()}}));
+ ASSERT_STRICT_EQ(
+ f.Tuple("T", {{f.Int64()}, {f.Int8()}}),
+ f.Tuple("T", {{f.Int64()}, {f.Int8()}}));
+ ASSERT_STRICT_EQ(
+ f.Variant(f.Struct("Inner", {{"x", f.Void()}})),
+ f.Variant(f.Struct("Inner", {{"x", f.Void()}})));
+ ASSERT_STRICT_EQ(
+ f.Variant("V", f.Struct("Inner", {{"x", f.Void()}})),
+ f.Variant("V", f.Struct("Inner", {{"x", f.Void()}})));
+ ASSERT_STRICT_EQ(
+ f.Variant(f.Tuple("Inner", {{f.Void()}})),
+ f.Variant(f.Tuple("Inner", {{f.Void()}})));
+ ASSERT_STRICT_EQ(
+ f.Variant("V", f.Tuple("Inner", {{f.Void()}})),
+ f.Variant("V", f.Tuple("Inner", {{f.Void()}})));
+ ASSERT_STRICT_EQ(
+ f.Tagged(f.Void(), "T"),
+ f.Tagged(f.Void(), "T"));
+}
+
+TEST_TF(TypeEquivalence, StrictNeOtherType) {
+ ASSERT_STRICT_NE(
+ f.Void(),
+ f.Bool());
+ ASSERT_STRICT_NE(
+ f.Bool(),
+ f.Int8());
+ ASSERT_STRICT_NE(
+ f.Int8(),
+ f.Int16());
+ ASSERT_STRICT_NE(
+ f.Int16(),
+ f.Int32());
+ ASSERT_STRICT_NE(
+ f.Int32(),
+ f.Int64());
+ ASSERT_STRICT_NE(
+ f.Int64(),
+ f.Uint8());
+ ASSERT_STRICT_NE(
+ f.Uint8(),
+ f.Uint16());
+ ASSERT_STRICT_NE(
+ f.Uint16(),
+ f.Uint32());
+ ASSERT_STRICT_NE(
+ f.Uint32(),
+ f.Uint64());
+ ASSERT_STRICT_NE(
+ f.Uint64(),
+ f.Float());
+ ASSERT_STRICT_NE(
+ f.Float(),
+ f.Double());
+ ASSERT_STRICT_NE(
+ f.Double(),
+ f.String());
+ ASSERT_STRICT_NE(
+ f.String(),
+ f.Utf8());
+ ASSERT_STRICT_NE(
+ f.Utf8(),
+ f.Date());
+ ASSERT_STRICT_NE(
+ f.Date(),
+ f.Datetime());
+ ASSERT_STRICT_NE(
+ f.Datetime(),
+ f.Timestamp());
+ ASSERT_STRICT_NE(
+ f.Timestamp(),
+ f.TzDate());
+ ASSERT_STRICT_NE(
+ f.TzDate(),
+ f.TzDatetime());
+ ASSERT_STRICT_NE(
+ f.TzDatetime(),
+ f.TzTimestamp());
+ ASSERT_STRICT_NE(
+ f.TzTimestamp(),
+ f.Interval());
+ ASSERT_STRICT_NE(
+ f.Interval(),
+ f.Decimal(20, 10));
+ ASSERT_STRICT_NE(
+ f.Decimal(20, 10),
+ f.Json());
+ ASSERT_STRICT_NE(
+ f.Json(),
+ f.Yson());
+ ASSERT_STRICT_NE(
+ f.Yson(),
+ f.Uuid());
+ ASSERT_STRICT_NE(
+ f.Uuid(),
+ f.Optional(f.Void()));
+ ASSERT_STRICT_NE(
+ f.Optional(f.Void()),
+ f.List(f.Void()));
+ ASSERT_STRICT_NE(
+ f.List(f.Void()),
+ f.Dict(f.Void(), f.String()));
+ ASSERT_STRICT_NE(
+ f.Dict(f.Void(), f.String()),
+ f.Struct({}));
+ ASSERT_STRICT_NE(
+ f.Struct({}),
+ f.Struct("S", {}));
+ ASSERT_STRICT_NE(
+ f.Struct("S", {}),
+ f.Struct({{"a", f.Int64()}, {"b", f.Int8()}}));
+ ASSERT_STRICT_NE(
+ f.Struct({{"a", f.Int64()}, {"b", f.Int8()}}),
+ f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}}));
+ ASSERT_STRICT_NE(
+ f.Struct("S", {{"a", f.Int64()}, {"b", f.Int8()}}),
+ f.Tuple({}));
+ ASSERT_STRICT_NE(
+ f.Tuple({}),
+ f.Tuple("T", {}));
+ ASSERT_STRICT_NE(
+ f.Tuple("T", {}),
+ f.Tuple({{f.Int64()}, {f.Int8()}}));
+ ASSERT_STRICT_NE(
+ f.Tuple({{f.Int64()}, {f.Int8()}}),
+ f.Tuple("T", {{f.Int64()}, {f.Int8()}}));
+ ASSERT_STRICT_NE(
+ f.Tuple("T", {{f.Int64()}, {f.Int8()}}),
+ f.Variant(f.Struct("Inner", {{"x", f.Void()}})));
+ ASSERT_STRICT_NE(
+ f.Variant(f.Struct("Inner", {{"x", f.Void()}})),
+ f.Variant("V", f.Struct("Inner", {{"x", f.Void()}})));
+ ASSERT_STRICT_NE(
+ f.Variant("V", f.Struct("Inner", {{"x", f.Void()}})),
+ f.Variant(f.Tuple("Inner", {{f.Void()}})));
+ ASSERT_STRICT_NE(
+ f.Variant(f.Tuple("Inner", {{f.Void()}})),
+ f.Variant("V", f.Tuple("Inner", {{f.Void()}})));
+ ASSERT_STRICT_NE(
+ f.Variant("V", f.Tuple("Inner", {{f.Void()}})),
+ f.Tagged(f.Void(), "T"));
+ ASSERT_STRICT_NE(
+ f.Tagged(f.Void(), "T"),
+ f.Void());
+}
+
+TEST_TF(TypeEquivalence, StrictNeDecimal) {
+ ASSERT_STRICT_NE(
+ f.Decimal(20, 10),
+ f.Decimal(21, 10));
+ ASSERT_STRICT_NE(
+ f.Decimal(20, 10),
+ f.Decimal(20, 11));
+}
+
+TEST_TF(TypeEquivalence, StrictNeStruct) {
+ ASSERT_STRICT_NE(
+ f.Struct({}),
+ f.Struct("", {}));
+ ASSERT_STRICT_NE(
+ f.Struct("name", {}),
+ f.Struct("other name", {}));
+ ASSERT_STRICT_NE(
+ f.Struct({}),
+ f.Struct({{"x", f.Void()}}));
+ ASSERT_STRICT_NE(
+ f.Struct({{"x", f.Void()}}),
+ f.Struct({{"y", f.Void()}}));
+ ASSERT_STRICT_NE(
+ f.Struct({{"x", f.Void()}}),
+ f.Struct({{"x", f.String()}}));
+ ASSERT_STRICT_NE(
+ f.Struct("name", {}),
+ f.Struct("name", {{"x", f.Void()}}));
+ ASSERT_STRICT_NE(
+ f.Struct("name", {{"x", f.Void()}}),
+ f.Struct("name", {{"y", f.Void()}}));
+ ASSERT_STRICT_NE(
+ f.Struct("name", {{"x", f.Void()}}),
+ f.Struct("name", {{"x", f.String()}}));
+ ASSERT_STRICT_NE(
+ f.Struct({{"x", f.Void()}, {"y", f.String()}}),
+ f.Struct({{"x", f.String()}, {"y", f.Void()}}));
+ ASSERT_STRICT_NE(
+ f.Struct({{"x", f.Void()}, {"y", f.Void()}}),
+ f.Struct({{"y", f.Void()}, {"x", f.Void()}}));
+ ASSERT_STRICT_NE(
+ f.Struct({{"x", f.Void()}, {"y", f.Void()}}),
+ f.Struct({{"x", f.Void()}, {"y", f.Void()}, {"z", f.Void()}}));
+}
+
+TEST_TF(TypeEquivalence, StrictNeTuple) {
+ ASSERT_STRICT_NE(
+ f.Tuple({}),
+ f.Tuple("", {}));
+ ASSERT_STRICT_NE(
+ f.Tuple("name", {}),
+ f.Tuple("other name", {}));
+ ASSERT_STRICT_NE(
+ f.Tuple({}),
+ f.Tuple({{f.Void()}}));
+ ASSERT_STRICT_NE(
+ f.Tuple({{f.Void()}}),
+ f.Tuple({{f.String()}}));
+ ASSERT_STRICT_NE(
+ f.Tuple("name", {}),
+ f.Tuple("name", {{f.Void()}}));
+ ASSERT_STRICT_NE(
+ f.Tuple("name", {{f.Void()}}),
+ f.Tuple("name", {{f.String()}}));
+ ASSERT_STRICT_NE(
+ f.Tuple({{f.String()}, {f.Void()}}),
+ f.Tuple({{f.Void()}, {f.String()}}));
+ ASSERT_STRICT_NE(
+ f.Tuple({{f.Void()}, {f.Void()}}),
+ f.Tuple({{f.Void()}, {f.Void()}, {f.Void()}}));
+}
+
+TEST_TF(TypeEquivalence, StrictNeVariant) {
+ ASSERT_STRICT_NE(
+ f.Variant(f.Tuple({{f.Void()}})),
+ f.Variant("", f.Tuple({{f.Void()}})));
+ ASSERT_STRICT_NE(
+ f.Variant("", f.Tuple({{f.Void()}})),
+ f.Variant("X", f.Tuple({{f.Void()}})));
+ ASSERT_STRICT_NE(
+ f.Variant(f.Tuple({{f.Void()}})),
+ f.Variant(f.Tuple({{f.String()}})));
+ ASSERT_STRICT_NE(
+ f.Variant("X", f.Tuple({{f.Utf8()}})),
+ f.Variant("X", f.Tuple({{f.String()}})));
+ ASSERT_STRICT_NE(
+ f.Variant(f.Tuple({{f.Utf8()}})),
+ f.Variant(f.Struct({{"_", f.Utf8()}})));
+ ASSERT_STRICT_NE(
+ f.Variant(f.Struct({{"item1", f.String()}})),
+ f.Variant(f.Struct({{"item2", f.String()}})));
+ ASSERT_STRICT_NE(
+ f.Variant("X", f.Struct({{"item2", f.String()}})),
+ f.Variant("X", f.Struct({{"item1", f.String()}})));
+}
+
+TEST_TF(TypeEquivalence, StrictNeTagged) {
+ ASSERT_STRICT_NE(
+ f.Tagged(f.String(), "Tag"),
+ f.Tagged(f.String(), "Other tag"));
+ ASSERT_STRICT_NE(
+ f.Tagged(f.String(), "Tag"),
+ f.Tagged(f.Utf8(), "Tag"));
+}
+
+TEST_TF(TypeEquivalence, StrictNeDeep) {
+ auto t1 = f.Struct({
+ {"i", f.Optional(f.String())},
+ {"don't", f.Utf8()},
+ {"have", f.Tuple("GeoCoordinates", {{f.Float()}, {f.Float()}})},
+ {"enough", f.List(f.Optional(f.String()))},
+ {"fantasy", f.Dict(f.String(), f.List(f.Utf8()))}, // < difference
+ {"to", f.Yson()},
+ {"think", f.Optional(f.Json())},
+ {"of", f.Optional(f.String())},
+ {"a", f.List(f.Tuple({{f.String()}, {f.Tuple("GeoCoordinates", {{f.Float()}, {f.Float()}})}}))},
+ {"meaningful", f.Optional(f.String())},
+ {"example", f.Dict(f.Optional(f.Decimal(10, 5)), f.List(f.Dict(f.Int32(), f.Float())))},
+ });
+
+ auto t2 = f.Struct({
+ {"i", f.Optional(f.String())},
+ {"don't", f.Utf8()},
+ {"have", f.Tuple("GeoCoordinates", {{f.Float()}, {f.Float()}})},
+ {"enough", f.List(f.Optional(f.String()))},
+ {"fantasy", f.Dict(f.String(), f.List(f.String()))}, // < difference
+ {"to", f.Yson()},
+ {"think", f.Optional(f.Json())},
+ {"of", f.Optional(f.String())},
+ {"a", f.List(f.Tuple({{f.String()}, {f.Tuple("GeoCoordinates", {{f.Float()}, {f.Float()}})}}))},
+ {"meaningful", f.Optional(f.String())},
+ {"example", f.Dict(f.Optional(f.Decimal(10, 5)), f.List(f.Dict(f.Int32(), f.Float())))},
+ });
+
+ ASSERT_STRICT_NE(t1, t2);
+}
diff --git a/library/cpp/type_info/ut/type_factory.cpp b/library/cpp/type_info/ut/type_factory.cpp
new file mode 100644
index 0000000000..0ab44129ad
--- /dev/null
+++ b/library/cpp/type_info/ut/type_factory.cpp
@@ -0,0 +1,121 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+TEST(TypeFactory, AdoptPoolToPool) {
+ auto f1 = NTi::PoolFactory();
+ auto f2 = NTi::PoolFactory();
+
+ auto t = f1->Optional(f1->List(f1->String()));
+ auto ta = f2->Adopt(t);
+
+ ASSERT_NE(t.Get(), ta.Get());
+ ASSERT_NE(t->GetItemTypeRaw(), ta->GetItemTypeRaw());
+ ASSERT_EQ(t->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw(), ta->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw());
+
+ f1 = nullptr;
+ f2 = nullptr;
+ t = nullptr;
+
+ // `ta` is still alive
+
+ ASSERT_TRUE(ta->IsOptional());
+ ASSERT_TRUE(ta->GetItemTypeRaw()->IsList());
+}
+
+TEST(TypeFactory, AdoptPoolToSamePool) {
+ auto f = NTi::PoolFactory();
+
+ auto t = f->Optional(f->List(f->String()));
+ auto ta = f->Adopt(t);
+
+ ASSERT_EQ(t.Get(), ta.Get());
+}
+
+TEST(TypeFactory, AdoptHeapToHeap) {
+ auto f = NTi::HeapFactory();
+
+ auto t = f->Optional(f->List(f->String()));
+ auto ta = f->Adopt(t);
+
+ ASSERT_EQ(t.Get(), ta.Get());
+}
+
+TEST(TypeFactory, AdoptHeapToPool) {
+ auto f1 = NTi::HeapFactory();
+ auto f2 = NTi::PoolFactory();
+
+ auto t = f1->Optional(f1->List(f1->String()));
+ auto ta = f2->Adopt(t);
+
+ ASSERT_NE(t.Get(), ta.Get());
+ ASSERT_NE(t->GetItemTypeRaw(), ta->GetItemTypeRaw());
+ ASSERT_EQ(t->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw(), ta->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw());
+
+ f1 = nullptr;
+ f2 = nullptr;
+ t = nullptr;
+
+ // `ta` is still alive
+
+ ASSERT_TRUE(ta->IsOptional());
+ ASSERT_TRUE(ta->GetItemTypeRaw()->IsList());
+}
+
+TEST(TypeFactory, AdoptPoolToHeap) {
+ auto f1 = NTi::PoolFactory();
+ auto f2 = NTi::HeapFactory();
+
+ auto t = f1->Optional(f1->List(f1->String()));
+ auto ta = f2->Adopt(t);
+
+ ASSERT_NE(t.Get(), ta.Get());
+ ASSERT_NE(t->GetItemTypeRaw(), ta->GetItemTypeRaw());
+ ASSERT_EQ(t->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw(), ta->GetItemTypeRaw()->AsListRaw()->GetItemTypeRaw());
+
+ f1 = nullptr;
+ f2 = nullptr;
+ t = nullptr;
+
+ // `ta` is still alive
+
+ ASSERT_TRUE(ta->IsOptional());
+ ASSERT_TRUE(ta->GetItemTypeRaw()->IsList());
+}
+
+TEST(TypeFactory, AdoptStaticToPool) {
+ auto f = NTi::PoolFactory();
+
+ auto t = NTi::Void();
+ auto ta = f->Adopt(t);
+
+ ASSERT_EQ(t.Get(), ta.Get());
+}
+
+TEST(TypeFactory, AdoptStaticToHeap) {
+ auto f = NTi::HeapFactory();
+
+ auto t = NTi::Void();
+ auto ta = f->Adopt(t);
+ ASSERT_EQ(t.Get(), ta.Get());
+}
+
+TEST(TypeFactory, Dedup) {
+ {
+ auto f = NTi::PoolFactory(/* deduplicate = */ false);
+
+ auto a = f->OptionalRaw(f->StringRaw());
+ auto b = f->OptionalRaw(f->StringRaw());
+
+ ASSERT_NE(a, b);
+ }
+
+ {
+ auto f = NTi::PoolFactory(/* deduplicate = */ true);
+
+ auto a = f->OptionalRaw(f->StringRaw());
+ auto b = f->OptionalRaw(f->StringRaw());
+
+ ASSERT_EQ(a, b);
+ }
+}
diff --git a/library/cpp/type_info/ut/type_factory_raw.cpp b/library/cpp/type_info/ut/type_factory_raw.cpp
new file mode 100644
index 0000000000..37f5d71aa7
--- /dev/null
+++ b/library/cpp/type_info/ut/type_factory_raw.cpp
@@ -0,0 +1,365 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+TEST(TypeFactoryRaw, Void) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->VoidRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Void);
+ ASSERT_TRUE(t->IsVoid());
+}
+
+TEST(TypeFactoryRaw, Bool) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->BoolRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Bool);
+ ASSERT_TRUE(t->IsBool());
+}
+
+TEST(TypeFactoryRaw, Int8) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->Int8Raw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int8);
+ ASSERT_TRUE(t->IsInt8());
+}
+
+TEST(TypeFactoryRaw, Int16) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->Int16Raw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int16);
+ ASSERT_TRUE(t->IsInt16());
+}
+
+TEST(TypeFactoryRaw, Int32) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->Int32Raw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int32);
+ ASSERT_TRUE(t->IsInt32());
+}
+
+TEST(TypeFactoryRaw, Int64) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->Int64Raw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Int64);
+ ASSERT_TRUE(t->IsInt64());
+}
+
+TEST(TypeFactoryRaw, Uint8) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->Uint8Raw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint8);
+ ASSERT_TRUE(t->IsUint8());
+}
+
+TEST(TypeFactoryRaw, Uint16) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->Uint16Raw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint16);
+ ASSERT_TRUE(t->IsUint16());
+}
+
+TEST(TypeFactoryRaw, Uint32) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->Uint32Raw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint32);
+ ASSERT_TRUE(t->IsUint32());
+}
+
+TEST(TypeFactoryRaw, Uint64) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->Uint64Raw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uint64);
+ ASSERT_TRUE(t->IsUint64());
+}
+
+TEST(TypeFactoryRaw, Float) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->FloatRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Float);
+ ASSERT_TRUE(t->IsFloat());
+}
+
+TEST(TypeFactoryRaw, Double) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->DoubleRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Double);
+ ASSERT_TRUE(t->IsDouble());
+}
+
+TEST(TypeFactoryRaw, String) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->StringRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::String);
+ ASSERT_TRUE(t->IsString());
+}
+
+TEST(TypeFactoryRaw, Utf8) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->Utf8Raw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Utf8);
+ ASSERT_TRUE(t->IsUtf8());
+}
+
+TEST(TypeFactoryRaw, Date) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->DateRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Date);
+ ASSERT_TRUE(t->IsDate());
+}
+
+TEST(TypeFactoryRaw, Datetime) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->DatetimeRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Datetime);
+ ASSERT_TRUE(t->IsDatetime());
+}
+
+TEST(TypeFactoryRaw, Timestamp) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->TimestampRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Timestamp);
+ ASSERT_TRUE(t->IsTimestamp());
+}
+
+TEST(TypeFactoryRaw, TzDate) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->TzDateRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzDate);
+ ASSERT_TRUE(t->IsTzDate());
+}
+
+TEST(TypeFactoryRaw, TzDatetime) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->TzDatetimeRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzDatetime);
+ ASSERT_TRUE(t->IsTzDatetime());
+}
+
+TEST(TypeFactoryRaw, TzTimestamp) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->TzTimestampRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::TzTimestamp);
+ ASSERT_TRUE(t->IsTzTimestamp());
+}
+
+TEST(TypeFactoryRaw, Interval) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->IntervalRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Interval);
+ ASSERT_TRUE(t->IsInterval());
+}
+
+TEST(TypeFactoryRaw, Decimal) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->DecimalRaw(20, 10);
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Decimal);
+ ASSERT_TRUE(t->IsDecimal());
+ ASSERT_EQ(t->GetPrecision(), 20);
+ ASSERT_EQ(t->GetScale(), 10);
+}
+
+TEST(TypeFactoryRaw, Json) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->JsonRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Json);
+ ASSERT_TRUE(t->IsJson());
+}
+
+TEST(TypeFactoryRaw, Yson) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->YsonRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Yson);
+ ASSERT_TRUE(t->IsYson());
+}
+
+TEST(TypeFactoryRaw, Uuid) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->UuidRaw();
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Uuid);
+ ASSERT_TRUE(t->IsUuid());
+}
+
+TEST(TypeFactoryRaw, Optional) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->OptionalRaw(f->VoidRaw());
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Optional);
+ ASSERT_TRUE(t->IsOptional());
+ ASSERT_TRUE(t->GetItemType()->IsVoid());
+ ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw());
+}
+
+TEST(TypeFactoryRaw, List) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->ListRaw(f->VoidRaw());
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::List);
+ ASSERT_TRUE(t->IsList());
+ ASSERT_TRUE(t->GetItemType()->IsVoid());
+ ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw());
+}
+
+TEST(TypeFactoryRaw, Dict) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->DictRaw(f->VoidRaw(), f->StringRaw());
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Dict);
+ ASSERT_TRUE(t->IsDict());
+ ASSERT_TRUE(t->GetKeyType()->IsVoid());
+ ASSERT_EQ(t->GetKeyType().Get(), t->GetKeyTypeRaw());
+ ASSERT_TRUE(t->GetValueType()->IsString());
+ ASSERT_EQ(t->GetValueType().Get(), t->GetValueTypeRaw());
+}
+
+TEST(TypeFactoryRaw, EmptyStruct) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->StructRaw({});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct);
+ ASSERT_TRUE(t->IsStruct());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_EQ(t->GetMembers().size(), 0);
+}
+
+TEST(TypeFactoryRaw, NamedEmptyStruct) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->StructRaw("S", {});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct);
+ ASSERT_TRUE(t->IsStruct());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "S");
+ ASSERT_EQ(t->GetMembers().size(), 0);
+}
+
+TEST(TypeFactoryRaw, Struct) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->StructRaw({{"a", f->Int64Raw()}, {"b", f->Int8Raw()}});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct);
+ ASSERT_TRUE(t->IsStruct());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_EQ(t->GetMembers().size(), 2);
+ ASSERT_EQ(t->GetMembers()[0].GetName(), "a");
+ ASSERT_TRUE(t->GetMembers()[0].GetTypeRaw()->IsInt64());
+ ASSERT_EQ(t->GetMembers()[0].GetType().Get(), t->GetMembers()[0].GetTypeRaw());
+ ASSERT_EQ(t->GetMembers()[1].GetName(), "b");
+ ASSERT_TRUE(t->GetMembers()[1].GetTypeRaw()->IsInt8());
+ ASSERT_EQ(t->GetMembers()[1].GetType().Get(), t->GetMembers()[1].GetTypeRaw());
+}
+
+TEST(TypeFactoryRaw, NamedStruct) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->StructRaw("S", {{"a", f->Int64Raw()}, {"b", f->Int8Raw()}});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Struct);
+ ASSERT_TRUE(t->IsStruct());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "S");
+ ASSERT_EQ(t->GetMembers().size(), 2);
+ ASSERT_EQ(t->GetMembers()[0].GetName(), "a");
+ ASSERT_TRUE(t->GetMembers()[0].GetTypeRaw()->IsInt64());
+ ASSERT_EQ(t->GetMembers()[0].GetType().Get(), t->GetMembers()[0].GetTypeRaw());
+ ASSERT_EQ(t->GetMembers()[1].GetName(), "b");
+ ASSERT_TRUE(t->GetMembers()[1].GetTypeRaw()->IsInt8());
+ ASSERT_EQ(t->GetMembers()[1].GetType().Get(), t->GetMembers()[1].GetTypeRaw());
+}
+
+TEST(TypeFactoryRaw, EmptyTuple) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->TupleRaw({});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple);
+ ASSERT_TRUE(t->IsTuple());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_EQ(t->GetElements().size(), 0);
+}
+
+TEST(TypeFactoryRaw, NamedEmptyTuple) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->TupleRaw("T", {});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple);
+ ASSERT_TRUE(t->IsTuple());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "T");
+ ASSERT_EQ(t->GetElements().size(), 0);
+}
+
+TEST(TypeFactoryRaw, Tuple) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->TupleRaw({{f->Int64Raw()}, {f->Int8Raw()}});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple);
+ ASSERT_TRUE(t->IsTuple());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_EQ(t->GetElements().size(), 2);
+ ASSERT_TRUE(t->GetElements()[0].GetTypeRaw()->IsInt64());
+ ASSERT_EQ(t->GetElements()[0].GetType().Get(), t->GetElements()[0].GetTypeRaw());
+ ASSERT_TRUE(t->GetElements()[1].GetTypeRaw()->IsInt8());
+ ASSERT_EQ(t->GetElements()[1].GetType().Get(), t->GetElements()[1].GetTypeRaw());
+}
+
+TEST(TypeFactoryRaw, NamedTuple) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->TupleRaw("T", {{f->Int64Raw()}, {f->Int8Raw()}});
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tuple);
+ ASSERT_TRUE(t->IsTuple());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "T");
+ ASSERT_EQ(t->GetElements().size(), 2);
+ ASSERT_TRUE(t->GetElements()[0].GetTypeRaw()->IsInt64());
+ ASSERT_EQ(t->GetElements()[0].GetType().Get(), t->GetElements()[0].GetTypeRaw());
+ ASSERT_TRUE(t->GetElements()[1].GetTypeRaw()->IsInt8());
+ ASSERT_EQ(t->GetElements()[1].GetType().Get(), t->GetElements()[1].GetTypeRaw());
+}
+
+TEST(TypeFactoryRaw, VariantOverStruct) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->VariantRaw(f->StructRaw("Inner", {{"x", f->VoidRaw()}}));
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant);
+ ASSERT_TRUE(t->IsVariant());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_TRUE(t->IsVariantOverStruct());
+ ASSERT_FALSE(t->IsVariantOverTuple());
+ ASSERT_EQ(t->GetUnderlyingType()->AsStruct()->GetName().GetRef(), "Inner");
+ ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw());
+}
+
+TEST(TypeFactoryRaw, NamedVariantOverStruct) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->VariantRaw("V", f->StructRaw("Inner", {{"x", f->VoidRaw()}}));
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant);
+ ASSERT_TRUE(t->IsVariant());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "V");
+ ASSERT_TRUE(t->IsVariantOverStruct());
+ ASSERT_FALSE(t->IsVariantOverTuple());
+ ASSERT_EQ(t->GetUnderlyingType()->AsStruct()->GetName().GetRef(), "Inner");
+ ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw());
+}
+
+TEST(TypeFactoryRaw, VariantOverTuple) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->VariantRaw(f->TupleRaw("Inner", {{f->VoidRaw()}}));
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant);
+ ASSERT_TRUE(t->IsVariant());
+ ASSERT_FALSE(t->GetName().Defined());
+ ASSERT_FALSE(t->IsVariantOverStruct());
+ ASSERT_TRUE(t->IsVariantOverTuple());
+ ASSERT_EQ(t->GetUnderlyingType()->AsTuple()->GetName().GetRef(), "Inner");
+ ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw());
+}
+
+TEST(TypeFactoryRaw, NamedVariantOverTuple) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->VariantRaw("V", f->TupleRaw("Inner", {{f->VoidRaw()}}));
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Variant);
+ ASSERT_TRUE(t->IsVariant());
+ ASSERT_TRUE(t->GetName().Defined());
+ ASSERT_EQ(t->GetName().GetRef(), "V");
+ ASSERT_FALSE(t->IsVariantOverStruct());
+ ASSERT_TRUE(t->IsVariantOverTuple());
+ ASSERT_EQ(t->GetUnderlyingType()->AsTuple()->GetName().GetRef(), "Inner");
+ ASSERT_EQ(t->GetUnderlyingType().Get(), t->GetUnderlyingTypeRaw());
+}
+
+TEST(TypeFactoryRaw, Tagged) {
+ auto f = NTi::PoolFactory(false);
+ auto t = f->TaggedRaw(f->VoidRaw(), "T");
+ ASSERT_EQ(t->GetTypeName(), NTi::ETypeName::Tagged);
+ ASSERT_TRUE(t->IsTagged());
+ ASSERT_EQ(t->GetTag(), "T");
+ ASSERT_TRUE(t->GetItemType()->IsVoid());
+ ASSERT_EQ(t->GetItemType().Get(), t->GetItemTypeRaw());
+}
diff --git a/library/cpp/type_info/ut/type_io.cpp b/library/cpp/type_info/ut/type_io.cpp
new file mode 100644
index 0000000000..4feb9b7d83
--- /dev/null
+++ b/library/cpp/type_info/ut/type_io.cpp
@@ -0,0 +1,535 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include "utils.h"
+
+TEST_TF(TypeIO, AsYqlType) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Void().Get()),
+ "[VoidType]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Null().Get()),
+ "[NullType]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Bool().Get()),
+ "[DataType; Bool]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Int8().Get()),
+ "[DataType; Int8]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Int16().Get()),
+ "[DataType; Int16]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Int32().Get()),
+ "[DataType; Int32]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Int64().Get()),
+ "[DataType; Int64]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Uint8().Get()),
+ "[DataType; Uint8]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Uint16().Get()),
+ "[DataType; Uint16]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Uint32().Get()),
+ "[DataType; Uint32]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Uint64().Get()),
+ "[DataType; Uint64]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Float().Get()),
+ "[DataType; Float]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Double().Get()),
+ "[DataType; Double]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.String().Get()),
+ "[DataType; String]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Utf8().Get()),
+ "[DataType; Utf8]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Date().Get()),
+ "[DataType; Date]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Datetime().Get()),
+ "[DataType; Datetime]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Timestamp().Get()),
+ "[DataType; Timestamp]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.TzDate().Get()),
+ "[DataType; TzDate]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.TzDatetime().Get()),
+ "[DataType; TzDatetime]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.TzTimestamp().Get()),
+ "[DataType; TzTimestamp]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Interval().Get()),
+ "[DataType; Interval]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Json().Get()),
+ "[DataType; Json]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Yson().Get()),
+ "[DataType; Yson]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Uuid().Get()),
+ "[DataType; Uuid]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Decimal(20, 10).Get()),
+ "[DataType; Decimal; \"20\"; \"10\"]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Decimal(35, 35).Get()),
+ "[DataType; Decimal; \"35\"; \"35\"]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(Optional(f.Bool()).Get()),
+ "[OptionalType; [DataType; Bool]]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.List(f.Bool()).Get()),
+ "[ListType; [DataType; Bool]]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Dict(f.Bool(), f.Int32()).Get()),
+ "[DictType; [DataType; Bool]; [DataType; Int32]]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Struct({}).Get()),
+ "[StructType; []]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Struct({{"a", f.Bool()}}).Get()),
+ "[StructType; [[a; [DataType; Bool]]]]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Struct({{"a", f.Yson()}, {"b", f.Bool()}}).Get()),
+ "[StructType; [[a; [DataType; Yson]]; [b; [DataType; Bool]]]]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Struct({{"a", f.Int32()}, {"b", f.Int32()}, {"c", f.Int64()}}).Get()),
+ "[StructType; [[a; [DataType; Int32]]; [b; [DataType; Int32]]; [c; [DataType; Int64]]]]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Tuple({}).Get()),
+ "[TupleType; []]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Tuple({{f.Bool()}}).Get()),
+ "[TupleType; [[DataType; Bool]]]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Tuple({{f.Yson()}, {f.Bool()}}).Get()),
+ "[TupleType; [[DataType; Yson]; [DataType; Bool]]]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Tuple({{f.Int32()}, {f.Int32()}, {f.Int64()}}).Get()),
+ "[TupleType; [[DataType; Int32]; [DataType; Int32]; [DataType; Int64]]]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Variant(f.Struct({{"a", f.Bool()}})).Get()),
+ "[VariantType; [StructType; [[a; [DataType; Bool]]]]]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Variant(f.Struct({{"a", f.Yson()}, {"b", f.Bool()}})).Get()),
+ "[VariantType; [StructType; [[a; [DataType; Yson]]; [b; [DataType; Bool]]]]]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Variant(f.Struct({{"a", f.Int32()}, {"b", f.Int32()}, {"c", f.Int64()}})).Get()),
+ "[VariantType; [StructType; [[a; [DataType; Int32]]; [b; [DataType; Int32]]; [c; [DataType; Int64]]]]]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Variant(f.Tuple({{f.Bool()}})).Get()),
+ "[VariantType; [TupleType; [[DataType; Bool]]]]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Variant(f.Tuple({{f.Yson()}, {f.Bool()}})).Get()),
+ "[VariantType; [TupleType; [[DataType; Yson]; [DataType; Bool]]]]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Variant(f.Tuple({{f.Int32()}, {f.Int32()}, {f.Int32()}})).Get()),
+ "[VariantType; [TupleType; [[DataType; Int32]; [DataType; Int32]; [DataType; Int32]]]]");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Tagged(f.String(), "Url").Get()),
+ "[TaggedType; Url; [DataType; String]]");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlType(f.Tagged(f.String(), "Url").Get(), /* includeTags = */ false),
+ "[DataType; String]");
+}
+
+TEST_TF(TypeIO, AsYqlRowSpec) {
+ {
+ auto type = f.Struct({});
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlRowSpec(type.Get()),
+ "{StrictSchema=%true; Type=[StructType; []]}");
+ }
+ {
+ auto type = f.Tagged(f.Struct({}), "Event");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlRowSpec(type.Get()),
+ "{StrictSchema=%true; Type=[StructType; []]}");
+ }
+ {
+ auto type = f.Struct({{"x", f.Tagged(f.String(), "Url")}});
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlRowSpec(type.Get()),
+ "{StrictSchema=%true; Type=[StructType; [[x; [TaggedType; Url; [DataType; String]]]]]}");
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlRowSpec(type.Get(), /* includeTags = */ false),
+ "{StrictSchema=%true; Type=[StructType; [[x; [DataType; String]]]]}");
+ }
+ {
+ auto type = f.Struct({{"a", f.Bool()}});
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlRowSpec(type.Get()),
+ "{StrictSchema=%true; Type=[StructType; [[a; [DataType; Bool]]]]}");
+ }
+ {
+ auto type = f.Struct({{"a", f.Yson()}, {"b", f.Bool()}});
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlRowSpec(type.Get()),
+ "{StrictSchema=%true; Type=[StructType; [[a; [DataType; Yson]]; [b; [DataType; Bool]]]]}");
+ }
+ {
+ auto type = f.Struct({{"a", f.Int32()}, {"b", f.Int32()}, {"c", f.Int32()}});
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYqlRowSpec(type.Get()),
+ "{StrictSchema=%true; Type=[StructType; [[a; [DataType; Int32]]; [b; [DataType; Int32]]; [c; [DataType; Int32]]]]}");
+ }
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ NTi::NIo::AsYqlRowSpec(f.Void().Get());
+ }(),
+ NTi::TApiException, "expected a struct type");
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ NTi::NIo::AsYqlRowSpec(Optional(f.Struct({})).Get());
+ }(),
+ NTi::TApiException, "expected a struct type");
+}
+
+TEST_TF(TypeIO, AsYtSchema) {
+ {
+ auto type = f.Struct({
+ {"void", f.Void()},
+ {"null", f.Null()},
+ {"bool", f.Bool()},
+ {"int8", f.Int8()},
+ {"int16", f.Int16()},
+ {"int32", f.Int32()},
+ {"int64", f.Int64()},
+ {"uint8", f.Uint8()},
+ {"uint16", f.Uint16()},
+ {"uint32", f.Uint32()},
+ {"uint64", f.Uint64()},
+ {"float", f.Float()},
+ {"double", f.Double()},
+ {"string", f.String()},
+ {"utf8", f.Utf8()},
+ {"date", f.Date()},
+ {"datetime", f.Datetime()},
+ {"timestamp", f.Timestamp()},
+ {"tzdate", f.TzDate()},
+ {"tzdatetime", f.TzDatetime()},
+ {"tztimestamp", f.TzTimestamp()},
+ {"interval", f.Interval()},
+ {"decimal", f.Decimal(20, 10)},
+ {"json", f.Json()},
+ {"yson", f.Yson()},
+ {"uuid", f.Uuid()},
+ {"list", f.List(f.Bool())},
+ {"dict", f.Dict(f.Bool(), f.Bool())},
+ {"struct", f.Struct({})},
+ {"tuple", f.Tuple({})},
+ {"variant", f.Variant(f.Struct({{"x", f.Bool()}}))},
+ });
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYtSchema(type.Get()),
+ R"(
+ <strict=%true; unique_keys=%false>
+ [
+ {name=void; required=%false; type=any };
+ {name=null; required=%false; type=any };
+ {name=bool; required=%true; type=boolean };
+ {name=int8; required=%true; type=int8 };
+ {name=int16; required=%true; type=int16 };
+ {name=int32; required=%true; type=int32 };
+ {name=int64; required=%true; type=int64 };
+ {name=uint8; required=%true; type=uint8 };
+ {name=uint16; required=%true; type=uint16 };
+ {name=uint32; required=%true; type=uint32 };
+ {name=uint64; required=%true; type=uint64 };
+ {name=float; required=%true; type=double };
+ {name=double; required=%true; type=double };
+ {name=string; required=%true; type=string };
+ {name=utf8; required=%true; type=utf8 };
+ {name=date; required=%true; type=uint16 };
+ {name=datetime; required=%true; type=uint32 };
+ {name=timestamp; required=%true; type=uint64 };
+ {name=tzdate; required=%true; type=string };
+ {name=tzdatetime; required=%true; type=string };
+ {name=tztimestamp; required=%true; type=string };
+ {name=interval; required=%true; type=int64 };
+ {name=decimal; required=%true; type=string };
+ {name=json; required=%true; type=string };
+ {name=yson; required=%false; type=any };
+ {name=uuid; required=%true; type=string };
+ {name=list; required=%false; type=any };
+ {name=dict; required=%false; type=any };
+ {name=struct; required=%false; type=any };
+ {name=tuple; required=%false; type=any };
+ {name=variant; required=%false; type=any };
+ ]
+ )");
+ }
+
+ {
+ auto type = f.Struct({
+ {"void", f.Tagged(f.Void(), "Tag")},
+ {"null", f.Tagged(f.Null(), "Tag")},
+ {"bool", f.Tagged(f.Bool(), "Tag")},
+ {"int8", f.Tagged(f.Int8(), "Tag")},
+ {"int16", f.Tagged(f.Int16(), "Tag")},
+ {"int32", f.Tagged(f.Int32(), "Tag")},
+ {"int64", f.Tagged(f.Int64(), "Tag")},
+ {"uint8", f.Tagged(f.Uint8(), "Tag")},
+ {"uint16", f.Tagged(f.Uint16(), "Tag")},
+ {"uint32", f.Tagged(f.Uint32(), "Tag")},
+ {"uint64", f.Tagged(f.Uint64(), "Tag")},
+ {"float", f.Tagged(f.Float(), "Tag")},
+ {"double", f.Tagged(f.Double(), "Tag")},
+ {"string", f.Tagged(f.String(), "Tag")},
+ {"utf8", f.Tagged(f.Utf8(), "Tag")},
+ {"date", f.Tagged(f.Date(), "Tag")},
+ {"datetime", f.Tagged(f.Datetime(), "Tag")},
+ {"timestamp", f.Tagged(f.Timestamp(), "Tag")},
+ {"tzdate", f.Tagged(f.TzDate(), "Tag")},
+ {"tzdatetime", f.Tagged(f.TzDatetime(), "Tag")},
+ {"tztimestamp", f.Tagged(f.TzTimestamp(), "Tag")},
+ {"interval", f.Tagged(f.Interval(), "Tag")},
+ {"decimal", f.Tagged(f.Decimal(20, 10), "Tag")},
+ {"json", f.Tagged(f.Json(), "Tag")},
+ {"yson", f.Tagged(f.Yson(), "Tag")},
+ {"uuid", f.Tagged(f.Uuid(), "Tag")},
+ {"list", f.Tagged(f.List(f.Bool()), "Tag")},
+ {"dict", f.Tagged(f.Dict(f.Bool(), f.Bool()), "Tag")},
+ {"struct", f.Tagged(f.Struct({}), "Tag")},
+ {"tuple", f.Tagged(f.Tuple({}), "Tag")},
+ {"variant", f.Tagged(f.Variant(f.Struct({{"x", f.Bool()}})), "Tag")},
+ });
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYtSchema(type.Get()),
+ R"(
+ <strict=%true; unique_keys=%false>
+ [
+ {name=void; required=%false; type=any };
+ {name=null; required=%false; type=any };
+ {name=bool; required=%true; type=boolean };
+ {name=int8; required=%true; type=int8 };
+ {name=int16; required=%true; type=int16 };
+ {name=int32; required=%true; type=int32 };
+ {name=int64; required=%true; type=int64 };
+ {name=uint8; required=%true; type=uint8 };
+ {name=uint16; required=%true; type=uint16 };
+ {name=uint32; required=%true; type=uint32 };
+ {name=uint64; required=%true; type=uint64 };
+ {name=float; required=%true; type=double };
+ {name=double; required=%true; type=double };
+ {name=string; required=%true; type=string };
+ {name=utf8; required=%true; type=utf8 };
+ {name=date; required=%true; type=uint16 };
+ {name=datetime; required=%true; type=uint32 };
+ {name=timestamp; required=%true; type=uint64 };
+ {name=tzdate; required=%true; type=string };
+ {name=tzdatetime; required=%true; type=string };
+ {name=tztimestamp; required=%true; type=string };
+ {name=interval; required=%true; type=int64 };
+ {name=decimal; required=%true; type=string };
+ {name=json; required=%true; type=string };
+ {name=yson; required=%false; type=any };
+ {name=uuid; required=%true; type=string };
+ {name=list; required=%false; type=any };
+ {name=dict; required=%false; type=any };
+ {name=struct; required=%false; type=any };
+ {name=tuple; required=%false; type=any };
+ {name=variant; required=%false; type=any };
+ ]
+ )");
+ }
+
+ {
+ auto type = f.Struct({
+ {"void", Optional(f.Void())},
+ {"null", Optional(f.Null())},
+ {"bool", Optional(f.Bool())},
+ {"int8", Optional(f.Int8())},
+ {"int16", Optional(f.Int16())},
+ {"int32", Optional(f.Int32())},
+ {"int64", Optional(f.Int64())},
+ {"uint8", Optional(f.Uint8())},
+ {"uint16", Optional(f.Uint16())},
+ {"uint32", Optional(f.Uint32())},
+ {"uint64", Optional(f.Uint64())},
+ {"float", Optional(f.Float())},
+ {"double", Optional(f.Double())},
+ {"string", Optional(f.String())},
+ {"utf8", Optional(f.Utf8())},
+ {"date", Optional(f.Date())},
+ {"datetime", Optional(f.Datetime())},
+ {"timestamp", Optional(f.Timestamp())},
+ {"tzdate", Optional(f.TzDate())},
+ {"tzdatetime", Optional(f.TzDatetime())},
+ {"tztimestamp", Optional(f.TzTimestamp())},
+ {"interval", Optional(f.Interval())},
+ {"decimal", Optional(f.Decimal(20, 10))},
+ {"json", Optional(f.Json())},
+ {"yson", Optional(f.Yson())},
+ {"uuid", Optional(f.Uuid())},
+ {"list", Optional(f.List(f.Bool()))},
+ {"dict", Optional(f.Dict(f.Bool(), f.Bool()))},
+ {"struct", Optional(f.Struct({}))},
+ {"tuple", Optional(f.Tuple({}))},
+ {"variant", Optional(f.Variant(f.Struct({{"x", f.Bool()}})))},
+ });
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYtSchema(type.Get()),
+ R"(
+ <strict=%true; unique_keys=%false>
+ [
+ {name=void; required=%false; type=any };
+ {name=null; required=%false; type=any };
+ {name=bool; required=%false; type=boolean };
+ {name=int8; required=%false; type=int8 };
+ {name=int16; required=%false; type=int16 };
+ {name=int32; required=%false; type=int32 };
+ {name=int64; required=%false; type=int64 };
+ {name=uint8; required=%false; type=uint8 };
+ {name=uint16; required=%false; type=uint16 };
+ {name=uint32; required=%false; type=uint32 };
+ {name=uint64; required=%false; type=uint64 };
+ {name=float; required=%false; type=double };
+ {name=double; required=%false; type=double };
+ {name=string; required=%false; type=string };
+ {name=utf8; required=%false; type=utf8 };
+ {name=date; required=%false; type=uint16 };
+ {name=datetime; required=%false; type=uint32 };
+ {name=timestamp; required=%false; type=uint64 };
+ {name=tzdate; required=%false; type=string };
+ {name=tzdatetime; required=%false; type=string };
+ {name=tztimestamp; required=%false; type=string };
+ {name=interval; required=%false; type=int64 };
+ {name=decimal; required=%false; type=string };
+ {name=json; required=%false; type=string };
+ {name=yson; required=%false; type=any };
+ {name=uuid; required=%false; type=string };
+ {name=list; required=%false; type=any };
+ {name=dict; required=%false; type=any };
+ {name=struct; required=%false; type=any };
+ {name=tuple; required=%false; type=any };
+ {name=variant; required=%false; type=any };
+ ]
+ )");
+ }
+
+ {
+ auto type = f.Struct({
+ {"void", Optional(Optional(f.Void()))},
+ {"null", Optional(Optional(f.Null()))},
+ {"bool", Optional(Optional(f.Bool()))},
+ {"int8", Optional(Optional(f.Int8()))},
+ {"int16", Optional(Optional(f.Int16()))},
+ {"int32", Optional(Optional(f.Int32()))},
+ {"int64", Optional(Optional(f.Int64()))},
+ {"uint8", Optional(Optional(f.Uint8()))},
+ {"uint16", Optional(Optional(f.Uint16()))},
+ {"uint32", Optional(Optional(f.Uint32()))},
+ {"uint64", Optional(Optional(f.Uint64()))},
+ {"float", Optional(Optional(f.Float()))},
+ {"double", Optional(Optional(f.Double()))},
+ {"string", Optional(Optional(f.String()))},
+ {"utf8", Optional(Optional(f.Utf8()))},
+ {"date", Optional(Optional(f.Date()))},
+ {"datetime", Optional(Optional(f.Datetime()))},
+ {"timestamp", Optional(Optional(f.Timestamp()))},
+ {"tzdate", Optional(Optional(f.TzDate()))},
+ {"tzdatetime", Optional(Optional(f.TzDatetime()))},
+ {"tztimestamp", Optional(Optional(f.TzTimestamp()))},
+ {"interval", Optional(Optional(f.Interval()))},
+ {"decimal", Optional(Optional(f.Decimal(20, 10)))},
+ {"json", Optional(Optional(f.Json()))},
+ {"yson", Optional(Optional(f.Yson()))},
+ {"uuid", Optional(Optional(f.Uuid()))},
+ {"list", Optional(Optional(f.List(f.Bool())))},
+ {"dict", Optional(Optional(f.Dict(f.Bool(), f.Bool())))},
+ {"struct", Optional(Optional(f.Struct({})))},
+ {"tuple", Optional(Optional(f.Tuple({})))},
+ {"variant", Optional(Optional(f.Variant(f.Struct({{"x", f.Bool()}}))))},
+ });
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYtSchema(type.Get()),
+ R"(
+ <strict=%true; unique_keys=%false>
+ [
+ {name=void; required=%false; type=any };
+ {name=null; required=%false; type=any };
+ {name=bool; required=%false; type=any };
+ {name=int8; required=%false; type=any };
+ {name=int16; required=%false; type=any };
+ {name=int32; required=%false; type=any };
+ {name=int64; required=%false; type=any };
+ {name=uint8; required=%false; type=any };
+ {name=uint16; required=%false; type=any };
+ {name=uint32; required=%false; type=any };
+ {name=uint64; required=%false; type=any };
+ {name=float; required=%false; type=any };
+ {name=double; required=%false; type=any };
+ {name=string; required=%false; type=any };
+ {name=utf8; required=%false; type=any };
+ {name=date; required=%false; type=any };
+ {name=datetime; required=%false; type=any };
+ {name=timestamp; required=%false; type=any };
+ {name=tzdate; required=%false; type=any };
+ {name=tzdatetime; required=%false; type=any };
+ {name=tztimestamp; required=%false; type=any };
+ {name=interval; required=%false; type=any };
+ {name=decimal; required=%false; type=any };
+ {name=json; required=%false; type=any };
+ {name=yson; required=%false; type=any };
+ {name=uuid; required=%false; type=any };
+ {name=list; required=%false; type=any };
+ {name=dict; required=%false; type=any };
+ {name=struct; required=%false; type=any };
+ {name=tuple; required=%false; type=any };
+ {name=variant; required=%false; type=any };
+ ]
+ )");
+ }
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ NTi::NIo::AsYtSchema(f.Void().Get());
+ }(),
+ NTi::TApiException, "expected a struct type");
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ NTi::NIo::AsYtSchema(Optional(f.Struct({})).Get());
+ }(),
+ NTi::TApiException, "expected a struct type");
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS([&f]() {
+ NTi::NIo::AsYtSchema(f.Struct({}).Get());
+ }(),
+ NTi::TApiException, "expected a non-empty struct");
+
+ ASSERT_YSON_EQ(
+ NTi::NIo::AsYtSchema(f.Struct({}).Get(), /* failOnEmptyf.Struct = */ false),
+ "<strict=%true; unique_keys=%false>[{name=_yql_fake_column; required=%false; type=boolean}]");
+}
diff --git a/library/cpp/type_info/ut/type_list.cpp b/library/cpp/type_info/ut/type_list.cpp
new file mode 100644
index 0000000000..56d9e16061
--- /dev/null
+++ b/library/cpp/type_info/ut/type_list.cpp
@@ -0,0 +1,64 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include <util/generic/serialized_enum.h>
+
+TEST(TypeList, PrimitiveTypeNameSequence) {
+ auto primitiveNames = GetEnumAllValues<NTi::EPrimitiveTypeName>();
+
+ ASSERT_GT(primitiveNames.size(), 0);
+
+ ASSERT_EQ(static_cast<i32>(primitiveNames[0]), 0);
+
+ for (size_t i = 0; i < primitiveNames.size() - 1; ++i) {
+ ASSERT_EQ(static_cast<i32>(primitiveNames[i]) + 1, static_cast<i32>(primitiveNames[i + 1]));
+ }
+}
+
+TEST(TypeList, PrimitiveTypeGroup) {
+ auto primitiveNames = GetEnumAllValues<NTi::EPrimitiveTypeName>();
+
+ for (auto typeName : primitiveNames) {
+ ASSERT_TRUE(NTi::IsPrimitive(static_cast<NTi::ETypeName>(typeName)));
+ }
+}
+
+TEST(TypeList, TypeNameInExactlyOneGroup) {
+ auto allNames = GetEnumAllValues<NTi::ETypeName>();
+
+ for (auto typeName : allNames) {
+ int groups = 0;
+ groups += NTi::IsPrimitive(typeName);
+ groups += NTi::IsSingular(typeName);
+ groups += NTi::IsContainer(typeName);
+ ASSERT_EQ(groups, 1);
+ }
+}
+
+TEST(TypeList, EnumCoherence) {
+ auto primitiveNames = GetEnumAllValues<NTi::EPrimitiveTypeName>();
+ auto allNames = GetEnumAllValues<NTi::ETypeName>();
+
+ ASSERT_GT(allNames.size(), primitiveNames.size());
+
+ size_t i = 0;
+
+ for (; i < primitiveNames.size(); ++i) {
+ ASSERT_EQ(static_cast<i32>(primitiveNames[i]), static_cast<i32>(allNames[i]));
+ ASSERT_EQ(ToString(primitiveNames[i]), ToString(allNames[i]));
+ ASSERT_TRUE(NTi::IsPrimitive(allNames[i]));
+ }
+
+ for (; i < allNames.size(); ++i) {
+ ASSERT_FALSE(NTi::IsPrimitive(allNames[i]));
+ }
+}
+
+TEST(TypeList, Cast) {
+ auto primitiveNames = GetEnumAllValues<NTi::EPrimitiveTypeName>();
+
+ for (auto typeName : primitiveNames) {
+ ASSERT_EQ(typeName, NTi::ToPrimitiveTypeName(NTi::ToTypeName(typeName)));
+ }
+}
diff --git a/library/cpp/type_info/ut/type_serialize.cpp b/library/cpp/type_info/ut/type_serialize.cpp
new file mode 100644
index 0000000000..0d7e184eca
--- /dev/null
+++ b/library/cpp/type_info/ut/type_serialize.cpp
@@ -0,0 +1,251 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include "utils.h"
+
+TEST(TypeSerialize, Void) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Void().Get()), R"(void)");
+}
+
+TEST(TypeSerialize, Null) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Null().Get()), R"(null)");
+}
+
+TEST(TypeSerialize, Bool) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Bool().Get()), R"(bool)");
+}
+
+TEST(TypeSerialize, Int8) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Int8().Get()), R"(int8)");
+}
+
+TEST(TypeSerialize, Int16) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Int16().Get()), R"(int16)");
+}
+
+TEST(TypeSerialize, Int32) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Int32().Get()), R"(int32)");
+}
+
+TEST(TypeSerialize, Int64) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Int64().Get()), R"(int64)");
+}
+
+TEST(TypeSerialize, Uint8) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uint8().Get()), R"(uint8)");
+}
+
+TEST(TypeSerialize, Uint16) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uint16().Get()), R"(uint16)");
+}
+
+TEST(TypeSerialize, Uint32) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uint32().Get()), R"(uint32)");
+}
+
+TEST(TypeSerialize, Uint64) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uint64().Get()), R"(uint64)");
+}
+
+TEST(TypeSerialize, Float) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Float().Get()), R"(float)");
+}
+
+TEST(TypeSerialize, Double) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Double().Get()), R"(double)");
+}
+
+TEST(TypeSerialize, String) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::String().Get()), R"(string)");
+}
+
+TEST(TypeSerialize, Utf8) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Utf8().Get()), R"(utf8)");
+}
+
+TEST(TypeSerialize, Date) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Date().Get()), R"(date)");
+}
+
+TEST(TypeSerialize, Datetime) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Datetime().Get()), R"(datetime)");
+}
+
+TEST(TypeSerialize, Timestamp) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Timestamp().Get()), R"(timestamp)");
+}
+
+TEST(TypeSerialize, TzDate) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::TzDate().Get()), R"(tz_date)");
+}
+
+TEST(TypeSerialize, TzDatetime) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::TzDatetime().Get()), R"(tz_datetime)");
+}
+
+TEST(TypeSerialize, TzTimestamp) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::TzTimestamp().Get()), R"(tz_timestamp)");
+}
+
+TEST(TypeSerialize, Interval) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Interval().Get()), R"(interval)");
+}
+
+TEST(TypeSerialize, Decimal) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Decimal(20, 10).Get()), R"({type_name=decimal; precision=20; scale=10})");
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Decimal(10, 10).Get()), R"({type_name=decimal; precision=10; scale=10})");
+}
+
+TEST(TypeSerialize, Json) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Json().Get()), R"(json)");
+}
+
+TEST(TypeSerialize, Yson) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Yson().Get()), R"(yson)");
+}
+
+TEST(TypeSerialize, Uuid) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Uuid().Get()), R"(uuid)");
+}
+
+TEST(TypeSerialize, Optional) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Optional(NTi::Void()).Get()), R"({type_name=optional; item=void})");
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Optional(NTi::String()).Get()), R"({type_name=optional; item=string})");
+}
+
+TEST(TypeSerialize, List) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::List(NTi::Void()).Get()), R"({type_name=list; item=void})");
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::List(NTi::String()).Get()), R"({type_name=list; item=string})");
+}
+
+TEST(TypeSerialize, Dict) {
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Dict(NTi::Void(), NTi::Void()).Get()), R"({type_name=dict; key=void; value=void})");
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(NTi::Dict(NTi::Int32(), NTi::String()).Get()), R"({type_name=dict; key=int32; value=string})");
+}
+
+TEST(TypeSerialize, StructEmpty) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Struct({}).Get()),
+ R"({type_name=struct; members=[]})");
+}
+
+TEST(TypeSerialize, Struct) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}}).Get()),
+ R"({type_name=struct; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})");
+}
+
+TEST(TypeSerialize, StructNamedEmpty) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Struct("S", {}).Get()),
+ R"({type_name=struct; members=[]})");
+}
+
+TEST(TypeSerialize, StructNamedEmptyNoNames) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Struct("S", {}).Get(),
+ /* binary = */ false,
+ /* includeTags = */ true),
+ R"({type_name=struct; members=[]})");
+}
+
+TEST(TypeSerialize, StructNamed) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Struct("S", {{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}}).Get()),
+ R"({type_name=struct; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})");
+}
+
+TEST(TypeSerialize, TupleEmpty) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Tuple({}).Get()),
+ R"({type_name=tuple; elements=[]})");
+}
+
+TEST(TypeSerialize, Tuple) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Tuple({{NTi::String()}, {NTi::List(NTi::Int64())}}).Get()),
+ R"({type_name=tuple; elements=[{type=string}; {type={type_name=list; item=int64}}]})");
+}
+
+TEST(TypeSerialize, TupleNamedEmpty) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Tuple("S", {}).Get()),
+ R"({type_name=tuple; elements=[]})");
+}
+
+TEST(TypeSerialize, TupleNamedEmptyNoNames) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Tuple("S", {}).Get(),
+ /* binary = */ false),
+ R"({type_name=tuple; elements=[]})");
+}
+
+TEST(TypeSerialize, VariantStruct) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Variant(
+ NTi::Struct({{"ItemB", NTi::String()}, {"ItemA", NTi::List(NTi::Int64())}}))
+ .Get()),
+ R"({type_name=variant; members=[{name=ItemB; type=string}; {name=ItemA; type={type_name=list; item=int64}}]})");
+}
+
+TEST(TypeSerialize, VariantTuple) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Variant(
+ NTi::Tuple({
+ {NTi::String()},
+ {NTi::List(NTi::Int64())},
+ }))
+ .Get()),
+ R"({type_name=variant; elements=[{type=string}; {type={type_name=list; item=int64}}]})");
+}
+
+TEST(TypeSerialize, Tagged) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Tagged(NTi::String(), "Url").Get()),
+ R"({type_name=tagged; tag=Url; item=string})");
+}
+
+TEST(TypeSerialize, TaggedNoTags) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Tagged(NTi::String(), "Url").Get(),
+ /* binary = */ false,
+ /* includeTags = */ false),
+ R"(string)");
+}
+
+TEST(TypeSerialize, MultipleType) {
+ auto result = TString();
+
+ {
+ auto writer = NYsonPull::MakeTextWriter(NYsonPull::NOutput::FromString(&result), NYsonPull::EStreamType::ListFragment);
+
+ writer.BeginStream();
+ NTi::NIo::SerializeYsonMultiple(NTi::Optional(NTi::String()).Get(), writer.GetConsumer());
+ NTi::NIo::SerializeYsonMultiple(NTi::List(NTi::Utf8()).Get(), writer.GetConsumer());
+ writer.EndStream();
+ }
+
+ ASSERT_YSON_EQ("[" + result + "]", R"([{type_name=optional; item=string}; {type_name=list; item=utf8}])");
+}
+
+TEST(TypeSerialize, Binary) {
+ ASSERT_YSON_EQ(
+ NTi::NIo::SerializeYson(
+ NTi::Tagged(NTi::String(), "Url").Get(),
+ /* binary = */ true),
+ R"({type_name=tagged; tag=Url; item=string})");
+}
diff --git a/library/cpp/type_info/ut/type_show.cpp b/library/cpp/type_info/ut/type_show.cpp
new file mode 100644
index 0000000000..cde0d9371a
--- /dev/null
+++ b/library/cpp/type_info/ut/type_show.cpp
@@ -0,0 +1,199 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include "utils.h"
+
+TEST_TF(TypeShow, Void) {
+ ASSERT_EQ(ToString(*f.Void()), "Void");
+}
+
+TEST_TF(TypeShow, Null) {
+ ASSERT_EQ(ToString(*f.Null()), "Null");
+}
+
+TEST_TF(TypeShow, Bool) {
+ ASSERT_EQ(ToString(*f.Bool()), "Bool");
+}
+
+TEST_TF(TypeShow, Int8) {
+ ASSERT_EQ(ToString(*f.Int8()), "Int8");
+}
+
+TEST_TF(TypeShow, Int16) {
+ ASSERT_EQ(ToString(*f.Int16()), "Int16");
+}
+
+TEST_TF(TypeShow, Int32) {
+ ASSERT_EQ(ToString(*f.Int32()), "Int32");
+}
+
+TEST_TF(TypeShow, Int64) {
+ ASSERT_EQ(ToString(*f.Int64()), "Int64");
+}
+
+TEST_TF(TypeShow, Uint8) {
+ ASSERT_EQ(ToString(*f.Uint8()), "Uint8");
+}
+
+TEST_TF(TypeShow, Uint16) {
+ ASSERT_EQ(ToString(*f.Uint16()), "Uint16");
+}
+
+TEST_TF(TypeShow, Uint32) {
+ ASSERT_EQ(ToString(*f.Uint32()), "Uint32");
+}
+
+TEST_TF(TypeShow, Uint64) {
+ ASSERT_EQ(ToString(*f.Uint64()), "Uint64");
+}
+
+TEST_TF(TypeShow, Float) {
+ ASSERT_EQ(ToString(*f.Float()), "Float");
+}
+
+TEST_TF(TypeShow, Double) {
+ ASSERT_EQ(ToString(*f.Double()), "Double");
+}
+
+TEST_TF(TypeShow, String) {
+ ASSERT_EQ(ToString(*f.String()), "String");
+}
+
+TEST_TF(TypeShow, Utf8) {
+ ASSERT_EQ(ToString(*f.Utf8()), "Utf8");
+}
+
+TEST_TF(TypeShow, Date) {
+ ASSERT_EQ(ToString(*f.Date()), "Date");
+}
+
+TEST_TF(TypeShow, Datetime) {
+ ASSERT_EQ(ToString(*f.Datetime()), "Datetime");
+}
+
+TEST_TF(TypeShow, Timestamp) {
+ ASSERT_EQ(ToString(*f.Timestamp()), "Timestamp");
+}
+
+TEST_TF(TypeShow, TzDate) {
+ ASSERT_EQ(ToString(*f.TzDate()), "TzDate");
+}
+
+TEST_TF(TypeShow, TzDatetime) {
+ ASSERT_EQ(ToString(*f.TzDatetime()), "TzDatetime");
+}
+
+TEST_TF(TypeShow, TzTimestamp) {
+ ASSERT_EQ(ToString(*f.TzTimestamp()), "TzTimestamp");
+}
+
+TEST_TF(TypeShow, Interval) {
+ ASSERT_EQ(ToString(*f.Interval()), "Interval");
+}
+
+TEST_TF(TypeShow, Decimal) {
+ ASSERT_EQ(ToString(*f.Decimal(20, 10)), "Decimal(20, 10)");
+}
+
+TEST_TF(TypeShow, Json) {
+ ASSERT_EQ(ToString(*f.Json()), "Json");
+}
+
+TEST_TF(TypeShow, Yson) {
+ ASSERT_EQ(ToString(*f.Yson()), "Yson");
+}
+
+TEST_TF(TypeShow, Uuid) {
+ ASSERT_EQ(ToString(*f.Uuid()), "Uuid");
+}
+
+TEST_TF(TypeShow, Optional) {
+ ASSERT_EQ(ToString(*f.Optional(f.String())), "Optional<String>");
+}
+
+TEST_TF(TypeShow, List) {
+ ASSERT_EQ(ToString(*f.List(f.String())), "List<String>");
+}
+
+TEST_TF(TypeShow, Dict) {
+ ASSERT_EQ(ToString(*f.Dict(f.String(), f.Int32())), "Dict<String, Int32>");
+}
+
+TEST_TF(TypeShow, Struct) {
+ ASSERT_EQ(
+ ToString(*f.Struct({})),
+ "Struct<>");
+ ASSERT_EQ(
+ ToString(*f.Struct({{"x1", f.Void()}})),
+ "Struct<'x1': Void>");
+ ASSERT_EQ(
+ ToString(*f.Struct({{"x1", f.Void()}, {"x2", f.String()}})),
+ "Struct<'x1': Void, 'x2': String>");
+ ASSERT_EQ(
+ ToString(*f.Struct("Name", {})),
+ "Struct['Name']<>");
+ ASSERT_EQ(
+ ToString(*f.Struct("Name", {{"x1", f.Void()}})),
+ "Struct['Name']<'x1': Void>");
+ ASSERT_EQ(
+ ToString(*f.Struct("Name", {{"x1", f.Void()}, {"x2", f.String()}})),
+ "Struct['Name']<'x1': Void, 'x2': String>");
+}
+
+TEST_TF(TypeShow, Tuple) {
+ ASSERT_EQ(
+ ToString(*f.Tuple({})),
+ "Tuple<>");
+ ASSERT_EQ(
+ ToString(*f.Tuple({{f.Void()}})),
+ "Tuple<Void>");
+ ASSERT_EQ(
+ ToString(*f.Tuple({{f.Void()}, {f.String()}})),
+ "Tuple<Void, String>");
+ ASSERT_EQ(
+ ToString(*f.Tuple("Name", {})),
+ "Tuple['Name']<>");
+ ASSERT_EQ(
+ ToString(*f.Tuple("Name", {{f.Void()}})),
+ "Tuple['Name']<Void>");
+ ASSERT_EQ(
+ ToString(*f.Tuple("Name", {{f.Void()}, {f.String()}})),
+ "Tuple['Name']<Void, String>");
+}
+
+TEST_TF(TypeShow, VariantStruct) {
+ ASSERT_EQ(
+ ToString(*f.Variant(f.Struct({{"x1", f.Void()}}))),
+ "Variant<'x1': Void>");
+ ASSERT_EQ(
+ ToString(*f.Variant(f.Struct({{"x1", f.Void()}, {"x2", f.String()}}))),
+ "Variant<'x1': Void, 'x2': String>");
+ ASSERT_EQ(
+ ToString(*f.Variant("Name", f.Struct({{"x1", f.Void()}}))),
+ "Variant['Name']<'x1': Void>");
+ ASSERT_EQ(
+ ToString(*f.Variant("Name", f.Struct({{"x1", f.Void()}, {"x2", f.String()}}))),
+ "Variant['Name']<'x1': Void, 'x2': String>");
+}
+
+TEST_TF(TypeShow, VariantTuple) {
+ ASSERT_EQ(
+ ToString(*f.Variant(f.Tuple({{f.Void()}}))),
+ "Variant<Void>");
+ ASSERT_EQ(
+ ToString(*f.Variant(f.Tuple({{f.Void()}, {f.String()}}))),
+ "Variant<Void, String>");
+ ASSERT_EQ(
+ ToString(*f.Variant("Name", f.Tuple({{f.Void()}}))),
+ "Variant['Name']<Void>");
+ ASSERT_EQ(
+ ToString(*f.Variant("Name", f.Tuple({{f.Void()}, {f.String()}}))),
+ "Variant['Name']<Void, String>");
+}
+
+TEST_TF(TypeShow, Tagged) {
+ ASSERT_EQ(
+ ToString(*f.Tagged(f.Void(), "Tag")),
+ "Tagged<Void, 'Tag'>");
+}
diff --git a/library/cpp/type_info/ut/type_strip_tags.cpp b/library/cpp/type_info/ut/type_strip_tags.cpp
new file mode 100644
index 0000000000..b3ef4c0825
--- /dev/null
+++ b/library/cpp/type_info/ut/type_strip_tags.cpp
@@ -0,0 +1,31 @@
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <library/cpp/type_info/type_info.h>
+
+#include "utils.h"
+
+TEST_TF(TypeStripTags, StripTags) {
+ auto t = f.List(f.Tagged(f.Void(), "Tag"));
+
+ ASSERT_EQ(t->StripTags().Get(), t.Get());
+ ASSERT_EQ(f.Tagged(t, "Tag")->StripTags().Get(), t.Get());
+ ASSERT_EQ(f.Tagged(f.Tagged(t, "Tag"), "Tag2")->StripTags().Get(), t.Get());
+}
+
+TEST_TF(TypeStripTags, StripOptionals) {
+ auto t = f.Tagged(f.Optional(f.Void()), "Tag");
+
+ ASSERT_EQ(t->StripOptionals().Get(), t.Get());
+ ASSERT_EQ(f.Optional(t)->StripOptionals().Get(), t.Get());
+ ASSERT_EQ(f.Optional(f.Optional(t))->StripOptionals().Get(), t.Get());
+}
+
+TEST_TF(TypeStripTags, StripTagsAndOptionals) {
+ auto t = f.Void();
+
+ ASSERT_EQ(t->StripTagsAndOptionals().Get(), t.Get());
+ ASSERT_EQ(f.Optional(t)->StripTagsAndOptionals().Get(), t.Get());
+ ASSERT_EQ(f.Optional(f.Optional(t))->StripTagsAndOptionals().Get(), t.Get());
+ ASSERT_EQ(f.Optional(f.Tagged(t, "Tag"))->StripTagsAndOptionals().Get(), t.Get());
+ ASSERT_EQ(f.Optional(f.Optional(f.Optional(t)))->StripTagsAndOptionals().Get(), t.Get());
+}
diff --git a/library/cpp/type_info/ut/utils.h b/library/cpp/type_info/ut/utils.h
new file mode 100644
index 0000000000..b35316cb2f
--- /dev/null
+++ b/library/cpp/type_info/ut/utils.h
@@ -0,0 +1,73 @@
+#pragma once
+
+//! @file utils.h
+//!
+//! Infrastructure for running type info tests with different factories and settings.
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <library/cpp/yson/consumer.h>
+#include <library/cpp/yson/node/node.h>
+#include <library/cpp/yson/node/node_builder.h>
+#include <library/cpp/yson/node/node_io.h>
+
+#include <library/cpp/type_info/fwd.h>
+
+template <typename T>
+inline TString ToCanonicalYson(const T& value) {
+ return NYT::NodeToCanonicalYsonString(NYT::NodeFromYsonString(TStringBuf(value)));
+}
+
+/// Assert that two YSON strings are equal.
+#define ASSERT_YSON_EQ(L, R) ASSERT_EQ(ToCanonicalYson(L), ToCanonicalYson(R))
+
+/// Assert that two types are strictly equal.
+#define ASSERT_STRICT_EQ(a, b) \
+ do { \
+ auto lhs = (a); \
+ auto rhs = (b); \
+ ASSERT_TRUE(NTi::NEq::TStrictlyEqual()(lhs, rhs)); \
+ ASSERT_TRUE(NTi::NEq::TStrictlyEqual().IgnoreHash(lhs, rhs)); \
+ ASSERT_EQ(lhs->GetHash(), NTi::NEq::TStrictlyEqualHash()(lhs)); \
+ ASSERT_EQ(rhs->GetHash(), NTi::NEq::TStrictlyEqualHash()(rhs)); \
+ ASSERT_EQ(lhs->GetHash(), rhs->GetHash()); \
+ } while (false)
+
+/// Assert that two types are strictly unequal.
+///
+/// Note: we check that, if types are not equal, their hashes are also not equal.
+/// While this is not guaranteed, we haven't seen any collisions so far.
+/// If some collision happen, check if hashing isn't broken before removing the assert.
+#define ASSERT_STRICT_NE(a, b) \
+ do { \
+ auto lhs = (a); \
+ auto rhs = (b); \
+ ASSERT_FALSE(NTi::NEq::TStrictlyEqual()(lhs, rhs)); \
+ ASSERT_FALSE(NTi::NEq::TStrictlyEqual().IgnoreHash(lhs, rhs)); \
+ ASSERT_EQ(lhs->GetHash(), NTi::NEq::TStrictlyEqualHash()(lhs)); \
+ ASSERT_EQ(rhs->GetHash(), NTi::NEq::TStrictlyEqualHash()(rhs)); \
+ ASSERT_NE(lhs->GetHash(), rhs->GetHash()); \
+ } while (false)
+
+/// Assert that a type string is equal to the given type after deserialization.
+#define ASSERT_DESERIALIZED_EQ(canonicalType, serializedType) \
+ do { \
+ auto reader = NYsonPull::TReader(NYsonPull::NInput::FromMemory(serializedType), NYsonPull::EStreamType::Node); \
+ auto deserializedType = NTi::NIo::DeserializeYson(*NTi::HeapFactory(), reader); \
+ ASSERT_STRICT_EQ(deserializedType, canonicalType); \
+ ASSERT_YSON_EQ(NTi::NIo::SerializeYson(canonicalType.Get()), NTi::NIo::SerializeYson(deserializedType.Get())); \
+ } while (false)
+
+/// Test parametrized over different type factories.
+#define TEST_TF(N, NN) \
+ void Test##N##NN(NTi::ITypeFactory& f); \
+ TEST(N, NN##_Heap) { \
+ Test##N##NN(*NTi::HeapFactory()); \
+ } \
+ TEST(N, NN##_Pool) { \
+ Test##N##NN(*NTi::PoolFactory(false)); \
+ } \
+ TEST(N, NN##_PoolDedup) { \
+ Test##N##NN(*NTi::PoolFactory(true)); \
+ } \
+ void Test##N##NN(NTi::ITypeFactory& f)
diff --git a/library/cpp/type_info/ut/ya.make b/library/cpp/type_info/ut/ya.make
new file mode 100644
index 0000000000..dc27e2a7db
--- /dev/null
+++ b/library/cpp/type_info/ut/ya.make
@@ -0,0 +1,32 @@
+UNITTEST()
+
+SRCS(
+ builder.cpp
+ type_basics.cpp
+ type_complexity_ut.cpp
+ type_constraints.cpp
+ type_deserialize.cpp
+ type_equivalence.cpp
+ type_factory.cpp
+ type_factory_raw.cpp
+ type_io.cpp
+ type_list.cpp
+ type_serialize.cpp
+ type_show.cpp
+ type_strip_tags.cpp
+ test_data.cpp
+)
+
+PEERDIR(
+ library/cpp/type_info
+ library/cpp/yson
+ library/cpp/yson/node
+ library/cpp/resource
+)
+
+RESOURCE(
+ ${ARCADIA_ROOT}/library/cpp/type_info/test-data/good-types.txt /good
+ ${ARCADIA_ROOT}/library/cpp/type_info/test-data/bad-types.txt /bad
+)
+
+END()
diff --git a/library/cpp/type_info/ya.make b/library/cpp/type_info/ya.make
new file mode 100644
index 0000000000..555f0d1433
--- /dev/null
+++ b/library/cpp/type_info/ya.make
@@ -0,0 +1,26 @@
+LIBRARY()
+
+SRCS(
+ type_info.cpp
+
+ builder.cpp
+ error.cpp
+ type.cpp
+ type_complexity.cpp
+ type_equivalence.cpp
+ type_factory.cpp
+ type_io.cpp
+ type_list.cpp
+)
+
+GENERATE_ENUM_SERIALIZATION(
+ type_list.h
+)
+
+PEERDIR(
+ library/cpp/yson_pull
+)
+
+END()
+
+RECURSE_FOR_TESTS(ut)
diff --git a/library/cpp/yson_pull/bridge.h b/library/cpp/yson_pull/bridge.h
new file mode 100644
index 0000000000..ac767dcba0
--- /dev/null
+++ b/library/cpp/yson_pull/bridge.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "consumer.h"
+#include "event.h"
+#include "writer.h"
+
+namespace NYsonPull {
+ //! \brief Connect YSON stream producer and consumer.
+ //!
+ //! Useful for writing YSON stream filters.
+ //! \p Producer must have a \p next_event() method (like \p NYsonPull::reader).
+ //! \p Consumer must be like \p NYsonPull::consumer interface.
+ template <typename Producer, typename Consumer>
+ inline void Bridge(Producer&& producer, Consumer&& consumer) {
+ for (;;) {
+ auto& event = producer.NextEvent();
+ consumer.OnEvent(event);
+ if (event.Type() == EEventType::EndStream) {
+ break;
+ }
+ }
+ }
+
+ template <typename Producer>
+ inline void Bridge(Producer&& producer, TWriter& writer_) {
+ Bridge(std::forward<Producer>(producer), writer_.GetConsumer());
+ }
+
+ template <typename Producer>
+ inline void Bridge(Producer&& producer, TWriter&& writer_) {
+ Bridge(std::forward<Producer>(producer), writer_.GetConsumer());
+ }
+
+}
diff --git a/library/cpp/yson_pull/yson.h b/library/cpp/yson_pull/yson.h
new file mode 100644
index 0000000000..a77eaa5c94
--- /dev/null
+++ b/library/cpp/yson_pull/yson.h
@@ -0,0 +1,14 @@
+#pragma once
+
+#include "bridge.h"
+#include "consumer.h"
+#include "event.h"
+#include "exceptions.h"
+#include "input.h"
+#include "output.h"
+#include "position_info.h"
+#include "range.h"
+#include "reader.h"
+#include "scalar.h"
+#include "stream_type.h"
+#include "writer.h"
diff --git a/library/cpp/yt/backtrace/backtrace-inl.h b/library/cpp/yt/backtrace/backtrace-inl.h
new file mode 100644
index 0000000000..b78eeffd75
--- /dev/null
+++ b/library/cpp/yt/backtrace/backtrace-inl.h
@@ -0,0 +1,36 @@
+#pragma once
+#ifndef BACKTRACE_INL_H_
+#error "Direct inclusion of this file is not allowed, include backtrace.h"
+// For the sake of sane code completion.
+#include "backtrace.h"
+#endif
+
+#include <util/system/compiler.h>
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class TCursor>
+Y_NO_INLINE TBacktrace GetBacktrace(
+ TCursor* cursor,
+ TBacktraceBuffer buffer,
+ int framesToSkip)
+{
+ // Account for the current frame.
+ ++framesToSkip;
+ size_t frameCount = 0;
+ while (frameCount < buffer.size() && !cursor->IsFinished()) {
+ if (framesToSkip > 0) {
+ --framesToSkip;
+ } else {
+ buffer[frameCount++] = cursor->GetCurrentIP();
+ }
+ cursor->MoveNext();
+ }
+ return {buffer.begin(), frameCount};
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/backtrace.cpp b/library/cpp/yt/backtrace/backtrace.cpp
new file mode 100644
index 0000000000..153a0a5dd0
--- /dev/null
+++ b/library/cpp/yt/backtrace/backtrace.cpp
@@ -0,0 +1,18 @@
+#include "backtrace.h"
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString SymbolizeBacktrace(TBacktrace backtrace)
+{
+ TString result;
+ SymbolizeBacktrace(
+ backtrace,
+ [&] (TStringBuf str) { result += str; });
+ return result;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/backtrace.h b/library/cpp/yt/backtrace/backtrace.h
new file mode 100644
index 0000000000..ea70d9558c
--- /dev/null
+++ b/library/cpp/yt/backtrace/backtrace.h
@@ -0,0 +1,45 @@
+#pragma once
+
+#include <library/cpp/yt/memory/range.h>
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+using TBacktrace = TRange<const void*>;
+using TBacktraceBuffer = TMutableRange<const void*>;
+
+//! Obtains a backtrace via a given cursor.
+/*!
+ * \param buffer is the buffer where the backtrace is written to
+ * \param framesToSkip is the number of top frames to skip
+ * \returns the portion of #buffer that has actually been filled
+ */
+template <class TCursor>
+TBacktrace GetBacktrace(
+ TCursor* cursor,
+ TBacktraceBuffer buffer,
+ int framesToSkip);
+
+//! Symbolizes a backtrace invoking a given callback for each frame.
+/*!
+ * \param backtrace Backtrace to symbolize
+ * \param frameCallback Callback to invoke per each frame
+ */
+void SymbolizeBacktrace(
+ TBacktrace backtrace,
+ const std::function<void(TStringBuf)>& frameCallback);
+
+//! Symbolizes a backtrace to a string.
+/*!
+ * \param backtrace Backtrace to symbolize
+ */
+TString SymbolizeBacktrace(TBacktrace backtrace);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
+
+#define BACKTRACE_INL_H_
+#include "backtrace-inl.h"
+#undef BACKTRACE_INL_H_
diff --git a/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp
new file mode 100644
index 0000000000..ea6e0bc08e
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp
@@ -0,0 +1,22 @@
+#include "dummy_cursor.h"
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool TDummyCursor::IsFinished() const
+{
+ return true;
+}
+
+const void* TDummyCursor::GetCurrentIP() const
+{
+ return nullptr;
+}
+
+void TDummyCursor::MoveNext()
+{ }
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h
new file mode 100644
index 0000000000..b47d7d2aba
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h
@@ -0,0 +1,17 @@
+#pragma once
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TDummyCursor
+{
+public:
+ bool IsFinished() const;
+ const void* GetCurrentIP() const;
+ void MoveNext();
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/cursors/dummy/ya.make b/library/cpp/yt/backtrace/cursors/dummy/ya.make
new file mode 100644
index 0000000000..49fd7be050
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/dummy/ya.make
@@ -0,0 +1,9 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+ dummy_cursor.cpp
+)
+
+END()
diff --git a/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp
new file mode 100644
index 0000000000..290d30c3ce
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp
@@ -0,0 +1,146 @@
+#include "frame_pointer_cursor.h"
+
+#include <util/generic/size_literals.h>
+
+#include <array>
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TFramePointerCursor::TFramePointerCursor(
+ TSafeMemoryReader* memoryReader,
+ const TFramePointerCursorContext& context)
+ : MemoryReader_(memoryReader)
+ , Rip_(reinterpret_cast<const void*>(context.Rip))
+ , Rbp_(reinterpret_cast<const void*>(context.Rbp))
+ , StartRsp_(reinterpret_cast<const void*>(context.Rsp))
+{ }
+
+bool TFramePointerCursor::IsFinished() const
+{
+ return Finished_;
+}
+
+const void* TFramePointerCursor::GetCurrentIP() const
+{
+ return Rip_;
+}
+
+void TFramePointerCursor::MoveNext()
+{
+ if (Finished_) {
+ return;
+ }
+
+ auto add = [] (auto ptr, auto delta) {
+ return reinterpret_cast<void*>(reinterpret_cast<intptr_t>(ptr) + delta);
+ };
+
+ auto checkPtr = [&] (auto ptr) {
+ ui8 data;
+ return MemoryReader_->Read(ptr, &data);
+ };
+
+ // We try unwinding stack manually by following frame pointers.
+ //
+ // We assume that stack does not span more than 4mb.
+
+ if (First_) {
+ First_ = false;
+
+ // For the first frame there are three special cases where naive
+ // unwinding would skip the caller frame.
+ //
+ // 1) Right after call instruction, rbp points to frame of a caller.
+ // 2) Right after "push rbp" instruction.
+ // 3) Right before ret instruction, rbp points to frame of a caller.
+ //
+ // We read current instruction and try to detect such cases.
+ //
+ // 55 push %rbp
+ // 48 89 e5 mov %rsp, %rbp
+ // c3 retq
+
+ std::array<ui8, 3> data;
+ if (!MemoryReader_->Read(Rip_, &data)) {
+ Finished_ = true;
+ return;
+ }
+
+ if (data[0] == 0xc3 || data[0] == 0x55) {
+ void* savedRip;
+ if (!MemoryReader_->Read(StartRsp_, &savedRip)) {
+ Finished_ = true;
+ return;
+ }
+
+ // Avoid infinite loop.
+ if (Rip_ == savedRip) {
+ Finished_ = true;
+ return;
+ }
+
+ // Detect garbage pointer.
+ if (!checkPtr(savedRip)) {
+ Finished_ = true;
+ return;
+ }
+
+ Rip_ = savedRip;
+ return;
+ }
+
+ if (data[0] == 0x48 && data[1] == 0x89 && data[2] == 0xe5) {
+ void* savedRip;
+ if (!MemoryReader_->Read(add(StartRsp_, 8), &savedRip)) {
+ Finished_ = true;
+ return;
+ }
+
+ // Avoid infinite loop.
+ if (Rip_ == savedRip) {
+ Finished_ = true;
+ return;
+ }
+
+ // Detect garbage pointer.
+ if (!checkPtr(savedRip)) {
+ Finished_ = true;
+ return;
+ }
+
+ Rip_ = savedRip;
+ return;
+ }
+ }
+
+ const void* savedRbp;
+ const void* savedRip;
+ if (!MemoryReader_->Read(Rbp_, &savedRbp) || !MemoryReader_->Read(add(Rbp_, 8), &savedRip)) {
+ Finished_ = true;
+ return;
+ }
+
+ if (!checkPtr(savedRbp)) {
+ Finished_ = true;
+ return;
+ }
+
+ if (!checkPtr(savedRip)) {
+ Finished_ = true;
+ return;
+ }
+
+ if (savedRbp < StartRsp_ || savedRbp > add(StartRsp_, 4_MB)) {
+ Finished_ = true;
+ return;
+ }
+
+ Rip_ = savedRip;
+ Rbp_ = savedRbp;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h
new file mode 100644
index 0000000000..7a6eaf431b
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#include <library/cpp/yt/memory/safe_memory_reader.h>
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TFramePointerCursorContext
+{
+ ui64 Rip;
+ ui64 Rsp;
+ ui64 Rbp;
+};
+
+class TFramePointerCursor
+{
+public:
+ TFramePointerCursor(
+ TSafeMemoryReader* memoryReader,
+ const TFramePointerCursorContext& context);
+
+ bool IsFinished() const;
+ const void* GetCurrentIP() const;
+ void MoveNext();
+
+private:
+ TSafeMemoryReader* MemoryReader_;
+ bool Finished_ = false;
+ bool First_ = true;
+
+ const void* Rip_ = nullptr;
+ const void* Rbp_ = nullptr;
+ const void* StartRsp_ = nullptr;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/cursors/frame_pointer/ya.make b/library/cpp/yt/backtrace/cursors/frame_pointer/ya.make
new file mode 100644
index 0000000000..cb85d70315
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/frame_pointer/ya.make
@@ -0,0 +1,9 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+ frame_pointer_cursor.cpp
+)
+
+END()
diff --git a/library/cpp/yt/backtrace/cursors/interop/interop.cpp b/library/cpp/yt/backtrace/cursors/interop/interop.cpp
new file mode 100644
index 0000000000..b4e6cfbe6e
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/interop/interop.cpp
@@ -0,0 +1,102 @@
+#include "interop.h"
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TFramePointerCursorContext FramePointerCursorContextFromUcontext(const ucontext_t& ucontext)
+{
+#if defined(_linux_)
+ return {
+ .Rip = static_cast<ui64>(ucontext.uc_mcontext.gregs[REG_RIP]),
+ .Rsp = static_cast<ui64>(ucontext.uc_mcontext.gregs[REG_RSP]),
+ .Rbp = static_cast<ui64>(ucontext.uc_mcontext.gregs[REG_RBP]),
+ };
+#elif defined(_darwin_)
+ return {
+ .Rip = static_cast<ui64>(ucontext.uc_mcontext->__ss.__rip),
+ .Rsp = static_cast<ui64>(ucontext.uc_mcontext->__ss.__rsp),
+ .Rbp = static_cast<ui64>(ucontext.uc_mcontext->__ss.__rbp),
+ };
+#else
+ #error Unsupported platform
+#endif
+}
+
+std::optional<unw_context_t> TrySynthesizeLibunwindContextFromMachineContext(
+ const TContMachineContext& machineContext)
+{
+ unw_context_t unwindContext;
+ if (unw_getcontext(&unwindContext) != 0) {
+ return {};
+ }
+
+ // Some dirty hacks follow.
+ struct TUnwindContextRegisters
+ {
+ ui64 Rax;
+ ui64 Rbx;
+ ui64 Rcx;
+ ui64 Rdx;
+ ui64 Rdi;
+ ui64 Rsi;
+ ui64 Rbp;
+ ui64 Rsp;
+ ui64 R8;
+ ui64 R9;
+ ui64 R10;
+ ui64 R11;
+ ui64 R12;
+ ui64 R13;
+ ui64 R14;
+ ui64 R15;
+ ui64 Rip;
+ ui64 Rflags;
+ ui64 CS;
+ ui64 FS;
+ ui64 GS;
+ };
+
+ struct TMachineContextRegisters
+ {
+ ui64 Rbx;
+ ui64 Rbp;
+ ui64 R12;
+ ui64 R13;
+ ui64 R14;
+ ui64 R15;
+ ui64 Rsp;
+ ui64 Rip;
+ };
+
+ static_assert(sizeof(TContMachineContext) >= sizeof(TMachineContextRegisters));
+ static_assert(sizeof(unw_context_t) >= sizeof(TUnwindContextRegisters));
+ const auto* machineContextRegisters = reinterpret_cast<const TMachineContextRegisters*>(&machineContext);
+ auto* unwindContextRegisters = reinterpret_cast<TUnwindContextRegisters*>(&unwindContext);
+ #define XX(register) unwindContextRegisters->register = machineContextRegisters->register;
+ XX(Rbx)
+ XX(Rbp)
+ XX(R12)
+ XX(R13)
+ XX(R14)
+ XX(R15)
+ XX(Rsp)
+ XX(Rip)
+ #undef XX
+ return unwindContext;
+}
+
+TFramePointerCursorContext FramePointerCursorContextFromLibunwindCursor(
+ const unw_cursor_t& cursor)
+{
+ TFramePointerCursorContext context{};
+ auto& mutableCursor = const_cast<unw_cursor_t&>(cursor);
+ YT_VERIFY(unw_get_reg(&mutableCursor, UNW_REG_IP, &context.Rip) == 0);
+ YT_VERIFY(unw_get_reg(&mutableCursor, UNW_X86_64_RSP, &context.Rsp) == 0);
+ YT_VERIFY(unw_get_reg(&mutableCursor, UNW_X86_64_RBP, &context.Rbp) == 0);
+ return context;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/cursors/interop/interop.h b/library/cpp/yt/backtrace/cursors/interop/interop.h
new file mode 100644
index 0000000000..62e7177107
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/interop/interop.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h>
+
+#include <contrib/libs/libunwind/include/libunwind.h>
+
+#include <util/system/context.h>
+
+#include <optional>
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TFramePointerCursorContext FramePointerCursorContextFromUcontext(const ucontext_t& ucontext);
+
+std::optional<unw_context_t> TrySynthesizeLibunwindContextFromMachineContext(
+ const TContMachineContext& machineContext);
+
+TFramePointerCursorContext FramePointerCursorContextFromLibunwindCursor(
+ const unw_cursor_t& uwCursor);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/cursors/interop/ya.make b/library/cpp/yt/backtrace/cursors/interop/ya.make
new file mode 100644
index 0000000000..6637f6a9b4
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/interop/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+ interop.cpp
+)
+
+PEERDIR(
+ library/cpp/yt/backtrace/cursors/frame_pointer
+ contrib/libs/libunwind
+)
+
+END()
diff --git a/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp
new file mode 100644
index 0000000000..f814753034
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp
@@ -0,0 +1,70 @@
+#include "libunwind_cursor.h"
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TLibunwindCursor::TLibunwindCursor()
+{
+ if (unw_getcontext(&Context_) != 0) {
+ Finished_ = true;
+ return;
+ }
+
+ Initialize();
+}
+
+TLibunwindCursor::TLibunwindCursor(const unw_context_t& context)
+ : Context_(context)
+{
+ Initialize();
+}
+
+void TLibunwindCursor::Initialize()
+{
+ if (unw_init_local(&Cursor_, &Context_) != 0) {
+ Finished_ = true;
+ return;
+ }
+
+ ReadCurrentIP();
+}
+
+bool TLibunwindCursor::IsFinished() const
+{
+ return Finished_;
+}
+
+const void* TLibunwindCursor::GetCurrentIP() const
+{
+ return CurrentIP_;
+}
+
+void TLibunwindCursor::MoveNext()
+{
+ if (Finished_) {
+ return;
+ }
+
+ if (unw_step(&Cursor_) <= 0) {
+ Finished_ = true;
+ return;
+ }
+
+ ReadCurrentIP();
+}
+
+void TLibunwindCursor::ReadCurrentIP()
+{
+ unw_word_t ip = 0;
+ if (unw_get_reg(&Cursor_, UNW_REG_IP, &ip) < 0) {
+ Finished_ = true;
+ return;
+ }
+
+ CurrentIP_ = reinterpret_cast<const void*>(ip);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h
new file mode 100644
index 0000000000..08b01d07ef
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h
@@ -0,0 +1,33 @@
+#pragma once
+
+#include <contrib/libs/libunwind/include/libunwind.h>
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TLibunwindCursor
+{
+public:
+ TLibunwindCursor();
+ explicit TLibunwindCursor(const unw_context_t& context);
+
+ bool IsFinished() const;
+ const void* GetCurrentIP() const;
+ void MoveNext();
+
+private:
+ unw_context_t Context_;
+ unw_cursor_t Cursor_;
+
+ bool Finished_ = false;
+
+ const void* CurrentIP_ = nullptr;
+
+ void Initialize();
+ void ReadCurrentIP();
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/cursors/libunwind/ya.make b/library/cpp/yt/backtrace/cursors/libunwind/ya.make
new file mode 100644
index 0000000000..8f3a8c5284
--- /dev/null
+++ b/library/cpp/yt/backtrace/cursors/libunwind/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+ libunwind_cursor.cpp
+)
+
+PEERDIR(
+ contrib/libs/libunwind
+)
+
+END()
diff --git a/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp b/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp
new file mode 100644
index 0000000000..19cb41e795
--- /dev/null
+++ b/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp
@@ -0,0 +1,25 @@
+#include <library/cpp/yt/backtrace/backtrace.h>
+
+#include <library/cpp/yt/string/raw_formatter.h>
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void SymbolizeBacktrace(
+ TBacktrace backtrace,
+ const std::function<void(TStringBuf)>& frameCallback)
+{
+ for (int index = 0; index < std::ssize(backtrace); ++index) {
+ TRawFormatter<1024> formatter;
+ formatter.AppendNumber(index + 1, 10, 2);
+ formatter.AppendString(". ");
+ formatter.AppendNumberAsHexWithPadding(reinterpret_cast<uintptr_t>(backtrace[index]), 12);
+ formatter.AppendString("\n");
+ frameCallback(formatter.GetBuffer());
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp b/library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp
new file mode 100644
index 0000000000..f5d02aaa33
--- /dev/null
+++ b/library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp
@@ -0,0 +1,64 @@
+#include <library/cpp/yt/backtrace/backtrace.h>
+
+#include <library/cpp/dwarf_backtrace/backtrace.h>
+
+#include <library/cpp/yt/string/raw_formatter.h>
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void SymbolizeBacktrace(
+ TBacktrace backtrace,
+ const std::function<void(TStringBuf)>& frameCallback)
+{
+ auto error = NDwarf::ResolveBacktrace({backtrace.begin(), backtrace.size()}, [&] (const NDwarf::TLineInfo& info) {
+ TRawFormatter<1024> formatter;
+ formatter.AppendNumber(info.Index + 1, 10, 2);
+ formatter.AppendString(". ");
+ formatter.AppendString("0x");
+ const int width = (sizeof(void*) == 8 ? 12 : 8);
+ // 12 for x86_64 because higher bits are always zeroed.
+ formatter.AppendNumber(info.Address, 16, width, '0');
+ formatter.AppendString(" in ");
+ formatter.AppendString(info.FunctionName);
+ const int bytesToAppendEstimate = 4 + info.FileName.Size() + 1 + 4 /* who cares about line numbers > 9999 */ + 1;
+ if (formatter.GetBytesRemaining() < bytesToAppendEstimate) {
+ const int offset = formatter.GetBytesRemaining() - bytesToAppendEstimate;
+ if (formatter.GetBytesWritten() + offset >= 0) {
+ formatter.Advance(offset);
+ }
+ }
+ formatter.AppendString(" at ");
+ formatter.AppendString(info.FileName);
+ formatter.AppendChar(':');
+ formatter.AppendNumber(info.Line);
+ if (formatter.GetBytesRemaining() == 0) {
+ formatter.Revert(1);
+ }
+ formatter.AppendString("\n");
+ frameCallback(formatter.GetBuffer());
+ // Call the callback exactly `frameCount` times,
+ // even if there are inline functions and one frame resolved to several lines.
+ // It needs for case when caller uses `frameCount` less than 100 for pretty formatting.
+ if (info.Index + 1 == std::ssize(backtrace)) {
+ return NDwarf::EResolving::Break;
+ }
+ return NDwarf::EResolving::Continue;
+ });
+ if (error) {
+ TRawFormatter<1024> formatter;
+ formatter.AppendString("*** Error symbolizing backtrace via Dwarf\n");
+ formatter.AppendString("*** Code: ");
+ formatter.AppendNumber(error->Code);
+ formatter.AppendString("\n");
+ formatter.AppendString("*** Message: ");
+ formatter.AppendString(error->Message);
+ formatter.AppendString("\n");
+ frameCallback(formatter.GetBuffer());
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/symbolizers/dwarf/ya.make b/library/cpp/yt/backtrace/symbolizers/dwarf/ya.make
new file mode 100644
index 0000000000..bffeb676d8
--- /dev/null
+++ b/library/cpp/yt/backtrace/symbolizers/dwarf/ya.make
@@ -0,0 +1,18 @@
+LIBRARY()
+
+SRCS(
+ GLOBAL dwarf_symbolizer.cpp
+)
+
+PEERDIR(
+ library/cpp/dwarf_backtrace
+ library/cpp/yt/backtrace
+)
+
+END()
+
+IF (BUILD_TYPE == "DEBUG" OR BUILD_TYPE == "PROFILE")
+ RECURSE_FOR_TESTS(
+ unittests
+ )
+ENDIF()
diff --git a/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp b/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp
new file mode 100644
index 0000000000..37ebda8e48
--- /dev/null
+++ b/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp
@@ -0,0 +1,113 @@
+#include <library/cpp/yt/backtrace/backtrace.h>
+
+#include <library/cpp/yt/string/raw_formatter.h>
+
+#include <util/system/compiler.h>
+
+#include <dlfcn.h>
+#include <cxxabi.h>
+
+namespace NYT::NBacktrace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace {
+
+int GetSymbolInfo(const void* pc, char* buffer, int length)
+{
+ TBaseFormatter formatter(buffer, length);
+
+ // See http://www.codesourcery.com/cxx-abi/abi.html#mangling
+ // And, yes, dladdr() is not async signal safe. We can substitute it
+ // with hand-written symbolization code from google-glog in case of any trouble.
+ Dl_info info;
+ if (!dladdr(pc, &info)) {
+ return 0;
+ }
+
+ /*
+ * typedef struct {
+ * const char *dli_fname; // Pathname of shared object that
+ * // contains address
+ * void *dli_fbase; // Address at which shared object
+ * // is loaded
+ * const char *dli_sname; // Name of nearest symbol with address
+ * // lower than addr
+ * void *dli_saddr; // Exact address of symbol named
+ * // in dli_sname
+ * } Dl_info;
+ *
+ * If no symbol matching addr could be found, then dli_sname and dli_saddr are set to NULL.
+ */
+
+ if (info.dli_sname && info.dli_saddr) {
+ formatter.AppendString("<");
+ int demangleStatus = 0;
+
+ if (info.dli_sname[0] == '_' && info.dli_sname[1] == 'Z') {
+ // This is also not async signal safe.
+ // But (ta-dah!) we can replace it with symbolization code from google-glob.
+ char* demangledName = abi::__cxa_demangle(info.dli_sname, 0, 0, &demangleStatus);
+ if (demangleStatus == 0) {
+ formatter.AppendString(demangledName);
+ } else {
+ formatter.AppendString(info.dli_sname);
+ }
+ free(demangledName);
+ } else {
+ formatter.AppendString(info.dli_sname);
+ }
+ formatter.AppendString("+");
+ formatter.AppendNumber((char*)pc - (char*)info.dli_saddr);
+ formatter.AppendString(">");
+ formatter.AppendString(" ");
+ }
+
+ if (info.dli_fname && info.dli_fbase) {
+ formatter.AppendString("(");
+ formatter.AppendString(info.dli_fname);
+ formatter.AppendString("+");
+ formatter.AppendNumber((char*)pc - (char*)info.dli_fbase);
+ formatter.AppendString(")");
+ }
+ return formatter.GetBytesWritten();
+}
+
+void DumpStackFrameInfo(TBaseFormatter* formatter, const void* pc)
+{
+ formatter->AppendString("@ ");
+ const int width = (sizeof(void*) == 8 ? 12 : 8) + 2;
+ // +2 for "0x"; 12 for x86_64 because higher bits are always zeroed.
+ formatter->AppendNumberAsHexWithPadding(reinterpret_cast<uintptr_t>(pc), width);
+ formatter->AppendString(" ");
+ // Get the symbol from the previous address of PC,
+ // because PC may be in the next function.
+ formatter->Advance(GetSymbolInfo(
+ reinterpret_cast<const char*>(pc) - 1,
+ formatter->GetCursor(),
+ formatter->GetBytesRemaining()));
+ if (formatter->GetBytesRemaining() == 0) {
+ formatter->Revert(1);
+ }
+ formatter->AppendString("\n");
+}
+
+} // namespace
+
+Y_WEAK void SymbolizeBacktrace(
+ TBacktrace backtrace,
+ const std::function<void(TStringBuf)>& frameCallback)
+{
+ for (int i = 0; i < std::ssize(backtrace); ++i) {
+ TRawFormatter<1024> formatter;
+ formatter.Reset();
+ formatter.AppendNumber(i + 1, 10, 2);
+ formatter.AppendString(". ");
+ DumpStackFrameInfo(&formatter, backtrace[i]);
+ frameCallback(formatter.GetBuffer());
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/unittests/backtrace_ut.cpp b/library/cpp/yt/backtrace/unittests/backtrace_ut.cpp
new file mode 100644
index 0000000000..5992b69277
--- /dev/null
+++ b/library/cpp/yt/backtrace/unittests/backtrace_ut.cpp
@@ -0,0 +1,61 @@
+#include <gtest/gtest.h>
+
+#include <gmock/gmock.h>
+
+#include <library/cpp/yt/memory/safe_memory_reader.h>
+
+#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h>
+
+#include <library/cpp/yt/backtrace/cursors/interop/interop.h>
+
+#include <util/system/compiler.h>
+
+#include <contrib/libs/libunwind/include/libunwind.h>
+
+namespace NYT::NBacktrace {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <int Depth, class TFn>
+Y_NO_INLINE void RunInDeepStack(TFn cb)
+{
+ if constexpr (Depth == 0) {
+ cb();
+ } else {
+ std::vector<int> touchMem;
+ touchMem.push_back(0);
+
+ RunInDeepStack<Depth-1>(cb);
+
+ DoNotOptimizeAway(touchMem);
+ }
+}
+
+TEST(TFramePointerCursor, FramePointerCursor)
+{
+ std::vector<const void*> backtrace;
+ RunInDeepStack<64>([&] {
+ unw_context_t unwContext;
+ ASSERT_TRUE(unw_getcontext(&unwContext) == 0);
+
+ unw_cursor_t unwCursor;
+ ASSERT_TRUE(unw_init_local(&unwCursor, &unwContext) == 0);
+
+ TSafeMemoryReader reader;
+ auto fpCursorContext = NBacktrace::FramePointerCursorContextFromLibunwindCursor(unwCursor);
+ NBacktrace::TFramePointerCursor fpCursor(&reader, fpCursorContext);
+
+ while (!fpCursor.IsFinished()) {
+ backtrace.push_back(fpCursor.GetCurrentIP());
+ fpCursor.MoveNext();
+ }
+ });
+
+ ASSERT_THAT(backtrace, testing::SizeIs(testing::Ge(64u)));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NBacktrace
diff --git a/library/cpp/yt/backtrace/unittests/ya.make b/library/cpp/yt/backtrace/unittests/ya.make
new file mode 100644
index 0000000000..89e55a95ef
--- /dev/null
+++ b/library/cpp/yt/backtrace/unittests/ya.make
@@ -0,0 +1,20 @@
+GTEST()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+PEERDIR(
+ library/cpp/testing/gtest
+ library/cpp/yt/backtrace
+ library/cpp/yt/backtrace/cursors/interop
+ library/cpp/yt/backtrace/cursors/frame_pointer
+ library/cpp/yt/backtrace/cursors/libunwind
+ library/cpp/yt/memory
+)
+
+IF (BUILD_TYPE == "DEBUG" OR BUILD_TYPE == "PROFILE")
+ SRCS(
+ backtrace_ut.cpp
+ )
+ENDIF()
+
+END()
diff --git a/library/cpp/yt/backtrace/ya.make b/library/cpp/yt/backtrace/ya.make
new file mode 100644
index 0000000000..d294082e06
--- /dev/null
+++ b/library/cpp/yt/backtrace/ya.make
@@ -0,0 +1,44 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+ backtrace.cpp
+)
+
+IF (OS_WINDOWS)
+ SRCS(
+ symbolizers/dummy/dummy_symbolizer.cpp
+ )
+ELSE()
+ SRCS(
+ symbolizers/dynload/dynload_symbolizer.cpp
+ )
+ENDIF()
+
+PEERDIR(
+ library/cpp/yt/string
+)
+
+END()
+
+RECURSE(
+ cursors/dummy
+ cursors/frame_pointer
+)
+
+IF (NOT OS_WINDOWS)
+ RECURSE(
+ cursors/libunwind
+ )
+ENDIF()
+
+IF (OS_LINUX)
+ RECURSE(
+ symbolizers/dwarf
+ )
+
+ RECURSE_FOR_TESTS(
+ unittests
+ )
+ENDIF()
diff --git a/library/cpp/yt/containers/sharded_set-inl.h b/library/cpp/yt/containers/sharded_set-inl.h
new file mode 100644
index 0000000000..67d5be58c6
--- /dev/null
+++ b/library/cpp/yt/containers/sharded_set-inl.h
@@ -0,0 +1,217 @@
+#ifndef SHARDED_SET_INL_H_
+#error "Direct inclusion of this file is not allowed, include sharded_set.h"
+// For the sake of sane code completion.
+#include "sharded_set.h"
+#endif
+
+#include <library/cpp/yt/assert/assert.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class T, int N, class F, class S>
+class TShardedSet<T, N, F, S>::const_iterator
+{
+private:
+ friend class TShardedSet<T, N, F, S>;
+
+ using TOwner = TShardedSet<T, N, F, S>;
+ using TShardIterator = typename S::const_iterator;
+
+ const TOwner* const Owner_;
+
+ int ShardIndex_;
+ TShardIterator ShardIterator_;
+
+ const_iterator(
+ const TOwner* owner,
+ int shardIndex,
+ TShardIterator shardIterator)
+ : Owner_(owner)
+ , ShardIndex_(shardIndex)
+ , ShardIterator_(shardIterator)
+ { }
+
+ bool IsValid() const
+ {
+ return ShardIterator_ != Owner_->Shards_[ShardIndex_].end();
+ }
+
+ void FastForward()
+ {
+ while (ShardIndex_ != N - 1 && !IsValid()) {
+ ++ShardIndex_;
+ ShardIterator_ = Owner_->Shards_[ShardIndex_].begin();
+ }
+ }
+
+public:
+ using difference_type = typename std::iterator_traits<TShardIterator>::difference_type;
+ using value_type = typename std::iterator_traits<TShardIterator>::value_type;
+ using pointer = typename std::iterator_traits<TShardIterator>::pointer;
+ using reference = typename std::iterator_traits<TShardIterator>::reference;
+ using iterator_category = std::forward_iterator_tag;
+
+ const_iterator& operator++()
+ {
+ ++ShardIterator_;
+ FastForward();
+
+ return *this;
+ }
+
+ const_iterator operator++(int)
+ {
+ auto result = *this;
+
+ ++ShardIterator_;
+ FastForward();
+
+ return result;
+ }
+
+ bool operator==(const const_iterator& rhs) const
+ {
+ return
+ ShardIndex_ == rhs.ShardIndex_ &&
+ ShardIterator_ == rhs.ShardIterator_;
+ }
+
+ bool operator!=(const const_iterator& rhs) const
+ {
+ return !(*this == rhs);
+ }
+
+ const T& operator*() const
+ {
+ return *ShardIterator_;
+ }
+
+ const T* operator->() const
+ {
+ return &operator*();
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class T, int N, class F, class S>
+TShardedSet<T, N, F, S>::TShardedSet(F elementToShard)
+ : ElementToShard_(elementToShard)
+{ }
+
+template <class T, int N, class F, class S>
+bool TShardedSet<T, N, F, S>::empty() const
+{
+ return size() == 0;
+}
+
+template <class T, int N, class F, class S>
+typename TShardedSet<T, N, F, S>::size_type TShardedSet<T, N, F, S>::size() const
+{
+ size_type result = 0;
+ for (const auto& shard : Shards_) {
+ result += shard.size();
+ }
+
+ return result;
+}
+
+template <class T, int N, class F, class S>
+const T& TShardedSet<T, N, F, S>::front() const
+{
+ return *begin();
+}
+
+template <class T, int N, class F, class S>
+typename TShardedSet<T, N, F, S>::size_type TShardedSet<T, N, F, S>::count(const T& value) const
+{
+ return GetShard(value).count(value);
+}
+
+template <class T, int N, class F, class S>
+bool TShardedSet<T, N, F, S>::contains(const T& value) const
+{
+ return GetShard(value).contains(value);
+}
+
+template <class T, int N, class F, class S>
+std::pair<typename TShardedSet<T, N, F, S>::const_iterator, bool> TShardedSet<T, N, F, S>::insert(const T& value)
+{
+ auto shardIndex = ElementToShard_(value);
+ auto& shard = Shards_[shardIndex];
+ auto [shardIterator, inserted] = shard.insert(value);
+
+ const_iterator iterator(this, shardIndex, shardIterator);
+ return {iterator, inserted};
+}
+
+template <class T, int N, class F, class S>
+bool TShardedSet<T, N, F, S>::erase(const T& value)
+{
+ return GetShard(value).erase(value);
+}
+
+template <class T, int N, class F, class S>
+void TShardedSet<T, N, F, S>::clear()
+{
+ for (auto& shard : Shards_) {
+ shard.clear();
+ }
+}
+
+template <class T, int N, class F, class S>
+typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::begin() const
+{
+ const_iterator iterator(this, /*shardIndex*/ 0, /*shardIterator*/ Shards_[0].begin());
+ iterator.FastForward();
+
+ return iterator;
+}
+
+template <class T, int N, class F, class S>
+typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::cbegin() const
+{
+ return begin();
+}
+
+template <class T, int N, class F, class S>
+typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::end() const
+{
+ return const_iterator(this, /*shardIndex*/ N - 1, /*shardIterator*/ Shards_[N - 1].end());
+}
+
+template <class T, int N, class F, class S>
+typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::cend() const
+{
+ return end();
+}
+
+template <class T, int N, class F, class S>
+const S& TShardedSet<T, N, F, S>::Shard(int shardIndex) const
+{
+ return Shards_[shardIndex];
+}
+
+template <class T, int N, class F, class S>
+S& TShardedSet<T, N, F, S>::MutableShard(int shardIndex)
+{
+ return Shards_[shardIndex];
+}
+
+template <class T, int N, class F, class S>
+S& TShardedSet<T, N, F, S>::GetShard(const T& value)
+{
+ return Shards_[ElementToShard_(value)];
+}
+
+template <class T, int N, class F, class S>
+const S& TShardedSet<T, N, F, S>::GetShard(const T& value) const
+{
+ return Shards_[ElementToShard_(value)];
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yt/containers/sharded_set.h b/library/cpp/yt/containers/sharded_set.h
new file mode 100644
index 0000000000..fa24893aa4
--- /dev/null
+++ b/library/cpp/yt/containers/sharded_set.h
@@ -0,0 +1,69 @@
+#pragma once
+
+#include <util/generic/hash_set.h>
+
+#include <array>
+#include <cstddef>
+#include <utility>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! A set that stores elements divided into fixed amount of shards.
+//! Provides access to whole set and particular shards.
+//! The interface is pretty minimalistic, feel free to extend it when needed.
+template <class T, int N, class F, class S = THashSet<T>>
+class TShardedSet
+{
+public:
+ using size_type = size_t;
+ using difference_type = ptrdiff_t;
+
+ using value_type = T;
+
+ class const_iterator;
+
+ explicit TShardedSet(F elementToShard = F());
+
+ [[nodiscard]] bool empty() const;
+
+ size_type size() const;
+
+ const T& front() const;
+
+ size_type count(const T& value) const;
+
+ bool contains(const T& value) const;
+
+ std::pair<const_iterator, bool> insert(const T& value);
+
+ bool erase(const T& value);
+
+ void clear();
+
+ const_iterator begin() const;
+ const_iterator cbegin() const;
+
+ const_iterator end() const;
+ const_iterator cend() const;
+
+ const S& Shard(int shardIndex) const;
+ S& MutableShard(int shardIndex);
+
+private:
+ std::array<S, N> Shards_;
+
+ const F ElementToShard_;
+
+ S& GetShard(const T& value);
+ const S& GetShard(const T& value) const;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
+
+#define SHARDED_SET_INL_H_
+#include "sharded_set-inl.h"
+#undef SHARDED_SET_INL_H_
diff --git a/library/cpp/yt/containers/unittests/sharded_set_ut.cpp b/library/cpp/yt/containers/unittests/sharded_set_ut.cpp
new file mode 100644
index 0000000000..2c4f8c5935
--- /dev/null
+++ b/library/cpp/yt/containers/unittests/sharded_set_ut.cpp
@@ -0,0 +1,121 @@
+#include <library/cpp/yt/containers/sharded_set.h>
+
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <random>
+
+namespace NYT {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TIntToShard
+{
+ int operator()(int value) const
+ {
+ return value % 16;
+ }
+};
+
+using TSet = TShardedSet<int, 16, TIntToShard>;
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(CompactSetTest, Insert)
+{
+ TSet set;
+
+ for (int i = 0; i < 4; i++) {
+ set.insert(i);
+ }
+
+ for (int i = 0; i < 4; i++) {
+ set.insert(i);
+ }
+
+ EXPECT_EQ(4u, set.size());
+
+ for (int i = 0; i < 4; i++)
+ EXPECT_EQ(1u, set.count(i));
+
+ EXPECT_EQ(0u, set.count(4));
+}
+
+TEST(CompactSetTest, Erase)
+{
+ TSet set;
+
+ for (int i = 0; i < 8; i++) {
+ set.insert(i);
+ }
+
+ EXPECT_EQ(8u, set.size());
+
+ // Remove elements one by one and check if all other elements are still there.
+ for (int i = 0; i < 8; i++) {
+ EXPECT_EQ(1u, set.count(i));
+ EXPECT_TRUE(set.erase(i));
+ EXPECT_EQ(0u, set.count(i));
+ EXPECT_EQ(8u - i - 1, set.size());
+ for (int j = i + 1; j < 8; j++) {
+ EXPECT_EQ(1u, set.count(j));
+ }
+ }
+
+ EXPECT_EQ(0u, set.count(8));
+}
+
+TEST(CompactSetTest, StressTest)
+{
+ TSet set;
+
+ constexpr int Iterations = 1'000'000;
+ constexpr int Values = 128;
+
+ THashSet<int> values;
+
+ auto checkEverything = [&] {
+ EXPECT_EQ(values.size(), set.size());
+ EXPECT_EQ(values.empty(), set.empty());
+ EXPECT_EQ(values, THashSet<int>(set.begin(), set.end()));
+
+ std::array<THashSet<int>, 16> shards;
+ for (int value : values) {
+ shards[value % 16].insert(value);
+ }
+ for (int shardIndex = 0; shardIndex < 16; ++shardIndex) {
+ EXPECT_EQ(shards[shardIndex], set.Shard(shardIndex));
+ }
+
+ for (int value = 0; value < Values; ++value) {
+ EXPECT_EQ(values.contains(value), set.contains(value));
+ EXPECT_EQ(values.count(value), set.count(value));
+ }
+ };
+
+ std::mt19937_64 rng(42);
+
+ for (int iteration = 0; iteration < Iterations; ++iteration) {
+ if (rng() % 100 == 0) {
+ set.clear();
+ values.clear();
+ checkEverything();
+ }
+
+ int value = rng() % Values;
+ if (rng() % 2 == 0) {
+ set.insert(value);
+ values.insert(value);
+ } else {
+ set.erase(value);
+ values.erase(value);
+ }
+
+ checkEverything();
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT
diff --git a/library/cpp/yt/containers/unittests/ya.make b/library/cpp/yt/containers/unittests/ya.make
new file mode 100644
index 0000000000..3e7cfd4311
--- /dev/null
+++ b/library/cpp/yt/containers/unittests/ya.make
@@ -0,0 +1,15 @@
+GTEST(unittester-containers)
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+ sharded_set_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/yt/containers
+
+ library/cpp/testing/gtest
+)
+
+END()
diff --git a/library/cpp/yt/cpu_clock/benchmark/benchmark.cpp b/library/cpp/yt/cpu_clock/benchmark/benchmark.cpp
new file mode 100644
index 0000000000..9d300b6726
--- /dev/null
+++ b/library/cpp/yt/cpu_clock/benchmark/benchmark.cpp
@@ -0,0 +1,41 @@
+#include "benchmark/benchmark.h"
+#include <benchmark/benchmark.h>
+
+#include <library/cpp/yt/cpu_clock/clock.h>
+
+namespace NYT {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void BM_GetCpuInstant(benchmark::State& state)
+{
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(GetCpuInstant());
+ }
+}
+
+BENCHMARK(BM_GetCpuInstant);
+
+void BM_GetCpuApproximateInstant(benchmark::State& state)
+{
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(GetApproximateCpuInstant());
+ }
+}
+
+BENCHMARK(BM_GetCpuApproximateInstant);
+
+void BM_InstantNow(benchmark::State& state)
+{
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(TInstant::Now());
+ }
+}
+
+BENCHMARK(BM_InstantNow);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT
diff --git a/library/cpp/yt/cpu_clock/benchmark/ya.make b/library/cpp/yt/cpu_clock/benchmark/ya.make
new file mode 100644
index 0000000000..4550bf5934
--- /dev/null
+++ b/library/cpp/yt/cpu_clock/benchmark/ya.make
@@ -0,0 +1,11 @@
+G_BENCHMARK()
+
+SRCS(
+ benchmark.cpp
+)
+
+PEERDIR(
+ library/cpp/yt/cpu_clock
+)
+
+END()
diff --git a/library/cpp/yt/cpu_clock/unittests/clock_ut.cpp b/library/cpp/yt/cpu_clock/unittests/clock_ut.cpp
new file mode 100644
index 0000000000..bd9cb6d4be
--- /dev/null
+++ b/library/cpp/yt/cpu_clock/unittests/clock_ut.cpp
@@ -0,0 +1,46 @@
+#include <gtest/gtest.h>
+
+#include <library/cpp/yt/cpu_clock/clock.h>
+
+namespace NYT {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class T>
+i64 DiffMS(T a, T b)
+{
+ return a >= b
+ ? static_cast<i64>(a.MilliSeconds()) - static_cast<i64>(b.MilliSeconds())
+ : DiffMS(b, a);
+}
+
+TEST(TTimingTest, GetInstant)
+{
+ GetInstant();
+
+ EXPECT_LE(DiffMS(GetInstant(), TInstant::Now()), 10);
+}
+
+TEST(TTimingTest, InstantVSCpuInstant)
+{
+ auto instant1 = TInstant::Now();
+ auto cpuInstant = InstantToCpuInstant(instant1);
+ auto instant2 = CpuInstantToInstant(cpuInstant);
+ EXPECT_LE(DiffMS(instant1, instant2), 10);
+}
+
+TEST(TTimingTest, DurationVSCpuDuration)
+{
+ auto cpuInstant1 = GetCpuInstant();
+ constexpr auto duration1 = TDuration::MilliSeconds(100);
+ Sleep(duration1);
+ auto cpuInstant2 = GetCpuInstant();
+ auto duration2 = CpuDurationToDuration(cpuInstant2 - cpuInstant1);
+ EXPECT_LE(DiffMS(duration1, duration2), 10);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT
diff --git a/library/cpp/yt/cpu_clock/unittests/ya.make b/library/cpp/yt/cpu_clock/unittests/ya.make
new file mode 100644
index 0000000000..921087c295
--- /dev/null
+++ b/library/cpp/yt/cpu_clock/unittests/ya.make
@@ -0,0 +1,13 @@
+GTEST()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+ clock_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/yt/cpu_clock
+)
+
+END()
diff --git a/library/cpp/yt/farmhash/farm_hash.h b/library/cpp/yt/farmhash/farm_hash.h
new file mode 100644
index 0000000000..fe4c8193a0
--- /dev/null
+++ b/library/cpp/yt/farmhash/farm_hash.h
@@ -0,0 +1,63 @@
+#pragma once
+
+#include <contrib/libs/farmhash/farmhash.h>
+
+#include <util/system/types.h>
+
+#include <util/generic/strbuf.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+using TFingerprint = ui64;
+
+static inline TFingerprint FarmHash(ui64 value)
+{
+ return ::util::Fingerprint(value);
+}
+
+static inline TFingerprint FarmHash(const void* buf, size_t len)
+{
+ return ::util::Hash64(static_cast<const char*>(buf), len);
+}
+
+static inline TFingerprint FarmHash(const void* buf, size_t len, ui64 seed)
+{
+ return ::util::Hash64WithSeed(static_cast<const char*>(buf), len, seed);
+}
+
+static inline TFingerprint FarmFingerprint(ui64 value)
+{
+ return ::util::Fingerprint(value);
+}
+
+static inline TFingerprint FarmFingerprint(const void* buf, size_t len)
+{
+ return ::util::Fingerprint64(static_cast<const char*>(buf), len);
+}
+
+static inline TFingerprint FarmFingerprint(TStringBuf buf)
+{
+ return FarmFingerprint(buf.Data(), buf.Size());
+}
+
+static inline TFingerprint FarmFingerprint(ui64 first, ui64 second)
+{
+ return ::util::Fingerprint(::util::Uint128(first, second));
+}
+
+// Forever-fixed Google FarmHash fingerprint.
+template <class T>
+TFingerprint FarmFingerprint(const T* begin, const T* end)
+{
+ ui64 result = 0xdeadc0de;
+ for (const auto* value = begin; value < end; ++value) {
+ result = FarmFingerprint(result, FarmFingerprint(*value));
+ }
+ return result ^ (end - begin);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h
new file mode 100644
index 0000000000..6f489da82d
--- /dev/null
+++ b/library/cpp/yt/logging/logger-inl.h
@@ -0,0 +1,303 @@
+#ifndef LOGGER_INL_H_
+#error "Direct inclusion of this file is not allowed, include logger.h"
+// For the sake of sane code completion.
+#include "logger.h"
+#endif
+#undef LOGGER_INL_H_
+
+#include <library/cpp/yt/yson_string/convert.h>
+#include <library/cpp/yt/yson_string/string.h>
+
+namespace NYT::NLogging {
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline bool TLogger::IsAnchorUpToDate(const TLoggingAnchor& position) const
+{
+ return
+ !Category_ ||
+ position.CurrentVersion == Category_->ActualVersion->load(std::memory_order::relaxed);
+}
+
+template <class... TArgs>
+void TLogger::AddTag(const char* format, TArgs&&... args)
+{
+ AddRawTag(Format(format, std::forward<TArgs>(args)...));
+}
+
+template <class TType>
+void TLogger::AddStructuredTag(TStringBuf key, TType value)
+{
+ StructuredTags_.emplace_back(key, NYson::ConvertToYsonString(value));
+}
+
+template <class... TArgs>
+TLogger TLogger::WithTag(const char* format, TArgs&&... args) const
+{
+ auto result = *this;
+ result.AddTag(format, std::forward<TArgs>(args)...);
+ return result;
+}
+
+template <class TType>
+TLogger TLogger::WithStructuredTag(TStringBuf key, TType value) const
+{
+ auto result = *this;
+ result.AddStructuredTag(key, value);
+ return result;
+}
+
+Y_FORCE_INLINE bool TLogger::IsLevelEnabled(ELogLevel level) const
+{
+ // This is the first check which is intended to be inlined next to
+ // logging invocation point. Check below is almost zero-cost due
+ // to branch prediction (which requires inlining for proper work).
+ if (level < MinLevel_) {
+ return false;
+ }
+
+ // Next check is heavier and requires full log manager definition which
+ // is undesirable in -inl.h header file. This is why we extract it
+ // to a separate method which is implemented in cpp file.
+ return IsLevelEnabledHeavy(level);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace NDetail {
+
+struct TMessageStringBuilderContext
+{
+ TSharedMutableRef Chunk;
+};
+
+struct TMessageBufferTag
+{ };
+
+class TMessageStringBuilder
+ : public TStringBuilderBase
+{
+public:
+ TSharedRef Flush();
+
+ // For testing only.
+ static void DisablePerThreadCache();
+
+protected:
+ void DoReset() override;
+ void DoReserve(size_t newLength) override;
+
+private:
+ struct TPerThreadCache
+ {
+ ~TPerThreadCache();
+
+ TSharedMutableRef Chunk;
+ size_t ChunkOffset = 0;
+ };
+
+ TSharedMutableRef Buffer_;
+
+ static thread_local TPerThreadCache* Cache_;
+ static thread_local bool CacheDestroyed_;
+ static TPerThreadCache* GetCache();
+
+ static constexpr size_t ChunkSize = 128_KB - 64;
+};
+
+inline bool HasMessageTags(
+ const TLoggingContext& loggingContext,
+ const TLogger& logger)
+{
+ if (logger.GetTag()) {
+ return true;
+ }
+ if (loggingContext.TraceLoggingTag) {
+ return true;
+ }
+ return false;
+}
+
+inline void AppendMessageTags(
+ TStringBuilderBase* builder,
+ const TLoggingContext& loggingContext,
+ const TLogger& logger)
+{
+ bool printComma = false;
+ if (const auto& loggerTag = logger.GetTag()) {
+ builder->AppendString(loggerTag);
+ printComma = true;
+ }
+ if (auto traceLoggingTag = loggingContext.TraceLoggingTag) {
+ if (printComma) {
+ builder->AppendString(TStringBuf(", "));
+ }
+ builder->AppendString(traceLoggingTag);
+ }
+}
+
+inline void AppendLogMessage(
+ TStringBuilderBase* builder,
+ const TLoggingContext& loggingContext,
+ const TLogger& logger,
+ TRef message)
+{
+ if (HasMessageTags(loggingContext, logger)) {
+ if (message.Size() >= 1 && message[message.Size() - 1] == ')') {
+ builder->AppendString(TStringBuf(message.Begin(), message.Size() - 1));
+ builder->AppendString(TStringBuf(", "));
+ } else {
+ builder->AppendString(TStringBuf(message.Begin(), message.Size()));
+ builder->AppendString(TStringBuf(" ("));
+ }
+ AppendMessageTags(builder, loggingContext, logger);
+ builder->AppendChar(')');
+ } else {
+ builder->AppendString(TStringBuf(message.Begin(), message.Size()));
+ }
+}
+
+template <class... TArgs>
+void AppendLogMessageWithFormat(
+ TStringBuilderBase* builder,
+ const TLoggingContext& loggingContext,
+ const TLogger& logger,
+ TStringBuf format,
+ TArgs&&... args)
+{
+ if (HasMessageTags(loggingContext, logger)) {
+ if (format.size() >= 2 && format[format.size() - 1] == ')') {
+ builder->AppendFormat(format.substr(0, format.size() - 1), std::forward<TArgs>(args)...);
+ builder->AppendString(TStringBuf(", "));
+ } else {
+ builder->AppendFormat(format, std::forward<TArgs>(args)...);
+ builder->AppendString(TStringBuf(" ("));
+ }
+ AppendMessageTags(builder, loggingContext, logger);
+ builder->AppendChar(')');
+ } else {
+ builder->AppendFormat(format, std::forward<TArgs>(args)...);
+ }
+}
+
+struct TLogMessage
+{
+ TSharedRef MessageRef;
+ TStringBuf Anchor;
+};
+
+template <size_t Length, class... TArgs>
+TLogMessage BuildLogMessage(
+ const TLoggingContext& loggingContext,
+ const TLogger& logger,
+ const char (&format)[Length],
+ TArgs&&... args)
+{
+ TMessageStringBuilder builder;
+ AppendLogMessageWithFormat(&builder, loggingContext, logger, format, std::forward<TArgs>(args)...);
+ return {builder.Flush(), format};
+}
+
+template <class T>
+TLogMessage BuildLogMessage(
+ const TLoggingContext& loggingContext,
+ const TLogger& logger,
+ const T& obj)
+{
+ TMessageStringBuilder builder;
+ FormatValue(&builder, obj, TStringBuf());
+ if (HasMessageTags(loggingContext, logger)) {
+ builder.AppendString(TStringBuf(" ("));
+ AppendMessageTags(&builder, loggingContext, logger);
+ builder.AppendChar(')');
+ }
+ return {builder.Flush(), TStringBuf()};
+}
+
+inline TLogMessage BuildLogMessage(
+ const TLoggingContext& loggingContext,
+ const TLogger& logger,
+ TStringBuf message)
+{
+ TMessageStringBuilder builder;
+ builder.AppendString(message);
+ if (HasMessageTags(loggingContext, logger)) {
+ builder.AppendString(TStringBuf(" ("));
+ AppendMessageTags(&builder, loggingContext, logger);
+ builder.AppendChar(')');
+ }
+ return {builder.Flush(), message};
+}
+
+template <size_t Length>
+TLogMessage BuildLogMessage(
+ const TLoggingContext& loggingContext,
+ const TLogger& logger,
+ const char (&message)[Length])
+{
+ return BuildLogMessage(
+ loggingContext,
+ logger,
+ TStringBuf(message));
+}
+
+inline TLogMessage BuildLogMessage(
+ const TLoggingContext& loggingContext,
+ const TLogger& logger,
+ TSharedRef&& message)
+{
+ if (HasMessageTags(loggingContext, logger)) {
+ TMessageStringBuilder builder;
+ AppendLogMessage(&builder, loggingContext, logger, message);
+ return {builder.Flush(), TStringBuf()};
+ } else {
+ return {std::move(message), TStringBuf()};
+ }
+}
+
+inline TLogEvent CreateLogEvent(
+ const TLoggingContext& loggingContext,
+ const TLogger& logger,
+ ELogLevel level)
+{
+ TLogEvent event;
+ event.Instant = loggingContext.Instant;
+ event.Category = logger.GetCategory();
+ event.Essential = logger.IsEssential();
+ event.Level = level;
+ event.ThreadId = loggingContext.ThreadId;
+ event.ThreadName = loggingContext.ThreadName;
+ event.FiberId = loggingContext.FiberId;
+ event.TraceId = loggingContext.TraceId;
+ event.RequestId = loggingContext.RequestId;
+ return event;
+}
+
+void OnCriticalLogEvent(
+ const TLogger& logger,
+ const TLogEvent& event);
+
+inline void LogEventImpl(
+ const TLoggingContext& loggingContext,
+ const TLogger& logger,
+ ELogLevel level,
+ ::TSourceLocation sourceLocation,
+ TSharedRef message)
+{
+ auto event = CreateLogEvent(loggingContext, logger, level);
+ event.MessageKind = ELogMessageKind::Unstructured;
+ event.MessageRef = std::move(message);
+ event.Family = ELogFamily::PlainText;
+ event.SourceFile = sourceLocation.File;
+ event.SourceLine = sourceLocation.Line;
+ logger.Write(std::move(event));
+ if (Y_UNLIKELY(event.Level >= ELogLevel::Alert)) {
+ OnCriticalLogEvent(logger, event);
+ }
+}
+
+} // namespace NDetail
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NLogging
diff --git a/library/cpp/yt/logging/logger.cpp b/library/cpp/yt/logging/logger.cpp
new file mode 100644
index 0000000000..4ee5c1a01b
--- /dev/null
+++ b/library/cpp/yt/logging/logger.cpp
@@ -0,0 +1,289 @@
+#include "logger.h"
+
+#include <library/cpp/yt/assert/assert.h>
+
+#include <library/cpp/yt/cpu_clock/clock.h>
+
+#include <library/cpp/yt/misc/thread_name.h>
+
+#include <util/system/compiler.h>
+#include <util/system/thread.h>
+
+namespace NYT::NLogging {
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace NDetail {
+
+void OnCriticalLogEvent(
+ const TLogger& logger,
+ const TLogEvent& event)
+{
+ if (event.Level == ELogLevel::Fatal ||
+ event.Level == ELogLevel::Alert && logger.GetAbortOnAlert())
+ {
+ fprintf(stderr, "*** Aborting on critical log event\n");
+ fwrite(event.MessageRef.begin(), 1, event.MessageRef.size(), stderr);
+ fprintf(stderr, "\n");
+ YT_ABORT();
+ }
+}
+
+TSharedRef TMessageStringBuilder::Flush()
+{
+ return Buffer_.Slice(0, GetLength());
+}
+
+void TMessageStringBuilder::DisablePerThreadCache()
+{
+ Cache_ = nullptr;
+ CacheDestroyed_ = true;
+}
+
+void TMessageStringBuilder::DoReset()
+{
+ Buffer_.Reset();
+}
+
+void TMessageStringBuilder::DoReserve(size_t newCapacity)
+{
+ auto oldLength = GetLength();
+ newCapacity = FastClp2(newCapacity);
+
+ auto newChunkSize = std::max(ChunkSize, newCapacity);
+ // Hold the old buffer until the data is copied.
+ auto oldBuffer = std::move(Buffer_);
+ auto* cache = GetCache();
+ if (Y_LIKELY(cache)) {
+ auto oldCapacity = End_ - Begin_;
+ auto deltaCapacity = newCapacity - oldCapacity;
+ if (End_ == cache->Chunk.Begin() + cache->ChunkOffset &&
+ cache->ChunkOffset + deltaCapacity <= cache->Chunk.Size())
+ {
+ // Resize inplace.
+ Buffer_ = cache->Chunk.Slice(cache->ChunkOffset - oldCapacity, cache->ChunkOffset + deltaCapacity);
+ cache->ChunkOffset += deltaCapacity;
+ End_ = Begin_ + newCapacity;
+ return;
+ }
+
+ if (Y_UNLIKELY(cache->ChunkOffset + newCapacity > cache->Chunk.Size())) {
+ cache->Chunk = TSharedMutableRef::Allocate<TMessageBufferTag>(newChunkSize, {.InitializeStorage = false});
+ cache->ChunkOffset = 0;
+ }
+
+ Buffer_ = cache->Chunk.Slice(cache->ChunkOffset, cache->ChunkOffset + newCapacity);
+ cache->ChunkOffset += newCapacity;
+ } else {
+ Buffer_ = TSharedMutableRef::Allocate<TMessageBufferTag>(newChunkSize, {.InitializeStorage = false});
+ newCapacity = newChunkSize;
+ }
+ if (oldLength > 0) {
+ ::memcpy(Buffer_.Begin(), Begin_, oldLength);
+ }
+ Begin_ = Buffer_.Begin();
+ End_ = Begin_ + newCapacity;
+}
+
+TMessageStringBuilder::TPerThreadCache* TMessageStringBuilder::GetCache()
+{
+ if (Y_LIKELY(Cache_)) {
+ return Cache_;
+ }
+ if (CacheDestroyed_) {
+ return nullptr;
+ }
+ static thread_local TPerThreadCache Cache;
+ Cache_ = &Cache;
+ return Cache_;
+}
+
+TMessageStringBuilder::TPerThreadCache::~TPerThreadCache()
+{
+ TMessageStringBuilder::DisablePerThreadCache();
+}
+
+thread_local TMessageStringBuilder::TPerThreadCache* TMessageStringBuilder::Cache_;
+thread_local bool TMessageStringBuilder::CacheDestroyed_;
+
+} // namespace NDetail
+
+////////////////////////////////////////////////////////////////////////////////
+
+Y_WEAK TLoggingContext GetLoggingContext()
+{
+ return {
+ .Instant = GetCpuInstant(),
+ .ThreadId = TThread::CurrentThreadId(),
+ .ThreadName = GetCurrentThreadName(),
+ };
+}
+
+Y_WEAK ILogManager* GetDefaultLogManager()
+{
+ return nullptr;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+thread_local ELogLevel ThreadMinLogLevel = ELogLevel::Minimum;
+
+void SetThreadMinLogLevel(ELogLevel minLogLevel)
+{
+ ThreadMinLogLevel = minLogLevel;
+}
+
+ELogLevel GetThreadMinLogLevel()
+{
+ return ThreadMinLogLevel;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TLogger::TLogger(ILogManager* logManager, TStringBuf categoryName)
+ : LogManager_(logManager)
+ , Category_(LogManager_ ? LogManager_->GetCategory(categoryName) : nullptr)
+ , MinLevel_(LogManager_ ? LoggerDefaultMinLevel : NullLoggerMinLevel)
+{ }
+
+TLogger::TLogger(TStringBuf categoryName)
+ : TLogger(GetDefaultLogManager(), categoryName)
+{ }
+
+TLogger::operator bool() const
+{
+ return LogManager_;
+}
+
+const TLoggingCategory* TLogger::GetCategory() const
+{
+ return Category_;
+}
+
+bool TLogger::IsLevelEnabledHeavy(ELogLevel level) const
+{
+ // Note that we managed to reach this point, i.e. level >= MinLevel_,
+ // which implies that MinLevel_ != ELogLevel::Maximum, so this logger was not
+ // default constructed, thus it has non-trivial category.
+ YT_ASSERT(Category_);
+
+ if (Category_->CurrentVersion != Category_->ActualVersion->load(std::memory_order::relaxed)) {
+ LogManager_->UpdateCategory(const_cast<TLoggingCategory*>(Category_));
+ }
+
+ return
+ level >= Category_->MinPlainTextLevel &&
+ level >= ThreadMinLogLevel;
+}
+
+bool TLogger::GetAbortOnAlert() const
+{
+ return LogManager_->GetAbortOnAlert();
+}
+
+bool TLogger::IsEssential() const
+{
+ return Essential_;
+}
+
+void TLogger::UpdateAnchor(TLoggingAnchor* anchor) const
+{
+ LogManager_->UpdateAnchor(anchor);
+}
+
+void TLogger::RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const
+{
+ LogManager_->RegisterStaticAnchor(anchor, sourceLocation, message);
+}
+
+void TLogger::Write(TLogEvent&& event) const
+{
+ LogManager_->Enqueue(std::move(event));
+}
+
+void TLogger::AddRawTag(const TString& tag)
+{
+ if (!Tag_.empty()) {
+ Tag_ += ", ";
+ }
+ Tag_ += tag;
+}
+
+TLogger TLogger::WithRawTag(const TString& tag) const
+{
+ auto result = *this;
+ result.AddRawTag(tag);
+ return result;
+}
+
+TLogger TLogger::WithEssential(bool essential) const
+{
+ auto result = *this;
+ result.Essential_ = essential;
+ return result;
+}
+
+TLogger TLogger::WithStructuredValidator(TStructuredValidator validator) const
+{
+ auto result = *this;
+ result.StructuredValidators_.push_back(std::move(validator));
+ return result;
+}
+
+TLogger TLogger::WithMinLevel(ELogLevel minLevel) const
+{
+ auto result = *this;
+ if (result) {
+ result.MinLevel_ = minLevel;
+ }
+ return result;
+}
+
+const TString& TLogger::GetTag() const
+{
+ return Tag_;
+}
+
+const TLogger::TStructuredTags& TLogger::GetStructuredTags() const
+{
+ return StructuredTags_;
+}
+
+const TLogger::TStructuredValidators& TLogger::GetStructuredValidators() const
+{
+ return StructuredValidators_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void LogStructuredEvent(
+ const TLogger& logger,
+ NYson::TYsonString message,
+ ELogLevel level)
+{
+ YT_VERIFY(message.GetType() == NYson::EYsonType::MapFragment);
+
+ if (!logger.GetStructuredValidators().empty()) {
+ auto samplingRate = logger.GetCategory()->StructuredValidationSamplingRate.load();
+ auto p = RandomNumber<double>();
+ if (p < samplingRate) {
+ for (const auto& validator : logger.GetStructuredValidators()) {
+ validator(message);
+ }
+ }
+ }
+
+ auto loggingContext = GetLoggingContext();
+ auto event = NDetail::CreateLogEvent(
+ loggingContext,
+ logger,
+ level);
+ event.MessageKind = ELogMessageKind::Structured;
+ event.MessageRef = message.ToSharedRef();
+ event.Family = ELogFamily::Structured;
+ logger.Write(std::move(event));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NLogging
diff --git a/library/cpp/yt/logging/logger.h b/library/cpp/yt/logging/logger.h
new file mode 100644
index 0000000000..cdb5584d29
--- /dev/null
+++ b/library/cpp/yt/logging/logger.h
@@ -0,0 +1,351 @@
+#pragma once
+
+#include "public.h"
+
+#include <library/cpp/yt/string/format.h>
+
+#include <library/cpp/yt/memory/ref.h>
+
+#include <library/cpp/yt/cpu_clock/public.h>
+
+#include <library/cpp/yt/yson_string/string.h>
+
+#include <library/cpp/yt/misc/guid.h>
+
+#include <library/cpp/yt/misc/thread_name.h>
+
+#include <library/cpp/yt/memory/leaky_singleton.h>
+
+#include <util/system/src_location.h>
+
+#include <util/generic/size_literals.h>
+
+#include <atomic>
+
+namespace NYT::NLogging {
+
+////////////////////////////////////////////////////////////////////////////////
+
+constexpr double DefaultStructuredValidationSamplingRate = 0.01;
+
+struct TLoggingCategory
+{
+ TString Name;
+ //! This value is used for early dropping of plaintext events in order
+ //! to reduce load on logging thread for events which are definitely going
+ //! to be dropped due to rule setup.
+ //! NB: this optimization is used only for plaintext events since structured
+ //! logging rate is negligible comparing to the plaintext logging rate.
+ std::atomic<ELogLevel> MinPlainTextLevel;
+ std::atomic<int> CurrentVersion;
+ std::atomic<int>* ActualVersion;
+ std::atomic<double> StructuredValidationSamplingRate = DefaultStructuredValidationSamplingRate;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TLoggingAnchor
+{
+ std::atomic<bool> Registered = false;
+ ::TSourceLocation SourceLocation = {TStringBuf{}, 0};
+ TString AnchorMessage;
+ TLoggingAnchor* NextAnchor = nullptr;
+
+ std::atomic<int> CurrentVersion = 0;
+ std::atomic<bool> Enabled = false;
+
+ struct TCounter
+ {
+ std::atomic<i64> Current = 0;
+ i64 Previous = 0;
+ };
+
+ TCounter MessageCounter;
+ TCounter ByteCounter;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Declare some type aliases to avoid circular dependencies.
+using TThreadId = size_t;
+using TFiberId = size_t;
+using TTraceId = TGuid;
+using TRequestId = TGuid;
+
+////////////////////////////////////////////////////////////////////////////////
+
+DEFINE_ENUM(ELogMessageKind,
+ (Unstructured)
+ (Structured)
+);
+
+struct TLogEvent
+{
+ const TLoggingCategory* Category = nullptr;
+ ELogLevel Level = ELogLevel::Minimum;
+ ELogFamily Family = ELogFamily::PlainText;
+ bool Essential = false;
+
+ ELogMessageKind MessageKind = ELogMessageKind::Unstructured;
+ TSharedRef MessageRef;
+
+ TCpuInstant Instant = 0;
+
+ TThreadId ThreadId = {};
+ TThreadName ThreadName = {};
+
+ TFiberId FiberId = {};
+
+ TTraceId TraceId;
+ TRequestId RequestId;
+
+ TStringBuf SourceFile;
+ int SourceLine = -1;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct ILogManager
+{
+ virtual ~ILogManager() = default;
+
+ virtual void RegisterStaticAnchor(
+ TLoggingAnchor* position,
+ ::TSourceLocation sourceLocation,
+ TStringBuf anchorMessage) = 0;
+ virtual void UpdateAnchor(TLoggingAnchor* position) = 0;
+
+ virtual void Enqueue(TLogEvent&& event) = 0;
+
+ virtual const TLoggingCategory* GetCategory(TStringBuf categoryName) = 0;
+ virtual void UpdateCategory(TLoggingCategory* category) = 0;
+
+ virtual bool GetAbortOnAlert() const = 0;
+};
+
+ILogManager* GetDefaultLogManager();
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TLoggingContext
+{
+ TCpuInstant Instant;
+ TThreadId ThreadId;
+ TThreadName ThreadName;
+ TFiberId FiberId;
+ TTraceId TraceId;
+ TRequestId RequestId;
+ TStringBuf TraceLoggingTag;
+};
+
+TLoggingContext GetLoggingContext();
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Sets the minimum logging level for messages in current thread.
+// NB: In fiber environment, min log level is attached to a fiber,
+// so after context switch thread min log level might change.
+void SetThreadMinLogLevel(ELogLevel minLogLevel);
+ELogLevel GetThreadMinLogLevel();
+
+////////////////////////////////////////////////////////////////////////////////
+
+static constexpr auto NullLoggerMinLevel = ELogLevel::Maximum;
+
+// Min level for non-null logger depends on whether we are in debug or release build.
+// - For release mode default behavior is to omit trace logging,
+// this is done by setting logger min level to Debug by default.
+// - For debug mode logger min level is set to trace by default, so that trace logging is
+// allowed by logger, but still may be discarded by category min level.
+#ifdef NDEBUG
+static constexpr auto LoggerDefaultMinLevel = ELogLevel::Debug;
+#else
+static constexpr auto LoggerDefaultMinLevel = ELogLevel::Trace;
+#endif
+
+class TLogger
+{
+public:
+ using TStructuredValidator = std::function<void(const NYson::TYsonString&)>;
+ using TStructuredValidators = std::vector<TStructuredValidator>;
+
+ using TStructuredTag = std::pair<TString, NYson::TYsonString>;
+ // TODO(max42): switch to TCompactVector after YT-15430.
+ using TStructuredTags = std::vector<TStructuredTag>;
+
+ TLogger() = default;
+ TLogger(const TLogger& other) = default;
+ TLogger& operator=(const TLogger& other) = default;
+
+ TLogger(ILogManager* logManager, TStringBuf categoryName);
+ explicit TLogger(TStringBuf categoryName);
+
+ explicit operator bool() const;
+
+ const TLoggingCategory* GetCategory() const;
+
+ //! Validate that level is admitted by logger's own min level
+ //! and by category's min level.
+ bool IsLevelEnabled(ELogLevel level) const;
+
+ bool GetAbortOnAlert() const;
+
+ bool IsEssential() const;
+
+ bool IsAnchorUpToDate(const TLoggingAnchor& anchor) const;
+ void UpdateAnchor(TLoggingAnchor* anchor) const;
+ void RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const;
+
+ void Write(TLogEvent&& event) const;
+
+ void AddRawTag(const TString& tag);
+ template <class... TArgs>
+ void AddTag(const char* format, TArgs&&... args);
+
+ template <class TType>
+ void AddStructuredTag(TStringBuf key, TType value);
+
+ TLogger WithRawTag(const TString& tag) const;
+ template <class... TArgs>
+ TLogger WithTag(const char* format, TArgs&&... args) const;
+
+ template <class TType>
+ TLogger WithStructuredTag(TStringBuf key, TType value) const;
+
+ TLogger WithStructuredValidator(TStructuredValidator validator) const;
+
+ TLogger WithMinLevel(ELogLevel minLevel) const;
+
+ TLogger WithEssential(bool essential = true) const;
+
+ const TString& GetTag() const;
+ const TStructuredTags& GetStructuredTags() const;
+
+ const TStructuredValidators& GetStructuredValidators() const;
+
+protected:
+ // These fields are set only during logger creation, so they are effectively const
+ // and accessing them is thread-safe.
+ ILogManager* LogManager_ = nullptr;
+ const TLoggingCategory* Category_ = nullptr;
+ bool Essential_ = false;
+ ELogLevel MinLevel_ = NullLoggerMinLevel;
+ TString Tag_;
+ TStructuredTags StructuredTags_;
+ TStructuredValidators StructuredValidators_;
+
+private:
+ //! This method checks level against category's min level.
+ //! Refer to comment in TLogger::IsLevelEnabled for more details.
+ bool IsLevelEnabledHeavy(ELogLevel level) const;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+void LogStructuredEvent(
+ const TLogger& logger,
+ NYson::TYsonString message,
+ ELogLevel level);
+
+////////////////////////////////////////////////////////////////////////////////
+
+#ifdef YT_ENABLE_TRACE_LOGGING
+#define YT_LOG_TRACE(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Trace, __VA_ARGS__)
+#define YT_LOG_TRACE_IF(condition, ...) if (condition) YT_LOG_TRACE(__VA_ARGS__)
+#define YT_LOG_TRACE_UNLESS(condition, ...) if (!(condition)) YT_LOG_TRACE(__VA_ARGS__)
+#else
+#define YT_LOG_UNUSED(...) if (true) { } else { YT_LOG_DEBUG(__VA_ARGS__); }
+#define YT_LOG_TRACE(...) YT_LOG_UNUSED(__VA_ARGS__)
+#define YT_LOG_TRACE_IF(condition, ...) YT_LOG_UNUSED(__VA_ARGS__)
+#define YT_LOG_TRACE_UNLESS(condition, ...) YT_LOG_UNUSED(__VA_ARGS__)
+#endif
+
+#define YT_LOG_DEBUG(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Debug, __VA_ARGS__)
+#define YT_LOG_DEBUG_IF(condition, ...) if (condition) YT_LOG_DEBUG(__VA_ARGS__)
+#define YT_LOG_DEBUG_UNLESS(condition, ...) if (!(condition)) YT_LOG_DEBUG(__VA_ARGS__)
+
+#define YT_LOG_INFO(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Info, __VA_ARGS__)
+#define YT_LOG_INFO_IF(condition, ...) if (condition) YT_LOG_INFO(__VA_ARGS__)
+#define YT_LOG_INFO_UNLESS(condition, ...) if (!(condition)) YT_LOG_INFO(__VA_ARGS__)
+
+#define YT_LOG_WARNING(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Warning, __VA_ARGS__)
+#define YT_LOG_WARNING_IF(condition, ...) if (condition) YT_LOG_WARNING(__VA_ARGS__)
+#define YT_LOG_WARNING_UNLESS(condition, ...) if (!(condition)) YT_LOG_WARNING(__VA_ARGS__)
+
+#define YT_LOG_ERROR(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Error, __VA_ARGS__)
+#define YT_LOG_ERROR_IF(condition, ...) if (condition) YT_LOG_ERROR(__VA_ARGS__)
+#define YT_LOG_ERROR_UNLESS(condition, ...) if (!(condition)) YT_LOG_ERROR(__VA_ARGS__)
+
+#define YT_LOG_ALERT(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Alert, __VA_ARGS__);
+#define YT_LOG_ALERT_IF(condition, ...) if (condition) YT_LOG_ALERT(__VA_ARGS__)
+#define YT_LOG_ALERT_UNLESS(condition, ...) if (!(condition)) YT_LOG_ALERT(__VA_ARGS__)
+
+#define YT_LOG_FATAL(...) \
+ do { \
+ YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Fatal, __VA_ARGS__); \
+ Y_UNREACHABLE(); \
+ } while(false)
+#define YT_LOG_FATAL_IF(condition, ...) if (Y_UNLIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__)
+#define YT_LOG_FATAL_UNLESS(condition, ...) if (!Y_LIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__)
+
+#define YT_LOG_EVENT(logger, level, ...) \
+ YT_LOG_EVENT_WITH_ANCHOR(logger, level, nullptr, __VA_ARGS__)
+
+#define YT_LOG_EVENT_WITH_ANCHOR(logger, level, anchor, ...) \
+ do { \
+ const auto& logger__##__LINE__ = (logger); \
+ auto level__##__LINE__ = (level); \
+ \
+ if (!logger__##__LINE__.IsLevelEnabled(level__##__LINE__)) { \
+ break; \
+ } \
+ \
+ auto location__##__LINE__ = __LOCATION__; \
+ \
+ ::NYT::NLogging::TLoggingAnchor* anchor__##__LINE__ = (anchor); \
+ if (!anchor__##__LINE__) { \
+ static ::NYT::TLeakyStorage<::NYT::NLogging::TLoggingAnchor> staticAnchor__##__LINE__; \
+ anchor__##__LINE__ = staticAnchor__##__LINE__.Get(); \
+ } \
+ \
+ bool anchorUpToDate__##__LINE__ = logger__##__LINE__.IsAnchorUpToDate(*anchor__##__LINE__); \
+ if (anchorUpToDate__##__LINE__ && !anchor__##__LINE__->Enabled.load(std::memory_order::relaxed)) { \
+ break; \
+ } \
+ \
+ auto loggingContext__##__LINE__ = ::NYT::NLogging::GetLoggingContext(); \
+ auto message__##__LINE__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__##__LINE__, logger__##__LINE__, __VA_ARGS__); \
+ \
+ if (!anchorUpToDate__##__LINE__) { \
+ logger__##__LINE__.RegisterStaticAnchor(anchor__##__LINE__, location__##__LINE__, message__##__LINE__.Anchor); \
+ logger__##__LINE__.UpdateAnchor(anchor__##__LINE__); \
+ } \
+ \
+ if (!anchor__##__LINE__->Enabled.load(std::memory_order::relaxed)) { \
+ break; \
+ } \
+ \
+ static thread_local i64 localByteCounter__##__LINE__; \
+ static thread_local ui8 localMessageCounter__##__LINE__; \
+ \
+ localByteCounter__##__LINE__ += message__##__LINE__.MessageRef.Size(); \
+ if (Y_UNLIKELY(++localMessageCounter__##__LINE__ == 0)) { \
+ anchor__##__LINE__->MessageCounter.Current += 256; \
+ anchor__##__LINE__->ByteCounter.Current += localByteCounter__##__LINE__; \
+ localByteCounter__##__LINE__ = 0; \
+ } \
+ \
+ ::NYT::NLogging::NDetail::LogEventImpl( \
+ loggingContext__##__LINE__, \
+ logger__##__LINE__, \
+ level__##__LINE__, \
+ location__##__LINE__, \
+ std::move(message__##__LINE__.MessageRef)); \
+ } while (false)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NLogging
+
+#define LOGGER_INL_H_
+#include "logger-inl.h"
+#undef LOGGER_INL_H_
diff --git a/library/cpp/yt/logging/public.h b/library/cpp/yt/logging/public.h
new file mode 100644
index 0000000000..1e2b59ca0d
--- /dev/null
+++ b/library/cpp/yt/logging/public.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#include <library/cpp/yt/misc/enum.h>
+
+namespace NYT::NLogging {
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Any change to this enum must be also propagated to FormatLevel.
+DEFINE_ENUM(ELogLevel,
+ (Minimum)
+ (Trace)
+ (Debug)
+ (Info)
+ (Warning)
+ (Error)
+ (Alert)
+ (Fatal)
+ (Maximum)
+);
+
+DEFINE_ENUM(ELogFamily,
+ (PlainText)
+ (Structured)
+);
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TLoggingCategory;
+struct TLoggingAnchor;
+struct TLogEvent;
+struct TLoggingContext;
+
+class TLogger;
+struct ILogManager;
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NLogging
diff --git a/library/cpp/yt/logging/unittests/logger_ut.cpp b/library/cpp/yt/logging/unittests/logger_ut.cpp
new file mode 100644
index 0000000000..7696ea4a83
--- /dev/null
+++ b/library/cpp/yt/logging/unittests/logger_ut.cpp
@@ -0,0 +1,38 @@
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <library/cpp/yt/logging/logger.h>
+
+namespace NYT::NLogging {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(TLogger, NullByDefault)
+{
+ {
+ TLogger logger;
+ EXPECT_FALSE(logger);
+ EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal));
+ }
+ {
+ TLogger logger{"Category"};
+ EXPECT_FALSE(logger);
+ EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal));
+ }
+}
+
+TEST(TLogger, CopyOfNullLogger)
+{
+ TLogger nullLogger{/*logManager*/ nullptr, "Category"};
+ ASSERT_FALSE(nullLogger);
+
+ auto logger = nullLogger.WithMinLevel(ELogLevel::Debug);
+
+ EXPECT_FALSE(logger);
+ EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NLogging
diff --git a/library/cpp/yt/logging/unittests/ya.make b/library/cpp/yt/logging/unittests/ya.make
new file mode 100644
index 0000000000..42268d3db2
--- /dev/null
+++ b/library/cpp/yt/logging/unittests/ya.make
@@ -0,0 +1,18 @@
+GTEST(unittester-library-logging)
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+IF (NOT OS_WINDOWS)
+ ALLOCATOR(YT)
+ENDIF()
+
+SRCS(
+ logger_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/testing/gtest
+ library/cpp/yt/logging
+)
+
+END()
diff --git a/library/cpp/yt/logging/ya.make b/library/cpp/yt/logging/ya.make
new file mode 100644
index 0000000000..cf629a24b6
--- /dev/null
+++ b/library/cpp/yt/logging/ya.make
@@ -0,0 +1,20 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+ logger.cpp
+)
+
+PEERDIR(
+ library/cpp/yt/assert
+ library/cpp/yt/memory
+ library/cpp/yt/misc
+ library/cpp/yt/yson_string
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ unittests
+)
diff --git a/library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h b/library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h
new file mode 100644
index 0000000000..1fba63c427
--- /dev/null
+++ b/library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h
@@ -0,0 +1,43 @@
+#ifndef LEAKY_REF_COUNTED_SINGLETON_INL_H_
+#error "Direct inclusion of this file is not allowed, include leaky_ref_counted_singleton.h"
+// For the sake of sane code completion.
+#include "leaky_ref_counted_singleton.h"
+#endif
+
+#include "new.h"
+
+#include <atomic>
+#include <mutex>
+
+#include <util/system/compiler.h>
+#include <util/system/sanitizers.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class T, class... TArgs>
+TIntrusivePtr<T> LeakyRefCountedSingleton(TArgs&&... args)
+{
+ static std::atomic<T*> Ptr;
+ auto* ptr = Ptr.load(std::memory_order::acquire);
+ if (Y_LIKELY(ptr)) {
+ return ptr;
+ }
+
+ static std::once_flag Initialized;
+ std::call_once(Initialized, [&] {
+ auto ptr = New<T>(std::forward<TArgs>(args)...);
+ Ref(ptr.Get());
+ Ptr.store(ptr.Get());
+#if defined(_asan_enabled_)
+ NSan::MarkAsIntentionallyLeaked(ptr.Get());
+#endif
+ });
+
+ return Ptr.load();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yt/memory/leaky_ref_counted_singleton.h b/library/cpp/yt/memory/leaky_ref_counted_singleton.h
new file mode 100644
index 0000000000..d77c3c9829
--- /dev/null
+++ b/library/cpp/yt/memory/leaky_ref_counted_singleton.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include "intrusive_ptr.h"
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+#define DECLARE_LEAKY_REF_COUNTED_SINGLETON_FRIEND() \
+ template <class T> \
+ friend struct ::NYT::TRefCountedWrapper;
+
+template <class T, class... TArgs>
+TIntrusivePtr<T> LeakyRefCountedSingleton(TArgs&&... args);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
+
+#define LEAKY_REF_COUNTED_SINGLETON_INL_H_
+#include "leaky_ref_counted_singleton-inl.h"
+#undef LEAKY_REF_COUNTED_SINGLETON_INL_H_
diff --git a/library/cpp/yt/misc/arcadia_enum-inl.h b/library/cpp/yt/misc/arcadia_enum-inl.h
new file mode 100644
index 0000000000..17a10bb3b2
--- /dev/null
+++ b/library/cpp/yt/misc/arcadia_enum-inl.h
@@ -0,0 +1,49 @@
+#pragma once
+#ifndef ARCADIA_ENUM_INL_H_
+#error "Direct inclusion of this file is not allowed, include arcadia_enum.h"
+// For the sake of sane code completion.
+#include "arcadia_enum.h"
+#endif
+
+#include <util/system/type_name.h>
+
+namespace NYT::NDetail {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class T>
+struct TArcadiaEnumTraitsImpl
+{
+ static constexpr bool IsBitEnum = false;
+ static constexpr bool IsStringSerializableEnum = false;
+
+ static TStringBuf GetTypeName()
+ {
+ static const auto Result = TypeName<T>();
+ return Result;
+ }
+
+ static std::optional<TStringBuf> FindLiteralByValue(T value)
+ {
+ auto names = GetEnumNames<T>();
+ auto it = names.find(value);
+ return it == names.end() ? std::nullopt : std::make_optional(TStringBuf(it->second));
+ }
+
+ static std::optional<T> FindValueByLiteral(TStringBuf literal)
+ {
+ static const auto LiteralToValue = [] {
+ THashMap<TString, T> result;
+ for (const auto& [value, name] : GetEnumNames<T>()) {
+ result.emplace(name, value);
+ }
+ return result;
+ }();
+ auto it = LiteralToValue.find(literal);
+ return it == LiteralToValue.end() ? std::nullopt : std::make_optional(it->second);
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NDetail
diff --git a/library/cpp/yt/misc/arcadia_enum.h b/library/cpp/yt/misc/arcadia_enum.h
new file mode 100644
index 0000000000..85ad182a6c
--- /dev/null
+++ b/library/cpp/yt/misc/arcadia_enum.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <util/generic/serialized_enum.h>
+
+////////////////////////////////////////////////////////////////////////////////
+// TEnumTraits interop for Arcadia enums
+
+#define YT_DEFINE_ARCADIA_ENUM_TRAITS(enumType) \
+ [[maybe_unused]] inline ::NYT::NDetail::TArcadiaEnumTraitsImpl<enumType> GetEnumTraitsImpl(enumType) \
+ { \
+ return {}; \
+ }
+
+////////////////////////////////////////////////////////////////////////////////
+
+#define ARCADIA_ENUM_INL_H_
+#include "arcadia_enum-inl.h"
+#undef ARCADIA_ENUM_INL_H_
diff --git a/library/cpp/yt/misc/property.h b/library/cpp/yt/misc/property.h
new file mode 100644
index 0000000000..d5c2a26c7a
--- /dev/null
+++ b/library/cpp/yt/misc/property.h
@@ -0,0 +1,306 @@
+#pragma once
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Declares a trivial public read-write property that is passed by reference.
+#define DECLARE_BYREF_RW_PROPERTY(type, name) \
+public: \
+ type& name(); \
+ const type& name() const
+
+//! Defines a trivial public read-write property that is passed by reference.
+//! All arguments after name are used as default value (via braced-init-list).
+#define DEFINE_BYREF_RW_PROPERTY(type, name, ...) \
+protected: \
+ type name##_ { __VA_ARGS__ }; \
+ \
+public: \
+ Y_FORCE_INLINE type& name() \
+ { \
+ return name##_; \
+ } \
+ \
+ Y_FORCE_INLINE const type& name() const \
+ { \
+ return name##_; \
+ } \
+ static_assert(true)
+
+//! Defines a trivial public read-write property that is passed by reference
+//! and is not inline-initialized.
+#define DEFINE_BYREF_RW_PROPERTY_NO_INIT(type, name) \
+protected: \
+ type name##_; \
+ \
+public: \
+ Y_FORCE_INLINE type& name() \
+ { \
+ return name##_; \
+ } \
+ \
+ Y_FORCE_INLINE const type& name() const \
+ { \
+ return name##_; \
+ } \
+ static_assert(true)
+
+//! Forwards a trivial public read-write property that is passed by reference.
+#define DELEGATE_BYREF_RW_PROPERTY(declaringType, type, name, delegateTo) \
+ type& declaringType::name() \
+ { \
+ return (delegateTo).name(); \
+ } \
+ \
+ const type& declaringType::name() const \
+ { \
+ return (delegateTo).name(); \
+ } \
+ static_assert(true)
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Declares a trivial public read-only property that is passed by reference.
+#define DECLARE_BYREF_RO_PROPERTY(type, name) \
+public: \
+ const type& name() const
+
+//! Defines a trivial public read-only property that is passed by reference.
+//! All arguments after name are used as default value (via braced-init-list).
+#define DEFINE_BYREF_RO_PROPERTY(type, name, ...) \
+protected: \
+ type name##_ { __VA_ARGS__ }; \
+ \
+public: \
+ Y_FORCE_INLINE const type& name() const \
+ { \
+ return name##_; \
+ } \
+ static_assert(true)
+
+//! Defines a trivial public read-only property that is passed by reference
+//! and is not inline-initialized.
+#define DEFINE_BYREF_RO_PROPERTY_NO_INIT(type, name) \
+protected: \
+ type name##_; \
+ \
+public: \
+ Y_FORCE_INLINE const type& name() const \
+ { \
+ return name##_; \
+ } \
+ static_assert(true)
+
+//! Forwards a trivial public read-only property that is passed by reference.
+#define DELEGATE_BYREF_RO_PROPERTY(declaringType, type, name, delegateTo) \
+ const type& declaringType::name() const \
+ { \
+ return (delegateTo).name(); \
+ } \
+ static_assert(true)
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Declares a trivial public read-write property that is passed by value.
+#define DECLARE_BYVAL_RW_PROPERTY(type, name) \
+public: \
+ type Get##name() const; \
+ void Set##name(type value)
+
+//! Defines a trivial public read-write property that is passed by value.
+//! All arguments after name are used as default value (via braced-init-list).
+#define DEFINE_BYVAL_RW_PROPERTY(type, name, ...) \
+protected: \
+ type name##_ { __VA_ARGS__ }; \
+ \
+public: \
+ Y_FORCE_INLINE type Get##name() const \
+ { \
+ return name##_; \
+ } \
+ \
+ Y_FORCE_INLINE void Set##name(type value) \
+ { \
+ name##_ = value; \
+ } \
+ static_assert(true)
+
+
+//! Defines a trivial public read-write property that is passed by value.
+//! All arguments after name are used as default value (via braced-init-list).
+#define DEFINE_BYVAL_RW_PROPERTY_WITH_FLUENT_SETTER(declaringType, type, name, ...) \
+protected: \
+ type name##_ { __VA_ARGS__ }; \
+ \
+public: \
+ Y_FORCE_INLINE type Get##name() const \
+ { \
+ return name##_; \
+ } \
+ \
+ Y_FORCE_INLINE void Set##name(type value) &\
+ { \
+ name##_ = value; \
+ } \
+ \
+ Y_FORCE_INLINE declaringType&& Set##name(type value) &&\
+ { \
+ name##_ = value; \
+ return std::move(*this); \
+ } \
+ static_assert(true)
+
+//! Defines a trivial public read-write property that is passed by value
+//! and is not inline-initialized.
+#define DEFINE_BYVAL_RW_PROPERTY_NO_INIT(type, name, ...) \
+protected: \
+ type name##_; \
+ \
+public: \
+ Y_FORCE_INLINE type Get##name() const \
+ { \
+ return name##_; \
+ } \
+ \
+ Y_FORCE_INLINE void Set##name(type value) \
+ { \
+ name##_ = value; \
+ } \
+ static_assert(true)
+
+//! Forwards a trivial public read-write property that is passed by value.
+#define DELEGATE_BYVAL_RW_PROPERTY(declaringType, type, name, delegateTo) \
+ type declaringType::Get##name() const \
+ { \
+ return (delegateTo).Get##name(); \
+ } \
+ \
+ void declaringType::Set##name(type value) \
+ { \
+ (delegateTo).Set##name(value); \
+ } \
+ static_assert(true)
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Declares a trivial public read-only property that is passed by value.
+#define DECLARE_BYVAL_RO_PROPERTY(type, name) \
+public: \
+ type Get##name() const
+
+//! Defines a trivial public read-only property that is passed by value.
+//! All arguments after name are used as default value (via braced-init-list).
+#define DEFINE_BYVAL_RO_PROPERTY(type, name, ...) \
+protected: \
+ type name##_ { __VA_ARGS__ }; \
+ \
+public: \
+ Y_FORCE_INLINE type Get##name() const \
+ { \
+ return name##_; \
+ } \
+ static_assert(true)
+
+
+//! Defines a trivial public read-only property that is passed by value
+//! and is not inline-initialized.
+#define DEFINE_BYVAL_RO_PROPERTY_NO_INIT(type, name) \
+protected: \
+ type name##_; \
+ \
+public: \
+ Y_FORCE_INLINE type Get##name() const \
+ { \
+ return name##_; \
+ } \
+ static_assert(true)
+
+//! Forwards a trivial public read-only property that is passed by value.
+#define DELEGATE_BYVAL_RO_PROPERTY(declaringType, type, name, delegateTo) \
+ type declaringType::Get##name() \
+ { \
+ return (delegateTo).Get##name(); \
+ } \
+ static_assert(true)
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Below are macro helpers for extra properties.
+//! Extra properties should be used for lazy memory allocation for properties that
+//! hold default values for the majority of objects.
+
+//! Initializes extra property holder if it is not initialized.
+#define INITIALIZE_EXTRA_PROPERTY_HOLDER(holder) \
+ if (!holder##_) { \
+ holder##_.reset(new decltype(holder##_)::element_type()); \
+ } \
+ static_assert(true)
+
+//! Declares an extra property holder. Holder contains extra properties values.
+//! Holder is not created until some property is set with a non-default value.
+//! If there is no holder property getter returns default value.
+#define DECLARE_EXTRA_PROPERTY_HOLDER(type, holder) \
+public: \
+ Y_FORCE_INLINE bool HasCustom##holder() const \
+ { \
+ return static_cast<bool>(holder##_); \
+ } \
+ Y_FORCE_INLINE const type* GetCustom##holder() const \
+ { \
+ return holder##_.get(); \
+ } \
+ Y_FORCE_INLINE type* GetCustom##holder() \
+ { \
+ return holder##_.get(); \
+ } \
+ Y_FORCE_INLINE void InitializeCustom##holder() \
+ { \
+ INITIALIZE_EXTRA_PROPERTY_HOLDER(holder); \
+ } \
+private: \
+ std::unique_ptr<type> holder##_; \
+ static const type Default##holder##_
+
+//! Defines a storage for extra properties default values.
+#define DEFINE_EXTRA_PROPERTY_HOLDER(class, type, holder) \
+ const type class::Default##holder##_
+
+//! Defines a public read-write extra property that is passed by value.
+#define DEFINE_BYVAL_RW_EXTRA_PROPERTY(holder, name) \
+public: \
+ Y_FORCE_INLINE decltype(holder##_->name) Get##name() const \
+ { \
+ if (!holder##_) { \
+ return Default##holder##_.name; \
+ } \
+ return holder##_->name; \
+ } \
+ Y_FORCE_INLINE void Set##name(decltype(holder##_->name) val) \
+ { \
+ if (!holder##_) { \
+ if (val == Default##holder##_.name) { \
+ return; \
+ } \
+ INITIALIZE_EXTRA_PROPERTY_HOLDER(holder); \
+ } \
+ holder##_->name = val; \
+ } \
+ static_assert(true)
+
+//! Defines a public read-write extra property that is passed by reference.
+#define DEFINE_BYREF_RW_EXTRA_PROPERTY(holder, name) \
+public: \
+ Y_FORCE_INLINE const decltype(holder##_->name)& name() const \
+ { \
+ if (!holder##_) { \
+ return Default##holder##_.name; \
+ } \
+ return holder##_->name; \
+ } \
+ Y_FORCE_INLINE decltype(holder##_->name)& Mutable##name() \
+ { \
+ INITIALIZE_EXTRA_PROPERTY_HOLDER(holder); \
+ return holder##_->name; \
+ } \
+ static_assert(true)
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/library/cpp/yt/string/raw_formatter.h b/library/cpp/yt/string/raw_formatter.h
new file mode 100644
index 0000000000..6956330883
--- /dev/null
+++ b/library/cpp/yt/string/raw_formatter.h
@@ -0,0 +1,212 @@
+#pragma once
+
+#include "guid.h"
+
+#include <algorithm>
+#include <array>
+
+#include <util/generic/strbuf.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! A dead-simple string formatter.
+/*!
+ * This formatter is intended to be as simple as possible and async signal safe.
+ * This is the reason we do not use printf(): it does not meet signal-safety
+ * requirements.
+ */
+
+class TBaseFormatter
+{
+public:
+ TBaseFormatter(char* buffer, int length)
+ : Begin_(buffer)
+ , Cursor_(buffer)
+ , End_(buffer + length)
+ { }
+
+ //! Returns an underlying cursor.
+ char* GetCursor()
+ {
+ return Cursor_;
+ }
+
+ //! Returns an pointer to the underlying buffer.
+ const char* GetData() const
+ {
+ return Begin_;
+ }
+
+ //! Returns the number of bytes written in the buffer.
+ int GetBytesWritten() const
+ {
+ return Cursor_ - Begin_;
+ }
+
+ //! Returns the number of bytes available in the buffer.
+ int GetBytesRemaining() const
+ {
+ return End_ - Cursor_;
+ }
+
+ //! Advances the internal cursor #count symbols forward (assuming the data is already present).
+ void Advance(int count)
+ {
+ Cursor_ += count;
+
+ if (Cursor_ > End_) {
+ Cursor_ = End_;
+ }
+ }
+
+ //! Drops trailing #count symbols (assuming these are present).
+ void Revert(int count)
+ {
+ Cursor_ -= count;
+ }
+
+ //! Appends the string and updates the internal cursor.
+ void AppendString(const char* string)
+ {
+ while (*string != '\0' && Cursor_ < End_) {
+ *Cursor_++ = *string++;
+ }
+ }
+
+ //! Appends the string and updates the internal cursor.
+ void AppendString(TStringBuf string)
+ {
+ size_t position = 0;
+ while (position < string.length() && Cursor_ < End_) {
+ *Cursor_++ = string[position++];
+ }
+ }
+
+ //! Appends a single character and updates the internal cursor.
+ void AppendChar(char ch)
+ {
+ if (Cursor_ < End_) {
+ *Cursor_++ = ch;
+ }
+ }
+
+ //! Formats |number| in base |radix| and updates the internal cursor.
+ void AppendNumber(uintptr_t number, int radix = 10, int width = 0, char ch = ' ')
+ {
+ int digits = 0;
+
+ if (radix == 16) {
+ // Optimize output of hex numbers.
+
+ uintptr_t reverse = 0;
+ int length = 0;
+ do {
+ reverse <<= 4;
+ reverse |= number & 0xf;
+ number >>= 4;
+ ++length;
+ } while (number > 0);
+
+ for (int index = 0; index < length && Cursor_ + digits < End_; ++index) {
+ unsigned int modulus = reverse & 0xf;
+ Cursor_[digits] = (modulus < 10 ? '0' + modulus : 'a' + modulus - 10);
+ ++digits;
+ reverse >>= 4;
+ }
+ } else {
+ while (Cursor_ + digits < End_) {
+ const int modulus = number % radix;
+ number /= radix;
+ Cursor_[digits] = (modulus < 10 ? '0' + modulus : 'a' + modulus - 10);
+ ++digits;
+ if (number == 0) {
+ break;
+ }
+ }
+
+ // Reverse the bytes written.
+ std::reverse(Cursor_, Cursor_ + digits);
+ }
+
+ if (digits < width) {
+ auto delta = width - digits;
+ std::copy(Cursor_, Cursor_ + digits, Cursor_ + delta);
+ std::fill(Cursor_, Cursor_ + delta, ch);
+ Cursor_ += width;
+ } else {
+ Cursor_ += digits;
+ }
+ }
+
+ //! Formats |number| as hexadecimal number and updates the internal cursor.
+ //! Padding will be added in front if needed.
+ void AppendNumberAsHexWithPadding(uintptr_t number, int width)
+ {
+ char* begin = Cursor_;
+ AppendString("0x");
+ AppendNumber(number, 16);
+ // Move to right and add padding in front if needed.
+ if (Cursor_ < begin + width) {
+ auto delta = begin + width - Cursor_;
+ std::copy(begin, Cursor_, begin + delta);
+ std::fill(begin, begin + delta, ' ');
+ Cursor_ = begin + width;
+ }
+ }
+
+ //! Formats |guid| and updates the internal cursor.
+ void AppendGuid(TGuid guid)
+ {
+ if (Y_LIKELY(End_ - Cursor_ >= MaxGuidStringSize)) {
+ // Fast path.
+ Cursor_ = WriteGuidToBuffer(Cursor_, guid);
+ } else {
+ // Slow path.
+ std::array<char, MaxGuidStringSize> buffer;
+ auto* end = WriteGuidToBuffer(buffer.data(), guid);
+ AppendString(TStringBuf(buffer.data(), end));
+ }
+ }
+
+ //! Resets the underlying cursor.
+ void Reset()
+ {
+ Cursor_ = Begin_;
+ }
+
+ TStringBuf GetBuffer() const
+ {
+ return {Begin_, Cursor_};
+ }
+
+private:
+ char* const Begin_;
+ char* Cursor_;
+ char* const End_;
+
+};
+
+template <size_t N>
+class TRawFormatter
+ : public TBaseFormatter
+{
+public:
+ TRawFormatter()
+ : TBaseFormatter(Buffer_, N)
+ { }
+
+ TRawFormatter(char* buffer, int length)
+ : TBaseFormatter(buffer, length)
+ { }
+
+private:
+ char Buffer_[N];
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
+
diff --git a/library/cpp/yt/threading/unittests/count_down_latch_ut.cpp b/library/cpp/yt/threading/unittests/count_down_latch_ut.cpp
new file mode 100644
index 0000000000..894bdab22a
--- /dev/null
+++ b/library/cpp/yt/threading/unittests/count_down_latch_ut.cpp
@@ -0,0 +1,78 @@
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <library/cpp/yt/threading/count_down_latch.h>
+
+#include <thread>
+
+namespace NYT::NThreading {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void WaitForLatch(const TCountDownLatch& latch)
+{
+ latch.Wait();
+ EXPECT_EQ(0, latch.GetCount());
+}
+
+TEST(TCountDownLatch, TwoThreads)
+{
+ TCountDownLatch latch(2);
+
+ std::thread t1(std::bind(&WaitForLatch, std::cref(latch)));
+ std::thread t2(std::bind(&WaitForLatch, std::cref(latch)));
+
+ EXPECT_EQ(2, latch.GetCount());
+ latch.CountDown();
+ EXPECT_EQ(1, latch.GetCount());
+ latch.CountDown();
+ EXPECT_EQ(0, latch.GetCount());
+
+ t1.join();
+ t2.join();
+}
+
+TEST(TCountDownLatch, TwoThreadsPredecremented)
+{
+ TCountDownLatch latch(2);
+
+ EXPECT_EQ(2, latch.GetCount());
+ latch.CountDown();
+ EXPECT_EQ(1, latch.GetCount());
+ latch.CountDown();
+ EXPECT_EQ(0, latch.GetCount());
+
+ std::thread t1(std::bind(&WaitForLatch, std::cref(latch)));
+ std::thread t2(std::bind(&WaitForLatch, std::cref(latch)));
+
+ t1.join();
+ t2.join();
+}
+
+TEST(TCountDownLatch, TwoThreadsTwoLatches)
+{
+ TCountDownLatch first(1);
+ TCountDownLatch second(1);
+
+ std::thread t1([&] () {
+ first.Wait();
+ second.CountDown();
+ EXPECT_EQ(0, first.GetCount());
+ EXPECT_EQ(0, second.GetCount());
+ });
+
+ std::thread t2([&] () {
+ first.CountDown();
+ second.Wait();
+ EXPECT_EQ(0, first.GetCount());
+ EXPECT_EQ(0, second.GetCount());
+ });
+
+ t1.join();
+ t2.join();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NThreading
diff --git a/library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp b/library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp
new file mode 100644
index 0000000000..9c2d8f16cb
--- /dev/null
+++ b/library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp
@@ -0,0 +1,88 @@
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <library/cpp/yt/threading/recursive_spin_lock.h>
+#include <library/cpp/yt/threading/event_count.h>
+
+#include <thread>
+
+namespace NYT::NThreading {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(TRecursiveSpinLockTest, SingleThread)
+{
+ TRecursiveSpinLock lock;
+ EXPECT_FALSE(lock.IsLocked());
+ EXPECT_TRUE(lock.TryAcquire());
+ EXPECT_TRUE(lock.IsLocked());
+ EXPECT_TRUE(lock.TryAcquire());
+ EXPECT_TRUE(lock.IsLocked());
+ lock.Release();
+ EXPECT_TRUE(lock.IsLocked());
+ lock.Release();
+ EXPECT_FALSE(lock.IsLocked());
+ EXPECT_TRUE(lock.TryAcquire());
+ EXPECT_TRUE(lock.IsLocked());
+ lock.Release();
+ lock.Acquire();
+ lock.Release();
+}
+
+TEST(TRecursiveSpinLockTest, TwoThreads)
+{
+ TRecursiveSpinLock lock;
+ TEvent e1, e2, e3, e4, e5, e6, e7;
+
+ std::thread t1([&] {
+ e1.Wait();
+ EXPECT_TRUE(lock.IsLocked());
+ EXPECT_FALSE(lock.IsLockedByCurrentThread());
+ EXPECT_FALSE(lock.TryAcquire());
+ e2.NotifyOne();
+ e3.Wait();
+ EXPECT_TRUE(lock.IsLocked());
+ EXPECT_FALSE(lock.IsLockedByCurrentThread());
+ EXPECT_FALSE(lock.TryAcquire());
+ e4.NotifyOne();
+ e5.Wait();
+ EXPECT_FALSE(lock.IsLocked());
+ EXPECT_FALSE(lock.IsLockedByCurrentThread());
+ EXPECT_TRUE(lock.TryAcquire());
+ e6.NotifyOne();
+ e7.Wait();
+ lock.Release();
+ });
+
+ std::thread t2([&] {
+ EXPECT_FALSE(lock.IsLocked());
+ EXPECT_TRUE(lock.TryAcquire());
+ EXPECT_TRUE(lock.IsLockedByCurrentThread());
+ e1.NotifyOne();
+ e2.Wait();
+ EXPECT_TRUE(lock.TryAcquire());
+ EXPECT_TRUE(lock.IsLockedByCurrentThread());
+ e3.NotifyOne();
+ e4.Wait();
+ lock.Release();
+ lock.Release();
+ EXPECT_FALSE(lock.IsLocked());
+ e5.NotifyOne();
+ e6.Wait();
+ EXPECT_TRUE(lock.IsLocked());
+ EXPECT_FALSE(lock.IsLockedByCurrentThread());
+ e7.NotifyOne();
+ lock.Acquire();
+ lock.Acquire();
+ lock.Release();
+ lock.Release();
+ });
+
+ t1.join();
+ t2.join();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NThreading
diff --git a/library/cpp/yt/threading/unittests/spin_wait_ut.cpp b/library/cpp/yt/threading/unittests/spin_wait_ut.cpp
new file mode 100644
index 0000000000..8469634f34
--- /dev/null
+++ b/library/cpp/yt/threading/unittests/spin_wait_ut.cpp
@@ -0,0 +1,48 @@
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <library/cpp/yt/threading/spin_wait.h>
+#include <library/cpp/yt/threading/spin_wait_hook.h>
+
+#include <thread>
+#include <mutex>
+
+namespace NYT::NThreading {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool SpinWaitSlowPathHookInvoked;
+
+void SpinWaitSlowPathHook(
+ TCpuDuration cpuDelay,
+ const TSourceLocation& /*location*/,
+ ESpinLockActivityKind /*activityKind*/)
+{
+ SpinWaitSlowPathHookInvoked = true;
+ auto delay = CpuDurationToDuration(cpuDelay);
+ EXPECT_GE(delay, TDuration::Seconds(1));
+ EXPECT_LE(delay, TDuration::Seconds(5));
+}
+
+TEST(TSpinWaitTest, SlowPathHook)
+{
+ static std::once_flag registerFlag;
+ std::call_once(
+ registerFlag,
+ [] {
+ RegisterSpinWaitSlowPathHook(SpinWaitSlowPathHook);
+ });
+ SpinWaitSlowPathHookInvoked = false;
+ {
+ TSpinWait spinWait(__LOCATION__, ESpinLockActivityKind::ReadWrite);
+ for (int i = 0; i < 1'000'000; ++i) {
+ spinWait.Wait();
+ }
+ }
+ EXPECT_TRUE(SpinWaitSlowPathHookInvoked);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NThreading
diff --git a/library/cpp/yt/threading/unittests/ya.make b/library/cpp/yt/threading/unittests/ya.make
new file mode 100644
index 0000000000..ef9b5d2995
--- /dev/null
+++ b/library/cpp/yt/threading/unittests/ya.make
@@ -0,0 +1,17 @@
+GTEST()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+ count_down_latch_ut.cpp
+ recursive_spin_lock_ut.cpp
+ spin_wait_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/yt/assert
+ library/cpp/yt/threading
+ library/cpp/testing/gtest
+)
+
+END()
diff --git a/library/cpp/yt/user_job_statistics/user_job_statistics.cpp b/library/cpp/yt/user_job_statistics/user_job_statistics.cpp
new file mode 100644
index 0000000000..b7fd71503d
--- /dev/null
+++ b/library/cpp/yt/user_job_statistics/user_job_statistics.cpp
@@ -0,0 +1,133 @@
+#include "user_job_statistics.h"
+#include <yt/cpp/mapreduce/common/helpers.h>
+#include <util/stream/null.h>
+#include <util/string/builder.h>
+#include <util/system/mutex.h>
+#include <util/system/env.h>
+
+using namespace NYtTools;
+
+static TMutex GlobalStatsWritingMutex;
+
+#if defined(_unix_)
+const FHANDLE TUserJobStatsProxy::JobStatisticsHandle = 5;
+#elif defined(_win_)
+const FHANDLE TUserJobStatsProxy::JobStatisticsHandle = nullptr;
+#endif
+
+static IOutputStream* CorrectHandle(const FHANDLE h) {
+#if defined(_unix_)
+ if (fcntl(h, F_GETFD) == -1) {
+ return &Cerr;
+ }
+ return nullptr;
+#elif defined(_win_)
+ return &Cerr;
+#endif
+}
+
+static TString PrintNodeSimple(const NYT::TNode& n) {
+ return NYT::NodeToYsonString(n, NYson::EYsonFormat::Text);
+}
+
+void TUserJobStatsProxy::Init(IOutputStream * usingStream) {
+ if (usingStream == nullptr) {
+ usingStream = CorrectHandle(JobStatisticsHandle);
+ }
+
+ if(usingStream == nullptr && GetEnv("YT_JOB_ID").empty()) {
+ usingStream = &Cerr;
+ }
+
+
+ if (usingStream == nullptr) {
+ TFileHandle fixedDesrc(JobStatisticsHandle);
+ FetchedOut = MakeHolder<TFixedBufferFileOutput>(TFile(fixedDesrc.Duplicate()));
+ UsingStream = FetchedOut.Get();
+ fixedDesrc.Release();
+ } else {
+ UsingStream = usingStream;
+ }
+}
+
+void TUserJobStatsProxy::InitChecked(IOutputStream* def) {
+ IOutputStream* usingStream = CorrectHandle(JobStatisticsHandle);
+
+ if (usingStream == nullptr && !GetEnv("YT_JOB_ID").empty()) {
+ TFileHandle fixedDesrc(JobStatisticsHandle);
+ FetchedOut = MakeHolder<TFixedBufferFileOutput>(TFile(fixedDesrc.Duplicate()));
+ UsingStream = FetchedOut.Get();
+ fixedDesrc.Release();
+ } else {
+ UsingStream = def;
+ }
+}
+
+void TUserJobStatsProxy::InitIfNotInited(IOutputStream * usingStream) {
+ if (UsingStream == nullptr) {
+ Init(usingStream);
+ }
+}
+
+void TUserJobStatsProxy::CommitStats() {
+ if (Stats.empty()) {
+ return;
+ }
+
+ auto res = NYT::TNode::CreateMap();
+ for (auto& p : Stats) {
+ res[p.first] = p.second;
+ }
+ for (auto& p : TimeStats) {
+ res[p.first] = p.second.MilliSeconds();
+ }
+ with_lock(GlobalStatsWritingMutex) {
+ *UsingStream << PrintNodeSimple(res) << ";" << Endl;
+ }
+ Stats.clear();
+}
+
+
+TTimeStatHolder TUserJobStatsProxy::TimerStart(TString name, bool commitOnFinish) {
+ return THolder(new TTimeStat(this, name, commitOnFinish));
+}
+
+void TUserJobStatsProxy::WriteStat(TString name, i64 val) {
+ auto res = NYT::TNode {} (name, val);
+ with_lock(GlobalStatsWritingMutex) {
+ *UsingStream << PrintNodeSimple(res) << ";" << Endl;
+ }
+}
+
+void TUserJobStatsProxy::WriteStatNoFlush(TString name, i64 val) {
+ auto res = NYT::TNode {} (name, val);
+ with_lock(GlobalStatsWritingMutex) {
+ *UsingStream << (TStringBuilder{} << PrintNodeSimple(res) << ";\n");
+ }
+}
+
+TTimeStat::TTimeStat(TUserJobStatsProxy* parent, TString name, bool commit)
+ : Parent(parent)
+ , Name(name)
+ , Commit(commit) {}
+
+TTimeStat::~TTimeStat() {
+ Finish();
+}
+
+void TTimeStat::Cancel() {
+ Parent = nullptr;
+}
+
+void TTimeStat::Finish() {
+ if (!Parent) {
+ return;
+ }
+
+ if (Commit) {
+ Parent->WriteStatNoFlush(Name, Timer.Get().MilliSeconds());
+ } else {
+ Parent->TimeStats[Name] += Timer.Get();
+ }
+ Cancel();
+}
diff --git a/library/cpp/yt/user_job_statistics/user_job_statistics.h b/library/cpp/yt/user_job_statistics/user_job_statistics.h
new file mode 100644
index 0000000000..6939d20417
--- /dev/null
+++ b/library/cpp/yt/user_job_statistics/user_job_statistics.h
@@ -0,0 +1,58 @@
+#pragma once
+
+#include <util/stream/file.h>
+#include <util/generic/hash.h>
+#include <util/datetime/cputimer.h>
+
+namespace NYtTools {
+ class TTimeStat;
+ using TTimeStatHolder = THolder<TTimeStat>;
+
+ class TUserJobStatsProxy {
+ public:
+ static const FHANDLE JobStatisticsHandle;
+ private:
+ THolder<IOutputStream> FetchedOut;
+ IOutputStream* UsingStream = &Cerr;
+ public:
+ // TODO: add inheritance
+ THashMap<TString, i64> Stats;//will be dumped in CommitStats or desctructor
+ THashMap<TString, TDuration> TimeStats;//will be dumped in CommitStats or desctructor
+
+ TUserJobStatsProxy() { Init(nullptr); }
+ ~TUserJobStatsProxy() {
+ CommitStats();
+ }
+ TUserJobStatsProxy (IOutputStream* usingStream) {Init(usingStream);}
+
+ void Init(IOutputStream* usingStream);
+ void InitChecked(IOutputStream* ifNotInJob);
+ void InitIfNotInited(IOutputStream* usingStream);
+ IOutputStream* GetStream() const { return UsingStream; }
+ void CommitStats();
+ void WriteStat(TString name, i64 val); //immidiatly wirtes stat
+ void WriteStatNoFlush(TString name, i64 val); //immidiatly wirtes stat but do not flush it
+
+ //@param name name of statistic to be written in millisecs from creation to destruction
+ //@param commitOnFinish if false: will update state/write on job finish; if true: write stat in destructor
+ TTimeStatHolder TimerStart(TString name, bool commitOnFinish = false);
+ };
+
+ class TTimeStat {
+ TUserJobStatsProxy* Parent;
+ TString Name;
+ bool Commit;
+
+ TTimeStat(TUserJobStatsProxy* parent, TString name, bool commit);
+ friend class TUserJobStatsProxy;
+
+ TSimpleTimer Timer;
+ public:
+ ~TTimeStat();
+ TDuration Get() const {
+ return Timer.Get();
+ }
+ void Cancel();
+ void Finish();
+ };
+}
diff --git a/library/cpp/yt/user_job_statistics/ya.make b/library/cpp/yt/user_job_statistics/ya.make
new file mode 100644
index 0000000000..7179660b31
--- /dev/null
+++ b/library/cpp/yt/user_job_statistics/ya.make
@@ -0,0 +1,11 @@
+LIBRARY()
+
+SRCS(
+ user_job_statistics.cpp
+)
+
+PEERDIR(
+ yt/cpp/mapreduce/common
+)
+
+END()