diff options
author | monster <monster@yandex-team.ru> | 2022-02-10 16:47:19 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:19 +0300 |
commit | b23c1d7a8015c2006a148fd93b84cdeb0aee17a3 (patch) | |
tree | 9814fbd1c3effac9b8377c5d604b367b14e2db55 | |
parent | dd76ae1f6213d065375ab296699f764faafbe5bd (diff) | |
download | ydb-b23c1d7a8015c2006a148fd93b84cdeb0aee17a3.tar.gz |
Restoring authorship annotation for <monster@yandex-team.ru>. Commit 2 of 2.
498 files changed, 30855 insertions, 30855 deletions
diff --git a/contrib/libs/tcmalloc/patches/userdata.patch b/contrib/libs/tcmalloc/patches/userdata.patch index 51012c7ef9..83373cebfe 100644 --- a/contrib/libs/tcmalloc/patches/userdata.patch +++ b/contrib/libs/tcmalloc/patches/userdata.patch @@ -1,220 +1,220 @@ ---- contrib/libs/tcmalloc/tcmalloc/internal/logging.h (index) -+++ contrib/libs/tcmalloc/tcmalloc/internal/logging.h (working tree) -@@ -67,6 +67,8 @@ struct StackTrace { - // between the previous sample and this one - size_t weight; - -+ void* user_data; -+ - template <typename H> - friend H AbslHashValue(H h, const StackTrace& t) { - // As we use StackTrace as a key-value node in StackTraceTable, we only ---- contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h (index) -+++ contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h (working tree) -@@ -120,6 +120,12 @@ ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_SetMaxTotalThreadCacheBytes( - ABSL_ATTRIBUTE_WEAK void - MallocExtension_EnableForkSupport(); - -+ABSL_ATTRIBUTE_WEAK void -+MallocExtension_SetSampleUserDataCallbacks( -+ tcmalloc::MallocExtension::CreateSampleUserDataCallback create, -+ tcmalloc::MallocExtension::CopySampleUserDataCallback copy, -+ tcmalloc::MallocExtension::DestroySampleUserDataCallback destroy); -+ - } - - #endif ---- contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc (index) -+++ contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc (working tree) -@@ -468,6 +468,21 @@ void MallocExtension::EnableForkSupport() { - #endif - } - -+void MallocExtension::SetSampleUserDataCallbacks( -+ CreateSampleUserDataCallback create, -+ CopySampleUserDataCallback copy, -+ DestroySampleUserDataCallback destroy) { -+#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS -+ if (&MallocExtension_SetSampleUserDataCallbacks != nullptr) { -+ MallocExtension_SetSampleUserDataCallbacks(create, copy, destroy); -+ } -+#else -+ (void)create; -+ (void)copy; -+ (void)destroy; -+#endif -+} -+ - } // namespace tcmalloc - - // Default implementation just returns size. The expectation is that ---- contrib/libs/tcmalloc/tcmalloc/malloc_extension.h (index) -+++ contrib/libs/tcmalloc/tcmalloc/malloc_extension.h (working tree) -@@ -94,6 +94,8 @@ class Profile final { - - int depth; - void* stack[kMaxStackDepth]; -+ -+ void* user_data; - }; - - void Iterate(absl::FunctionRef<void(const Sample&)> f) const; -@@ -472,6 +474,16 @@ class MallocExtension final { - // Enables fork support. - // Allocator will continue to function correctly in the child, after calling fork(). - static void EnableForkSupport(); -+ -+ using CreateSampleUserDataCallback = void*(); -+ using CopySampleUserDataCallback = void*(void*); -+ using DestroySampleUserDataCallback = void(void*); -+ -+ // Sets callbacks for lifetime control of custom user data attached to allocation samples -+ static void SetSampleUserDataCallbacks( -+ CreateSampleUserDataCallback create, -+ CopySampleUserDataCallback copy, -+ DestroySampleUserDataCallback destroy); - }; - - } // namespace tcmalloc ---- contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc (index) -+++ contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc (working tree) -@@ -55,6 +55,7 @@ void PeakHeapTracker::MaybeSaveSample() { - StackTrace *t = peak_sampled_span_stacks_, *next = nullptr; - while (t != nullptr) { - next = reinterpret_cast<StackTrace*>(t->stack[kMaxStackDepth - 1]); -+ Static::DestroySampleUserData(t->user_data); - Static::stacktrace_allocator().Delete(t); - t = next; - } -@@ -63,7 +64,9 @@ void PeakHeapTracker::MaybeSaveSample() { - for (Span* s : Static::sampled_objects_) { - t = Static::stacktrace_allocator().New(); - -- *t = *s->sampled_stack(); -+ StackTrace* sampled_stack = s->sampled_stack(); -+ *t = *sampled_stack; -+ t->user_data = Static::CopySampleUserData(sampled_stack->user_data); - if (t->depth == kMaxStackDepth) { - t->depth = kMaxStackDepth - 1; - } ---- contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc (index) -+++ contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc (working tree) -@@ -73,6 +73,7 @@ StackTraceTable::~StackTraceTable() { - Bucket* b = table_[i]; - while (b != nullptr) { - Bucket* next = b->next; -+ Static::DestroySampleUserData(b->trace.user_data); - Static::bucket_allocator().Delete(b); - b = next; - } -@@ -104,6 +105,7 @@ void StackTraceTable::AddTrace(double count, const StackTrace& t) { - b = Static::bucket_allocator().New(); - b->hash = h; - b->trace = t; -+ b->trace.user_data = Static::CopySampleUserData(t.user_data); - b->count = count; - b->total_weight = t.weight * count; - b->next = table_[idx]; -@@ -135,6 +137,8 @@ void StackTraceTable::Iterate( - e.requested_alignment = b->trace.requested_alignment; - e.allocated_size = allocated_size; - -+ e.user_data = b->trace.user_data; -+ - e.depth = b->trace.depth; - static_assert(kMaxStackDepth <= Profile::Sample::kMaxStackDepth, - "Profile stack size smaller than internal stack sizes"); ---- contrib/libs/tcmalloc/tcmalloc/static_vars.cc (index) -+++ contrib/libs/tcmalloc/tcmalloc/static_vars.cc (working tree) -@@ -60,6 +60,12 @@ ABSL_CONST_INIT PageHeapAllocator<StackTraceTable::Bucket> - ABSL_CONST_INIT std::atomic<bool> Static::inited_{false}; - ABSL_CONST_INIT bool Static::cpu_cache_active_ = false; - ABSL_CONST_INIT bool Static::fork_support_enabled_ = false; -+ABSL_CONST_INIT Static::CreateSampleUserDataCallback* -+ Static::create_sample_user_data_callback_ = nullptr; -+ABSL_CONST_INIT Static::CopySampleUserDataCallback* -+ Static::copy_sample_user_data_callback_ = nullptr; -+ABSL_CONST_INIT Static::DestroySampleUserDataCallback* -+ Static::destroy_sample_user_data_callback_ = nullptr; - ABSL_CONST_INIT Static::PageAllocatorStorage Static::page_allocator_; - ABSL_CONST_INIT PageMap Static::pagemap_; - ABSL_CONST_INIT absl::base_internal::SpinLock guarded_page_lock( ---- contrib/libs/tcmalloc/tcmalloc/static_vars.h (index) -+++ contrib/libs/tcmalloc/tcmalloc/static_vars.h (working tree) -@@ -130,6 +130,34 @@ class Static { - static bool ForkSupportEnabled() { return fork_support_enabled_; } - static void EnableForkSupport() { fork_support_enabled_ = true; } - -+ using CreateSampleUserDataCallback = void*(); -+ using CopySampleUserDataCallback = void*(void*); -+ using DestroySampleUserDataCallback = void(void*); -+ -+ static void SetSampleUserDataCallbacks( -+ CreateSampleUserDataCallback create, -+ CopySampleUserDataCallback copy, -+ DestroySampleUserDataCallback destroy) { -+ create_sample_user_data_callback_ = create; -+ copy_sample_user_data_callback_ = copy; -+ destroy_sample_user_data_callback_ = destroy; -+ } -+ -+ static void* CreateSampleUserData() { -+ if (create_sample_user_data_callback_) -+ return create_sample_user_data_callback_(); -+ return nullptr; -+ } -+ static void* CopySampleUserData(void* user_data) { -+ if (copy_sample_user_data_callback_) -+ return copy_sample_user_data_callback_(user_data); -+ return nullptr; -+ } -+ static void DestroySampleUserData(void* user_data) { -+ if (destroy_sample_user_data_callback_) -+ destroy_sample_user_data_callback_(user_data); -+ } -+ - static bool ABSL_ATTRIBUTE_ALWAYS_INLINE IsOnFastPath() { - return - #ifndef TCMALLOC_DEPRECATED_PERTHREAD -@@ -176,6 +204,9 @@ class Static { - ABSL_CONST_INIT static std::atomic<bool> inited_; - static bool cpu_cache_active_; - static bool fork_support_enabled_; -+ static CreateSampleUserDataCallback* create_sample_user_data_callback_; -+ static CopySampleUserDataCallback* copy_sample_user_data_callback_; -+ static DestroySampleUserDataCallback* destroy_sample_user_data_callback_; - ABSL_CONST_INIT static PeakHeapTracker peak_heap_tracker_; - ABSL_CONST_INIT static NumaTopology<kNumaPartitions, kNumBaseClasses> - numa_topology_; ---- contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc (index) -+++ contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc (working tree) -@@ -1151,6 +1151,13 @@ void TCMallocPostFork() { - } - } - -+extern "C" void MallocExtension_SetSampleUserDataCallbacks( -+ MallocExtension::CreateSampleUserDataCallback create, -+ MallocExtension::CopySampleUserDataCallback copy, -+ MallocExtension::DestroySampleUserDataCallback destroy) { -+ Static::SetSampleUserDataCallbacks(create, copy, destroy); -+} -+ - // nallocx slow path. - // Moved to a separate function because size_class_with_alignment is not inlined - // which would cause nallocx to become non-leaf function with stack frame and -@@ -1500,6 +1507,7 @@ static void* SampleifyAllocation(size_t requested_size, size_t weight, - tmp.requested_alignment = requested_alignment; - tmp.allocated_size = allocated_size; - tmp.weight = weight; -+ tmp.user_data = Static::CreateSampleUserData(); - - { - absl::base_internal::SpinLockHolder h(&pageheap_lock); -@@ -1629,6 +1637,7 @@ static void do_free_pages(void* ptr, const PageId p) { - 1); - } - notify_sampled_alloc = true; -+ Static::DestroySampleUserData(st->user_data); - Static::stacktrace_allocator().Delete(st); - } - if (IsSampledMemory(ptr)) { +--- contrib/libs/tcmalloc/tcmalloc/internal/logging.h (index) ++++ contrib/libs/tcmalloc/tcmalloc/internal/logging.h (working tree) +@@ -67,6 +67,8 @@ struct StackTrace { + // between the previous sample and this one + size_t weight; + ++ void* user_data; ++ + template <typename H> + friend H AbslHashValue(H h, const StackTrace& t) { + // As we use StackTrace as a key-value node in StackTraceTable, we only +--- contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h (index) ++++ contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h (working tree) +@@ -120,6 +120,12 @@ ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_SetMaxTotalThreadCacheBytes( + ABSL_ATTRIBUTE_WEAK void + MallocExtension_EnableForkSupport(); + ++ABSL_ATTRIBUTE_WEAK void ++MallocExtension_SetSampleUserDataCallbacks( ++ tcmalloc::MallocExtension::CreateSampleUserDataCallback create, ++ tcmalloc::MallocExtension::CopySampleUserDataCallback copy, ++ tcmalloc::MallocExtension::DestroySampleUserDataCallback destroy); ++ + } + + #endif +--- contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc (index) ++++ contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc (working tree) +@@ -468,6 +468,21 @@ void MallocExtension::EnableForkSupport() { + #endif + } + ++void MallocExtension::SetSampleUserDataCallbacks( ++ CreateSampleUserDataCallback create, ++ CopySampleUserDataCallback copy, ++ DestroySampleUserDataCallback destroy) { ++#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS ++ if (&MallocExtension_SetSampleUserDataCallbacks != nullptr) { ++ MallocExtension_SetSampleUserDataCallbacks(create, copy, destroy); ++ } ++#else ++ (void)create; ++ (void)copy; ++ (void)destroy; ++#endif ++} ++ + } // namespace tcmalloc + + // Default implementation just returns size. The expectation is that +--- contrib/libs/tcmalloc/tcmalloc/malloc_extension.h (index) ++++ contrib/libs/tcmalloc/tcmalloc/malloc_extension.h (working tree) +@@ -94,6 +94,8 @@ class Profile final { + + int depth; + void* stack[kMaxStackDepth]; ++ ++ void* user_data; + }; + + void Iterate(absl::FunctionRef<void(const Sample&)> f) const; +@@ -472,6 +474,16 @@ class MallocExtension final { + // Enables fork support. + // Allocator will continue to function correctly in the child, after calling fork(). + static void EnableForkSupport(); ++ ++ using CreateSampleUserDataCallback = void*(); ++ using CopySampleUserDataCallback = void*(void*); ++ using DestroySampleUserDataCallback = void(void*); ++ ++ // Sets callbacks for lifetime control of custom user data attached to allocation samples ++ static void SetSampleUserDataCallbacks( ++ CreateSampleUserDataCallback create, ++ CopySampleUserDataCallback copy, ++ DestroySampleUserDataCallback destroy); + }; + + } // namespace tcmalloc +--- contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc (index) ++++ contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc (working tree) +@@ -55,6 +55,7 @@ void PeakHeapTracker::MaybeSaveSample() { + StackTrace *t = peak_sampled_span_stacks_, *next = nullptr; + while (t != nullptr) { + next = reinterpret_cast<StackTrace*>(t->stack[kMaxStackDepth - 1]); ++ Static::DestroySampleUserData(t->user_data); + Static::stacktrace_allocator().Delete(t); + t = next; + } +@@ -63,7 +64,9 @@ void PeakHeapTracker::MaybeSaveSample() { + for (Span* s : Static::sampled_objects_) { + t = Static::stacktrace_allocator().New(); + +- *t = *s->sampled_stack(); ++ StackTrace* sampled_stack = s->sampled_stack(); ++ *t = *sampled_stack; ++ t->user_data = Static::CopySampleUserData(sampled_stack->user_data); + if (t->depth == kMaxStackDepth) { + t->depth = kMaxStackDepth - 1; + } +--- contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc (index) ++++ contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc (working tree) +@@ -73,6 +73,7 @@ StackTraceTable::~StackTraceTable() { + Bucket* b = table_[i]; + while (b != nullptr) { + Bucket* next = b->next; ++ Static::DestroySampleUserData(b->trace.user_data); + Static::bucket_allocator().Delete(b); + b = next; + } +@@ -104,6 +105,7 @@ void StackTraceTable::AddTrace(double count, const StackTrace& t) { + b = Static::bucket_allocator().New(); + b->hash = h; + b->trace = t; ++ b->trace.user_data = Static::CopySampleUserData(t.user_data); + b->count = count; + b->total_weight = t.weight * count; + b->next = table_[idx]; +@@ -135,6 +137,8 @@ void StackTraceTable::Iterate( + e.requested_alignment = b->trace.requested_alignment; + e.allocated_size = allocated_size; + ++ e.user_data = b->trace.user_data; ++ + e.depth = b->trace.depth; + static_assert(kMaxStackDepth <= Profile::Sample::kMaxStackDepth, + "Profile stack size smaller than internal stack sizes"); +--- contrib/libs/tcmalloc/tcmalloc/static_vars.cc (index) ++++ contrib/libs/tcmalloc/tcmalloc/static_vars.cc (working tree) +@@ -60,6 +60,12 @@ ABSL_CONST_INIT PageHeapAllocator<StackTraceTable::Bucket> + ABSL_CONST_INIT std::atomic<bool> Static::inited_{false}; + ABSL_CONST_INIT bool Static::cpu_cache_active_ = false; + ABSL_CONST_INIT bool Static::fork_support_enabled_ = false; ++ABSL_CONST_INIT Static::CreateSampleUserDataCallback* ++ Static::create_sample_user_data_callback_ = nullptr; ++ABSL_CONST_INIT Static::CopySampleUserDataCallback* ++ Static::copy_sample_user_data_callback_ = nullptr; ++ABSL_CONST_INIT Static::DestroySampleUserDataCallback* ++ Static::destroy_sample_user_data_callback_ = nullptr; + ABSL_CONST_INIT Static::PageAllocatorStorage Static::page_allocator_; + ABSL_CONST_INIT PageMap Static::pagemap_; + ABSL_CONST_INIT absl::base_internal::SpinLock guarded_page_lock( +--- contrib/libs/tcmalloc/tcmalloc/static_vars.h (index) ++++ contrib/libs/tcmalloc/tcmalloc/static_vars.h (working tree) +@@ -130,6 +130,34 @@ class Static { + static bool ForkSupportEnabled() { return fork_support_enabled_; } + static void EnableForkSupport() { fork_support_enabled_ = true; } + ++ using CreateSampleUserDataCallback = void*(); ++ using CopySampleUserDataCallback = void*(void*); ++ using DestroySampleUserDataCallback = void(void*); ++ ++ static void SetSampleUserDataCallbacks( ++ CreateSampleUserDataCallback create, ++ CopySampleUserDataCallback copy, ++ DestroySampleUserDataCallback destroy) { ++ create_sample_user_data_callback_ = create; ++ copy_sample_user_data_callback_ = copy; ++ destroy_sample_user_data_callback_ = destroy; ++ } ++ ++ static void* CreateSampleUserData() { ++ if (create_sample_user_data_callback_) ++ return create_sample_user_data_callback_(); ++ return nullptr; ++ } ++ static void* CopySampleUserData(void* user_data) { ++ if (copy_sample_user_data_callback_) ++ return copy_sample_user_data_callback_(user_data); ++ return nullptr; ++ } ++ static void DestroySampleUserData(void* user_data) { ++ if (destroy_sample_user_data_callback_) ++ destroy_sample_user_data_callback_(user_data); ++ } ++ + static bool ABSL_ATTRIBUTE_ALWAYS_INLINE IsOnFastPath() { + return + #ifndef TCMALLOC_DEPRECATED_PERTHREAD +@@ -176,6 +204,9 @@ class Static { + ABSL_CONST_INIT static std::atomic<bool> inited_; + static bool cpu_cache_active_; + static bool fork_support_enabled_; ++ static CreateSampleUserDataCallback* create_sample_user_data_callback_; ++ static CopySampleUserDataCallback* copy_sample_user_data_callback_; ++ static DestroySampleUserDataCallback* destroy_sample_user_data_callback_; + ABSL_CONST_INIT static PeakHeapTracker peak_heap_tracker_; + ABSL_CONST_INIT static NumaTopology<kNumaPartitions, kNumBaseClasses> + numa_topology_; +--- contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc (index) ++++ contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc (working tree) +@@ -1151,6 +1151,13 @@ void TCMallocPostFork() { + } + } + ++extern "C" void MallocExtension_SetSampleUserDataCallbacks( ++ MallocExtension::CreateSampleUserDataCallback create, ++ MallocExtension::CopySampleUserDataCallback copy, ++ MallocExtension::DestroySampleUserDataCallback destroy) { ++ Static::SetSampleUserDataCallbacks(create, copy, destroy); ++} ++ + // nallocx slow path. + // Moved to a separate function because size_class_with_alignment is not inlined + // which would cause nallocx to become non-leaf function with stack frame and +@@ -1500,6 +1507,7 @@ static void* SampleifyAllocation(size_t requested_size, size_t weight, + tmp.requested_alignment = requested_alignment; + tmp.allocated_size = allocated_size; + tmp.weight = weight; ++ tmp.user_data = Static::CreateSampleUserData(); + + { + absl::base_internal::SpinLockHolder h(&pageheap_lock); +@@ -1629,6 +1637,7 @@ static void do_free_pages(void* ptr, const PageId p) { + 1); + } + notify_sampled_alloc = true; ++ Static::DestroySampleUserData(st->user_data); + Static::stacktrace_allocator().Delete(st); + } + if (IsSampledMemory(ptr)) { diff --git a/contrib/libs/tcmalloc/tcmalloc/internal/logging.h b/contrib/libs/tcmalloc/tcmalloc/internal/logging.h index 2722c2fccc..4d42aa40a9 100644 --- a/contrib/libs/tcmalloc/tcmalloc/internal/logging.h +++ b/contrib/libs/tcmalloc/tcmalloc/internal/logging.h @@ -67,8 +67,8 @@ struct StackTrace { // between the previous sample and this one size_t weight; - void* user_data; - + void* user_data; + template <typename H> friend H AbslHashValue(H h, const StackTrace& t) { // As we use StackTrace as a key-value node in StackTraceTable, we only diff --git a/contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h b/contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h index 8654588aa9..66027418ed 100644 --- a/contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h +++ b/contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h @@ -120,12 +120,12 @@ ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_SetMaxTotalThreadCacheBytes( ABSL_ATTRIBUTE_WEAK void MallocExtension_EnableForkSupport(); -ABSL_ATTRIBUTE_WEAK void -MallocExtension_SetSampleUserDataCallbacks( - tcmalloc::MallocExtension::CreateSampleUserDataCallback create, - tcmalloc::MallocExtension::CopySampleUserDataCallback copy, - tcmalloc::MallocExtension::DestroySampleUserDataCallback destroy); - +ABSL_ATTRIBUTE_WEAK void +MallocExtension_SetSampleUserDataCallbacks( + tcmalloc::MallocExtension::CreateSampleUserDataCallback create, + tcmalloc::MallocExtension::CopySampleUserDataCallback copy, + tcmalloc::MallocExtension::DestroySampleUserDataCallback destroy); + } #endif diff --git a/contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc b/contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc index ae958f5bb7..ad3205fcdc 100644 --- a/contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc +++ b/contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc @@ -468,21 +468,21 @@ void MallocExtension::EnableForkSupport() { #endif } -void MallocExtension::SetSampleUserDataCallbacks( - CreateSampleUserDataCallback create, - CopySampleUserDataCallback copy, - DestroySampleUserDataCallback destroy) { -#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS - if (&MallocExtension_SetSampleUserDataCallbacks != nullptr) { - MallocExtension_SetSampleUserDataCallbacks(create, copy, destroy); - } -#else - (void)create; - (void)copy; - (void)destroy; -#endif -} - +void MallocExtension::SetSampleUserDataCallbacks( + CreateSampleUserDataCallback create, + CopySampleUserDataCallback copy, + DestroySampleUserDataCallback destroy) { +#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS + if (&MallocExtension_SetSampleUserDataCallbacks != nullptr) { + MallocExtension_SetSampleUserDataCallbacks(create, copy, destroy); + } +#else + (void)create; + (void)copy; + (void)destroy; +#endif +} + } // namespace tcmalloc // Default implementation just returns size. The expectation is that diff --git a/contrib/libs/tcmalloc/tcmalloc/malloc_extension.h b/contrib/libs/tcmalloc/tcmalloc/malloc_extension.h index 77255a8772..fcbd347ca1 100644 --- a/contrib/libs/tcmalloc/tcmalloc/malloc_extension.h +++ b/contrib/libs/tcmalloc/tcmalloc/malloc_extension.h @@ -94,8 +94,8 @@ class Profile final { int depth; void* stack[kMaxStackDepth]; - - void* user_data; + + void* user_data; }; void Iterate(absl::FunctionRef<void(const Sample&)> f) const; @@ -474,16 +474,16 @@ class MallocExtension final { // Enables fork support. // Allocator will continue to function correctly in the child, after calling fork(). static void EnableForkSupport(); - - using CreateSampleUserDataCallback = void*(); - using CopySampleUserDataCallback = void*(void*); - using DestroySampleUserDataCallback = void(void*); - - // Sets callbacks for lifetime control of custom user data attached to allocation samples - static void SetSampleUserDataCallbacks( - CreateSampleUserDataCallback create, - CopySampleUserDataCallback copy, - DestroySampleUserDataCallback destroy); + + using CreateSampleUserDataCallback = void*(); + using CopySampleUserDataCallback = void*(void*); + using DestroySampleUserDataCallback = void(void*); + + // Sets callbacks for lifetime control of custom user data attached to allocation samples + static void SetSampleUserDataCallbacks( + CreateSampleUserDataCallback create, + CopySampleUserDataCallback copy, + DestroySampleUserDataCallback destroy); }; } // namespace tcmalloc diff --git a/contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc b/contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc index 7bb756885b..0dcc0df536 100644 --- a/contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc +++ b/contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc @@ -55,7 +55,7 @@ void PeakHeapTracker::MaybeSaveSample() { StackTrace *t = peak_sampled_span_stacks_, *next = nullptr; while (t != nullptr) { next = reinterpret_cast<StackTrace*>(t->stack[kMaxStackDepth - 1]); - Static::DestroySampleUserData(t->user_data); + Static::DestroySampleUserData(t->user_data); Static::stacktrace_allocator().Delete(t); t = next; } @@ -64,9 +64,9 @@ void PeakHeapTracker::MaybeSaveSample() { for (Span* s : Static::sampled_objects_) { t = Static::stacktrace_allocator().New(); - StackTrace* sampled_stack = s->sampled_stack(); - *t = *sampled_stack; - t->user_data = Static::CopySampleUserData(sampled_stack->user_data); + StackTrace* sampled_stack = s->sampled_stack(); + *t = *sampled_stack; + t->user_data = Static::CopySampleUserData(sampled_stack->user_data); if (t->depth == kMaxStackDepth) { t->depth = kMaxStackDepth - 1; } diff --git a/contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc b/contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc index da3cbc4d8f..5b5741b6a8 100644 --- a/contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc +++ b/contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc @@ -73,7 +73,7 @@ StackTraceTable::~StackTraceTable() { Bucket* b = table_[i]; while (b != nullptr) { Bucket* next = b->next; - Static::DestroySampleUserData(b->trace.user_data); + Static::DestroySampleUserData(b->trace.user_data); Static::bucket_allocator().Delete(b); b = next; } @@ -105,7 +105,7 @@ void StackTraceTable::AddTrace(double count, const StackTrace& t) { b = Static::bucket_allocator().New(); b->hash = h; b->trace = t; - b->trace.user_data = Static::CopySampleUserData(t.user_data); + b->trace.user_data = Static::CopySampleUserData(t.user_data); b->count = count; b->total_weight = t.weight * count; b->next = table_[idx]; @@ -137,8 +137,8 @@ void StackTraceTable::Iterate( e.requested_alignment = b->trace.requested_alignment; e.allocated_size = allocated_size; - e.user_data = b->trace.user_data; - + e.user_data = b->trace.user_data; + e.depth = b->trace.depth; static_assert(kMaxStackDepth <= Profile::Sample::kMaxStackDepth, "Profile stack size smaller than internal stack sizes"); diff --git a/contrib/libs/tcmalloc/tcmalloc/static_vars.cc b/contrib/libs/tcmalloc/tcmalloc/static_vars.cc index c517daa393..08a70de493 100644 --- a/contrib/libs/tcmalloc/tcmalloc/static_vars.cc +++ b/contrib/libs/tcmalloc/tcmalloc/static_vars.cc @@ -60,12 +60,12 @@ ABSL_CONST_INIT PageHeapAllocator<StackTraceTable::Bucket> ABSL_CONST_INIT std::atomic<bool> Static::inited_{false}; ABSL_CONST_INIT bool Static::cpu_cache_active_ = false; ABSL_CONST_INIT bool Static::fork_support_enabled_ = false; -ABSL_CONST_INIT Static::CreateSampleUserDataCallback* - Static::create_sample_user_data_callback_ = nullptr; -ABSL_CONST_INIT Static::CopySampleUserDataCallback* - Static::copy_sample_user_data_callback_ = nullptr; -ABSL_CONST_INIT Static::DestroySampleUserDataCallback* - Static::destroy_sample_user_data_callback_ = nullptr; +ABSL_CONST_INIT Static::CreateSampleUserDataCallback* + Static::create_sample_user_data_callback_ = nullptr; +ABSL_CONST_INIT Static::CopySampleUserDataCallback* + Static::copy_sample_user_data_callback_ = nullptr; +ABSL_CONST_INIT Static::DestroySampleUserDataCallback* + Static::destroy_sample_user_data_callback_ = nullptr; ABSL_CONST_INIT Static::PageAllocatorStorage Static::page_allocator_; ABSL_CONST_INIT PageMap Static::pagemap_; ABSL_CONST_INIT absl::base_internal::SpinLock guarded_page_lock( diff --git a/contrib/libs/tcmalloc/tcmalloc/static_vars.h b/contrib/libs/tcmalloc/tcmalloc/static_vars.h index 9985c3a18c..be68edc189 100644 --- a/contrib/libs/tcmalloc/tcmalloc/static_vars.h +++ b/contrib/libs/tcmalloc/tcmalloc/static_vars.h @@ -130,34 +130,34 @@ class Static { static bool ForkSupportEnabled() { return fork_support_enabled_; } static void EnableForkSupport() { fork_support_enabled_ = true; } - using CreateSampleUserDataCallback = void*(); - using CopySampleUserDataCallback = void*(void*); - using DestroySampleUserDataCallback = void(void*); - - static void SetSampleUserDataCallbacks( - CreateSampleUserDataCallback create, - CopySampleUserDataCallback copy, - DestroySampleUserDataCallback destroy) { - create_sample_user_data_callback_ = create; - copy_sample_user_data_callback_ = copy; - destroy_sample_user_data_callback_ = destroy; - } - - static void* CreateSampleUserData() { - if (create_sample_user_data_callback_) - return create_sample_user_data_callback_(); - return nullptr; - } - static void* CopySampleUserData(void* user_data) { - if (copy_sample_user_data_callback_) - return copy_sample_user_data_callback_(user_data); - return nullptr; - } - static void DestroySampleUserData(void* user_data) { - if (destroy_sample_user_data_callback_) - destroy_sample_user_data_callback_(user_data); - } - + using CreateSampleUserDataCallback = void*(); + using CopySampleUserDataCallback = void*(void*); + using DestroySampleUserDataCallback = void(void*); + + static void SetSampleUserDataCallbacks( + CreateSampleUserDataCallback create, + CopySampleUserDataCallback copy, + DestroySampleUserDataCallback destroy) { + create_sample_user_data_callback_ = create; + copy_sample_user_data_callback_ = copy; + destroy_sample_user_data_callback_ = destroy; + } + + static void* CreateSampleUserData() { + if (create_sample_user_data_callback_) + return create_sample_user_data_callback_(); + return nullptr; + } + static void* CopySampleUserData(void* user_data) { + if (copy_sample_user_data_callback_) + return copy_sample_user_data_callback_(user_data); + return nullptr; + } + static void DestroySampleUserData(void* user_data) { + if (destroy_sample_user_data_callback_) + destroy_sample_user_data_callback_(user_data); + } + static bool ABSL_ATTRIBUTE_ALWAYS_INLINE IsOnFastPath() { return #ifndef TCMALLOC_DEPRECATED_PERTHREAD @@ -204,9 +204,9 @@ class Static { ABSL_CONST_INIT static std::atomic<bool> inited_; static bool cpu_cache_active_; static bool fork_support_enabled_; - static CreateSampleUserDataCallback* create_sample_user_data_callback_; - static CopySampleUserDataCallback* copy_sample_user_data_callback_; - static DestroySampleUserDataCallback* destroy_sample_user_data_callback_; + static CreateSampleUserDataCallback* create_sample_user_data_callback_; + static CopySampleUserDataCallback* copy_sample_user_data_callback_; + static DestroySampleUserDataCallback* destroy_sample_user_data_callback_; ABSL_CONST_INIT static PeakHeapTracker peak_heap_tracker_; ABSL_CONST_INIT static NumaTopology<kNumaPartitions, kNumBaseClasses> numa_topology_; diff --git a/contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc b/contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc index 8c3a03f610..8e62ba91b9 100644 --- a/contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc +++ b/contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc @@ -1151,13 +1151,13 @@ void TCMallocPostFork() { } } -extern "C" void MallocExtension_SetSampleUserDataCallbacks( - MallocExtension::CreateSampleUserDataCallback create, - MallocExtension::CopySampleUserDataCallback copy, - MallocExtension::DestroySampleUserDataCallback destroy) { - Static::SetSampleUserDataCallbacks(create, copy, destroy); -} - +extern "C" void MallocExtension_SetSampleUserDataCallbacks( + MallocExtension::CreateSampleUserDataCallback create, + MallocExtension::CopySampleUserDataCallback copy, + MallocExtension::DestroySampleUserDataCallback destroy) { + Static::SetSampleUserDataCallbacks(create, copy, destroy); +} + // nallocx slow path. // Moved to a separate function because size_class_with_alignment is not inlined // which would cause nallocx to become non-leaf function with stack frame and @@ -1507,7 +1507,7 @@ static void* SampleifyAllocation(size_t requested_size, size_t weight, tmp.requested_alignment = requested_alignment; tmp.allocated_size = allocated_size; tmp.weight = weight; - tmp.user_data = Static::CreateSampleUserData(); + tmp.user_data = Static::CreateSampleUserData(); { absl::base_internal::SpinLockHolder h(&pageheap_lock); @@ -1637,7 +1637,7 @@ static void do_free_pages(void* ptr, const PageId p) { 1); } notify_sampled_alloc = true; - Static::DestroySampleUserData(st->user_data); + Static::DestroySampleUserData(st->user_data); Static::stacktrace_allocator().Delete(st); } if (IsSampledMemory(ptr)) { diff --git a/library/cpp/actors/core/README.md b/library/cpp/actors/core/README.md index 3cd4354c6d..439a8dd459 100644 --- a/library/cpp/actors/core/README.md +++ b/library/cpp/actors/core/README.md @@ -1,99 +1,99 @@ -## Memory tracker - -https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/actors/core/memory_track.h - -Использование: - -* отслеживание аллокаций экземпляров конкретного класса через new/delete и new[]/delete[] -* отслеживание аллокаций в контейнерах -* ручное отслеживание моментов аллокации/деаллокации - ----- - -### Отслеживание аллокаций класса через new/delete - -Использование с автоматически генерируемой меткой: - -```cpp -#include <library/cpp/actors/core/memory_track.h> - -struct TTypeLabeled - : public NActors::NMemory::TTrack<TTypeLabeled> -{ - char payload[16]; -}; -``` - -Использование с пользовательским именем метки: - -```cpp -#include <library/cpp/actors/core/memory_track.h> - -static const char NamedLabel[] = "NamedLabel"; - -struct TNameLabeled - : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel> -{ - char payload[32]; -}; -``` - ----- - -### Отслеживание аллокаций в контейнерах - -```cpp -#include <library/cpp/actors/core/memory_track.h> - -static const char InContainerLabel[] = "InContainerLabel"; - -struct TInContainer { - char payload[16]; -}; - -std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT; - -std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN; - -using TKey = int; - -std::map<TKey, TInContainer, std::less<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT; - -std::map<TKey, TInContainer, std::less<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN; - -std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT; - -std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN; -``` - ----- - -### Ручное отслеживание аллокаций/деаллокаций - -```cpp -#include <library/cpp/actors/core/memory_track.h> - -static const char ManualLabel[] = "ManualLabel"; - -... -NActors::NMemory::TLabel<ManualLabel>::Add(size); - -... -NActors::NMemory::TLabel<ManualLabel>::Sub(size); -``` - ----- - -### Собираемые метрики - -Сервис **utils**, пользовательская метка **label**, сенсоры: - -- MT/Count: количество аллокаций в моменте -- MT/Memory: аллоцированная память в моменте -- MT/PeakCount: пиковое значение количества аллокаций (сэмплится с фиксированной частотой) -- MT/PeakMemory: пиковое значение аллоцированной памяти - +## Memory tracker + +https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/actors/core/memory_track.h + +Использование: + +* отслеживание аллокаций экземпляров конкретного класса через new/delete и new[]/delete[] +* отслеживание аллокаций в контейнерах +* ручное отслеживание моментов аллокации/деаллокации + +---- + +### Отслеживание аллокаций класса через new/delete + +Использование с автоматически генерируемой меткой: + +```cpp +#include <library/cpp/actors/core/memory_track.h> + +struct TTypeLabeled + : public NActors::NMemory::TTrack<TTypeLabeled> +{ + char payload[16]; +}; +``` + +Использование с пользовательским именем метки: + +```cpp +#include <library/cpp/actors/core/memory_track.h> + +static const char NamedLabel[] = "NamedLabel"; + +struct TNameLabeled + : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel> +{ + char payload[32]; +}; +``` + +---- + +### Отслеживание аллокаций в контейнерах + +```cpp +#include <library/cpp/actors/core/memory_track.h> + +static const char InContainerLabel[] = "InContainerLabel"; + +struct TInContainer { + char payload[16]; +}; + +std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT; + +std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN; + +using TKey = int; + +std::map<TKey, TInContainer, std::less<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT; + +std::map<TKey, TInContainer, std::less<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN; + +std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT; + +std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN; +``` + +---- + +### Ручное отслеживание аллокаций/деаллокаций + +```cpp +#include <library/cpp/actors/core/memory_track.h> + +static const char ManualLabel[] = "ManualLabel"; + +... +NActors::NMemory::TLabel<ManualLabel>::Add(size); + +... +NActors::NMemory::TLabel<ManualLabel>::Sub(size); +``` + +---- + +### Собираемые метрики + +Сервис **utils**, пользовательская метка **label**, сенсоры: + +- MT/Count: количество аллокаций в моменте +- MT/Memory: аллоцированная память в моменте +- MT/PeakCount: пиковое значение количества аллокаций (сэмплится с фиксированной частотой) +- MT/PeakMemory: пиковое значение аллоцированной памяти + diff --git a/library/cpp/actors/core/actorsystem.h b/library/cpp/actors/core/actorsystem.h index 7df0a9f633..40499d7586 100644 --- a/library/cpp/actors/core/actorsystem.h +++ b/library/cpp/actors/core/actorsystem.h @@ -332,8 +332,8 @@ namespace NActors { TActorId LookupLocalService(const TActorId& x) const; TActorId RegisterLocalService(const TActorId& serviceId, const TActorId& actorId); - ui32 GetMaxActivityType() const { - return SystemSetup ? SystemSetup->MaxActivityType : 1; + ui32 GetMaxActivityType() const { + return SystemSetup ? SystemSetup->MaxActivityType : 1; } TInstant Timestamp() const { diff --git a/library/cpp/actors/core/event.h b/library/cpp/actors/core/event.h index dbdb45daac..6ff02aaf94 100644 --- a/library/cpp/actors/core/event.h +++ b/library/cpp/actors/core/event.h @@ -46,8 +46,8 @@ namespace NActors { }; // fat handle - class IEventHandle : TNonCopyable { - struct TOnNondelivery { + class IEventHandle : TNonCopyable { + struct TOnNondelivery { TActorId Recipient; TOnNondelivery(const TActorId& recipient) diff --git a/library/cpp/actors/core/executor_thread.cpp b/library/cpp/actors/core/executor_thread.cpp index ae5f111dff..446b651efd 100644 --- a/library/cpp/actors/core/executor_thread.cpp +++ b/library/cpp/actors/core/executor_thread.cpp @@ -137,7 +137,7 @@ namespace NActors { NHPTimer::STime hpprev = hpstart; IActor* actor = nullptr; - ui32 prevActivityType = std::numeric_limits<ui32>::max(); + ui32 prevActivityType = std::numeric_limits<ui32>::max(); TActorId recipient; for (ui32 executed = 0; executed < Ctx.EventsPerMailbox; ++executed) { TAutoPtr<IEventHandle> ev(mailbox->Pop()); @@ -170,14 +170,14 @@ namespace NActors { ui32 evTypeForTracing = ev->Type; - ui32 activityType = actor->GetActivityType(); - if (activityType != prevActivityType) { - prevActivityType = activityType; - NProfiling::TMemoryTagScope::Reset(ActorSystem->MemProfActivityBase + activityType); + ui32 activityType = actor->GetActivityType(); + if (activityType != prevActivityType) { + prevActivityType = activityType; + NProfiling::TMemoryTagScope::Reset(ActorSystem->MemProfActivityBase + activityType); } - actor->Receive(ev, ctx); - + actor->Receive(ev, ctx); + size_t dyingActorsCnt = DyingActors.size(); Ctx.UpdateActorsStats(dyingActorsCnt); if (dyingActorsCnt) { @@ -273,7 +273,7 @@ namespace NActors { } } - NProfiling::TMemoryTagScope::Reset(0); + NProfiling::TMemoryTagScope::Reset(0); TlsActivationContext = nullptr; UnlockFromExecution(mailbox, Ctx.Executor, reclaimAsFree, hint, Ctx.WorkerId, RevolvingWriteCounter); } diff --git a/library/cpp/actors/core/mailbox.h b/library/cpp/actors/core/mailbox.h index 277a259932..0bd9c4d314 100644 --- a/library/cpp/actors/core/mailbox.h +++ b/library/cpp/actors/core/mailbox.h @@ -397,7 +397,7 @@ namespace NActors { static_assert(sizeof(TRevolvingMailbox) == 128, "expect sizeof(TRevolvingMailbox) == 128"); struct THTSwapMailbox: public TMailboxHeader { - using TQueueType = NThreading::THTSwapQueue<IEventHandle*>; + using TQueueType = NThreading::THTSwapQueue<IEventHandle*>; TQueueType Queue; NHPTimer::STime ScheduleMoment; @@ -447,7 +447,7 @@ namespace NActors { "expect sizeof(THTSwapMailbox) == 64"); struct TReadAsFilledMailbox: public TMailboxHeader { - using TQueueType = NThreading::TReadAsFilledQueue<IEventHandle>; + using TQueueType = NThreading::TReadAsFilledQueue<IEventHandle>; TQueueType Queue; NHPTimer::STime ScheduleMoment; diff --git a/library/cpp/actors/core/memory_track.cpp b/library/cpp/actors/core/memory_track.cpp index 79392ef686..5f422116be 100644 --- a/library/cpp/actors/core/memory_track.cpp +++ b/library/cpp/actors/core/memory_track.cpp @@ -1,38 +1,38 @@ -#include "memory_track.h" -#include "memory_tracker.h" - -namespace NActors { -namespace NMemory { - -namespace NPrivate { - -TThreadLocalInfo::TThreadLocalInfo() - : Metrics(TMemoryTracker::Instance()->GetCount()) -{ - TMemoryTracker::Instance()->OnCreateThread(this); -} - -TThreadLocalInfo::~TThreadLocalInfo() { - TMemoryTracker::Instance()->OnDestroyThread(this); -} - -TMetric* TThreadLocalInfo::GetMetric(size_t index) { - if (Y_UNLIKELY(index >= Metrics.size())) { - return &Null; - } - return &Metrics[index]; -} - -const std::vector<TMetric>& TThreadLocalInfo::GetMetrics() const { - return Metrics; -} - -size_t TBaseLabel::RegisterStaticMemoryLabel(const char* name, bool hasSensor) { - return TMemoryTracker::Instance()->RegisterStaticMemoryLabel(name, hasSensor); -} - -} - -} -} - +#include "memory_track.h" +#include "memory_tracker.h" + +namespace NActors { +namespace NMemory { + +namespace NPrivate { + +TThreadLocalInfo::TThreadLocalInfo() + : Metrics(TMemoryTracker::Instance()->GetCount()) +{ + TMemoryTracker::Instance()->OnCreateThread(this); +} + +TThreadLocalInfo::~TThreadLocalInfo() { + TMemoryTracker::Instance()->OnDestroyThread(this); +} + +TMetric* TThreadLocalInfo::GetMetric(size_t index) { + if (Y_UNLIKELY(index >= Metrics.size())) { + return &Null; + } + return &Metrics[index]; +} + +const std::vector<TMetric>& TThreadLocalInfo::GetMetrics() const { + return Metrics; +} + +size_t TBaseLabel::RegisterStaticMemoryLabel(const char* name, bool hasSensor) { + return TMemoryTracker::Instance()->RegisterStaticMemoryLabel(name, hasSensor); +} + +} + +} +} + diff --git a/library/cpp/actors/core/memory_track.h b/library/cpp/actors/core/memory_track.h index 97a7c5dfae..6035333eeb 100644 --- a/library/cpp/actors/core/memory_track.h +++ b/library/cpp/actors/core/memory_track.h @@ -1,293 +1,293 @@ -#pragma once - -#include <vector> - +#pragma once + +#include <vector> + #include <util/system/type_name.h> -#include <util/thread/singleton.h> - -#define ENABLE_MEMORY_TRACKING - -namespace NActors { -namespace NMemory { - -namespace NPrivate { - -class TMetric { - std::atomic<ssize_t> Memory; - std::atomic<ssize_t> Count; - - void Copy(const TMetric& other) { - Memory.store(other.GetMemory(), std::memory_order_relaxed); - Count.store(other.GetCount(), std::memory_order_relaxed); - } - -public: - TMetric() - : Memory(0) - , Count(0) - {} - - inline TMetric(const TMetric& other) { - Copy(other); - } - - inline TMetric(TMetric&& other) { - Copy(other); - } - - inline TMetric& operator=(const TMetric& other) { - Copy(other); - return *this; - } - - inline TMetric& operator=(TMetric&& other) { - Copy(other); - return *this; - } - - inline ssize_t GetMemory() const { - return Memory.load(std::memory_order_relaxed); - } - inline void SetMemory(ssize_t value) { - Memory.store(value, std::memory_order_relaxed); - } - - inline ssize_t GetCount() const { - return Count.load(std::memory_order_relaxed); - } - inline void SetCount(ssize_t value) { - Count.store(value, std::memory_order_relaxed); - } - - inline void operator+=(const TMetric& other) { - SetMemory(GetMemory() + other.GetMemory()); - SetCount(GetCount() + other.GetCount()); - } - - inline void CalculatePeak(const TMetric& other) { - SetMemory(Max(GetMemory(), other.GetMemory())); - SetCount(Max(GetCount(), other.GetCount())); - } - - inline void Add(size_t size) { - SetMemory(GetMemory() + size); - SetCount(GetCount() + 1); - } - - inline void Sub(size_t size) { - SetMemory(GetMemory() - size); - SetCount(GetCount() - 1); - } -}; - - -class TThreadLocalInfo { -public: - TThreadLocalInfo(); - ~TThreadLocalInfo(); - - TMetric* GetMetric(size_t index); - const std::vector<TMetric>& GetMetrics() const; - -private: - std::vector<TMetric> Metrics; - - inline static TMetric Null = {}; -}; - - -class TBaseLabel { -protected: - static size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor); - - inline static TMetric* GetLocalMetric(size_t index) { - return FastTlsSingleton<TThreadLocalInfo>()->GetMetric(index); - } -}; - - -template <const char* Name> -class TNameLabel - : TBaseLabel -{ -public: - static void Add(size_t size) { -#if defined(ENABLE_MEMORY_TRACKING) - Y_UNUSED(MetricInit); - - if (Y_UNLIKELY(!Metric)) { - Metric = GetLocalMetric(Index); - } - - Metric->Add(size); -#else - Y_UNUSED(size); -#endif - } - - static void Sub(size_t size) { -#if defined(ENABLE_MEMORY_TRACKING) - Y_UNUSED(MetricInit); - - if (Y_UNLIKELY(!Metric)) { - Metric = GetLocalMetric(Index); - } - - Metric->Sub(size); -#else - Y_UNUSED(size); -#endif - } - -private: -#if defined(ENABLE_MEMORY_TRACKING) - inline static size_t Index = Max<size_t>(); - inline static struct TMetricInit { - TMetricInit() { - Index = RegisterStaticMemoryLabel(Name, true); - } - } MetricInit; - - inline static thread_local TMetric* Metric = nullptr; -#endif -}; - - -template <typename TType> -class TTypeLabel - : TBaseLabel -{ -public: - static void Add(size_t size) { -#if defined(ENABLE_MEMORY_TRACKING) - Y_UNUSED(MetricInit); - - if (Y_UNLIKELY(!Metric)) { - Metric = GetLocalMetric(Index); - } - - Metric->Add(size); -#else - Y_UNUSED(size); -#endif - } - - static void Sub(size_t size) { -#if defined(ENABLE_MEMORY_TRACKING) - Y_UNUSED(MetricInit); - - if (Y_UNLIKELY(!Metric)) { - Metric = GetLocalMetric(Index); - } - - Metric->Sub(size); -#else - Y_UNUSED(size); -#endif - } - -private: -#if defined(ENABLE_MEMORY_TRACKING) - inline static size_t Index = Max<size_t>(); - inline static struct TMetricInit { - TMetricInit() { - Index = RegisterStaticMemoryLabel(TypeName<TType>().c_str(), false); - } - } MetricInit; - - inline static thread_local TMetric* Metric = nullptr; -#endif -}; - - -template <typename T> -struct TTrackHelper { -#if defined(ENABLE_MEMORY_TRACKING) - void* operator new(size_t size) { - T::Add(size); - return malloc(size); - } - - void* operator new[](size_t size) { - T::Add(size); - return malloc(size); - } - - void operator delete(void* ptr, size_t size) { - T::Sub(size); - free(ptr); - } - - void operator delete[](void* ptr, size_t size) { - T::Sub(size); - free(ptr); - } -#endif -}; - -template <typename TType, typename T> -struct TAllocHelper { - typedef size_t size_type; - typedef TType value_type; - typedef TType* pointer; - typedef const TType* const_pointer; - - struct propagate_on_container_copy_assignment : public std::false_type {}; - struct propagate_on_container_move_assignment : public std::false_type {}; - struct propagate_on_container_swap : public std::false_type {}; - - pointer allocate(size_type n, const void* hint = nullptr) { - Y_UNUSED(hint); - auto size = n * sizeof(TType); - T::Add(size); - return (pointer)malloc(size); - } - - void deallocate(pointer ptr, size_t n) { - auto size = n * sizeof(TType); - T::Sub(size); - free((void*)ptr); - } -}; - -} // NPrivate - - -template <const char* Name> -using TLabel = NPrivate::TNameLabel<Name>; - -template <typename TType, const char* Name = nullptr> -struct TTrack - : public NPrivate::TTrackHelper<NPrivate::TNameLabel<Name>> -{ -}; - -template <typename TType> -struct TTrack<TType, nullptr> - : public NPrivate::TTrackHelper<NPrivate::TTypeLabel<TType>> -{ -}; - -template <typename TType, const char* Name = nullptr> -struct TAlloc - : public NPrivate::TAllocHelper<TType, NPrivate::TNameLabel<Name>> -{ - template<typename U> - struct rebind { - typedef TAlloc<U, Name> other; - }; -}; - -template <typename TType> -struct TAlloc<TType, nullptr> - : public NPrivate::TAllocHelper<TType, NPrivate::TTypeLabel<TType>> -{ - template<typename U> - struct rebind { - typedef TAlloc<U> other; - }; -}; - -} -} - +#include <util/thread/singleton.h> + +#define ENABLE_MEMORY_TRACKING + +namespace NActors { +namespace NMemory { + +namespace NPrivate { + +class TMetric { + std::atomic<ssize_t> Memory; + std::atomic<ssize_t> Count; + + void Copy(const TMetric& other) { + Memory.store(other.GetMemory(), std::memory_order_relaxed); + Count.store(other.GetCount(), std::memory_order_relaxed); + } + +public: + TMetric() + : Memory(0) + , Count(0) + {} + + inline TMetric(const TMetric& other) { + Copy(other); + } + + inline TMetric(TMetric&& other) { + Copy(other); + } + + inline TMetric& operator=(const TMetric& other) { + Copy(other); + return *this; + } + + inline TMetric& operator=(TMetric&& other) { + Copy(other); + return *this; + } + + inline ssize_t GetMemory() const { + return Memory.load(std::memory_order_relaxed); + } + inline void SetMemory(ssize_t value) { + Memory.store(value, std::memory_order_relaxed); + } + + inline ssize_t GetCount() const { + return Count.load(std::memory_order_relaxed); + } + inline void SetCount(ssize_t value) { + Count.store(value, std::memory_order_relaxed); + } + + inline void operator+=(const TMetric& other) { + SetMemory(GetMemory() + other.GetMemory()); + SetCount(GetCount() + other.GetCount()); + } + + inline void CalculatePeak(const TMetric& other) { + SetMemory(Max(GetMemory(), other.GetMemory())); + SetCount(Max(GetCount(), other.GetCount())); + } + + inline void Add(size_t size) { + SetMemory(GetMemory() + size); + SetCount(GetCount() + 1); + } + + inline void Sub(size_t size) { + SetMemory(GetMemory() - size); + SetCount(GetCount() - 1); + } +}; + + +class TThreadLocalInfo { +public: + TThreadLocalInfo(); + ~TThreadLocalInfo(); + + TMetric* GetMetric(size_t index); + const std::vector<TMetric>& GetMetrics() const; + +private: + std::vector<TMetric> Metrics; + + inline static TMetric Null = {}; +}; + + +class TBaseLabel { +protected: + static size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor); + + inline static TMetric* GetLocalMetric(size_t index) { + return FastTlsSingleton<TThreadLocalInfo>()->GetMetric(index); + } +}; + + +template <const char* Name> +class TNameLabel + : TBaseLabel +{ +public: + static void Add(size_t size) { +#if defined(ENABLE_MEMORY_TRACKING) + Y_UNUSED(MetricInit); + + if (Y_UNLIKELY(!Metric)) { + Metric = GetLocalMetric(Index); + } + + Metric->Add(size); +#else + Y_UNUSED(size); +#endif + } + + static void Sub(size_t size) { +#if defined(ENABLE_MEMORY_TRACKING) + Y_UNUSED(MetricInit); + + if (Y_UNLIKELY(!Metric)) { + Metric = GetLocalMetric(Index); + } + + Metric->Sub(size); +#else + Y_UNUSED(size); +#endif + } + +private: +#if defined(ENABLE_MEMORY_TRACKING) + inline static size_t Index = Max<size_t>(); + inline static struct TMetricInit { + TMetricInit() { + Index = RegisterStaticMemoryLabel(Name, true); + } + } MetricInit; + + inline static thread_local TMetric* Metric = nullptr; +#endif +}; + + +template <typename TType> +class TTypeLabel + : TBaseLabel +{ +public: + static void Add(size_t size) { +#if defined(ENABLE_MEMORY_TRACKING) + Y_UNUSED(MetricInit); + + if (Y_UNLIKELY(!Metric)) { + Metric = GetLocalMetric(Index); + } + + Metric->Add(size); +#else + Y_UNUSED(size); +#endif + } + + static void Sub(size_t size) { +#if defined(ENABLE_MEMORY_TRACKING) + Y_UNUSED(MetricInit); + + if (Y_UNLIKELY(!Metric)) { + Metric = GetLocalMetric(Index); + } + + Metric->Sub(size); +#else + Y_UNUSED(size); +#endif + } + +private: +#if defined(ENABLE_MEMORY_TRACKING) + inline static size_t Index = Max<size_t>(); + inline static struct TMetricInit { + TMetricInit() { + Index = RegisterStaticMemoryLabel(TypeName<TType>().c_str(), false); + } + } MetricInit; + + inline static thread_local TMetric* Metric = nullptr; +#endif +}; + + +template <typename T> +struct TTrackHelper { +#if defined(ENABLE_MEMORY_TRACKING) + void* operator new(size_t size) { + T::Add(size); + return malloc(size); + } + + void* operator new[](size_t size) { + T::Add(size); + return malloc(size); + } + + void operator delete(void* ptr, size_t size) { + T::Sub(size); + free(ptr); + } + + void operator delete[](void* ptr, size_t size) { + T::Sub(size); + free(ptr); + } +#endif +}; + +template <typename TType, typename T> +struct TAllocHelper { + typedef size_t size_type; + typedef TType value_type; + typedef TType* pointer; + typedef const TType* const_pointer; + + struct propagate_on_container_copy_assignment : public std::false_type {}; + struct propagate_on_container_move_assignment : public std::false_type {}; + struct propagate_on_container_swap : public std::false_type {}; + + pointer allocate(size_type n, const void* hint = nullptr) { + Y_UNUSED(hint); + auto size = n * sizeof(TType); + T::Add(size); + return (pointer)malloc(size); + } + + void deallocate(pointer ptr, size_t n) { + auto size = n * sizeof(TType); + T::Sub(size); + free((void*)ptr); + } +}; + +} // NPrivate + + +template <const char* Name> +using TLabel = NPrivate::TNameLabel<Name>; + +template <typename TType, const char* Name = nullptr> +struct TTrack + : public NPrivate::TTrackHelper<NPrivate::TNameLabel<Name>> +{ +}; + +template <typename TType> +struct TTrack<TType, nullptr> + : public NPrivate::TTrackHelper<NPrivate::TTypeLabel<TType>> +{ +}; + +template <typename TType, const char* Name = nullptr> +struct TAlloc + : public NPrivate::TAllocHelper<TType, NPrivate::TNameLabel<Name>> +{ + template<typename U> + struct rebind { + typedef TAlloc<U, Name> other; + }; +}; + +template <typename TType> +struct TAlloc<TType, nullptr> + : public NPrivate::TAllocHelper<TType, NPrivate::TTypeLabel<TType>> +{ + template<typename U> + struct rebind { + typedef TAlloc<U> other; + }; +}; + +} +} + diff --git a/library/cpp/actors/core/memory_tracker.cpp b/library/cpp/actors/core/memory_tracker.cpp index 42b93a2868..8a12452c71 100644 --- a/library/cpp/actors/core/memory_tracker.cpp +++ b/library/cpp/actors/core/memory_tracker.cpp @@ -1,103 +1,103 @@ -#include "memory_tracker.h" - -#include <util/generic/xrange.h> - -namespace NActors { -namespace NMemory { - -namespace NPrivate { - -TMemoryTracker* TMemoryTracker::Instance() { - return SingletonWithPriority<TMemoryTracker, 0>(); -} - -void TMemoryTracker::Initialize() { - GlobalMetrics.resize(Indices.size()); -} - -const std::map<TString, size_t>& TMemoryTracker::GetMetricIndices() const { - return Indices; -} - -const std::unordered_set<size_t>& TMemoryTracker::GetSensors() const { - return Sensors; -} - -TString TMemoryTracker::GetName(size_t index) const { - return Names[index]; -} - -size_t TMemoryTracker::GetCount() const { - return Indices.size(); -} - -void TMemoryTracker::GatherMetrics(std::vector<TMetric>& metrics) const { - metrics.resize(0); - auto count = GetCount(); - - if (!count || GlobalMetrics.size() != count) { - return; - } - - TReadGuard guard(LockThreadInfo); - - metrics.resize(count); - for (size_t i : xrange(count)) { - metrics[i] += GlobalMetrics[i]; - } - - for (auto info : ThreadInfo) { - auto& localMetrics = info->GetMetrics(); - if (localMetrics.size() == count) { - for (size_t i : xrange(count)) { - metrics[i] += localMetrics[i]; - } - } - } -} - -size_t TMemoryTracker::RegisterStaticMemoryLabel(const char* name, bool hasSensor) { - size_t index = 0; - auto found = Indices.find(name); - if (found == Indices.end()) { - TString str(name); - auto next = Names.size(); - Indices.emplace(str, next); - Names.push_back(str); - index = next; - } else { - index = found->second; - } - - if (hasSensor) { - Sensors.emplace(index); - } - return index; -} - -void TMemoryTracker::OnCreateThread(TThreadLocalInfo* info) { - TWriteGuard guard(LockThreadInfo); - ThreadInfo.insert(info); -} - -void TMemoryTracker::OnDestroyThread(TThreadLocalInfo* info) { - TWriteGuard guard(LockThreadInfo); - - auto count = GetCount(); - if (count && GlobalMetrics.size() == count) { - const auto& localMetrics = info->GetMetrics(); - if (localMetrics.size() == count) { - for (size_t i : xrange(count)) { - GlobalMetrics[i] += localMetrics[i]; - } - } - } - - ThreadInfo.erase(info); -} - -} - -} -} - +#include "memory_tracker.h" + +#include <util/generic/xrange.h> + +namespace NActors { +namespace NMemory { + +namespace NPrivate { + +TMemoryTracker* TMemoryTracker::Instance() { + return SingletonWithPriority<TMemoryTracker, 0>(); +} + +void TMemoryTracker::Initialize() { + GlobalMetrics.resize(Indices.size()); +} + +const std::map<TString, size_t>& TMemoryTracker::GetMetricIndices() const { + return Indices; +} + +const std::unordered_set<size_t>& TMemoryTracker::GetSensors() const { + return Sensors; +} + +TString TMemoryTracker::GetName(size_t index) const { + return Names[index]; +} + +size_t TMemoryTracker::GetCount() const { + return Indices.size(); +} + +void TMemoryTracker::GatherMetrics(std::vector<TMetric>& metrics) const { + metrics.resize(0); + auto count = GetCount(); + + if (!count || GlobalMetrics.size() != count) { + return; + } + + TReadGuard guard(LockThreadInfo); + + metrics.resize(count); + for (size_t i : xrange(count)) { + metrics[i] += GlobalMetrics[i]; + } + + for (auto info : ThreadInfo) { + auto& localMetrics = info->GetMetrics(); + if (localMetrics.size() == count) { + for (size_t i : xrange(count)) { + metrics[i] += localMetrics[i]; + } + } + } +} + +size_t TMemoryTracker::RegisterStaticMemoryLabel(const char* name, bool hasSensor) { + size_t index = 0; + auto found = Indices.find(name); + if (found == Indices.end()) { + TString str(name); + auto next = Names.size(); + Indices.emplace(str, next); + Names.push_back(str); + index = next; + } else { + index = found->second; + } + + if (hasSensor) { + Sensors.emplace(index); + } + return index; +} + +void TMemoryTracker::OnCreateThread(TThreadLocalInfo* info) { + TWriteGuard guard(LockThreadInfo); + ThreadInfo.insert(info); +} + +void TMemoryTracker::OnDestroyThread(TThreadLocalInfo* info) { + TWriteGuard guard(LockThreadInfo); + + auto count = GetCount(); + if (count && GlobalMetrics.size() == count) { + const auto& localMetrics = info->GetMetrics(); + if (localMetrics.size() == count) { + for (size_t i : xrange(count)) { + GlobalMetrics[i] += localMetrics[i]; + } + } + } + + ThreadInfo.erase(info); +} + +} + +} +} + diff --git a/library/cpp/actors/core/memory_tracker.h b/library/cpp/actors/core/memory_tracker.h index 7b29c673cf..e74508191b 100644 --- a/library/cpp/actors/core/memory_tracker.h +++ b/library/cpp/actors/core/memory_tracker.h @@ -1,53 +1,53 @@ -#pragma once - -#include "memory_track.h" - -#include <map> -#include <unordered_map> -#include <unordered_set> - -#include <util/system/rwlock.h> - -namespace NActors { -namespace NMemory { - -namespace NPrivate { - -class TMemoryTracker { -public: - static TMemoryTracker* Instance(); - - void Initialize(); - - const std::map<TString, size_t>& GetMetricIndices() const; - const std::unordered_set<size_t>& GetSensors() const; - TString GetName(size_t index) const; - size_t GetCount() const; - - void GatherMetrics(std::vector<TMetric>& metrics) const; - -private: - size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor); - - void OnCreateThread(TThreadLocalInfo* info); - void OnDestroyThread(TThreadLocalInfo* info); - -private: - std::map<TString, size_t> Indices; - std::vector<TString> Names; - - std::vector<TMetric> GlobalMetrics; - - std::unordered_set<size_t> Sensors; - - std::unordered_set<TThreadLocalInfo*> ThreadInfo; - TRWMutex LockThreadInfo; - - friend class TThreadLocalInfo; - friend class TBaseLabel; -}; - -} - -} -} +#pragma once + +#include "memory_track.h" + +#include <map> +#include <unordered_map> +#include <unordered_set> + +#include <util/system/rwlock.h> + +namespace NActors { +namespace NMemory { + +namespace NPrivate { + +class TMemoryTracker { +public: + static TMemoryTracker* Instance(); + + void Initialize(); + + const std::map<TString, size_t>& GetMetricIndices() const; + const std::unordered_set<size_t>& GetSensors() const; + TString GetName(size_t index) const; + size_t GetCount() const; + + void GatherMetrics(std::vector<TMetric>& metrics) const; + +private: + size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor); + + void OnCreateThread(TThreadLocalInfo* info); + void OnDestroyThread(TThreadLocalInfo* info); + +private: + std::map<TString, size_t> Indices; + std::vector<TString> Names; + + std::vector<TMetric> GlobalMetrics; + + std::unordered_set<size_t> Sensors; + + std::unordered_set<TThreadLocalInfo*> ThreadInfo; + TRWMutex LockThreadInfo; + + friend class TThreadLocalInfo; + friend class TBaseLabel; +}; + +} + +} +} diff --git a/library/cpp/actors/core/memory_tracker_ut.cpp b/library/cpp/actors/core/memory_tracker_ut.cpp index a9d3f502f1..d168214da6 100644 --- a/library/cpp/actors/core/memory_tracker_ut.cpp +++ b/library/cpp/actors/core/memory_tracker_ut.cpp @@ -1,262 +1,262 @@ -#include "memory_tracker.h" - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/system/hp_timer.h> -#include <util/system/thread.h> - -namespace NActors { -namespace NMemory { - -Y_UNIT_TEST_SUITE(TMemoryTrackerTest) { - -#if defined(ENABLE_MEMORY_TRACKING) - -using namespace NPrivate; - -size_t FindLabelIndex(const char* label) { - auto indices = TMemoryTracker::Instance()->GetMetricIndices(); - auto it = indices.find(label); - UNIT_ASSERT(it != indices.end()); - return it->second; -} - - -struct TTypeLabeled - : public NActors::NMemory::TTrack<TTypeLabeled> -{ - char payload[16]; -}; - -static constexpr char NamedLabel[] = "NamedLabel"; - -struct TNameLabeled - : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel> -{ - char payload[32]; -}; - -Y_UNIT_TEST(Gathering) -{ - TMemoryTracker::Instance()->Initialize(); - - auto* typed = new TTypeLabeled; - auto* typedArray = new TTypeLabeled[3]; - - auto* named = new TNameLabeled; - auto* namedArray = new TNameLabeled[5]; - NActors::NMemory::TLabel<NamedLabel>::Add(100); - - std::vector<TMetric> metrics; - TMemoryTracker::Instance()->GatherMetrics(metrics); - - auto typeIndex = FindLabelIndex(TypeName<TTypeLabeled>().c_str()); - UNIT_ASSERT(typeIndex < metrics.size()); - UNIT_ASSERT(metrics[typeIndex].GetMemory() == sizeof(TTypeLabeled) * 4 + sizeof(size_t)); - UNIT_ASSERT(metrics[typeIndex].GetCount() == 2); - - auto nameIndex = FindLabelIndex(NamedLabel); - UNIT_ASSERT(nameIndex < metrics.size()); - UNIT_ASSERT(metrics[nameIndex].GetMemory() == sizeof(TNameLabeled) * 6 + sizeof(size_t) + 100); - UNIT_ASSERT(metrics[nameIndex].GetCount() == 3); - - NActors::NMemory::TLabel<NamedLabel>::Sub(100); - delete [] namedArray; - delete named; - - delete [] typedArray; - delete typed; - - TMemoryTracker::Instance()->GatherMetrics(metrics); - - UNIT_ASSERT(metrics[typeIndex].GetMemory() == 0); - UNIT_ASSERT(metrics[typeIndex].GetCount() == 0); - - UNIT_ASSERT(metrics[nameIndex].GetMemory() == 0); - UNIT_ASSERT(metrics[nameIndex].GetCount() == 0); -} - - -static constexpr char InContainerLabel[] = "InContainerLabel"; - -struct TInContainer { - char payload[16]; -}; - -Y_UNIT_TEST(Containers) { - TMemoryTracker::Instance()->Initialize(); - - std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT; - vecT.resize(5); - - std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN; - vecN.resize(7); - - using TKey = int; - - std::map<TKey, TInContainer, std::less<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT; - mapT.emplace(0, TInContainer()); - mapT.emplace(1, TInContainer()); - - std::map<TKey, TInContainer, std::less<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN; - mapN.emplace(0, TInContainer()); - - std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT; - umapT.emplace(0, TInContainer()); - - std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN; - umapN.emplace(0, TInContainer()); - umapN.emplace(1, TInContainer()); - - std::vector<TMetric> metrics; - TMemoryTracker::Instance()->GatherMetrics(metrics); - - auto indices = TMemoryTracker::Instance()->GetMetricIndices(); - for (auto& [name, index] : indices) { - Cerr << "---- " << name - << ": memory = " << metrics[index].GetMemory() - << ", count = " << metrics[index].GetCount() << Endl; - } - - auto vecTIndex = FindLabelIndex(TypeName<TInContainer>().c_str()); - UNIT_ASSERT(metrics[vecTIndex].GetMemory() >= ssize_t(sizeof(TInContainer) * 5)); - UNIT_ASSERT(metrics[vecTIndex].GetCount() == 1); - - auto labelIndex = FindLabelIndex(InContainerLabel); - UNIT_ASSERT(metrics[labelIndex].GetCount() == 5); - UNIT_ASSERT(metrics[labelIndex].GetMemory() >= ssize_t( - sizeof(TInContainer) * 7 + - sizeof(decltype(mapN)::value_type) + - sizeof(decltype(umapN)::value_type) * 2)); -} - - -static constexpr char InThreadLabel[] = "InThreadLabel"; - -struct TInThread - : public NActors::NMemory::TTrack<TInThread, InThreadLabel> -{ - char payload[16]; -}; - -void* ThreadProc(void*) { - return new TInThread; -} - -Y_UNIT_TEST(Threads) { - TMemoryTracker::Instance()->Initialize(); - - auto index = FindLabelIndex(InThreadLabel); - - auto* object1 = new TInThread; - - std::vector<TMetric> metrics; - TMemoryTracker::Instance()->GatherMetrics(metrics); - UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread)); - UNIT_ASSERT(metrics[index].GetCount() == 1); - - TThread thread(&ThreadProc, nullptr); - thread.Start(); - auto* object2 = static_cast<TInThread*>(thread.Join()); - - TMemoryTracker::Instance()->GatherMetrics(metrics); - UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread) * 2); - UNIT_ASSERT(metrics[index].GetCount() == 2); - - delete object2; - - TMemoryTracker::Instance()->GatherMetrics(metrics); - UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread)); - UNIT_ASSERT(metrics[index].GetCount() == 1); - - delete object1; -} - - -struct TNotTracked { - char payload[16]; -}; - -struct TTracked - : public NActors::NMemory::TTrack<TTracked> -{ - char payload[16]; -}; - -template <typename T> -double MeasureAllocations() { - constexpr size_t objectsCount = 4 << 20; - - std::vector<T*> objects; - objects.resize(objectsCount); - - THPTimer timer; - - for (size_t i = 0; i < objectsCount; ++i) { - objects[i] = new T; - } - - for (size_t i = 0; i < objectsCount; ++i) { - delete objects[i]; - } - - auto seconds = timer.Passed(); - Cerr << "---- objects: " << objectsCount << ", time: " << seconds << Endl; - return seconds; -} - -Y_UNIT_TEST(Performance) { - TMemoryTracker::Instance()->Initialize(); - - constexpr size_t Runs = 16; - - Cerr << "---- warmup" << Endl; - MeasureAllocations<TNotTracked>(); - MeasureAllocations<TTracked>(); - - std::vector<double> noTrack; - std::vector<double> track; - - for (size_t run = 0; run < Runs; ++run) { - Cerr << "---- no track" << Endl; - auto time = MeasureAllocations<TNotTracked>(); - noTrack.push_back(time); - - Cerr << "---- track" << Endl; - time = MeasureAllocations<TTracked>(); - track.push_back(time); - } - - double meanNoTrack = 0, stddevNoTrack = 0; - double meanTrack = 0, stddevTrack = 0; - for (size_t i = 0; i < Runs; ++i) { - meanNoTrack += noTrack[i]; - meanTrack += track[i]; - } - meanNoTrack /= Runs; - meanTrack /= Runs; - - auto sqr = [](double val) { return val * val; }; - - for (size_t i = 0; i < Runs; ++i) { - stddevNoTrack += sqr(noTrack[i] - meanNoTrack); - stddevTrack += sqr(track[i] - meanTrack); - } - stddevNoTrack = sqrt(stddevNoTrack / (Runs - 1)); - stddevTrack = sqrt(stddevTrack / (Runs - 1)); - - Cerr << "---- no track - mean: " << meanNoTrack << ", stddev: " << stddevNoTrack << Endl; - Cerr << "---- track - mean: " << meanTrack << ", stddev: " << stddevTrack << Endl; - Cerr << "---- tracking is slower by " << int((meanTrack / meanNoTrack - 1.0) * 100) << "%" << Endl; -} - -#endif - -} - -} -} +#include "memory_tracker.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/hp_timer.h> +#include <util/system/thread.h> + +namespace NActors { +namespace NMemory { + +Y_UNIT_TEST_SUITE(TMemoryTrackerTest) { + +#if defined(ENABLE_MEMORY_TRACKING) + +using namespace NPrivate; + +size_t FindLabelIndex(const char* label) { + auto indices = TMemoryTracker::Instance()->GetMetricIndices(); + auto it = indices.find(label); + UNIT_ASSERT(it != indices.end()); + return it->second; +} + + +struct TTypeLabeled + : public NActors::NMemory::TTrack<TTypeLabeled> +{ + char payload[16]; +}; + +static constexpr char NamedLabel[] = "NamedLabel"; + +struct TNameLabeled + : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel> +{ + char payload[32]; +}; + +Y_UNIT_TEST(Gathering) +{ + TMemoryTracker::Instance()->Initialize(); + + auto* typed = new TTypeLabeled; + auto* typedArray = new TTypeLabeled[3]; + + auto* named = new TNameLabeled; + auto* namedArray = new TNameLabeled[5]; + NActors::NMemory::TLabel<NamedLabel>::Add(100); + + std::vector<TMetric> metrics; + TMemoryTracker::Instance()->GatherMetrics(metrics); + + auto typeIndex = FindLabelIndex(TypeName<TTypeLabeled>().c_str()); + UNIT_ASSERT(typeIndex < metrics.size()); + UNIT_ASSERT(metrics[typeIndex].GetMemory() == sizeof(TTypeLabeled) * 4 + sizeof(size_t)); + UNIT_ASSERT(metrics[typeIndex].GetCount() == 2); + + auto nameIndex = FindLabelIndex(NamedLabel); + UNIT_ASSERT(nameIndex < metrics.size()); + UNIT_ASSERT(metrics[nameIndex].GetMemory() == sizeof(TNameLabeled) * 6 + sizeof(size_t) + 100); + UNIT_ASSERT(metrics[nameIndex].GetCount() == 3); + + NActors::NMemory::TLabel<NamedLabel>::Sub(100); + delete [] namedArray; + delete named; + + delete [] typedArray; + delete typed; + + TMemoryTracker::Instance()->GatherMetrics(metrics); + + UNIT_ASSERT(metrics[typeIndex].GetMemory() == 0); + UNIT_ASSERT(metrics[typeIndex].GetCount() == 0); + + UNIT_ASSERT(metrics[nameIndex].GetMemory() == 0); + UNIT_ASSERT(metrics[nameIndex].GetCount() == 0); +} + + +static constexpr char InContainerLabel[] = "InContainerLabel"; + +struct TInContainer { + char payload[16]; +}; + +Y_UNIT_TEST(Containers) { + TMemoryTracker::Instance()->Initialize(); + + std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT; + vecT.resize(5); + + std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN; + vecN.resize(7); + + using TKey = int; + + std::map<TKey, TInContainer, std::less<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT; + mapT.emplace(0, TInContainer()); + mapT.emplace(1, TInContainer()); + + std::map<TKey, TInContainer, std::less<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN; + mapN.emplace(0, TInContainer()); + + std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT; + umapT.emplace(0, TInContainer()); + + std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN; + umapN.emplace(0, TInContainer()); + umapN.emplace(1, TInContainer()); + + std::vector<TMetric> metrics; + TMemoryTracker::Instance()->GatherMetrics(metrics); + + auto indices = TMemoryTracker::Instance()->GetMetricIndices(); + for (auto& [name, index] : indices) { + Cerr << "---- " << name + << ": memory = " << metrics[index].GetMemory() + << ", count = " << metrics[index].GetCount() << Endl; + } + + auto vecTIndex = FindLabelIndex(TypeName<TInContainer>().c_str()); + UNIT_ASSERT(metrics[vecTIndex].GetMemory() >= ssize_t(sizeof(TInContainer) * 5)); + UNIT_ASSERT(metrics[vecTIndex].GetCount() == 1); + + auto labelIndex = FindLabelIndex(InContainerLabel); + UNIT_ASSERT(metrics[labelIndex].GetCount() == 5); + UNIT_ASSERT(metrics[labelIndex].GetMemory() >= ssize_t( + sizeof(TInContainer) * 7 + + sizeof(decltype(mapN)::value_type) + + sizeof(decltype(umapN)::value_type) * 2)); +} + + +static constexpr char InThreadLabel[] = "InThreadLabel"; + +struct TInThread + : public NActors::NMemory::TTrack<TInThread, InThreadLabel> +{ + char payload[16]; +}; + +void* ThreadProc(void*) { + return new TInThread; +} + +Y_UNIT_TEST(Threads) { + TMemoryTracker::Instance()->Initialize(); + + auto index = FindLabelIndex(InThreadLabel); + + auto* object1 = new TInThread; + + std::vector<TMetric> metrics; + TMemoryTracker::Instance()->GatherMetrics(metrics); + UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread)); + UNIT_ASSERT(metrics[index].GetCount() == 1); + + TThread thread(&ThreadProc, nullptr); + thread.Start(); + auto* object2 = static_cast<TInThread*>(thread.Join()); + + TMemoryTracker::Instance()->GatherMetrics(metrics); + UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread) * 2); + UNIT_ASSERT(metrics[index].GetCount() == 2); + + delete object2; + + TMemoryTracker::Instance()->GatherMetrics(metrics); + UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread)); + UNIT_ASSERT(metrics[index].GetCount() == 1); + + delete object1; +} + + +struct TNotTracked { + char payload[16]; +}; + +struct TTracked + : public NActors::NMemory::TTrack<TTracked> +{ + char payload[16]; +}; + +template <typename T> +double MeasureAllocations() { + constexpr size_t objectsCount = 4 << 20; + + std::vector<T*> objects; + objects.resize(objectsCount); + + THPTimer timer; + + for (size_t i = 0; i < objectsCount; ++i) { + objects[i] = new T; + } + + for (size_t i = 0; i < objectsCount; ++i) { + delete objects[i]; + } + + auto seconds = timer.Passed(); + Cerr << "---- objects: " << objectsCount << ", time: " << seconds << Endl; + return seconds; +} + +Y_UNIT_TEST(Performance) { + TMemoryTracker::Instance()->Initialize(); + + constexpr size_t Runs = 16; + + Cerr << "---- warmup" << Endl; + MeasureAllocations<TNotTracked>(); + MeasureAllocations<TTracked>(); + + std::vector<double> noTrack; + std::vector<double> track; + + for (size_t run = 0; run < Runs; ++run) { + Cerr << "---- no track" << Endl; + auto time = MeasureAllocations<TNotTracked>(); + noTrack.push_back(time); + + Cerr << "---- track" << Endl; + time = MeasureAllocations<TTracked>(); + track.push_back(time); + } + + double meanNoTrack = 0, stddevNoTrack = 0; + double meanTrack = 0, stddevTrack = 0; + for (size_t i = 0; i < Runs; ++i) { + meanNoTrack += noTrack[i]; + meanTrack += track[i]; + } + meanNoTrack /= Runs; + meanTrack /= Runs; + + auto sqr = [](double val) { return val * val; }; + + for (size_t i = 0; i < Runs; ++i) { + stddevNoTrack += sqr(noTrack[i] - meanNoTrack); + stddevTrack += sqr(track[i] - meanTrack); + } + stddevNoTrack = sqrt(stddevNoTrack / (Runs - 1)); + stddevTrack = sqrt(stddevTrack / (Runs - 1)); + + Cerr << "---- no track - mean: " << meanNoTrack << ", stddev: " << stddevNoTrack << Endl; + Cerr << "---- track - mean: " << meanTrack << ", stddev: " << stddevTrack << Endl; + Cerr << "---- tracking is slower by " << int((meanTrack / meanNoTrack - 1.0) * 100) << "%" << Endl; +} + +#endif + +} + +} +} diff --git a/library/cpp/actors/core/mon_stats.h b/library/cpp/actors/core/mon_stats.h index e58429ad0a..d55552af0c 100644 --- a/library/cpp/actors/core/mon_stats.h +++ b/library/cpp/actors/core/mon_stats.h @@ -2,13 +2,13 @@ #include "defs.h" #include "actor.h" -#include <library/cpp/monlib/metrics/histogram_snapshot.h> +#include <library/cpp/monlib/metrics/histogram_snapshot.h> #include <util/system/hp_timer.h> namespace NActors { - struct TLogHistogram : public NMonitoring::IHistogramSnapshot { + struct TLogHistogram : public NMonitoring::IHistogramSnapshot { TLogHistogram() { - memset(Buckets, 0, sizeof(Buckets)); + memset(Buckets, 0, sizeof(Buckets)); } inline void Add(ui64 val, ui64 inc = 1) { @@ -17,8 +17,8 @@ namespace NActors { asm volatile("" :: : "memory"); #endif - if (val > 1) { - ind = GetValueBitCount(val - 1); + if (val > 1) { + ind = GetValueBitCount(val - 1); } #if defined(__clang__) && __clang_major__ == 3 && __clang_minor__ == 7 asm volatile("" :: @@ -36,25 +36,25 @@ namespace NActors { } } - // IHistogramSnapshot - ui32 Count() const override { - return Y_ARRAY_SIZE(Buckets); - } - - NMonitoring::TBucketBound UpperBound(ui32 index) const override { - Y_ASSERT(index < Y_ARRAY_SIZE(Buckets)); - if (index == 0) { - return 1; - } - return NMonitoring::TBucketBound(1ull << (index - 1)) * 2.0; - } - - NMonitoring::TBucketValue Value(ui32 index) const override { - Y_ASSERT(index < Y_ARRAY_SIZE(Buckets)); - return Buckets[index]; - } - - ui64 TotalSamples = 0; + // IHistogramSnapshot + ui32 Count() const override { + return Y_ARRAY_SIZE(Buckets); + } + + NMonitoring::TBucketBound UpperBound(ui32 index) const override { + Y_ASSERT(index < Y_ARRAY_SIZE(Buckets)); + if (index == 0) { + return 1; + } + return NMonitoring::TBucketBound(1ull << (index - 1)) * 2.0; + } + + NMonitoring::TBucketValue Value(ui32 index) const override { + Y_ASSERT(index < Y_ARRAY_SIZE(Buckets)); + return Buckets[index]; + } + + ui64 TotalSamples = 0; ui64 Buckets[65]; }; diff --git a/library/cpp/actors/core/process_stats.cpp b/library/cpp/actors/core/process_stats.cpp index 2cc73216b4..0e1dbd0031 100644 --- a/library/cpp/actors/core/process_stats.cpp +++ b/library/cpp/actors/core/process_stats.cpp @@ -181,9 +181,9 @@ namespace { class TDynamicCounterCollector: public TProcStatCollectingActor<TDynamicCounterCollector> { using TBase = TProcStatCollectingActor<TDynamicCounterCollector>; public: - TDynamicCounterCollector( - ui32 intervalSeconds, - NMonitoring::TDynamicCounterPtr counters) + TDynamicCounterCollector( + ui32 intervalSeconds, + NMonitoring::TDynamicCounterPtr counters) : TBase{TDuration::Seconds(intervalSeconds)} { ProcStatGroup = counters->GetSubgroup("counters", "utils"); @@ -218,7 +218,7 @@ namespace { } private: - NMonitoring::TDynamicCounterPtr ProcStatGroup; + NMonitoring::TDynamicCounterPtr ProcStatGroup; NMonitoring::TDynamicCounters::TCounterPtr VmSize; NMonitoring::TDynamicCounters::TCounterPtr AnonRssSize; NMonitoring::TDynamicCounters::TCounterPtr FileRssSize; @@ -293,8 +293,8 @@ namespace { }; } // namespace - IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters) { - return new TDynamicCounterCollector(intervalSec, counters); + IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters) { + return new TDynamicCounterCollector(intervalSec, counters); } IActor* CreateProcStatCollector(TDuration interval, NMonitoring::TMetricRegistry& registry) { diff --git a/library/cpp/actors/core/process_stats.h b/library/cpp/actors/core/process_stats.h index 236ea829ee..66346d0b5a 100644 --- a/library/cpp/actors/core/process_stats.h +++ b/library/cpp/actors/core/process_stats.h @@ -61,6 +61,6 @@ namespace NActors { long ObtainPageSize(); }; - IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters); + IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters); IActor* CreateProcStatCollector(TDuration interval, NMonitoring::TMetricRegistry& registry); } diff --git a/library/cpp/actors/core/scheduler_queue.h b/library/cpp/actors/core/scheduler_queue.h index 000321e7d3..3b8fac28f0 100644 --- a/library/cpp/actors/core/scheduler_queue.h +++ b/library/cpp/actors/core/scheduler_queue.h @@ -13,7 +13,7 @@ namespace NActors { ISchedulerCookie* Cookie; }; - struct TChunk : TQueueChunkDerived<TEntry, 512, TChunk> {}; + struct TChunk : TQueueChunkDerived<TEntry, 512, TChunk> {}; class TReader; class TWriter; diff --git a/library/cpp/actors/core/ut/ya.make b/library/cpp/actors/core/ut/ya.make index fc3195905b..3ee28d5850 100644 --- a/library/cpp/actors/core/ut/ya.make +++ b/library/cpp/actors/core/ut/ya.make @@ -39,7 +39,7 @@ SRCS( executor_pool_basic_ut.cpp executor_pool_united_ut.cpp log_ut.cpp - memory_tracker_ut.cpp + memory_tracker_ut.cpp scheduler_actor_ut.cpp ) diff --git a/library/cpp/actors/core/ya.make b/library/cpp/actors/core/ya.make index 633de9edc1..880a9d00db 100644 --- a/library/cpp/actors/core/ya.make +++ b/library/cpp/actors/core/ya.make @@ -75,10 +75,10 @@ SRCS( mailbox.h mailbox_queue_revolving.h mailbox_queue_simple.h - memory_track.cpp - memory_track.h - memory_tracker.cpp - memory_tracker.h + memory_track.cpp + memory_track.h + memory_tracker.cpp + memory_tracker.h mon.h mon_stats.h monotonic.cpp diff --git a/library/cpp/actors/helpers/mon_histogram_helper.h b/library/cpp/actors/helpers/mon_histogram_helper.h index 1da0eec1db..a9a57e3823 100644 --- a/library/cpp/actors/helpers/mon_histogram_helper.h +++ b/library/cpp/actors/helpers/mon_histogram_helper.h @@ -16,9 +16,9 @@ namespace NActors { THistogramCounterHelper(const THistogramCounterHelper&) = default; - void Init(NMonitoring::TDynamicCounters* group, const TString& baseName, const TString& unit, - ui64 firstBucket, ui64 bucketCnt, bool useSensorLabelName = true) - { + void Init(NMonitoring::TDynamicCounters* group, const TString& baseName, const TString& unit, + ui64 firstBucket, ui64 bucketCnt, bool useSensorLabelName = true) + { Y_ASSERT(FirstBucketVal == 0); Y_ASSERT(BucketCount == 0); @@ -28,8 +28,8 @@ namespace NActors { Buckets.reserve(BucketCount); for (size_t i = 0; i < BucketCount; ++i) { TString bucketName = GetBucketName(i) + " " + unit; - auto labelName = useSensorLabelName ? "sensor" : "name"; - BucketsHolder.push_back(group->GetSubgroup(labelName, baseName)->GetNamedCounter("range", bucketName, true)); + auto labelName = useSensorLabelName ? "sensor" : "name"; + BucketsHolder.push_back(group->GetSubgroup(labelName, baseName)->GetNamedCounter("range", bucketName, true)); Buckets.push_back(BucketsHolder.back().Get()); } } @@ -48,20 +48,20 @@ namespace NActors { Buckets[ind]->Inc(); } - ui64 GetBucketCount() const { - return BucketCount; - } - - ui64 GetBucketValue(size_t index) const { - Y_ASSERT(index < BucketCount); - return Buckets[index]->Val(); - } - - void SetBucketValue(ui64 index, ui64 value) { - Y_ASSERT(index < BucketCount); - *Buckets[index] = value; - } - + ui64 GetBucketCount() const { + return BucketCount; + } + + ui64 GetBucketValue(size_t index) const { + Y_ASSERT(index < BucketCount); + return Buckets[index]->Val(); + } + + void SetBucketValue(ui64 index, ui64 value) { + Y_ASSERT(index < BucketCount); + *Buckets[index] = value; + } + private: TString GetBucketName(size_t ind) const { Y_ASSERT(FirstBucketVal != 0); diff --git a/library/cpp/actors/interconnect/interconnect_counters.cpp b/library/cpp/actors/interconnect/interconnect_counters.cpp index e4b1412f62..224160d4b4 100644 --- a/library/cpp/actors/interconnect/interconnect_counters.cpp +++ b/library/cpp/actors/interconnect/interconnect_counters.cpp @@ -278,11 +278,11 @@ namespace { InflyLimitReach = AdaptiveCounters->GetCounter("InflyLimitReach", true); InflightDataAmount = AdaptiveCounters->GetCounter("Inflight_Data"); - LegacyPingTimeHist = {}; - LegacyPingTimeHist.Init(AdaptiveCounters.Get(), "PingTimeHist", "mks", 125, 18); - - PingTimeHistogram = AdaptiveCounters->GetHistogram( - "PingTimeUs", NMonitoring::ExponentialHistogram(18, 2, 125)); + LegacyPingTimeHist = {}; + LegacyPingTimeHist.Init(AdaptiveCounters.Get(), "PingTimeHist", "mks", 125, 18); + + PingTimeHistogram = AdaptiveCounters->GetHistogram( + "PingTimeUs", NMonitoring::ExponentialHistogram(18, 2, 125)); } if (updateGlobal) { @@ -299,7 +299,7 @@ namespace { auto disconnectReasonGroup = Counters->GetSubgroup("subsystem", "disconnectReason"); for (const char *reason : TDisconnectReason::Reasons) { - DisconnectByReason[reason] = disconnectReasonGroup->GetCounter(reason, true); + DisconnectByReason[reason] = disconnectReasonGroup->GetCounter(reason, true); } } @@ -329,9 +329,9 @@ namespace { NMonitoring::TDynamicCounters::TCounterPtr SpuriousReadWakeups; NMonitoring::TDynamicCounters::TCounterPtr UsefulWriteWakeups; NMonitoring::TDynamicCounters::TCounterPtr SpuriousWriteWakeups; - NMon::THistogramCounterHelper LegacyPingTimeHist; - NMonitoring::THistogramPtr PingTimeHistogram; - + NMon::THistogramCounterHelper LegacyPingTimeHist; + NMonitoring::THistogramPtr PingTimeHistogram; + std::unordered_map<ui16, TOutputChannel> OutputChannels; TOutputChannel OtherOutputChannel; TInputChannels InputChannels; diff --git a/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp b/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp index 16f0223502..0abe9fe659 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp @@ -196,7 +196,7 @@ namespace NActors { if (AtomicGet(Context->ControlPacketId) <= HeaderConfirm && !NewPingProtocol) { ui64 sendTime = AtomicGet(Context->ControlPacketSendTimer); TDuration duration = CyclesToDuration(GetCycleCountFast() - sendTime); - const auto durationUs = duration.MicroSeconds(); + const auto durationUs = duration.MicroSeconds(); Metrics->UpdateLegacyPingTimeHist(durationUs); PingQ.push_back(duration); if (PingQ.size() > 16) { @@ -448,8 +448,8 @@ namespace NActors { PingQ.pop_front(); } const TDuration ping = *std::min_element(PingQ.begin(), PingQ.end()); - const auto pingUs = ping.MicroSeconds(); - Context->PingRTT_us = pingUs; + const auto pingUs = ping.MicroSeconds(); + Context->PingRTT_us = pingUs; NewPingProtocol = true; Metrics->UpdateLegacyPingTimeHist(pingUs); } diff --git a/library/cpp/actors/prof/tag.cpp b/library/cpp/actors/prof/tag.cpp index caf516812b..9ccf03e1a9 100644 --- a/library/cpp/actors/prof/tag.cpp +++ b/library/cpp/actors/prof/tag.cpp @@ -1,9 +1,9 @@ #include "tag.h" -#include "tcmalloc.h" +#include "tcmalloc.h" #include <library/cpp/charset/ci_string.h> #include <library/cpp/containers/atomizer/atomizer.h> -#include <library/cpp/malloc/api/malloc.h> +#include <library/cpp/malloc/api/malloc.h> #if defined(PROFILE_MEMORY_ALLOCATIONS) #include <library/cpp/lfalloc/dbg_info/dbg_info.h> @@ -76,11 +76,11 @@ namespace NProfiling { return TStringAtoms::Instance().GetTagsCount(); } - static ui32 SetThreadAllocTag_Default(ui32 tag) { - Y_UNUSED(tag); - return 0; - } - + static ui32 SetThreadAllocTag_Default(ui32 tag) { + Y_UNUSED(tag); + return 0; + } + #if defined(PROFILE_MEMORY_ALLOCATIONS) static ui32 SetThreadAllocTag_YT(ui32 tag) { auto prev = NYT::NYTAlloc::GetCurrentMemoryTag(); @@ -96,24 +96,24 @@ namespace NProfiling { return (TSetThreadAllocTag*)NAllocDbg::SetThreadAllocTag; } else if (name.StartsWith("yt")) { return SetThreadAllocTag_YT; - } else if (name.StartsWith("tc")) { - return SetTCMallocThreadAllocTag; + } else if (name.StartsWith("tc")) { + return SetTCMallocThreadAllocTag; + } else { + return SetThreadAllocTag_Default; + } + } +#else + static TSetThreadAllocTag* SetThreadAllocTagFn() { + const auto& info = NMalloc::MallocInfo(); + + TStringBuf name(info.Name); + if (name.StartsWith("tc")) { + return SetTCMallocThreadAllocTag; } else { return SetThreadAllocTag_Default; } } -#else - static TSetThreadAllocTag* SetThreadAllocTagFn() { - const auto& info = NMalloc::MallocInfo(); - - TStringBuf name(info.Name); - if (name.StartsWith("tc")) { - return SetTCMallocThreadAllocTag; - } else { - return SetThreadAllocTag_Default; - } - } -#endif - +#endif + TSetThreadAllocTag* SetThreadAllocTag = SetThreadAllocTagFn(); } diff --git a/library/cpp/actors/prof/tag.h b/library/cpp/actors/prof/tag.h index 81815b7289..357e264a22 100644 --- a/library/cpp/actors/prof/tag.h +++ b/library/cpp/actors/prof/tag.h @@ -5,7 +5,7 @@ /* Common registry for tagging memory profiler. Register a new tag with MakeTag using a unique string. - Use registered tags with SetThreadAllocTag function in allocator API. + Use registered tags with SetThreadAllocTag function in allocator API. */ namespace NProfiling { @@ -22,43 +22,43 @@ namespace NProfiling { class TMemoryTagScope { public: - explicit TMemoryTagScope(ui32 tag) + explicit TMemoryTagScope(ui32 tag) : RestoreTag(SetThreadAllocTag(tag)) { } - explicit TMemoryTagScope(const char* tagName) { + explicit TMemoryTagScope(const char* tagName) { ui32 newTag = MakeTag(tagName); RestoreTag = SetThreadAllocTag(newTag); } TMemoryTagScope(TMemoryTagScope&& move) : RestoreTag(move.RestoreTag) - , Released(move.Released) + , Released(move.Released) { - move.Released = true; + move.Released = true; } TMemoryTagScope& operator=(TMemoryTagScope&& move) { RestoreTag = move.RestoreTag; - Released = move.Released; - move.Released = true; + Released = move.Released; + move.Released = true; return *this; } - static void Reset(ui32 tag) { + static void Reset(ui32 tag) { SetThreadAllocTag(tag); } void Release() { - if (!Released) { + if (!Released) { SetThreadAllocTag(RestoreTag); - Released = true; + Released = true; } } ~TMemoryTagScope() { - if (!Released) { + if (!Released) { SetThreadAllocTag(RestoreTag); } } @@ -67,7 +67,7 @@ namespace NProfiling { TMemoryTagScope(const TMemoryTagScope&) = delete; void operator=(const TMemoryTagScope&) = delete; - ui32 RestoreTag = 0; - bool Released = false; + ui32 RestoreTag = 0; + bool Released = false; }; } diff --git a/library/cpp/actors/prof/tcmalloc.cpp b/library/cpp/actors/prof/tcmalloc.cpp index 331181e80b..3d4f203dbb 100644 --- a/library/cpp/actors/prof/tcmalloc.cpp +++ b/library/cpp/actors/prof/tcmalloc.cpp @@ -1,32 +1,32 @@ -#include "tcmalloc.h" - -#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h> - -namespace NProfiling { - -static thread_local ui32 AllocationTag = 0; - -static struct TInitTCMallocCallbacks { - static void* CreateTag() { - return reinterpret_cast<void*>(AllocationTag); - } - static void* CopyTag(void* tag) { - return tag; - } - static void DestroyTag(void* tag) { - Y_UNUSED(tag); - } - - TInitTCMallocCallbacks() { - tcmalloc::MallocExtension::SetSampleUserDataCallbacks( - CreateTag, CopyTag, DestroyTag); - } -} InitTCMallocCallbacks; - -ui32 SetTCMallocThreadAllocTag(ui32 tag) { - ui32 prev = AllocationTag; - AllocationTag = tag; - return prev; -} - -} +#include "tcmalloc.h" + +#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h> + +namespace NProfiling { + +static thread_local ui32 AllocationTag = 0; + +static struct TInitTCMallocCallbacks { + static void* CreateTag() { + return reinterpret_cast<void*>(AllocationTag); + } + static void* CopyTag(void* tag) { + return tag; + } + static void DestroyTag(void* tag) { + Y_UNUSED(tag); + } + + TInitTCMallocCallbacks() { + tcmalloc::MallocExtension::SetSampleUserDataCallbacks( + CreateTag, CopyTag, DestroyTag); + } +} InitTCMallocCallbacks; + +ui32 SetTCMallocThreadAllocTag(ui32 tag) { + ui32 prev = AllocationTag; + AllocationTag = tag; + return prev; +} + +} diff --git a/library/cpp/actors/prof/tcmalloc.h b/library/cpp/actors/prof/tcmalloc.h index 0da31a2024..659fb4eaf3 100644 --- a/library/cpp/actors/prof/tcmalloc.h +++ b/library/cpp/actors/prof/tcmalloc.h @@ -1,9 +1,9 @@ -#pragma once - -#include <util/generic/fwd.h> - -namespace NProfiling { - -ui32 SetTCMallocThreadAllocTag(ui32 tag); - -} +#pragma once + +#include <util/generic/fwd.h> + +namespace NProfiling { + +ui32 SetTCMallocThreadAllocTag(ui32 tag); + +} diff --git a/library/cpp/actors/prof/tcmalloc_null.cpp b/library/cpp/actors/prof/tcmalloc_null.cpp index 26e6f61bb9..75c0013154 100644 --- a/library/cpp/actors/prof/tcmalloc_null.cpp +++ b/library/cpp/actors/prof/tcmalloc_null.cpp @@ -1,10 +1,10 @@ -#include "tcmalloc.h" - -namespace NProfiling { - -ui32 SetTCMallocThreadAllocTag(ui32 tag) { - Y_UNUSED(tag); - return 0; -} - -} +#include "tcmalloc.h" + +namespace NProfiling { + +ui32 SetTCMallocThreadAllocTag(ui32 tag) { + Y_UNUSED(tag); + return 0; +} + +} diff --git a/library/cpp/actors/prof/ya.make b/library/cpp/actors/prof/ya.make index d20ee798c4..b5e2497563 100644 --- a/library/cpp/actors/prof/ya.make +++ b/library/cpp/actors/prof/ya.make @@ -23,11 +23,11 @@ IF (PROFILE_MEMORY_ALLOCATIONS) ) ENDIF() -IF(ALLOCATOR == "TCMALLOC_256K") - SRCS(tcmalloc.cpp) - PEERDIR(contrib/libs/tcmalloc) -ELSE() - SRCS(tcmalloc_null.cpp) -ENDIF() - +IF(ALLOCATOR == "TCMALLOC_256K") + SRCS(tcmalloc.cpp) + PEERDIR(contrib/libs/tcmalloc) +ELSE() + SRCS(tcmalloc_null.cpp) +ENDIF() + END() diff --git a/library/cpp/binsaver/bin_saver.cpp b/library/cpp/binsaver/bin_saver.cpp index ca215fa809..fe0775af9f 100644 --- a/library/cpp/binsaver/bin_saver.cpp +++ b/library/cpp/binsaver/bin_saver.cpp @@ -42,10 +42,10 @@ void IBinSaver::StoreObject(IObjectBase* pObject) { ObjectQueue.push_back(pObject); (*Objects)[ptrId]; int typeId = pSaverClasses->GetObjectTypeID(pObject); - if (typeId == -1) { - fprintf(stderr, "IBinSaver: trying to save unregistered object\n"); - abort(); - } + if (typeId == -1) { + fprintf(stderr, "IBinSaver: trying to save unregistered object\n"); + abort(); + } DataChunk(&typeId, sizeof(typeId)); } } @@ -64,9 +64,9 @@ IObjectBase* IBinSaver::LoadObject() { IObjectBase* pObj = pSaverClasses->CreateObject(typeId); Y_ASSERT(pObj != nullptr); if (pObj == nullptr) { - fprintf(stderr, "IBinSaver: trying to load unregistered object\n"); - abort(); - } + fprintf(stderr, "IBinSaver: trying to load unregistered object\n"); + abort(); + } (*Objects)[ptrId] = pObj; ObjectQueue.push_back(pObj); return pObj; diff --git a/library/cpp/binsaver/bin_saver.h b/library/cpp/binsaver/bin_saver.h index 0e9e46a2e5..412424889f 100644 --- a/library/cpp/binsaver/bin_saver.h +++ b/library/cpp/binsaver/bin_saver.h @@ -156,13 +156,13 @@ private: } } - // hash_multimap + // hash_multimap template <class AMM> void DoAnyMultiMap(AMM& data) { - if (IsReading()) { - data.clear(); + if (IsReading()) { + data.clear(); TStoredSize nSize; - Add(3, &nSize); + Add(3, &nSize); TVector<typename AMM::key_type, typename std::allocator_traits<typename AMM::allocator_type>::template rebind_alloc<typename AMM::key_type>> indices; indices.resize(nSize); for (TStoredSize i = 0; i < nSize; ++i) @@ -170,19 +170,19 @@ private: for (TStoredSize i = 0; i < nSize; ++i) { std::pair<typename AMM::key_type, typename AMM::mapped_type> valToInsert; valToInsert.first = indices[i]; - Add(2, &valToInsert.second); - data.insert(valToInsert); - } - } else { + Add(2, &valToInsert.second); + data.insert(valToInsert); + } + } else { TStoredSize nSize = data.size(); CheckOverflow(nSize, data.size()); - Add(3, &nSize); + Add(3, &nSize); for (auto pos = data.begin(); pos != data.end(); ++pos) Add(1, (typename AMM::key_type*)(&pos->first)); for (auto pos = data.begin(); pos != data.end(); ++pos) - Add(2, &pos->second); - } - } + Add(2, &pos->second); + } + } template <class T> void DoAnySet(T& data) { @@ -270,7 +270,7 @@ private: IObjectBase* LoadObject(); bool bRead; - TBufferedStream<> File; + TBufferedStream<> File; // maps objects addresses during save(first) to addresses during load(second) - during loading // or serves as a sign that some object has been already stored - during storing bool StableOutput; @@ -357,7 +357,7 @@ public: int Add(const chunk_id, THashMultiMap<T1, T2, T3, T4, T5>* pHash) { DoAnyMultiMap(*pHash); return 0; - } + } template <class K, class L, class A> int Add(const chunk_id, TSet<K, L, A>* pSet) { DoAnySet(*pSet); diff --git a/library/cpp/binsaver/buffered_io.h b/library/cpp/binsaver/buffered_io.h index 659bc9f62b..75465c9c5c 100644 --- a/library/cpp/binsaver/buffered_io.h +++ b/library/cpp/binsaver/buffered_io.h @@ -26,7 +26,7 @@ struct IBinaryStream { } virtual bool IsValid() const = 0; - virtual bool IsFailed() const = 0; + virtual bool IsFailed() const = 0; private: virtual int WriteImpl(const void* userBuffer, int size) = 0; @@ -36,49 +36,49 @@ private: i64 LongWrite(const void* userBuffer, i64 size); }; -template <int N_SIZE = 16384> +template <int N_SIZE = 16384> class TBufferedStream { char Buf[N_SIZE]; i64 Pos, BufSize; IBinaryStream& Stream; - bool bIsReading, bIsEof, bFailed; + bool bIsReading, bIsEof, bFailed; void ReadComplex(void* userBuffer, i64 size) { - if (bIsEof) { - memset(userBuffer, 0, size); - return; - } + if (bIsEof) { + memset(userBuffer, 0, size); + return; + } char* dst = (char*)userBuffer; i64 leftBytes = BufSize - Pos; - memcpy(dst, Buf + Pos, leftBytes); - dst += leftBytes; - size -= leftBytes; - Pos = BufSize = 0; - if (size > N_SIZE) { + memcpy(dst, Buf + Pos, leftBytes); + dst += leftBytes; + size -= leftBytes; + Pos = BufSize = 0; + if (size > N_SIZE) { i64 n = Stream.Read(dst, size); - bFailed = Stream.IsFailed(); - if (n != size) { - bIsEof = true; - memset(dst + n, 0, size - n); - } - } else { - BufSize = Stream.Read(Buf, N_SIZE); - bFailed = Stream.IsFailed(); - if (BufSize == 0) - bIsEof = true; - Read(dst, size); - } - } - + bFailed = Stream.IsFailed(); + if (n != size) { + bIsEof = true; + memset(dst + n, 0, size - n); + } + } else { + BufSize = Stream.Read(Buf, N_SIZE); + bFailed = Stream.IsFailed(); + if (BufSize == 0) + bIsEof = true; + Read(dst, size); + } + } + void WriteComplex(const void* userBuffer, i64 size) { - Flush(); - if (size >= N_SIZE) { - Stream.Write(userBuffer, size); - bFailed = Stream.IsFailed(); + Flush(); + if (size >= N_SIZE) { + Stream.Write(userBuffer, size); + bFailed = Stream.IsFailed(); } else - Write(userBuffer, size); - } - + Write(userBuffer, size); + } + void operator=(const TBufferedStream&) { } @@ -94,16 +94,16 @@ public: } ~TBufferedStream() { if (!bIsReading) - Flush(); + Flush(); } - void Flush() { + void Flush() { Y_ASSERT(!bIsReading); - if (bIsReading) - return; - Stream.Write(Buf, Pos); - bFailed = Stream.IsFailed(); - Pos = 0; - } + if (bIsReading) + return; + Stream.Write(Buf, Pos); + bFailed = Stream.IsFailed(); + Pos = 0; + } bool IsEof() const { return bIsEof; } diff --git a/library/cpp/binsaver/class_factory.h b/library/cpp/binsaver/class_factory.h index 4bfb225b28..e83512331b 100644 --- a/library/cpp/binsaver/class_factory.h +++ b/library/cpp/binsaver/class_factory.h @@ -15,12 +15,12 @@ public: typedef const std::type_info* VFT; private: - typedef T* (*newFunc)(); + typedef T* (*newFunc)(); typedef THashMap<int, newFunc> CTypeNewHash; // typeID->newFunc() typedef THashMap<VFT, int> CTypeIndexHash; // vftable->typeID - CTypeIndexHash typeIndex; - CTypeNewHash typeInfo; + CTypeIndexHash typeIndex; + CTypeNewHash typeInfo; void RegisterTypeBase(int nTypeID, newFunc func, VFT vft); static VFT GetObjectType(T* pObject) { @@ -29,15 +29,15 @@ private: int VFT2TypeID(VFT t) { CTypeIndexHash::iterator i = typeIndex.find(t); if (i != typeIndex.end()) - return i->second; + return i->second; for (i = typeIndex.begin(); i != typeIndex.end(); ++i) { if (*i->first == *t) { - typeIndex[t] = i->second; - return i->second; - } - } - return -1; - } + typeIndex[t] = i->second; + return i->second; + } + } + return -1; + } public: template <class TT> @@ -45,10 +45,10 @@ public: RegisterTypeBase(nTypeID, func, &typeid(TT)); } void RegisterTypeSafe(int nTypeID, newFunc func) { - TPtr<T> pObj = func(); + TPtr<T> pObj = func(); VFT vft = GetObjectType(pObj); RegisterTypeBase(nTypeID, func, vft); - } + } T* CreateObject(int nTypeID) { newFunc f = typeInfo[nTypeID]; if (f) diff --git a/library/cpp/binsaver/mem_io.h b/library/cpp/binsaver/mem_io.h index ca5cf0598a..2a9e36fe68 100644 --- a/library/cpp/binsaver/mem_io.h +++ b/library/cpp/binsaver/mem_io.h @@ -6,7 +6,7 @@ namespace NMemIoInternals { class TMemoryStream: public IBinaryStream { TVector<char>& Data; ui64 Pos; - + public: TMemoryStream(TVector<char>* data, ui64 pos = 0) : Data(*data) @@ -15,7 +15,7 @@ namespace NMemIoInternals { } ~TMemoryStream() override { } // keep gcc happy - + bool IsValid() const override { return true; } diff --git a/library/cpp/digest/sfh/sfh.h b/library/cpp/digest/sfh/sfh.h index 6535013201..372938654c 100644 --- a/library/cpp/digest/sfh/sfh.h +++ b/library/cpp/digest/sfh/sfh.h @@ -1,51 +1,51 @@ #pragma once - + #include <util/system/defaults.h> #include <util/system/unaligned_mem.h> - + inline ui32 SuperFastHash(const void* d, size_t l) noexcept { ui32 hash = (ui32)l; ui32 tmp; if (!l || !d) return 0; - + TUnalignedMemoryIterator<ui16, 4> iter(d, l); - + while (!iter.AtEnd()) { hash += (ui32)iter.Next(); tmp = ((ui32)iter.Next() << 11) ^ hash; hash = (hash << 16) ^ tmp; hash += hash >> 11; - } - + } + switch (iter.Left()) { case 3: hash += (ui32)iter.Next(); - hash ^= hash << 16; + hash ^= hash << 16; hash ^= ((ui32)(i32) * (const i8*)iter.Last()) << 18; - hash += hash >> 11; - break; + hash += hash >> 11; + break; case 2: hash += (ui32)iter.Cur(); - hash ^= hash << 11; - hash += hash >> 17; - break; + hash ^= hash << 11; + hash += hash >> 17; + break; case 1: hash += *((const i8*)iter.Last()); - hash ^= hash << 10; - hash += hash >> 1; - } - - /* Force "avalanching" of final 127 bits */ - hash ^= hash << 3; - hash += hash >> 5; - hash ^= hash << 4; - hash += hash >> 17; - hash ^= hash << 25; - hash += hash >> 6; - - return hash; -} + hash ^= hash << 10; + hash += hash >> 1; + } + + /* Force "avalanching" of final 127 bits */ + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + + return hash; +} diff --git a/library/cpp/grpc/server/grpc_counters.h b/library/cpp/grpc/server/grpc_counters.h index 29600c7d96..0b6c36c84c 100644 --- a/library/cpp/grpc/server/grpc_counters.h +++ b/library/cpp/grpc/server/grpc_counters.h @@ -18,13 +18,13 @@ struct ICounterBlock : public TThrRefBase { virtual void CountRequestsWithoutDatabase() {} virtual void CountRequestsWithoutToken() {} virtual void CountRequestWithoutTls() {} - - virtual TIntrusivePtr<ICounterBlock> Clone() { return this; } - virtual void UseDatabase(const TString& database) { Y_UNUSED(database); } + + virtual TIntrusivePtr<ICounterBlock> Clone() { return this; } + virtual void UseDatabase(const TString& database) { Y_UNUSED(database); } }; -using ICounterBlockPtr = TIntrusivePtr<ICounterBlock>; - +using ICounterBlockPtr = TIntrusivePtr<ICounterBlock>; + class TCounterBlock final : public ICounterBlock { NMonitoring::TDynamicCounters::TCounterPtr TotalCounter; NMonitoring::TDynamicCounters::TCounterPtr InflyCounter; @@ -113,10 +113,10 @@ public: } } - ICounterBlockPtr Clone() override { - return this; - } - + ICounterBlockPtr Clone() override { + return this; + } + void Update() { if (Percentile) { RequestHistMs.Update(); diff --git a/library/cpp/grpc/server/grpc_request.h b/library/cpp/grpc/server/grpc_request.h index aeb8d31f29..5bd8d3902b 100644 --- a/library/cpp/grpc/server/grpc_request.h +++ b/library/cpp/grpc/server/grpc_request.h @@ -224,10 +224,10 @@ public: return &Arena_; } - void UseDatabase(const TString& database) override { - Counters_->UseDatabase(database); - } - + void UseDatabase(const TString& database) override { + Counters_->UseDatabase(database); + } + private: void Clone() { if (!Server_->IsShuttingDown()) { diff --git a/library/cpp/grpc/server/grpc_request_base.h b/library/cpp/grpc/server/grpc_request_base.h index c6d64ed648..fcfce1c181 100644 --- a/library/cpp/grpc/server/grpc_request_base.h +++ b/library/cpp/grpc/server/grpc_request_base.h @@ -90,9 +90,9 @@ public: //! The metadata will be send at the time of rpc finish virtual void AddTrailingMetadata(const TString& key, const TString& value) = 0; - //! Use validated database name for counters - virtual void UseDatabase(const TString& database) = 0; - + //! Use validated database name for counters + virtual void UseDatabase(const TString& database) = 0; + // Streaming part //! Set callback. The callback will be called when response deliverid to the client diff --git a/library/cpp/grpc/server/grpc_server.cpp b/library/cpp/grpc/server/grpc_server.cpp index 579ee24f06..7437b7a8f5 100644 --- a/library/cpp/grpc/server/grpc_server.cpp +++ b/library/cpp/grpc/server/grpc_server.cpp @@ -147,7 +147,7 @@ void TGRpcServer::Start() { Server_ = builder.BuildAndStart(); if (!Server_) { - ythrow yexception() << "can't start grpc server on " << server_address; + ythrow yexception() << "can't start grpc server on " << server_address; } size_t index = 0; diff --git a/library/cpp/monlib/dynamic_counters/contention_ut.cpp b/library/cpp/monlib/dynamic_counters/contention_ut.cpp index d142397e1a..8798044ee3 100644 --- a/library/cpp/monlib/dynamic_counters/contention_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/contention_ut.cpp @@ -33,7 +33,7 @@ Y_UNIT_TEST_SUITE(TDynamicCountersContentionTest) { Response.Wait(); } - void OnHistogram(const TString& /*labelName*/, const TString& /*labelValue*/, IHistogramSnapshotPtr /*snapshot*/, bool /*derivative*/) override { + void OnHistogram(const TString& /*labelName*/, const TString& /*labelValue*/, IHistogramSnapshotPtr /*snapshot*/, bool /*derivative*/) override { } void OnGroupBegin(const TString& /*labelName*/, const TString& /*labelValue*/, const TDynamicCounters* /*group*/) override { diff --git a/library/cpp/monlib/dynamic_counters/counters.cpp b/library/cpp/monlib/dynamic_counters/counters.cpp index 977c8ea7b0..3635d87d0d 100644 --- a/library/cpp/monlib/dynamic_counters/counters.cpp +++ b/library/cpp/monlib/dynamic_counters/counters.cpp @@ -70,38 +70,38 @@ TDynamicCounters::TCounterPtr TDynamicCounters::GetNamedCounter(const TString& n return AsCounterRef(GetNamedCounterImpl<false, TCounterForPtr>(name, value, derivative, vis)); } -THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { - return GetNamedHistogram("sensor", value, std::move(collector), derivative, vis); +THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return GetNamedHistogram("sensor", value, std::move(collector), derivative, vis); } -THistogramPtr TDynamicCounters::GetNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { - return AsHistogramRef(GetNamedCounterImpl<false, THistogramCounter>(name, value, std::move(collector), derivative, vis)); +THistogramPtr TDynamicCounters::GetNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return AsHistogramRef(GetNamedCounterImpl<false, THistogramCounter>(name, value, std::move(collector), derivative, vis)); } -THistogramPtr TDynamicCounters::GetExpiringHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { - return GetExpiringNamedHistogram("sensor", value, std::move(collector), derivative, vis); +THistogramPtr TDynamicCounters::GetExpiringHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return GetExpiringNamedHistogram("sensor", value, std::move(collector), derivative, vis); } -THistogramPtr TDynamicCounters::GetExpiringNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { - return AsHistogramRef(GetNamedCounterImpl<true, TExpiringHistogramCounter>(name, value, std::move(collector), derivative, vis)); +THistogramPtr TDynamicCounters::GetExpiringNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return AsHistogramRef(GetNamedCounterImpl<true, TExpiringHistogramCounter>(name, value, std::move(collector), derivative, vis)); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::FindCounter(const TString& value) const { + return FindNamedCounter("sensor", value); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::FindNamedCounter(const TString& name, const TString& value) const { + return AsCounterRef(FindNamedCounterImpl<TCounterForPtr>(name, value)); +} + +THistogramPtr TDynamicCounters::FindHistogram(const TString& value) const { + return FindNamedHistogram("sensor", value); +} + +THistogramPtr TDynamicCounters::FindNamedHistogram(const TString& name,const TString& value) const { + return AsHistogramRef(FindNamedCounterImpl<THistogramCounter>(name, value)); } -TDynamicCounters::TCounterPtr TDynamicCounters::FindCounter(const TString& value) const { - return FindNamedCounter("sensor", value); -} - -TDynamicCounters::TCounterPtr TDynamicCounters::FindNamedCounter(const TString& name, const TString& value) const { - return AsCounterRef(FindNamedCounterImpl<TCounterForPtr>(name, value)); -} - -THistogramPtr TDynamicCounters::FindHistogram(const TString& value) const { - return FindNamedHistogram("sensor", value); -} - -THistogramPtr TDynamicCounters::FindNamedHistogram(const TString& name,const TString& value) const { - return AsHistogramRef(FindNamedCounterImpl<THistogramCounter>(name, value)); -} - void TDynamicCounters::RemoveCounter(const TString &value) { RemoveNamedCounter("sensor", value); } @@ -298,11 +298,11 @@ TDynamicCounters::TCountablePtr TDynamicCounters::GetNamedCounterImpl(const TStr } return it->second; } - -template <class TCounterType> -TDynamicCounters::TCountablePtr TDynamicCounters::FindNamedCounterImpl(const TString& name, const TString& value) const { - TReadGuard g(Lock); - auto it = Counters.find({name, value}); - return it != Counters.end() ? it->second : nullptr; -} - + +template <class TCounterType> +TDynamicCounters::TCountablePtr TDynamicCounters::FindNamedCounterImpl(const TString& name, const TString& value) const { + TReadGuard g(Lock); + auto it = Counters.find({name, value}); + return it != Counters.end() ? it->second : nullptr; +} + diff --git a/library/cpp/monlib/dynamic_counters/counters.h b/library/cpp/monlib/dynamic_counters/counters.h index 9882001de5..dc178cfbe0 100644 --- a/library/cpp/monlib/dynamic_counters/counters.h +++ b/library/cpp/monlib/dynamic_counters/counters.h @@ -66,7 +66,7 @@ namespace NMonitoring { virtual void OnHistogram( const TString& labelName, const TString& labelValue, - IHistogramSnapshotPtr snapshot, bool derivative) = 0; + IHistogramSnapshotPtr snapshot, bool derivative) = 0; virtual void OnGroupBegin( const TString& labelName, const TString& labelValue, @@ -93,8 +93,8 @@ namespace NMonitoring { Visibility_ = vis; } - TCounterForPtr(const TCounterForPtr&) = delete; - TCounterForPtr& operator=(const TCounterForPtr& other) = delete; + TCounterForPtr(const TCounterForPtr&) = delete; + TCounterForPtr& operator=(const TCounterForPtr& other) = delete; void Accept( const TString& labelName, const TString& labelValue, @@ -129,10 +129,10 @@ namespace NMonitoring { }; struct THistogramCounter: public TCountableBase { - explicit THistogramCounter( - IHistogramCollectorPtr collector, bool derivative = true, EVisibility vis = EVisibility::Public) + explicit THistogramCounter( + IHistogramCollectorPtr collector, bool derivative = true, EVisibility vis = EVisibility::Public) : Collector_(std::move(collector)) - , Derivative_(derivative) + , Derivative_(derivative) { Visibility_ = vis; } @@ -145,34 +145,34 @@ namespace NMonitoring { Collector_->Collect(value, count); } - void Collect(double value, ui32 count) { - Collector_->Collect(value, count); - } - - void Collect(const IHistogramSnapshot& snapshot) { - Collector_->Collect(snapshot); - } - + void Collect(double value, ui32 count) { + Collector_->Collect(value, count); + } + + void Collect(const IHistogramSnapshot& snapshot) { + Collector_->Collect(snapshot); + } + void Accept( const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const override { if (IsVisible(Visibility(), consumer.Visibility())) { - consumer.OnHistogram(labelName, labelValue, Collector_->Snapshot(), Derivative_); + consumer.OnHistogram(labelName, labelValue, Collector_->Snapshot(), Derivative_); } } - void Reset() { - Collector_->Reset(); - } - + void Reset() { + Collector_->Reset(); + } + IHistogramSnapshotPtr Snapshot() const { return Collector_->Snapshot(); } private: IHistogramCollectorPtr Collector_; - bool Derivative_; + bool Derivative_; }; struct TExpiringHistogramCounter: public THistogramCounter { @@ -284,14 +284,14 @@ namespace NMonitoring { THistogramPtr GetHistogram( const TString& value, IHistogramCollectorPtr collector, - bool derivative = true, + bool derivative = true, TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); THistogramPtr GetNamedHistogram( const TString& name, const TString& value, IHistogramCollectorPtr collector, - bool derivative = true, + bool derivative = true, TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); // These counters will be automatically removed from the registry @@ -310,22 +310,22 @@ namespace NMonitoring { THistogramPtr GetExpiringHistogram( const TString& value, IHistogramCollectorPtr collector, - bool derivative = true, + bool derivative = true, TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); THistogramPtr GetExpiringNamedHistogram( const TString& name, const TString& value, IHistogramCollectorPtr collector, - bool derivative = true, + bool derivative = true, TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - TCounterPtr FindCounter(const TString& value) const; - TCounterPtr FindNamedCounter(const TString& name, const TString& value) const; - - THistogramPtr FindHistogram(const TString& value) const; - THistogramPtr FindNamedHistogram(const TString& name,const TString& value) const; - + TCounterPtr FindCounter(const TString& value) const; + TCounterPtr FindNamedCounter(const TString& name, const TString& value) const; + + THistogramPtr FindHistogram(const TString& value) const; + THistogramPtr FindNamedHistogram(const TString& name,const TString& value) const; + void RemoveCounter(const TString &value); void RemoveNamedCounter(const TString& name, const TString &value); @@ -366,9 +366,9 @@ namespace NMonitoring { template <bool expiring, class TCounterType, class... TArgs> TCountablePtr GetNamedCounterImpl(const TString& name, const TString& value, TArgs&&... args); - - template <class TCounterType> - TCountablePtr FindNamedCounterImpl(const TString& name, const TString& value) const; + + template <class TCounterType> + TCountablePtr FindNamedCounterImpl(const TString& name, const TString& value) const; }; } diff --git a/library/cpp/monlib/dynamic_counters/counters_ut.cpp b/library/cpp/monlib/dynamic_counters/counters_ut.cpp index 04285ae5e3..3591037e0a 100644 --- a/library/cpp/monlib/dynamic_counters/counters_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/counters_ut.cpp @@ -23,7 +23,7 @@ private: void OnHistogram( const TString& labelName, const TString& labelValue, - IHistogramSnapshotPtr snapshot, bool /*derivative*/) override { + IHistogramSnapshotPtr snapshot, bool /*derivative*/) override { Indent(Out_, Level_) << labelName << ':' << labelValue << " = " << *snapshot << '\n'; @@ -311,32 +311,32 @@ Y_UNIT_TEST_SUITE(TDynamicCountersTest) { UNIT_ASSERT_VALUES_EQUAL(subGroupLookups->Val(), 1); UNIT_ASSERT_VALUES_EQUAL(lookups->Val(), 5); } - - Y_UNIT_TEST(FindCounters) { - TDynamicCounterPtr rootGroup(new TDynamicCounters()); - - auto counter = rootGroup->FindCounter("counter1"); - UNIT_ASSERT(!counter); - rootGroup->GetCounter("counter1"); - counter = rootGroup->FindCounter("counter1"); - UNIT_ASSERT(counter); - - counter = rootGroup->FindNamedCounter("name", "counter2"); - UNIT_ASSERT(!counter); - rootGroup->GetNamedCounter("name", "counter2"); - counter = rootGroup->FindNamedCounter("name", "counter2"); - UNIT_ASSERT(counter); - - auto histogram = rootGroup->FindHistogram("histogram1"); - UNIT_ASSERT(!histogram); - rootGroup->GetHistogram("histogram1", ExponentialHistogram(4, 2)); - histogram = rootGroup->FindHistogram("histogram1"); - UNIT_ASSERT(histogram); - - histogram = rootGroup->FindNamedHistogram("name", "histogram2"); - UNIT_ASSERT(!histogram); - rootGroup->GetNamedHistogram("name", "histogram2", ExponentialHistogram(4, 2)); - histogram = rootGroup->FindNamedHistogram("name", "histogram2"); - UNIT_ASSERT(histogram); - } + + Y_UNIT_TEST(FindCounters) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + + auto counter = rootGroup->FindCounter("counter1"); + UNIT_ASSERT(!counter); + rootGroup->GetCounter("counter1"); + counter = rootGroup->FindCounter("counter1"); + UNIT_ASSERT(counter); + + counter = rootGroup->FindNamedCounter("name", "counter2"); + UNIT_ASSERT(!counter); + rootGroup->GetNamedCounter("name", "counter2"); + counter = rootGroup->FindNamedCounter("name", "counter2"); + UNIT_ASSERT(counter); + + auto histogram = rootGroup->FindHistogram("histogram1"); + UNIT_ASSERT(!histogram); + rootGroup->GetHistogram("histogram1", ExponentialHistogram(4, 2)); + histogram = rootGroup->FindHistogram("histogram1"); + UNIT_ASSERT(histogram); + + histogram = rootGroup->FindNamedHistogram("name", "histogram2"); + UNIT_ASSERT(!histogram); + rootGroup->GetNamedHistogram("name", "histogram2", ExponentialHistogram(4, 2)); + histogram = rootGroup->FindNamedHistogram("name", "histogram2"); + UNIT_ASSERT(histogram); + } } diff --git a/library/cpp/monlib/dynamic_counters/encode.cpp b/library/cpp/monlib/dynamic_counters/encode.cpp index 6f5005f71a..ffa48d276e 100644 --- a/library/cpp/monlib/dynamic_counters/encode.cpp +++ b/library/cpp/monlib/dynamic_counters/encode.cpp @@ -3,7 +3,7 @@ #include <library/cpp/monlib/encode/encoder.h> #include <library/cpp/monlib/encode/json/json.h> #include <library/cpp/monlib/encode/spack/spack_v1.h> -#include <library/cpp/monlib/encode/prometheus/prometheus.h> +#include <library/cpp/monlib/encode/prometheus/prometheus.h> #include <util/stream/str.h> @@ -41,10 +41,10 @@ namespace NMonitoring { void OnHistogram( const TString& labelName, const TString& labelValue, - IHistogramSnapshotPtr snapshot, bool derivative) override { - NMonitoring::EMetricType metricType = derivative ? EMetricType::HIST_RATE : EMetricType::HIST; - - EncoderImpl_->OnMetricBegin(metricType); + IHistogramSnapshotPtr snapshot, bool derivative) override { + NMonitoring::EMetricType metricType = derivative ? EMetricType::HIST_RATE : EMetricType::HIST; + + EncoderImpl_->OnMetricBegin(metricType); EncodeLabels(labelName, labelValue); EncoderImpl_->OnHistogram(ZERO_TIME, snapshot); EncoderImpl_->OnMetricEnd(); @@ -104,9 +104,9 @@ namespace NMonitoring { out, NMonitoring::ETimePrecision::SECONDS, NMonitoring::ECompression::ZSTD), vis); - case EFormat::PROMETHEUS: - return MakeHolder<TConsumer>(NMonitoring::EncoderPrometheus( - out), vis); + case EFormat::PROMETHEUS: + return MakeHolder<TConsumer>(NMonitoring::EncoderPrometheus( + out), vis); default: ythrow yexception() << "unsupported metric encoding format: " << format; break; diff --git a/library/cpp/monlib/dynamic_counters/golovan_page.cpp b/library/cpp/monlib/dynamic_counters/golovan_page.cpp index 6d76956234..49cf2d39bb 100644 --- a/library/cpp/monlib/dynamic_counters/golovan_page.cpp +++ b/library/cpp/monlib/dynamic_counters/golovan_page.cpp @@ -39,7 +39,7 @@ public: out << "\"," << counter->Val() << "]"; } - void OnHistogram(const TString&, const TString&, IHistogramSnapshotPtr, bool) override { + void OnHistogram(const TString&, const TString&, IHistogramSnapshotPtr, bool) override { } void OnGroupBegin(const TString&, const TString& value, const TDynamicCounters*) override { diff --git a/library/cpp/monlib/dynamic_counters/page.cpp b/library/cpp/monlib/dynamic_counters/page.cpp index 812ee6b4ca..5124a47bb3 100644 --- a/library/cpp/monlib/dynamic_counters/page.cpp +++ b/library/cpp/monlib/dynamic_counters/page.cpp @@ -19,8 +19,8 @@ TMaybe<EFormat> ParseFormat(TStringBuf str) { return EFormat::JSON; } else if (str == TStringBuf("spack")) { return EFormat::SPACK; - } else if (str == TStringBuf("prometheus")) { - return EFormat::PROMETHEUS; + } else if (str == TStringBuf("prometheus")) { + return EFormat::PROMETHEUS; } else { return Nothing(); } @@ -84,8 +84,8 @@ void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { out << HTTPOKJSON; } else if (*format == EFormat::SPACK) { out << HTTPOKSPACK; - } else if (*format == EFormat::PROMETHEUS) { - out << HTTPOKPROMETHEUS; + } else if (*format == EFormat::PROMETHEUS) { + out << HTTPOKPROMETHEUS; } else { ythrow yexception() << "unsupported metric encoding format: " << *format; } diff --git a/library/cpp/monlib/dynamic_counters/ya.make b/library/cpp/monlib/dynamic_counters/ya.make index 0b1e97fa4e..aafe1c34be 100644 --- a/library/cpp/monlib/dynamic_counters/ya.make +++ b/library/cpp/monlib/dynamic_counters/ya.make @@ -18,7 +18,7 @@ PEERDIR( library/cpp/containers/stack_vector library/cpp/monlib/encode/json library/cpp/monlib/encode/spack - library/cpp/monlib/encode/prometheus + library/cpp/monlib/encode/prometheus library/cpp/monlib/service/pages library/cpp/string_utils/quote library/cpp/threading/light_rw_lock diff --git a/library/cpp/monlib/service/pages/mon_page.h b/library/cpp/monlib/service/pages/mon_page.h index ad7afb867d..e396612bb0 100644 --- a/library/cpp/monlib/service/pages/mon_page.h +++ b/library/cpp/monlib/service/pages/mon_page.h @@ -12,7 +12,7 @@ namespace NMonitoring { static const char HTTPOKHTML[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/html\r\nConnection: Close\r\n\r\n"; static const char HTTPOKJSON[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/json\r\nConnection: Close\r\n\r\n"; static const char HTTPOKSPACK[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/x-solomon-spack\r\nConnection: Close\r\n\r\n"; - static const char HTTPOKPROMETHEUS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/plain\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKPROMETHEUS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/plain\r\nConnection: Close\r\n\r\n"; static const char HTTPOKJAVASCRIPT[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/javascript\r\nConnection: Close\r\n\r\n"; static const char HTTPOKCSS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/css\r\nConnection: Close\r\n\r\n"; static const char HTTPNOCONTENT[] = "HTTP/1.1 204 No content\r\nConnection: Close\r\n\r\n"; diff --git a/library/cpp/yson/consumer.cpp b/library/cpp/yson/consumer.cpp index 33e9ee6327..40ae452978 100644 --- a/library/cpp/yson/consumer.cpp +++ b/library/cpp/yson/consumer.cpp @@ -1,15 +1,15 @@ -#include "consumer.h" +#include "consumer.h" #include "string.h" -#include "parser.h" - +#include "parser.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + void TYsonConsumerBase::OnRaw(TStringBuf str, NYT::NYson::EYsonType type) { ParseYsonStringBuffer(str, this, type); } - + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/consumer.h b/library/cpp/yson/consumer.h index fe1902c8f0..d5a9d66335 100644 --- a/library/cpp/yson/consumer.h +++ b/library/cpp/yson/consumer.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <library/cpp/yt/yson/consumer.h> - -#include <util/generic/strbuf.h> -#include <util/system/defaults.h> - + +#include <util/generic/strbuf.h> +#include <util/system/defaults.h> + namespace NYson { struct TYsonConsumerBase : public virtual NYT::NYson::IYsonConsumer { diff --git a/library/cpp/yson/detail.h b/library/cpp/yson/detail.h index d2a4a6308e..27f5e8ffff 100644 --- a/library/cpp/yson/detail.h +++ b/library/cpp/yson/detail.h @@ -1,47 +1,47 @@ -#pragma once - -#include "public.h" -#include "zigzag.h" - -#include <util/generic/vector.h> -#include <util/generic/maybe.h> -#include <util/generic/buffer.h> -#include <util/string/escape.h> -#include <util/string/cast.h> -#include <util/stream/input.h> - +#pragma once + +#include "public.h" +#include "zigzag.h" + +#include <util/generic/vector.h> +#include <util/generic/maybe.h> +#include <util/generic/buffer.h> +#include <util/string/escape.h> +#include <util/string/cast.h> +#include <util/stream/input.h> + namespace NYson { namespace NDetail { //////////////////////////////////////////////////////////////////////////////// - + //! Indicates the beginning of a list. const char BeginListSymbol = '['; //! Indicates the end of a list. const char EndListSymbol = ']'; - + //! Indicates the beginning of a map. const char BeginMapSymbol = '{'; //! Indicates the end of a map. const char EndMapSymbol = '}'; - + //! Indicates the beginning of an attribute map. const char BeginAttributesSymbol = '<'; //! Indicates the end of an attribute map. const char EndAttributesSymbol = '>'; - + //! Separates items in lists. const char ListItemSeparatorSymbol = ';'; //! Separates items in maps, attributes. const char KeyedItemSeparatorSymbol = ';'; //! Separates keys from values in maps. const char KeyValueSeparatorSymbol = '='; - + //! Indicates an entity. const char EntitySymbol = '#'; - + //! Indicates end of stream. const char EndSymbol = '\0'; - + //! Marks the beginning of a binary string literal. const char StringMarker = '\x01'; //! Marks the beginning of a binary i64 literal. @@ -53,19 +53,19 @@ namespace NYson { const char TrueMarker = '\x05'; //! Marks the beginning of a binary ui64 literal. const char Uint64Marker = '\x06'; - + //////////////////////////////////////////////////////////////////////////////// - + template <bool EnableLinePositionInfo> class TPositionInfo; - + template <> class TPositionInfo<true> { private: int Offset; int Line; int Column; - + public: TPositionInfo() : Offset(0) @@ -73,7 +73,7 @@ namespace NYson { , Column(1) { } - + void OnRangeConsumed(const char* begin, const char* end) { Offset += end - begin; for (auto current = begin; current != end; ++current) { @@ -85,7 +85,7 @@ namespace NYson { } } }; - + template <> class TPositionInfo<false> { private: @@ -95,13 +95,13 @@ namespace NYson { TPositionInfo() : Offset(0) { - } - + } + void OnRangeConsumed(const char* begin, const char* end) { Offset += end - begin; } }; - + template <class TBlockStream, class TPositionBase> class TCharStream : public TBlockStream, @@ -111,11 +111,11 @@ namespace NYson { : TBlockStream(blockStream) { } - + bool IsEmpty() const { return TBlockStream::Begin() == TBlockStream::End(); } - + template <bool AllowFinish> void Refresh() { while (IsEmpty() && !TBlockStream::IsFinished()) { @@ -125,46 +125,46 @@ namespace NYson { ythrow TYsonException() << "Premature end of yson stream"; } } - + void Refresh() { return Refresh<false>(); } - + template <bool AllowFinish> char GetChar() { Refresh<AllowFinish>(); return !IsEmpty() ? *TBlockStream::Begin() : '\0'; } - + char GetChar() { return GetChar<false>(); } - + void Advance(size_t bytes) { TPositionBase::OnRangeConsumed(TBlockStream::Begin(), TBlockStream::Begin() + bytes); TBlockStream::Advance(bytes); } - + size_t Length() const { return TBlockStream::End() - TBlockStream::Begin(); } }; - + template <class TBaseStream> class TCodedStream : public TBaseStream { private: static const int MaxVarintBytes = 10; static const int MaxVarint32Bytes = 5; - + const ui8* BeginByte() const { return reinterpret_cast<const ui8*>(TBaseStream::Begin()); } - + const ui8* EndByte() const { return reinterpret_cast<const ui8*>(TBaseStream::End()); } - + // Following functions is an adaptation Protobuf code from coded_stream.cc bool ReadVarint32FromArray(ui32* value) { // Fast path: We have enough bytes left in the buffer to guarantee that @@ -172,7 +172,7 @@ namespace NYson { const ui8* ptr = BeginByte(); ui32 b; ui32 result; - + b = *(ptr++); result = (b & 0x7F); if (!(b & 0x80)) @@ -193,26 +193,26 @@ namespace NYson { result |= b << 28; if (!(b & 0x80)) goto done; - + // If the input is larger than 32 bits, we still need to read it all // and discard the high-order bits. - + for (int i = 0; i < MaxVarintBytes - MaxVarint32Bytes; i++) { b = *(ptr++); if (!(b & 0x80)) goto done; } - + // We have overrun the maximum size of a Varint (10 bytes). Assume // the data is corrupt. return false; - + done: TBaseStream::Advance(ptr - BeginByte()); *value = result; return true; } - + bool ReadVarint32Fallback(ui32* value) { if (BeginByte() + MaxVarint32Bytes <= EndByte() || // Optimization: If the Varint ends at exactly the end of the buffer, @@ -227,7 +227,7 @@ namespace NYson { return ReadVarint32Slow(value); } } - + bool ReadVarint32Slow(ui32* value) { ui64 result; // Directly invoke ReadVarint64Fallback, since we already tried to optimize @@ -239,15 +239,15 @@ namespace NYson { return false; } } - + bool ReadVarint64Slow(ui64* value) { // Slow path: This read might cross the end of the buffer, so we // need to check and refresh the buffer if and when it does. - + ui64 result = 0; int count = 0; ui32 b; - + do { if (count == MaxVarintBytes) { return false; @@ -260,11 +260,11 @@ namespace NYson { TBaseStream::Advance(1); ++count; } while (b & 0x80); - + *value = result; return true; - } - + } + bool ReadVarint64Fallback(ui64* value) { if (BeginByte() + MaxVarintBytes <= EndByte() || // Optimization: If the Varint ends at exactly the end of the buffer, @@ -273,14 +273,14 @@ namespace NYson { { // Fast path: We have enough bytes left in the buffer to guarantee that // this read won't cross the end, so we can skip the checks. - + const ui8* ptr = BeginByte(); ui32 b; - + // Splitting into 32-bit pieces gives better performance on 32-bit // processors. ui32 part0 = 0, part1 = 0, part2 = 0; - + b = *(ptr++); part0 = (b & 0x7F); if (!(b & 0x80)) @@ -321,11 +321,11 @@ namespace NYson { part2 |= (b & 0x7F) << 7; if (!(b & 0x80)) goto done; - + // We have overrun the maximum size of a Varint (10 bytes). The data // must be corrupt. return false; - + done: TBaseStream::Advance(ptr - BeginByte()); *value = (static_cast<ui64>(part0)) | @@ -336,13 +336,13 @@ namespace NYson { return ReadVarint64Slow(value); } } - + public: TCodedStream(const TBaseStream& baseStream) : TBaseStream(baseStream) { } - + bool ReadVarint64(ui64* value) { if (BeginByte() < EndByte() && *BeginByte() < 0x80) { *value = *BeginByte(); @@ -352,7 +352,7 @@ namespace NYson { return ReadVarint64Fallback(value); } } - + bool ReadVarint32(ui32* value) { if (BeginByte() < EndByte() && *BeginByte() < 0x80) { *value = *BeginByte(); @@ -363,13 +363,13 @@ namespace NYson { } } }; - + enum ENumericResult { Int64 = 0, Uint64 = 1, Double = 2 }; - + template <class TBlockStream, bool EnableLinePositionInfo> class TLexerBase : public TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>> { @@ -377,7 +377,7 @@ namespace NYson { using TBaseStream = TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>>; TVector<char> Buffer_; TMaybe<ui64> MemoryLimit_; - + void CheckMemoryLimit() { if (MemoryLimit_ && Buffer_.capacity() > *MemoryLimit_) { ythrow TYsonException() @@ -385,17 +385,17 @@ namespace NYson { << Buffer_.capacity() << ", limit " << (*MemoryLimit_); } } - + public: TLexerBase(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) : TBaseStream(blockStream) , MemoryLimit_(memoryLimit) { } - + protected: /// Lexer routines - + template <bool AllowFinish> ENumericResult ReadNumeric(TStringBuf* value) { Buffer_.clear(); @@ -418,18 +418,18 @@ namespace NYson { CheckMemoryLimit(); TBaseStream::Advance(1); } - + *value = TStringBuf(Buffer_.data(), Buffer_.size()); return result; - } - + } + template <bool AllowFinish> double ReadNanOrInf() { static const TStringBuf nanString = "nan"; static const TStringBuf infString = "inf"; static const TStringBuf plusInfString = "+inf"; static const TStringBuf minusInfString = "-inf"; - + TStringBuf expectedString; double expectedValue; char ch = TBaseStream::template GetChar<AllowFinish>(); @@ -515,25 +515,25 @@ namespace NYson { } CheckMemoryLimit(); TBaseStream::Advance(1); - } + } *value = TStringBuf(Buffer_.data(), Buffer_.size()); - } - + } + void ReadUnquotedString(TStringBuf* value) { return ReadUnquotedString<false>(value); - } - + } + void ReadBinaryString(TStringBuf* value) { ui32 ulength = 0; if (!TBaseStream::ReadVarint32(&ulength)) { ythrow TYsonException() << "Error parsing varint value"; } - + i32 length = ZigZagDecode32(ulength); if (length < 0) { ythrow TYsonException() << "Negative binary string literal length " << length; } - + if (TBaseStream::Begin() + length <= TBaseStream::End()) { *value = TStringBuf(TBaseStream::Begin(), length); TBaseStream::Advance(length); @@ -546,29 +546,29 @@ namespace NYson { continue; } size_t readingBytes = Min(needToRead, TBaseStream::Length()); - + Buffer_.insert(Buffer_.end(), TBaseStream::Begin(), TBaseStream::Begin() + readingBytes); CheckMemoryLimit(); needToRead -= readingBytes; TBaseStream::Advance(readingBytes); } *value = TStringBuf(Buffer_.data(), Buffer_.size()); - } - } - + } + } + template <bool AllowFinish> bool ReadBoolean() { Buffer_.clear(); - + static TStringBuf trueString = "true"; static TStringBuf falseString = "false"; - + auto throwIncorrectBoolean = [&]() { ythrow TYsonException() << "Incorrect boolean string " << TString(Buffer_.data(), Buffer_.size()); }; - - Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); - TBaseStream::Advance(1); + + Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); + TBaseStream::Advance(1); if (Buffer_[0] == trueString[0]) { for (size_t i = 1; i < trueString.size(); ++i) { Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); @@ -588,21 +588,21 @@ namespace NYson { } return false; } else { - throwIncorrectBoolean(); - } + throwIncorrectBoolean(); + } Y_FAIL("unreachable"); ; - } + } void ReadBinaryInt64(i64* result) { ui64 uvalue; if (!TBaseStream::ReadVarint64(&uvalue)) { ythrow TYsonException() << "Error parsing varint value"; - } + } *result = ZigZagDecode64(uvalue); - } - + } + void ReadBinaryUint64(ui64* result) { ui64 uvalue; if (!TBaseStream::ReadVarint64(&uvalue)) { @@ -610,16 +610,16 @@ namespace NYson { } *result = uvalue; } - + void ReadBinaryDouble(double* value) { size_t needToRead = sizeof(double); - + while (needToRead != 0) { if (TBaseStream::IsEmpty()) { TBaseStream::Refresh(); continue; } - + size_t chunkSize = Min(needToRead, TBaseStream::Length()); if (chunkSize == 0) { ythrow TYsonException() << "Error parsing binary double literal"; @@ -631,8 +631,8 @@ namespace NYson { needToRead -= chunkSize; TBaseStream::Advance(chunkSize); } - } - + } + /// Helpers void SkipCharToken(char symbol) { char ch = SkipSpaceAndGetChar(); @@ -641,8 +641,8 @@ namespace NYson { } TBaseStream::Advance(1); - } - + } + static bool IsSpaceFast(char ch) { static const ui8 lookupTable[] = { @@ -650,24 +650,24 @@ namespace NYson { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; return lookupTable[static_cast<ui8>(ch)]; } - + template <bool AllowFinish> char SkipSpaceAndGetChar() { if (!TBaseStream::IsEmpty()) { @@ -678,11 +678,11 @@ namespace NYson { } return SkipSpaceAndGetCharFallback<AllowFinish>(); } - + char SkipSpaceAndGetChar() { return SkipSpaceAndGetChar<false>(); - } - + } + template <bool AllowFinish> char SkipSpaceAndGetCharFallback() { while (true) { @@ -697,63 +697,63 @@ namespace NYson { break; } TBaseStream::Advance(1); - } + } return TBaseStream::template GetChar<AllowFinish>(); - } + } }; - + //////////////////////////////////////////////////////////////////////////////// - + } - + //////////////////////////////////////////////////////////////////////////////// - + class TStringReader { private: const char* BeginPtr; const char* EndPtr; - + public: TStringReader() : BeginPtr(nullptr) , EndPtr(nullptr) { } - + TStringReader(const char* begin, const char* end) : BeginPtr(begin) , EndPtr(end) { } - + const char* Begin() const { return BeginPtr; } - + const char* End() const { return EndPtr; } - + void RefreshBlock() { Y_FAIL("unreachable"); } - + void Advance(size_t bytes) { BeginPtr += bytes; } - + bool IsFinished() const { return true; } - + void SetBuffer(const char* begin, const char* end) { BeginPtr = begin; EndPtr = end; } }; - + //////////////////////////////////////////////////////////////////////////////// - + class TStreamReader { public: TStreamReader( @@ -767,40 +767,40 @@ namespace NYson { BeginPtr = EndPtr = Buffer; FinishFlag = false; } - + const char* Begin() const { return BeginPtr; } - + const char* End() const { return EndPtr; } - + void RefreshBlock() { size_t bytes = Stream->Read(Buffer, BufferSize); BeginPtr = Buffer; EndPtr = Buffer + bytes; FinishFlag = (bytes == 0); } - + void Advance(size_t bytes) { BeginPtr += bytes; } - + bool IsFinished() const { return FinishFlag; } - + private: IInputStream* Stream; char* Buffer; size_t BufferSize; - + const char* BeginPtr; const char* EndPtr; bool FinishFlag; }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/format.h b/library/cpp/yson/format.h index c2bad91a6a..2ff6dc9f6e 100644 --- a/library/cpp/yson/format.h +++ b/library/cpp/yson/format.h @@ -1,25 +1,25 @@ -#pragma once - -#include "token.h" - +#pragma once + +#include "token.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + const ETokenType BeginListToken = LeftBracket; const ETokenType EndListToken = RightBracket; - + const ETokenType BeginMapToken = LeftBrace; const ETokenType EndMapToken = RightBrace; - + const ETokenType BeginAttributesToken = LeftAngle; const ETokenType EndAttributesToken = RightAngle; - + const ETokenType ListItemSeparatorToken = Semicolon; const ETokenType KeyedItemSeparatorToken = Semicolon; const ETokenType KeyValueSeparatorToken = Equals; - + const ETokenType EntityToken = Hash; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/json/json_writer.cpp b/library/cpp/yson/json/json_writer.cpp index 2c961a02fa..87481256ec 100644 --- a/library/cpp/yson/json/json_writer.cpp +++ b/library/cpp/yson/json/json_writer.cpp @@ -1,16 +1,16 @@ -#include "json_writer.h" - +#include "json_writer.h" + #include <library/cpp/json/json_writer.h> - -namespace NYT { + +namespace NYT { //////////////////////////////////////////////////////////////////////////////// - + static bool IsSpecialJsonKey(const TStringBuf& key) { return key.size() > 0 && key[0] == '$'; } - + //////////////////////////////////////////////////////////////////////////////// - + TJsonWriter::TJsonWriter( IOutputStream* output, ::NYson::EYsonType type, @@ -41,15 +41,15 @@ namespace NYT { if (Type == ::NYson::EYsonType::MapFragment) { ythrow ::NYson::TYsonException() << ("Map fragments are not supported by Json"); } - + UnderlyingJsonWriter.Reset(new NJson::TJsonWriter( output, config)); JsonWriter = UnderlyingJsonWriter.Get(); HasAttributes = false; InAttributesBalance = 0; - } - + } + void TJsonWriter::EnterNode() { if (AttributesMode == JAM_NEVER) { HasAttributes = false; @@ -65,15 +65,15 @@ namespace NYT { HasAttributes = true; } HasUnfoldedStructureStack.push_back(HasAttributes); - + if (HasAttributes) { JsonWriter->Write("$value"); HasAttributes = false; - } - + } + Depth += 1; - } - + } + void TJsonWriter::LeaveNode() { Y_ASSERT(!HasUnfoldedStructureStack.empty()); if (HasUnfoldedStructureStack.back()) { @@ -81,54 +81,54 @@ namespace NYT { JsonWriter->CloseMap(); } HasUnfoldedStructureStack.pop_back(); - + Depth -= 1; if (Depth == 0 && Type == ::NYson::EYsonType::ListFragment && InAttributesBalance == 0) { JsonWriter->Flush(); Output->Write("\n"); } - } - + } + bool TJsonWriter::IsWriteAllowed() { if (AttributesMode == JAM_NEVER) { return InAttributesBalance == 0; } return true; - } - + } + void TJsonWriter::OnStringScalar(TStringBuf value) { if (IsWriteAllowed()) { EnterNode(); WriteStringScalar(value); LeaveNode(); } - } - + } + void TJsonWriter::OnInt64Scalar(i64 value) { if (IsWriteAllowed()) { EnterNode(); JsonWriter->Write(value); LeaveNode(); } - } - + } + void TJsonWriter::OnUint64Scalar(ui64 value) { if (IsWriteAllowed()) { EnterNode(); JsonWriter->Write(value); LeaveNode(); } - } - + } + void TJsonWriter::OnDoubleScalar(double value) { if (IsWriteAllowed()) { EnterNode(); JsonWriter->Write(value); LeaveNode(); } - } - + } + void TJsonWriter::OnBooleanScalar(bool value) { if (IsWriteAllowed()) { if (BooleanFormat == SBF_STRING) { @@ -139,40 +139,40 @@ namespace NYT { LeaveNode(); } } - } - + } + void TJsonWriter::OnEntity() { if (IsWriteAllowed()) { EnterNode(); JsonWriter->WriteNull(); LeaveNode(); } - } - + } + void TJsonWriter::OnBeginList() { if (IsWriteAllowed()) { EnterNode(); JsonWriter->OpenArray(); } - } - + } + void TJsonWriter::OnListItem() { - } - + } + void TJsonWriter::OnEndList() { if (IsWriteAllowed()) { JsonWriter->CloseArray(); LeaveNode(); } } - + void TJsonWriter::OnBeginMap() { if (IsWriteAllowed()) { EnterNode(); JsonWriter->OpenMap(); } - } - + } + void TJsonWriter::OnKeyedItem(TStringBuf name) { if (IsWriteAllowed()) { if (IsSpecialJsonKey(name)) { @@ -181,15 +181,15 @@ namespace NYT { WriteStringScalar(name); } } - } - + } + void TJsonWriter::OnEndMap() { if (IsWriteAllowed()) { JsonWriter->CloseMap(); LeaveNode(); - } - } - + } + } + void TJsonWriter::OnBeginAttributes() { InAttributesBalance += 1; if (AttributesMode != JAM_NEVER) { @@ -197,24 +197,24 @@ namespace NYT { JsonWriter->Write("$attributes"); JsonWriter->OpenMap(); } - } - + } + void TJsonWriter::OnEndAttributes() { InAttributesBalance -= 1; if (AttributesMode != JAM_NEVER) { HasAttributes = true; JsonWriter->CloseMap(); } - } - + } + void TJsonWriter::WriteStringScalar(const TStringBuf& value) { JsonWriter->Write(value); - } - + } + void TJsonWriter::Flush() { JsonWriter->Flush(); } - + //////////////////////////////////////////////////////////////////////////////// -} +} diff --git a/library/cpp/yson/json/json_writer.h b/library/cpp/yson/json/json_writer.h index 43e5293b0d..d84ac0de53 100644 --- a/library/cpp/yson/json/json_writer.h +++ b/library/cpp/yson/json/json_writer.h @@ -1,31 +1,31 @@ -#pragma once - +#pragma once + #include <library/cpp/yson/public.h> #include <library/cpp/yson/consumer.h> - + #include <library/cpp/json/json_writer.h> - -#include <util/generic/vector.h> - -namespace NYT { + +#include <util/generic/vector.h> + +namespace NYT { //////////////////////////////////////////////////////////////////////////////// - + enum EJsonFormat { JF_TEXT, JF_PRETTY }; - + enum EJsonAttributesMode { JAM_NEVER, JAM_ON_DEMAND, JAM_ALWAYS }; - + enum ESerializedBoolFormat { SBF_BOOLEAN, SBF_STRING }; - + class TJsonWriter : public ::NYson::TYsonConsumerBase { public: @@ -44,26 +44,26 @@ namespace NYT { ESerializedBoolFormat booleanFormat = SBF_STRING); void Flush(); - + void OnStringScalar(TStringBuf value) override; void OnInt64Scalar(i64 value) override; void OnUint64Scalar(ui64 value) override; void OnDoubleScalar(double value) override; void OnBooleanScalar(bool value) override; - + void OnEntity() override; - + void OnBeginList() override; void OnListItem() override; void OnEndList() override; - + void OnBeginMap() override; void OnKeyedItem(TStringBuf key) override; void OnEndMap() override; - + void OnBeginAttributes() override; void OnEndAttributes() override; - + private: THolder<NJson::TJsonWriter> UnderlyingJsonWriter; NJson::TJsonWriter* JsonWriter; @@ -71,19 +71,19 @@ namespace NYT { ::NYson::EYsonType Type; EJsonAttributesMode AttributesMode; ESerializedBoolFormat BooleanFormat; - + void WriteStringScalar(const TStringBuf& value); - + void EnterNode(); void LeaveNode(); bool IsWriteAllowed(); - + TVector<bool> HasUnfoldedStructureStack; int InAttributesBalance; bool HasAttributes; int Depth; }; - + //////////////////////////////////////////////////////////////////////////////// - + } diff --git a/library/cpp/yson/lexer.cpp b/library/cpp/yson/lexer.cpp index 6864991c24..5eae94273b 100644 --- a/library/cpp/yson/lexer.cpp +++ b/library/cpp/yson/lexer.cpp @@ -1,16 +1,16 @@ -#include "lexer.h" -#include "lexer_detail.h" -#include "token.h" - -#include <util/generic/ptr.h> - +#include "lexer.h" +#include "lexer_detail.h" +#include "token.h" + +#include <util/generic/ptr.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TStatelessLexer::TImpl { private: THolder<TStatelessYsonLexerImplBase> Impl; - + public: TImpl(bool enableLinePositionInfo = false) : Impl(enableLinePositionInfo @@ -18,26 +18,26 @@ namespace NYson { : static_cast<TStatelessYsonLexerImplBase*>(new TStatelesYsonLexerImpl<false>())) { } - + size_t GetToken(const TStringBuf& data, TToken* token) { return Impl->GetToken(data, token); } }; - + //////////////////////////////////////////////////////////////////////////////// TStatelessLexer::TStatelessLexer() : Impl(new TImpl()) - { - } - + { + } + TStatelessLexer::~TStatelessLexer() { } - + size_t TStatelessLexer::GetToken(const TStringBuf& data, TToken* token) { return Impl->GetToken(data, token); } - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/lexer.h b/library/cpp/yson/lexer.h index 17cb87ec6d..d9d701874d 100644 --- a/library/cpp/yson/lexer.h +++ b/library/cpp/yson/lexer.h @@ -1,26 +1,26 @@ -#pragma once - -#include "public.h" -#include "token.h" - -#include <util/generic/ptr.h> - +#pragma once + +#include "public.h" +#include "token.h" + +#include <util/generic/ptr.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TStatelessLexer { public: TStatelessLexer(); - + ~TStatelessLexer(); - + size_t GetToken(const TStringBuf& data, TToken* token); - + private: class TImpl; THolder<TImpl> Impl; }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/lexer_detail.h b/library/cpp/yson/lexer_detail.h index 5bb5dccc06..0bba30acdd 100644 --- a/library/cpp/yson/lexer_detail.h +++ b/library/cpp/yson/lexer_detail.h @@ -1,15 +1,15 @@ -#pragma once - -#include "detail.h" -#include "token.h" - +#pragma once + +#include "detail.h" +#include "token.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + namespace NDetail { /*! \internal */ //////////////////////////////////////////////////////////////////////////////// - + // EReadStartCase tree representation: // Root = xb // BinaryStringOrOtherSpecialToken = x0b @@ -29,17 +29,17 @@ namespace NYson { // Plus = 10011b // None = 10111b // Percent = 11011b - + enum EReadStartCase : unsigned { BinaryString = 0, // = 00b OtherSpecialToken = 2, // = 10b - + BinaryInt64 = 1, // = 001b BinaryDouble = 5, // = 101b BinaryFalse = 9, // = 1001b BinaryTrue = 13, // = 1101b BinaryUint64 = 17, // = 10001b - + Quote = 3, // = 00011b DigitOrMinus = 7, // = 00111b String = 11, // = 01011b @@ -48,34 +48,34 @@ namespace NYson { None = 23, // = 10111b Percent = 27 // = 11011b }; - + template <class TBlockStream, bool EnableLinePositionInfo> class TLexer : public TLexerBase<TBlockStream, EnableLinePositionInfo> { private: using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; - + static EReadStartCase GetStartState(char ch) { -#define NN EReadStartCase::None -#define BS EReadStartCase::BinaryString -#define BI EReadStartCase::BinaryInt64 -#define BD EReadStartCase::BinaryDouble -#define BF EReadStartCase::BinaryFalse -#define BT EReadStartCase::BinaryTrue -#define BU EReadStartCase::BinaryUint64 -#define SP NN // EReadStartCase::Space -#define DM EReadStartCase::DigitOrMinus -#define ST EReadStartCase::String -#define PL EReadStartCase::Plus -#define QU EReadStartCase::Quote -#define PC EReadStartCase::Percent -#define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken) - +#define NN EReadStartCase::None +#define BS EReadStartCase::BinaryString +#define BI EReadStartCase::BinaryInt64 +#define BD EReadStartCase::BinaryDouble +#define BF EReadStartCase::BinaryFalse +#define BT EReadStartCase::BinaryTrue +#define BU EReadStartCase::BinaryUint64 +#define SP NN // EReadStartCase::Space +#define DM EReadStartCase::DigitOrMinus +#define ST EReadStartCase::String +#define PL EReadStartCase::Plus +#define QU EReadStartCase::Quote +#define PC EReadStartCase::Percent +#define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken) + static const ui8 lookupTable[] = { NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - + // 32 SP, // ' ' NN, // '!' @@ -93,7 +93,7 @@ namespace NYson { DM, // '-' NN, // '.' NN, // '/' - + // 48 DM, DM, DM, DM, DM, DM, DM, DM, DM, DM, // '0' - '9' TT(Colon), // ':' @@ -102,7 +102,7 @@ namespace NYson { TT(Equals), // '=' TT(RightAngle), // '>' NN, // '?' - + // 64 NN, // '@' ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M' @@ -112,10 +112,10 @@ namespace NYson { TT(RightBracket), // ']' NN, // '^' ST, // '_' - + // 96 NN, // '`' - + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm' ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z' TT(LeftBrace), // '{' @@ -128,41 +128,41 @@ namespace NYson { NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN}; - -#undef NN -#undef BS -#undef BI -#undef BD -#undef SP -#undef DM -#undef ST -#undef PL -#undef QU -#undef TT + +#undef NN +#undef BS +#undef BI +#undef BD +#undef SP +#undef DM +#undef ST +#undef PL +#undef QU +#undef TT return static_cast<EReadStartCase>(lookupTable[static_cast<ui8>(ch)]); } - + public: TLexer(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) : TBase(blockStream, memoryLimit) { } - + void GetToken(TToken* token) { char ch1 = TBase::SkipSpaceAndGetChar(); auto state = GetStartState(ch1); auto stateBits = static_cast<unsigned>(state); - + if (ch1 == '\0') { *token = TToken::EndOfStream; return; } - + if (stateBits & 1) { // Other = x1b if (stateBits & 1 << 1) { // Other = xxx11b if (state == EReadStartCase::Quote) { @@ -174,9 +174,9 @@ namespace NYson { ReadNumeric<true>(token); } else if (state == EReadStartCase::Plus) { TBase::Advance(1); - + char ch2 = TBase::template GetChar<true>(); - + if (!isdigit(ch2)) { *token = TToken(ETokenType::Plus); } else { @@ -219,9 +219,9 @@ namespace NYson { } else { Y_FAIL("unreachable"); } - } + } } else { // BinaryStringOrOtherSpecialToken = x0b - TBase::Advance(1); + TBase::Advance(1); if (stateBits & 1 << 1) { // OtherSpecialToken = 10b Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::OtherSpecialToken)); *token = TToken(ETokenType(stateBits >> 2)); @@ -231,14 +231,14 @@ namespace NYson { TBase::ReadBinaryString(&value); *token = TToken(value); } - } - } - + } + } + template <bool AllowFinish> void ReadNumeric(TToken* token) { TStringBuf valueBuffer; ENumericResult numericResult = TBase::template ReadNumeric<AllowFinish>(&valueBuffer); - + if (numericResult == ENumericResult::Double) { try { *token = TToken(FromString<double>(valueBuffer)); @@ -258,39 +258,39 @@ namespace NYson { ythrow TYsonException() << "Error parsing uint64 literal " << valueBuffer; } } - } + } }; //////////////////////////////////////////////////////////////////////////////// /*! \endinternal */ - } - + } + class TStatelessYsonLexerImplBase { public: virtual size_t GetToken(const TStringBuf& data, TToken* token) = 0; - + virtual ~TStatelessYsonLexerImplBase() { } }; - + template <bool EnableLinePositionInfo> class TStatelesYsonLexerImpl: public TStatelessYsonLexerImplBase { private: using TLexer = NDetail::TLexer<TStringReader, EnableLinePositionInfo>; TLexer Lexer; - + public: TStatelesYsonLexerImpl() : Lexer(TStringReader(), Nothing()) { } - + size_t GetToken(const TStringBuf& data, TToken* token) override { Lexer.SetBuffer(data.begin(), data.end()); Lexer.GetToken(token); return Lexer.Begin() - data.begin(); } }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/node/node.cpp b/library/cpp/yson/node/node.cpp index a533e34fa6..b39e070718 100644 --- a/library/cpp/yson/node/node.cpp +++ b/library/cpp/yson/node/node.cpp @@ -1,5 +1,5 @@ #include "node.h" - + #include "node_io.h" #include <library/cpp/yson/writer.h> @@ -13,15 +13,15 @@ namespace NYT { bool TNode::TNull::operator==(const TNull&) const { return true; } - -//////////////////////////////////////////////////////////////////////////////// - + +//////////////////////////////////////////////////////////////////////////////// + bool TNode::TUndefined::operator==(const TUndefined&) const { return true; } - + //////////////////////////////////////////////////////////////////////////////// - + namespace NNodeCmp { bool IsComparableType(const TNode::EType type) { @@ -94,15 +94,15 @@ bool operator>=(const TNode& lhs, const TNode& rhs) TNode::TNode() : Value_(TUndefined{}) { } - + TNode::TNode(const char* s) : Value_(TString(s)) { } - + TNode::TNode(TStringBuf s) : Value_(TString(s)) { } - + TNode::TNode(std::string_view s) : Value_(TString(s)) { } @@ -114,12 +114,12 @@ TNode::TNode(const std::string& s) TNode::TNode(TString s) : Value_(std::move(s)) { } - + TNode::TNode(int i) : Value_(static_cast<i64>(i)) { } - - + + TNode::TNode(unsigned int ui) : Value_(static_cast<ui64>(ui)) { } @@ -127,27 +127,27 @@ TNode::TNode(unsigned int ui) TNode::TNode(long i) : Value_(static_cast<i64>(i)) { } - + TNode::TNode(unsigned long ui) : Value_(static_cast<ui64>(ui)) { } - + TNode::TNode(long long i) : Value_(static_cast<i64>(i)) { } - + TNode::TNode(unsigned long long ui) : Value_(static_cast<ui64>(ui)) { } - + TNode::TNode(double d) : Value_(d) { } - + TNode::TNode(bool b) : Value_(b) { } - + TNode::TNode(TMapType map) : Value_(std::move(map)) { } @@ -161,39 +161,39 @@ TNode::TNode(const TNode& rhs) } Value_ = rhs.Value_; } - + TNode& TNode::operator=(const TNode& rhs) { if (this != &rhs) { TNode tmp = rhs; Move(std::move(tmp)); - } + } return *this; } - + TNode::TNode(TNode&& rhs) noexcept : TNode() { Move(std::move(rhs)); } - + TNode& TNode::operator=(TNode&& rhs) noexcept { if (this != &rhs) { TNode tmp = std::move(rhs); Move(std::move(tmp)); - } + } return *this; } - + TNode::~TNode() = default; - + void TNode::Clear() { ClearAttributes(); Value_ = TUndefined(); } - + bool TNode::IsString() const { return std::holds_alternative<TString>(Value_); @@ -203,37 +203,37 @@ bool TNode::IsInt64() const { return std::holds_alternative<i64>(Value_); } - + bool TNode::IsUint64() const { return std::holds_alternative<ui64>(Value_); } - + bool TNode::IsDouble() const { return std::holds_alternative<double>(Value_); } - + bool TNode::IsBool() const { return std::holds_alternative<bool>(Value_); } - + bool TNode::IsList() const { return std::holds_alternative<TListType>(Value_); } - + bool TNode::IsMap() const { return std::holds_alternative<TMapType>(Value_); } - + bool TNode::IsEntity() const { return IsNull(); } - + bool TNode::IsNull() const { return std::holds_alternative<TNull>(Value_); @@ -260,9 +260,9 @@ bool TNode::Empty() const return std::get<TMapType>(Value_).empty(); default: ythrow TTypeError() << "Empty() called for type " << GetType(); - } + } } - + size_t TNode::Size() const { switch (GetType()) { @@ -274,9 +274,9 @@ size_t TNode::Size() const return std::get<TMapType>(Value_).size(); default: ythrow TTypeError() << "Size() called for type " << GetType(); - } + } } - + TNode::EType TNode::GetType() const { return std::visit(TOverloaded{ @@ -291,61 +291,61 @@ TNode::EType TNode::GetType() const [](const TNull&) { return Null; } }, Value_); } - + const TString& TNode::AsString() const { CheckType(String); return std::get<TString>(Value_); } - + i64 TNode::AsInt64() const { CheckType(Int64); return std::get<i64>(Value_); } - + ui64 TNode::AsUint64() const { CheckType(Uint64); return std::get<ui64>(Value_); } - + double TNode::AsDouble() const { CheckType(Double); return std::get<double>(Value_); } - + bool TNode::AsBool() const { CheckType(Bool); return std::get<bool>(Value_); } - + const TNode::TListType& TNode::AsList() const { CheckType(List); return std::get<TListType>(Value_); } - + const TNode::TMapType& TNode::AsMap() const { CheckType(Map); return std::get<TMapType>(Value_); } - + TNode::TListType& TNode::AsList() { CheckType(List); return std::get<TListType>(Value_); } - + TNode::TMapType& TNode::AsMap() { CheckType(Map); return std::get<TMapType>(Value_); } - + const TString& TNode::UncheckedAsString() const noexcept { return std::get<TString>(Value_); @@ -397,7 +397,7 @@ TNode TNode::CreateList() node.Value_ = TListType{}; return node; } - + TNode TNode::CreateList(TListType list) { TNode node; @@ -411,7 +411,7 @@ TNode TNode::CreateMap() node.Value_ = TMapType{}; return node; } - + TNode TNode::CreateMap(TMapType map) { TNode node; @@ -425,19 +425,19 @@ TNode TNode::CreateEntity() node.Value_ = TNull{}; return node; } - + const TNode& TNode::operator[](size_t index) const { CheckType(List); return std::get<TListType>(Value_)[index]; } - + TNode& TNode::operator[](size_t index) { CheckType(List); return std::get<TListType>(Value_)[index]; } - + const TNode& TNode::At(size_t index) const { CheckType(List); const auto& list = std::get<TListType>(Value_); @@ -461,7 +461,7 @@ TNode& TNode::Add() & AssureList(); return std::get<TListType>(Value_).emplace_back(); } - + TNode TNode::Add() && { return std::move(Add()); @@ -473,7 +473,7 @@ TNode& TNode::Add(const TNode& node) & std::get<TListType>(Value_).emplace_back(node); return *this; } - + TNode TNode::Add(const TNode& node) && { return std::move(Add(node)); @@ -485,7 +485,7 @@ TNode& TNode::Add(TNode&& node) & std::get<TListType>(Value_).emplace_back(std::move(node)); return *this; } - + TNode TNode::Add(TNode&& node) && { return std::move(Add(std::move(node))); @@ -496,14 +496,14 @@ bool TNode::HasKey(const TStringBuf key) const CheckType(Map); return std::get<TMapType>(Value_).contains(key); } - + TNode& TNode::operator()(const TString& key, const TNode& value) & { AssureMap(); std::get<TMapType>(Value_)[key] = value; return *this; } - + TNode TNode::operator()(const TString& key, const TNode& value) && { return std::move(operator()(key, value)); @@ -515,7 +515,7 @@ TNode& TNode::operator()(const TString& key, TNode&& value) & std::get<TMapType>(Value_)[key] = std::move(value); return *this; } - + TNode TNode::operator()(const TString& key, TNode&& value) && { return std::move(operator()(key, std::move(value))); @@ -531,15 +531,15 @@ const TNode& TNode::operator[](const TStringBuf key) const return notFound; } else { return i->second; - } + } } - + TNode& TNode::operator[](const TStringBuf key) { AssureMap(); return std::get<TMapType>(Value_)[key]; } - + const TNode& TNode::At(const TStringBuf key) const { CheckType(Map); const auto& map = std::get<TMapType>(Value_); @@ -782,68 +782,68 @@ bool TNode::HasAttributes() const { return Attributes_ && !Attributes_->Empty(); } - + void TNode::ClearAttributes() { if (Attributes_) { Attributes_.Destroy(); - } + } } - + const TNode& TNode::GetAttributes() const { static TNode notFound = TNode::CreateMap(); if (!Attributes_) { return notFound; - } + } return *Attributes_; } - + TNode& TNode::Attributes() { if (!Attributes_) { CreateAttributes(); - } + } return *Attributes_; } - + void TNode::MoveWithoutAttributes(TNode&& rhs) { Value_ = std::move(rhs.Value_); rhs.Clear(); } - + void TNode::Move(TNode&& rhs) { Value_ = std::move(rhs.Value_); Attributes_ = std::move(rhs.Attributes_); } - + void TNode::CheckType(EType type) const { Y_ENSURE_EX(GetType() == type, TTypeError() << "TNode type " << type << " expected, actual type " << GetType(); ); } - + void TNode::AssureMap() { if (std::holds_alternative<TUndefined>(Value_)) { Value_ = TMapType(); } else { CheckType(Map); - } + } } - + void TNode::AssureList() { if (std::holds_alternative<TUndefined>(Value_)) { Value_ = TListType(); } else { CheckType(List); - } + } } - + void TNode::CreateAttributes() { Attributes_ = MakeHolder<TNode>(); @@ -862,54 +862,54 @@ void TNode::Load(IInputStream* in) } //////////////////////////////////////////////////////////////////////////////// - + bool operator==(const TNode& lhs, const TNode& rhs) { if (std::holds_alternative<TNode::TUndefined>(lhs.Value_) || std::holds_alternative<TNode::TUndefined>(rhs.Value_)) - { + { // TODO: should try to remove this behaviour if nobody uses it. return false; - } - + } + if (lhs.GetType() != rhs.GetType()) { - return false; - } - - if (lhs.Attributes_) { - if (rhs.Attributes_) { + return false; + } + + if (lhs.Attributes_) { + if (rhs.Attributes_) { if (*lhs.Attributes_ != *rhs.Attributes_) { return false; } - } else { - return false; - } - } else { - if (rhs.Attributes_) { - return false; - } - } - + } else { + return false; + } + } else { + if (rhs.Attributes_) { + return false; + } + } + return rhs.Value_ == lhs.Value_; -} - +} + bool operator!=(const TNode& lhs, const TNode& rhs) -{ - return !(lhs == rhs); -} - +{ + return !(lhs == rhs); +} + bool GetBool(const TNode& node) -{ - if (node.IsBool()) { - return node.AsBool(); - } else if (node.IsString()) { - return node.AsString() == "true"; - } else { +{ + if (node.IsBool()) { + return node.AsBool(); + } else if (node.IsString()) { + return node.AsString() == "true"; + } else { ythrow TNode::TTypeError() << "GetBool(): not a boolean or string type"; - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node.h b/library/cpp/yson/node/node.h index bc7609e6da..5f90f95df0 100644 --- a/library/cpp/yson/node/node.h +++ b/library/cpp/yson/node/node.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <util/generic/bt_exception.h> #include <util/generic/cast.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/variant.h> -#include <util/generic/vector.h> -#include <util/generic/yexception.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> #include <util/generic/ylimits.h> #include <util/string/cast.h> @@ -14,23 +14,23 @@ class IInputStream; class IOutputStream; - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TNode -{ -public: + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TNode +{ +public: class TLookupError : public TWithBackTrace<yexception> { }; - class TTypeError + class TTypeError : public TWithBackTrace<yexception> - { }; - - enum EType { + { }; + + enum EType { Undefined = 0 /*"undefined"*/, // NOTE: string representation of all node types @@ -43,20 +43,20 @@ public: List = 6 /*"list_node"*/, Map = 7 /*"map_node"*/, Null = 8 /*"null"*/, - }; - + }; + using TListType = TVector<TNode>; using TMapType = THashMap<TString, TNode>; - + private: struct TNull { bool operator==(const TNull&) const; }; - + struct TUndefined { bool operator==(const TUndefined&) const; }; - + using TValue = std::variant< bool, i64, @@ -68,9 +68,9 @@ private: TNull, TUndefined >; - + public: - + TNode(); TNode(const char* s); TNode(TStringBuf s); @@ -78,7 +78,7 @@ public: explicit TNode(const std::string& s); TNode(TString s); TNode(int i); - + //this case made speccially for prevent mess cast of EType into TNode through TNode(int) constructor //usual case of error SomeNode == TNode::Undefined <-- SomeNode indeed will be compared with TNode(0) without this method //correct way is SomeNode.GetType() == TNode::Undefined @@ -102,17 +102,17 @@ public: TNode(double d); TNode(bool b); TNode(TMapType map); - + TNode(const TNode& rhs); TNode& operator=(const TNode& rhs); - + TNode(TNode&& rhs) noexcept; TNode& operator=(TNode&& rhs) noexcept; - + ~TNode(); - + void Clear(); - + bool IsString() const; bool IsInt64() const; bool IsUint64() const; @@ -127,7 +127,7 @@ public: bool IsUndefined() const; // Returns true if TNode is neither Null, nor Undefined bool HasValue() const; - + template<typename T> bool IsOfType() const noexcept; @@ -136,9 +136,9 @@ public: bool Empty() const; size_t Size() const; - + EType GetType() const; - + const TString& AsString() const; i64 AsInt64() const; ui64 AsUint64() const; @@ -148,7 +148,7 @@ public: const TMapType& AsMap() const; TListType& AsList(); TMapType& AsMap(); - + const TString& UncheckedAsString() const noexcept; i64 UncheckedAsInt64() const noexcept; ui64 UncheckedAsUint64() const noexcept; @@ -180,31 +180,31 @@ public: static TNode CreateMap(); static TNode CreateMap(TMapType map); static TNode CreateEntity(); - + const TNode& operator[](size_t index) const; TNode& operator[](size_t index); const TNode& At(size_t index) const; TNode& At(size_t index); - + TNode& Add() &; TNode Add() &&; TNode& Add(const TNode& node) &; TNode Add(const TNode& node) &&; TNode& Add(TNode&& node) &; TNode Add(TNode&& node) &&; - + bool HasKey(const TStringBuf key) const; TNode& operator()(const TString& key, const TNode& value) &; TNode operator()(const TString& key, const TNode& value) &&; TNode& operator()(const TString& key, TNode&& value) &; TNode operator()(const TString& key, TNode&& value) &&; - + const TNode& operator[](const TStringBuf key) const; TNode& operator[](const TStringBuf key); const TNode& At(const TStringBuf key) const; TNode& At(const TStringBuf key); - + // map getters // works the same way like simple getters const TString& ChildAsString(const TStringBuf key) const; @@ -254,42 +254,42 @@ public: T& ChildAs(size_t index); - // attributes + // attributes bool HasAttributes() const; void ClearAttributes(); const TNode& GetAttributes() const; TNode& Attributes(); - + void MoveWithoutAttributes(TNode&& rhs); - + // Serialize TNode using binary yson format. // Methods for ysaveload. void Save(IOutputStream* output) const; void Load(IInputStream* input); -private: +private: void Move(TNode&& rhs); - + void CheckType(EType type) const; - + void AssureMap(); void AssureList(); - + void CreateAttributes(); - -private: + +private: TValue Value_; THolder<TNode> Attributes_; - - friend bool operator==(const TNode& lhs, const TNode& rhs); - friend bool operator!=(const TNode& lhs, const TNode& rhs); -}; - + + friend bool operator==(const TNode& lhs, const TNode& rhs); + friend bool operator!=(const TNode& lhs, const TNode& rhs); +}; + bool operator==(const TNode& lhs, const TNode& rhs); bool operator!=(const TNode& lhs, const TNode& rhs); - + bool GetBool(const TNode& node); - + inline bool TNode::IsArithmetic() const { return IsInt64() || IsUint64() || IsDouble() || IsBool(); } @@ -508,8 +508,8 @@ inline const T& TNode::As() const { return std::get<T>(Value_); } -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + namespace NNodeCmp { bool operator<(const TNode& lhs, const TNode& rhs); bool operator<=(const TNode& lhs, const TNode& rhs); @@ -520,4 +520,4 @@ namespace NNodeCmp { //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT +} // namespace NYT diff --git a/library/cpp/yson/node/node_builder.cpp b/library/cpp/yson/node/node_builder.cpp index b21bc4502e..b4431bc77a 100644 --- a/library/cpp/yson/node/node_builder.cpp +++ b/library/cpp/yson/node/node_builder.cpp @@ -1,96 +1,96 @@ -#include "node_builder.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -TNodeBuilder::TNodeBuilder(TNode* node) -{ - Stack_.push(node); -} - +#include "node_builder.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +TNodeBuilder::TNodeBuilder(TNode* node) +{ + Stack_.push(node); +} + void TNodeBuilder::OnStringScalar(TStringBuf value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnInt64Scalar(i64 value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnUint64Scalar(ui64 value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnDoubleScalar(double value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnBooleanScalar(bool value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnEntity() -{ - AddNode(TNode::CreateEntity(), true); -} - -void TNodeBuilder::OnBeginList() -{ - AddNode(TNode::CreateList(), false); -} - -void TNodeBuilder::OnListItem() -{ - Stack_.push(&Stack_.top()->Add()); -} - -void TNodeBuilder::OnEndList() -{ - Stack_.pop(); -} - -void TNodeBuilder::OnBeginMap() -{ - AddNode(TNode::CreateMap(), false); -} - +{ + AddNode(value, true); +} + +void TNodeBuilder::OnInt64Scalar(i64 value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnUint64Scalar(ui64 value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnDoubleScalar(double value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnBooleanScalar(bool value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnEntity() +{ + AddNode(TNode::CreateEntity(), true); +} + +void TNodeBuilder::OnBeginList() +{ + AddNode(TNode::CreateList(), false); +} + +void TNodeBuilder::OnListItem() +{ + Stack_.push(&Stack_.top()->Add()); +} + +void TNodeBuilder::OnEndList() +{ + Stack_.pop(); +} + +void TNodeBuilder::OnBeginMap() +{ + AddNode(TNode::CreateMap(), false); +} + void TNodeBuilder::OnKeyedItem(TStringBuf key) -{ +{ Stack_.push(&(*Stack_.top())[TString(key)]); -} - -void TNodeBuilder::OnEndMap() -{ - Stack_.pop(); -} - -void TNodeBuilder::OnBeginAttributes() -{ - Stack_.push(&Stack_.top()->Attributes()); -} - -void TNodeBuilder::OnEndAttributes() -{ - Stack_.pop(); -} - +} + +void TNodeBuilder::OnEndMap() +{ + Stack_.pop(); +} + +void TNodeBuilder::OnBeginAttributes() +{ + Stack_.push(&Stack_.top()->Attributes()); +} + +void TNodeBuilder::OnEndAttributes() +{ + Stack_.pop(); +} + void TNodeBuilder::OnNode(TNode node) { AddNode(std::move(node), true); } -void TNodeBuilder::AddNode(TNode value, bool pop) -{ +void TNodeBuilder::AddNode(TNode value, bool pop) +{ Stack_.top()->MoveWithoutAttributes(std::move(value)); - if (pop) - Stack_.pop(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + if (pop) + Stack_.pop(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_builder.h b/library/cpp/yson/node/node_builder.h index 35bb256a5e..69800016e0 100644 --- a/library/cpp/yson/node/node_builder.h +++ b/library/cpp/yson/node/node_builder.h @@ -1,23 +1,23 @@ -#pragma once - +#pragma once + #include "node.h" - + #include <library/cpp/json/json_reader.h> #include <library/cpp/yson/consumer.h> -#include <util/generic/stack.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TNodeBuilder +#include <util/generic/stack.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TNodeBuilder : public ::NYson::TYsonConsumerBase -{ -public: - TNodeBuilder(TNode* node); - +{ +public: + TNodeBuilder(TNode* node); + void OnStringScalar(TStringBuf) override; void OnInt64Scalar(i64) override; void OnUint64Scalar(ui64) override; @@ -33,14 +33,14 @@ public: void OnBeginAttributes() override; void OnEndAttributes() override; void OnNode(TNode node); - -private: + +private: TStack<TNode*> Stack_; - -private: - inline void AddNode(TNode node, bool pop); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + +private: + inline void AddNode(TNode node, bool pop); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_io.cpp b/library/cpp/yson/node/node_io.cpp index ea53e4d8b7..294a7f7217 100644 --- a/library/cpp/yson/node/node_io.cpp +++ b/library/cpp/yson/node/node_io.cpp @@ -1,8 +1,8 @@ #include "node_io.h" - -#include "node_builder.h" -#include "node_visitor.h" - + +#include "node_builder.h" +#include "node_visitor.h" + #include <library/cpp/yson/json/json_writer.h> #include <library/cpp/yson/parser.h> #include <library/cpp/yson/writer.h> @@ -11,13 +11,13 @@ #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_value.h> -#include <util/stream/input.h> -#include <util/stream/output.h> -#include <util/stream/str.h> +#include <util/stream/input.h> +#include <util/stream/output.h> +#include <util/stream/str.h> #include <util/stream/mem.h> - -namespace NYT { - + +namespace NYT { + static void WalkJsonTree(const NJson::TJsonValue& jsonValue, NJson::TJsonCallbacks* callbacks) { using namespace NJson; @@ -68,20 +68,20 @@ static void WalkJsonTree(const NJson::TJsonValue& jsonValue, NJson::TJsonCallbac static TNode CreateEmptyNodeByType(::NYson::EYsonType type) { - TNode result; - switch (type) { + TNode result; + switch (type) { case ::NYson::EYsonType::ListFragment: - result = TNode::CreateList(); - break; + result = TNode::CreateList(); + break; case ::NYson::EYsonType::MapFragment: - result = TNode::CreateMap(); - break; - default: - break; - } + result = TNode::CreateMap(); + break; + default: + break; + } return result; } - + TNode NodeFromYsonString(const TStringBuf input, ::NYson::EYsonType type) { TMemoryInput stream(input); @@ -106,19 +106,19 @@ TNode NodeFromYsonStream(IInputStream* input, ::NYson::EYsonType type) { TNode result = CreateEmptyNodeByType(type); - TNodeBuilder builder(&result); + TNodeBuilder builder(&result); ::NYson::TYsonParser parser(&builder, input, type); - parser.Parse(); - return result; -} - + parser.Parse(); + return result; +} + void NodeToYsonStream(const TNode& node, IOutputStream* output, NYson::EYsonFormat format) -{ +{ ::NYson::TYsonWriter writer(output, format); - TNodeVisitor visitor(&writer); - visitor.Visit(node); -} - + TNodeVisitor visitor(&writer); + visitor.Visit(node); +} + void NodeToCanonicalYsonStream(const TNode& node, IOutputStream* output, NYson::EYsonFormat format) { ::NYson::TYsonWriter writer(output, format); @@ -149,6 +149,6 @@ TNode NodeFromJsonValue(const NJson::TJsonValue& input) return result; } -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_io.h b/library/cpp/yson/node/node_io.h index 5b63243317..2ad23b658f 100644 --- a/library/cpp/yson/node/node_io.h +++ b/library/cpp/yson/node/node_io.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include "node.h" #include <library/cpp/yson/public.h> - + namespace NJson { class TJsonValue; } // namespace NJson -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + // Parse TNode from string in YSON format TNode NodeFromYsonString(const TStringBuf input, ::NYson::EYsonType type = ::NYson::EYsonType::Node); // Serialize TNode to string in one of YSON formats with random order of maps' keys (don't use in tests) TString NodeToYsonString(const TNode& node, ::NYson::EYsonFormat format = ::NYson::EYsonFormat::Text); - + // Same as the latter, but maps' keys are sorted lexicographically (to be used in tests) TString NodeToCanonicalYsonString(const TNode& node, ::NYson::EYsonFormat format = ::NYson::EYsonFormat::Text); @@ -35,6 +35,6 @@ TNode NodeFromJsonString(const TStringBuf input); // Convert TJsonValue to TNode TNode NodeFromJsonValue(const NJson::TJsonValue& input); -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_visitor.cpp b/library/cpp/yson/node/node_visitor.cpp index 824d934867..899fbfa02a 100644 --- a/library/cpp/yson/node/node_visitor.cpp +++ b/library/cpp/yson/node/node_visitor.cpp @@ -1,12 +1,12 @@ -#include "node_visitor.h" - +#include "node_visitor.h" + #include <util/generic/algorithm.h> #include <util/string/printf.h> -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + namespace { template <typename Fun> @@ -31,122 +31,122 @@ void Iterate(const TNode::TMapType& nodeMap, bool sortByKey, Fun action) //////////////////////////////////////////////////////////////////////////////// TNodeVisitor::TNodeVisitor(NYson::IYsonConsumer* consumer, bool sortMapKeys) - : Consumer_(consumer) + : Consumer_(consumer) , SortMapKeys_(sortMapKeys) -{ } - -void TNodeVisitor::Visit(const TNode& node) -{ - VisitAny(node); -} - -void TNodeVisitor::VisitAny(const TNode& node) -{ - if (node.HasAttributes()) { - Consumer_->OnBeginAttributes(); +{ } + +void TNodeVisitor::Visit(const TNode& node) +{ + VisitAny(node); +} + +void TNodeVisitor::VisitAny(const TNode& node) +{ + if (node.HasAttributes()) { + Consumer_->OnBeginAttributes(); Iterate(node.GetAttributes().AsMap(), SortMapKeys_, [&](const std::pair<TString, TNode>& item) { - Consumer_->OnKeyedItem(item.first); + Consumer_->OnKeyedItem(item.first); if (item.second.IsUndefined()) { ythrow TNode::TTypeError() << "unable to visit attribute value of type " << TNode::EType::Undefined << "; attribute name: `" << item.first << '\'' ; } - VisitAny(item.second); + VisitAny(item.second); }); - Consumer_->OnEndAttributes(); - } - - switch (node.GetType()) { + Consumer_->OnEndAttributes(); + } + + switch (node.GetType()) { case TNode::String: - VisitString(node); - break; + VisitString(node); + break; case TNode::Int64: - VisitInt64(node); - break; + VisitInt64(node); + break; case TNode::Uint64: - VisitUint64(node); - break; + VisitUint64(node); + break; case TNode::Double: - VisitDouble(node); - break; + VisitDouble(node); + break; case TNode::Bool: - VisitBool(node); - break; + VisitBool(node); + break; case TNode::List: VisitList(node.AsList()); - break; + break; case TNode::Map: VisitMap(node.AsMap()); - break; + break; case TNode::Null: - VisitEntity(); - break; + VisitEntity(); + break; case TNode::Undefined: ythrow TNode::TTypeError() << "unable to visit TNode of type " << node.GetType(); default: Y_FAIL("Unexpected type: %d", node.GetType()); - } -} - -void TNodeVisitor::VisitString(const TNode& node) -{ - Consumer_->OnStringScalar(node.AsString()); -} - -void TNodeVisitor::VisitInt64(const TNode& node) -{ - Consumer_->OnInt64Scalar(node.AsInt64()); -} - -void TNodeVisitor::VisitUint64(const TNode& node) -{ - Consumer_->OnUint64Scalar(node.AsUint64()); -} - -void TNodeVisitor::VisitDouble(const TNode& node) -{ - Consumer_->OnDoubleScalar(node.AsDouble()); -} - -void TNodeVisitor::VisitBool(const TNode& node) -{ - Consumer_->OnBooleanScalar(node.AsBool()); -} - + } +} + +void TNodeVisitor::VisitString(const TNode& node) +{ + Consumer_->OnStringScalar(node.AsString()); +} + +void TNodeVisitor::VisitInt64(const TNode& node) +{ + Consumer_->OnInt64Scalar(node.AsInt64()); +} + +void TNodeVisitor::VisitUint64(const TNode& node) +{ + Consumer_->OnUint64Scalar(node.AsUint64()); +} + +void TNodeVisitor::VisitDouble(const TNode& node) +{ + Consumer_->OnDoubleScalar(node.AsDouble()); +} + +void TNodeVisitor::VisitBool(const TNode& node) +{ + Consumer_->OnBooleanScalar(node.AsBool()); +} + void TNodeVisitor::VisitList(const TNode::TListType& nodeList) -{ - Consumer_->OnBeginList(); +{ + Consumer_->OnBeginList(); size_t index = 0; for (const auto& item : nodeList) { - Consumer_->OnListItem(); + Consumer_->OnListItem(); if (item.IsUndefined()) { ythrow TNode::TTypeError() << "unable to visit list node child of type " << TNode::EType::Undefined << "; list index: " << index; } - VisitAny(item); + VisitAny(item); ++index; - } - Consumer_->OnEndList(); -} - + } + Consumer_->OnEndList(); +} + void TNodeVisitor::VisitMap(const TNode::TMapType& nodeMap) -{ - Consumer_->OnBeginMap(); +{ + Consumer_->OnBeginMap(); Iterate(nodeMap, SortMapKeys_, [&](const std::pair<TString, TNode>& item) { - Consumer_->OnKeyedItem(item.first); + Consumer_->OnKeyedItem(item.first); if (item.second.IsUndefined()) { ythrow TNode::TTypeError() << "unable to visit map node child of type " << TNode::EType::Undefined << "; map key: `" << item.first << '\'' ; } - VisitAny(item.second); + VisitAny(item.second); }); - Consumer_->OnEndMap(); -} - -void TNodeVisitor::VisitEntity() -{ - Consumer_->OnEntity(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + Consumer_->OnEndMap(); +} + +void TNodeVisitor::VisitEntity() +{ + Consumer_->OnEntity(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_visitor.h b/library/cpp/yson/node/node_visitor.h index d4eb8e5667..db25832309 100644 --- a/library/cpp/yson/node/node_visitor.h +++ b/library/cpp/yson/node/node_visitor.h @@ -1,37 +1,37 @@ -#pragma once - +#pragma once + #include "node.h" #include <library/cpp/yson/consumer.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TNodeVisitor -{ -public: + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TNodeVisitor +{ +public: TNodeVisitor(NYson::IYsonConsumer* consumer, bool sortMapKeys = false); - - void Visit(const TNode& node); + + void Visit(const TNode& node); void VisitMap(const TNode::TMapType& nodeMap); void VisitList(const TNode::TListType& nodeMap); - -private: + +private: NYson::IYsonConsumer* Consumer_; bool SortMapKeys_; - -private: - void VisitAny(const TNode& node); - - void VisitString(const TNode& node); - void VisitInt64(const TNode& node); - void VisitUint64(const TNode& node); - void VisitDouble(const TNode& node); - void VisitBool(const TNode& node); - void VisitEntity(); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + +private: + void VisitAny(const TNode& node); + + void VisitString(const TNode& node); + void VisitInt64(const TNode& node); + void VisitUint64(const TNode& node); + void VisitDouble(const TNode& node); + void VisitBool(const TNode& node); + void VisitEntity(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/parser.cpp b/library/cpp/yson/parser.cpp index 3af222ed4e..783f9b9047 100644 --- a/library/cpp/yson/parser.cpp +++ b/library/cpp/yson/parser.cpp @@ -1,14 +1,14 @@ -#include "parser.h" -#include "consumer.h" -#include "format.h" -#include "parser_detail.h" - -#include <util/stream/input.h> -#include <util/generic/buffer.h> - +#include "parser.h" +#include "consumer.h" +#include "format.h" +#include "parser_detail.h" + +#include <util/stream/input.h> +#include <util/generic/buffer.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TYsonParser::TImpl { public: TImpl( @@ -24,7 +24,7 @@ namespace NYson { , MemoryLimit_(memoryLimit) { } - + void Parse() { TBuffer buffer(64 << 10); ParseYsonStreamImpl<NYT::NYson::IYsonConsumer, TStreamReader>( @@ -48,26 +48,26 @@ namespace NYson { TYsonParser::TYsonParser( NYT::NYson::IYsonConsumer* consumer, IInputStream* stream, - EYsonType type, - bool enableLinePositionInfo, + EYsonType type, + bool enableLinePositionInfo, TMaybe<ui64> memoryLimit) : Impl(new TImpl(consumer, stream, type, enableLinePositionInfo, memoryLimit)) - { - } - + { + } + TYsonParser::~TYsonParser() { } - + void TYsonParser::Parse() { Impl->Parse(); } - + //////////////////////////////////////////////////////////////////////////////// - + class TStatelessYsonParser::TImpl { private: THolder<TStatelessYsonParserImplBase> Impl; - + public: TImpl( NYT::NYson::IYsonConsumer* consumer, @@ -79,31 +79,31 @@ namespace NYson { : static_cast<TStatelessYsonParserImplBase*>(new TStatelessYsonParserImpl<NYT::NYson::IYsonConsumer, false>(consumer, memoryLimit))) { } - + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) { Impl->Parse(data, type); } }; - + //////////////////////////////////////////////////////////////////////////////// - + TStatelessYsonParser::TStatelessYsonParser( NYT::NYson::IYsonConsumer* consumer, - bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit) + bool enableLinePositionInfo, + TMaybe<ui64> memoryLimit) : Impl(new TImpl(consumer, enableLinePositionInfo, memoryLimit)) { } - + TStatelessYsonParser::~TStatelessYsonParser() { } void TStatelessYsonParser::Parse(const TStringBuf& data, EYsonType type) { - Impl->Parse(data, type); - } - + Impl->Parse(data, type); + } + //////////////////////////////////////////////////////////////////////////////// - + void ParseYsonStringBuffer( const TStringBuf& buffer, NYT::NYson::IYsonConsumer* consumer, @@ -117,9 +117,9 @@ namespace NYson { enableLinePositionInfo, memoryLimit); } - + //////////////////////////////////////////////////////////////////////////////// - + class TYsonListParser::TImpl { public: TImpl( @@ -135,7 +135,7 @@ namespace NYson { , Reader_(Stream_, Buffer_.Data(), Buffer_.Capacity()) { } - + bool Parse() { if (!Impl_) { Impl_.Reset( @@ -145,7 +145,7 @@ namespace NYson { } return Impl_->Parse(); } - + private: NYT::NYson::IYsonConsumer* Consumer_; IInputStream* Stream_; @@ -155,9 +155,9 @@ namespace NYson { TStreamReader Reader_; THolder<TYsonListParserImplBase> Impl_; }; - + //////////////////////////////////////////////////////////////////////////////// - + TYsonListParser::TYsonListParser( NYT::NYson::IYsonConsumer* consumer, IInputStream* stream, diff --git a/library/cpp/yson/parser.h b/library/cpp/yson/parser.h index 715341d5b5..dce35a8cd4 100644 --- a/library/cpp/yson/parser.h +++ b/library/cpp/yson/parser.h @@ -1,19 +1,19 @@ -#pragma once - -#include "public.h" - -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> - +#pragma once + +#include "public.h" + +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> + class IInputStream; - + namespace NYT::NYson { struct IYsonConsumer; } // namespace NYT::NYson namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TYsonParser { public: TYsonParser( @@ -22,36 +22,36 @@ namespace NYson { EYsonType type = ::NYson::EYsonType::Node, bool enableLinePositionInfo = false, TMaybe<ui64> memoryLimit = Nothing()); - + ~TYsonParser(); - + void Parse(); - + private: class TImpl; THolder<TImpl> Impl; }; - + //////////////////////////////////////////////////////////////////////////////// - + class TStatelessYsonParser { public: TStatelessYsonParser( NYT::NYson::IYsonConsumer* consumer, bool enableLinePositionInfo = false, TMaybe<ui64> memoryLimit = Nothing()); - + ~TStatelessYsonParser(); - + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node); - + private: class TImpl; THolder<TImpl> Impl; }; - + //////////////////////////////////////////////////////////////////////////////// - + class TYsonListParser { public: TYsonListParser( @@ -59,7 +59,7 @@ namespace NYson { IInputStream* stream, bool enableLinePositionInfo = false, TMaybe<ui64> memoryLimit = Nothing()); - + ~TYsonListParser(); bool Parse(); // Returns false, if there is no more list items @@ -79,5 +79,5 @@ namespace NYson { TMaybe<ui64> memoryLimit = Nothing()); //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/parser_detail.h b/library/cpp/yson/parser_detail.h index b6eee05664..44223caf12 100644 --- a/library/cpp/yson/parser_detail.h +++ b/library/cpp/yson/parser_detail.h @@ -1,43 +1,43 @@ -#pragma once - -#include "detail.h" - +#pragma once + +#include "detail.h" + namespace NYson { namespace NDetail { //////////////////////////////////////////////////////////////////////////////// - + template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> class TParser : public TLexerBase<TBlockStream, EnableLinePositionInfo> { private: using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; TConsumer* Consumer; - + public: TParser(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) : TBase(blockStream, memoryLimit) , Consumer(consumer) { } - + void DoParse(EYsonType ysonType) { switch (ysonType) { case ::NYson::EYsonType::Node: ParseNode<true>(); break; - + case ::NYson::EYsonType::ListFragment: ParseListFragment<true>(EndSymbol); break; - + case ::NYson::EYsonType::MapFragment: ParseMapFragment<true>(EndSymbol); break; - + default: Y_FAIL("unreachable"); } - + while (!(TBase::IsFinished() && TBase::IsEmpty())) { if (TBase::template SkipSpaceAndGetChar<true>() != EndSymbol) { ythrow TYsonException() << "Stray '" << (*TBase::Begin()) << "' found"; @@ -46,12 +46,12 @@ namespace NYson { } } } - + bool DoParseListFragment(bool first) { bool ret = first ? first : ParseListSeparator<true>(EndSymbol); return ret && ParseListItem<true>(EndSymbol); - } - + } + void ParseAttributes() { Consumer->OnBeginAttributes(); ParseMapFragment(EndAttributesSymbol); @@ -65,19 +65,19 @@ namespace NYson { TBase::SkipCharToken(EndMapSymbol); Consumer->OnEndMap(); } - + void ParseList() { Consumer->OnBeginList(); ParseListFragment(EndListSymbol); TBase::SkipCharToken(EndListSymbol); Consumer->OnEndList(); } - + template <bool AllowFinish> void ParseNode() { return ParseNode<AllowFinish>(TBase::SkipSpaceAndGetChar()); } - + template <bool AllowFinish> void ParseNode(char ch) { if (ch == BeginAttributesSymbol) { @@ -85,18 +85,18 @@ namespace NYson { ParseAttributes(); ch = TBase::SkipSpaceAndGetChar(); } - + switch (ch) { case BeginMapSymbol: TBase::Advance(1); ParseMap(); break; - + case BeginListSymbol: TBase::Advance(1); ParseList(); break; - + case '"': { TBase::Advance(1); TStringBuf value; @@ -146,7 +146,7 @@ namespace NYson { TBase::Advance(1); Consumer->OnEntity(); break; - + default: { if (isdigit((unsigned char)ch) || ch == '-' || ch == '+') { // case of '+' is handled in AfterPlus state ReadNumeric<AllowFinish>(); @@ -167,11 +167,11 @@ namespace NYson { } } } - } + } void ParseKey() { return ParseKey(TBase::SkipSpaceAndGetChar()); - } + } void ParseKey(char ch) { switch (ch) { @@ -199,8 +199,8 @@ namespace NYson { } } } - } - + } + template <bool AllowFinish> void ParseMapFragment(char endSymbol) { char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); @@ -221,13 +221,13 @@ namespace NYson { ythrow TYsonException() << "Expected '" << KeyedItemSeparatorSymbol << "' or '" << endSymbol << "' but '" << ch << "' found"; } - } - } - + } + } + void ParseMapFragment(char endSymbol) { ParseMapFragment<false>(endSymbol); } - + template <bool AllowFinish> bool ParseListItem(char endSymbol) { char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); @@ -237,7 +237,7 @@ namespace NYson { return true; } return false; - } + } template <bool AllowFinish> bool ParseListSeparator(char endSymbol) { @@ -250,17 +250,17 @@ namespace NYson { << "' or '" << endSymbol << "' but '" << ch << "' found"; } return false; - } + } template <bool AllowFinish> void ParseListFragment(char endSymbol) { while (ParseListItem<AllowFinish>(endSymbol) && ParseListSeparator<AllowFinish>(endSymbol)) { - } - } - + } + } + void ParseListFragment(char endSymbol) { ParseListFragment<false>(endSymbol); - } + } template <bool AllowFinish> void ReadNumeric() { @@ -295,13 +295,13 @@ namespace NYson { } Consumer->OnUint64Scalar(value); } - } + } }; - + //////////////////////////////////////////////////////////////////////////////// - - } - + + } + template <class TConsumer, class TBlockStream> void ParseYsonStreamImpl( const TBlockStream& blockStream, @@ -317,9 +317,9 @@ namespace NYson { using TImpl = NDetail::TParser<TConsumer, TBlockStream, false>; TImpl impl(blockStream, consumer, memoryLimit); impl.DoParse(parsingMode); - } - } - + } + } + class TStatelessYsonParserImplBase { public: virtual void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) = 0; @@ -334,27 +334,27 @@ namespace NYson { private: using TParser = NDetail::TParser<TConsumer, TStringReader, EnableLinePositionInfo>; TParser Parser; - + public: TStatelessYsonParserImpl(TConsumer* consumer, TMaybe<ui64> memoryLimit) : Parser(TStringReader(), consumer, memoryLimit) { } - + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) override { Parser.SetBuffer(data.begin(), data.end()); Parser.DoParse(type); - } + } }; - + class TYsonListParserImplBase { public: virtual bool Parse() = 0; - + virtual ~TYsonListParserImplBase() { } }; - + template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> class TYsonListParserImpl : public TYsonListParserImplBase { @@ -362,20 +362,20 @@ namespace NYson { using TParser = NDetail::TParser<TConsumer, TBlockStream, EnableLinePositionInfo>; TParser Parser; bool First = true; - + public: TYsonListParserImpl(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) : Parser(blockStream, consumer, memoryLimit) { } - + bool Parse() override { bool ret = Parser.DoParseListFragment(First); First = false; return ret; } }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/public.h b/library/cpp/yson/public.h index 217fff9c8b..1ed793592b 100644 --- a/library/cpp/yson/public.h +++ b/library/cpp/yson/public.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <library/cpp/yt/misc/enum.h> -#include <util/generic/yexception.h> - +#include <util/generic/yexception.h> + #include <library/cpp/yt/yson_string/public.h> #include <library/cpp/yt/yson/public.h> namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + using NYT::NYson::EYsonFormat; using NYT::NYson::EYsonType; - + class TYsonStringBuf; struct TYsonConsumerBase; - + class TYsonWriter; class TYsonParser; class TStatelessYsonParser; @@ -24,7 +24,7 @@ namespace NYson { class TYsonException : public yexception {}; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/token.cpp b/library/cpp/yson/token.cpp index ae9fd741b8..c8584c8c2e 100644 --- a/library/cpp/yson/token.cpp +++ b/library/cpp/yson/token.cpp @@ -1,11 +1,11 @@ -#include "token.h" - -#include <util/string/vector.h> -#include <util/string/printf.h> - +#include "token.h" + +#include <util/string/vector.h> +#include <util/string/printf.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + ETokenType CharToTokenType(char ch) { switch (ch) { case ';': @@ -40,7 +40,7 @@ namespace NYson { return ETokenType::EndOfStream; } } - + char TokenTypeToChar(ETokenType type) { switch (type) { case ETokenType::Semicolon: @@ -74,16 +74,16 @@ namespace NYson { default: Y_FAIL("unreachable"); } - } - + } + TString TokenTypeToString(ETokenType type) { return TString(1, TokenTypeToChar(type)); - } - + } + //////////////////////////////////////////////////////////////////////////////// - + const TToken TToken::EndOfStream; - + TToken::TToken() : Type_(ETokenType::EndOfStream) , Int64Value(0) @@ -92,7 +92,7 @@ namespace NYson { , BooleanValue(false) { } - + TToken::TToken(ETokenType type) : Type_(type) , Int64Value(0) @@ -111,7 +111,7 @@ namespace NYson { break; } } - + TToken::TToken(const TStringBuf& stringValue) : Type_(ETokenType::String) , StringValue(stringValue) @@ -120,8 +120,8 @@ namespace NYson { , DoubleValue(0.0) , BooleanValue(false) { - } - + } + TToken::TToken(i64 int64Value) : Type_(ETokenType::Int64) , Int64Value(int64Value) @@ -129,7 +129,7 @@ namespace NYson { , DoubleValue(0.0) { } - + TToken::TToken(ui64 uint64Value) : Type_(ETokenType::Uint64) , Int64Value(0) @@ -138,7 +138,7 @@ namespace NYson { , BooleanValue(false) { } - + TToken::TToken(double doubleValue) : Type_(ETokenType::Double) , Int64Value(0) @@ -147,7 +147,7 @@ namespace NYson { , BooleanValue(false) { } - + TToken::TToken(bool booleanValue) : Type_(ETokenType::Boolean) , Int64Value(0) @@ -155,36 +155,36 @@ namespace NYson { , BooleanValue(booleanValue) { } - + bool TToken::IsEmpty() const { return Type_ == ETokenType::EndOfStream; } - + const TStringBuf& TToken::GetStringValue() const { CheckType(ETokenType::String); return StringValue; } - + i64 TToken::GetInt64Value() const { CheckType(ETokenType::Int64); return Int64Value; } - + ui64 TToken::GetUint64Value() const { CheckType(ETokenType::Uint64); return Uint64Value; } - + double TToken::GetDoubleValue() const { CheckType(ETokenType::Double); return DoubleValue; } - + bool TToken::GetBooleanValue() const { CheckType(ETokenType::Boolean); return BooleanValue; } - + void TToken::CheckType(ETokenType expectedType) const { if (Type_ != expectedType) { if (Type_ == ETokenType::EndOfStream) { @@ -194,9 +194,9 @@ namespace NYson { << "', Type: " << TokenTypeToString(Type_) << ", ExpectedType: " << TokenTypeToString(expectedType) << ")"; } - } - } - + } + } + void TToken::Reset() { Type_ = ETokenType::EndOfStream; Int64Value = 0; @@ -205,32 +205,32 @@ namespace NYson { StringValue = TStringBuf(); BooleanValue = false; } - + TString ToString(const TToken& token) { switch (token.GetType()) { case ETokenType::EndOfStream: return TString(); - + case ETokenType::String: return TString(token.GetStringValue()); - + case ETokenType::Int64: return ::ToString(token.GetInt64Value()); - + case ETokenType::Uint64: return ::ToString(token.GetUint64Value()); - + case ETokenType::Double: return ::ToString(token.GetDoubleValue()); - + case ETokenType::Boolean: return token.GetBooleanValue() ? "true" : "false"; - + default: return TokenTypeToString(token.GetType()); } - } - + } + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/token.h b/library/cpp/yson/token.h index ee455a29ca..7283e56950 100644 --- a/library/cpp/yson/token.h +++ b/library/cpp/yson/token.h @@ -1,21 +1,21 @@ -#pragma once - -#include "public.h" - -#include <util/generic/strbuf.h> - +#pragma once + +#include "public.h" + +#include <util/generic/strbuf.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + enum ETokenType { EndOfStream, - + String, Int64, Uint64, Double, Boolean, - + // Special values: // YSON Semicolon, // ; @@ -27,7 +27,7 @@ namespace NYson { RightBrace, // } LeftAngle, // < RightAngle, // > - + // Table ranges LeftParenthesis, // ( RightParenthesis, // ) @@ -35,23 +35,23 @@ namespace NYson { Colon, // : Comma, // , }; - + //////////////////////////////////////////////////////////////////////////////// - + ETokenType CharToTokenType(char ch); char TokenTypeToChar(ETokenType type); TString TokenTypeToString(ETokenType type); - + //////////////////////////////////////////////////////////////////////////////// - + class TLexerImpl; - + //////////////////////////////////////////////////////////////////////////////// - + class TToken { public: static const TToken EndOfStream; - + TToken(); TToken(ETokenType type); explicit TToken(const TStringBuf& stringValue); @@ -59,35 +59,35 @@ namespace NYson { explicit TToken(ui64 int64Value); explicit TToken(double doubleValue); explicit TToken(bool booleanValue); - + ETokenType GetType() const { return Type_; } - + bool IsEmpty() const; const TStringBuf& GetStringValue() const; i64 GetInt64Value() const; ui64 GetUint64Value() const; double GetDoubleValue() const; bool GetBooleanValue() const; - + void CheckType(ETokenType expectedType) const; void Reset(); - + private: friend class TLexerImpl; - + ETokenType Type_; - + TStringBuf StringValue; i64 Int64Value; ui64 Uint64Value; double DoubleValue; bool BooleanValue; }; - + TString ToString(const TToken& token); - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/tokenizer.cpp b/library/cpp/yson/tokenizer.cpp index 595700e62f..06760170d4 100644 --- a/library/cpp/yson/tokenizer.cpp +++ b/library/cpp/yson/tokenizer.cpp @@ -1,37 +1,37 @@ -#include "tokenizer.h" - +#include "tokenizer.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + TTokenizer::TTokenizer(const TStringBuf& input) : Input(input) , Parsed(0) { } - + bool TTokenizer::ParseNext() { Input = Input.Tail(Parsed); Token.Reset(); Parsed = Lexer.GetToken(Input, &Token); return !CurrentToken().IsEmpty(); } - + const TToken& TTokenizer::CurrentToken() const { return Token; } - + ETokenType TTokenizer::GetCurrentType() const { return CurrentToken().GetType(); } - + TStringBuf TTokenizer::GetCurrentSuffix() const { return Input.Tail(Parsed); } - + const TStringBuf& TTokenizer::CurrentInput() const { return Input; } - + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/tokenizer.h b/library/cpp/yson/tokenizer.h index 25e42ab9fc..0576aace95 100644 --- a/library/cpp/yson/tokenizer.h +++ b/library/cpp/yson/tokenizer.h @@ -1,28 +1,28 @@ -#pragma once - -#include "public.h" -#include "lexer.h" - +#pragma once + +#include "public.h" +#include "lexer.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TTokenizer { public: explicit TTokenizer(const TStringBuf& input); - + bool ParseNext(); const TToken& CurrentToken() const; ETokenType GetCurrentType() const; TStringBuf GetCurrentSuffix() const; const TStringBuf& CurrentInput() const; - + private: TStringBuf Input; TToken Token; TStatelessLexer Lexer; size_t Parsed; }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/varint.cpp b/library/cpp/yson/varint.cpp index b962dcede7..d538ee3cff 100644 --- a/library/cpp/yson/varint.cpp +++ b/library/cpp/yson/varint.cpp @@ -1,12 +1,12 @@ #include "varint.h" -#include "zigzag.h" - +#include "zigzag.h" + #include <util/generic/yexception.h> - + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + int WriteVarUInt64(IOutputStream* output, ui64 value) { bool stop = false; int bytesWritten = 0; @@ -19,22 +19,22 @@ namespace NYson { byte &= 0x7F; } output->Write(byte); - } + } return bytesWritten; - } - + } + int WriteVarInt32(IOutputStream* output, i32 value) { return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode32(value))); } - + int WriteVarInt64(IOutputStream* output, i64 value) { return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode64(value))); } - + int ReadVarUInt64(IInputStream* input, ui64* value) { size_t count = 0; ui64 result = 0; - + ui8 byte = 0; do { if (7 * count > 8 * sizeof(ui64)) { @@ -56,16 +56,16 @@ namespace NYson { int bytesRead = ReadVarUInt64(input, &varInt); if (varInt > Max<ui32>()) { ythrow yexception() << "The data is too long to read ui64"; - } + } *value = ZigZagDecode32(static_cast<ui32>(varInt)); return bytesRead; } - + int ReadVarInt64(IInputStream* input, i64* value) { ui64 varInt; int bytesRead = ReadVarUInt64(input, &varInt); *value = ZigZagDecode64(varInt); return bytesRead; - } - + } + } // namespace NYson diff --git a/library/cpp/yson/writer.cpp b/library/cpp/yson/writer.cpp index 9f0386e213..054459f9f5 100644 --- a/library/cpp/yson/writer.cpp +++ b/library/cpp/yson/writer.cpp @@ -1,18 +1,18 @@ -#include "writer.h" +#include "writer.h" #include "detail.h" -#include "format.h" +#include "format.h" #include "parser.h" #include "varint.h" -#include "zigzag.h" - -#include <util/string/cast.h> - +#include "zigzag.h" + +#include <util/string/cast.h> + #include <cmath> namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + // Copied from <util/string/escape.cpp> namespace { inline char HexDigit(char value) { @@ -22,26 +22,26 @@ namespace NYson { else return 'A' + value - 10; } - + inline char OctDigit(char value) { Y_ASSERT(value < 8); return '0' + value; } - + inline bool IsPrintable(char c) { return c >= 32 && c <= 126; } - + inline bool IsHexDigit(char c) { return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); } - + inline bool IsOctDigit(char c) { return c >= '0' && c <= '7'; } - + const size_t ESCAPE_C_BUFFER_SIZE = 4; - + inline size_t EscapeC(unsigned char c, char next, char r[ESCAPE_C_BUFFER_SIZE]) { // (1) Printable characters go as-is, except backslash and double quote. // (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible). @@ -87,33 +87,33 @@ namespace NYson { return 4; } } - + void EscapeC(const char* str, size_t len, IOutputStream& output) { char buffer[ESCAPE_C_BUFFER_SIZE]; - + size_t i, j; for (i = 0, j = 0; i < len; ++i) { size_t rlen = EscapeC(str[i], (i + 1 < len ? str[i + 1] : 0), buffer); - + if (rlen > 1) { output.Write(str + j, i - j); j = i + 1; output.Write(buffer, rlen); } } - + if (j > 0) { output.Write(str + j, len - j); } else { output.Write(str, len); } - } - + } + TString FloatToStringWithNanInf(double value) { if (std::isfinite(value)) { return ::ToString(value); } - + static const TStringBuf nanLiteral = "%nan"; static const TStringBuf infLiteral = "%inf"; static const TStringBuf negativeInfLiteral = "%-inf"; @@ -130,7 +130,7 @@ namespace NYson { } } - + //////////////////////////////////////////////////////////////////////////////// TYsonWriter::TYsonWriter( @@ -147,17 +147,17 @@ namespace NYson { { Y_ASSERT(stream); } - + void TYsonWriter::WriteIndent() { for (int i = 0; i < IndentSize * Depth; ++i) { Stream->Write(' '); } } - + bool TYsonWriter::IsTopLevelFragmentContext() const { return Depth == 0 && (Type == ::NYson::EYsonType::ListFragment || Type == ::NYson::EYsonType::MapFragment); - } - + } + void TYsonWriter::EndNode() { if (IsTopLevelFragmentContext()) { ETokenType separatorToken = @@ -168,15 +168,15 @@ namespace NYson { if (Format == EYsonFormat::Text || Format == EYsonFormat::Pretty) { Stream->Write('\n'); } - } - } - + } + } + void TYsonWriter::BeginCollection(ETokenType beginToken) { Stream->Write(TokenTypeToChar(beginToken)); ++Depth; BeforeFirstItem = true; } - + void TYsonWriter::CollectionItem(ETokenType separatorToken) { if (!IsTopLevelFragmentContext()) { if (!BeforeFirstItem) { @@ -187,21 +187,21 @@ namespace NYson { Stream->Write('\n'); WriteIndent(); } - } - + } + BeforeFirstItem = false; } void TYsonWriter::EndCollection(ETokenType endToken) { --Depth; if (Format == EYsonFormat::Pretty && !BeforeFirstItem) { - Stream->Write('\n'); - WriteIndent(); - } + Stream->Write('\n'); + WriteIndent(); + } Stream->Write(TokenTypeToChar(endToken)); BeforeFirstItem = false; - } - + } + void TYsonWriter::WriteStringScalar(const TStringBuf& value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::StringMarker); @@ -213,12 +213,12 @@ namespace NYson { Stream->Write('"'); } } - + void TYsonWriter::OnStringScalar(TStringBuf value) { WriteStringScalar(value); EndNode(); - } - + } + void TYsonWriter::OnInt64Scalar(i64 value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::Int64Marker); @@ -227,8 +227,8 @@ namespace NYson { Stream->Write(::ToString(value)); } EndNode(); - } - + } + void TYsonWriter::OnUint64Scalar(ui64 value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::Uint64Marker); @@ -238,8 +238,8 @@ namespace NYson { Stream->Write("u"); } EndNode(); - } - + } + void TYsonWriter::OnDoubleScalar(double value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::DoubleMarker); @@ -252,44 +252,44 @@ namespace NYson { } } EndNode(); - } - + } + void TYsonWriter::OnBooleanScalar(bool value) { if (Format == EYsonFormat::Binary) { Stream->Write(value ? NDetail::TrueMarker : NDetail::FalseMarker); } else { Stream->Write(value ? "%true" : "%false"); - } + } EndNode(); - } - + } + void TYsonWriter::OnEntity() { Stream->Write(TokenTypeToChar(EntityToken)); EndNode(); - } - + } + void TYsonWriter::OnBeginList() { BeginCollection(BeginListToken); } - + void TYsonWriter::OnListItem() { CollectionItem(ListItemSeparatorToken); } - + void TYsonWriter::OnEndList() { EndCollection(EndListToken); EndNode(); } - + void TYsonWriter::OnBeginMap() { BeginCollection(BeginMapToken); } - + void TYsonWriter::OnKeyedItem(TStringBuf key) { CollectionItem(KeyedItemSeparatorToken); - + WriteStringScalar(key); - + if (Format == NYson::EYsonFormat::Pretty) { Stream->Write(' '); } @@ -297,26 +297,26 @@ namespace NYson { if (Format == NYson::EYsonFormat::Pretty) { Stream->Write(' '); } - + BeforeFirstItem = false; - } + } void TYsonWriter::OnEndMap() { EndCollection(EndMapToken); EndNode(); - } - + } + void TYsonWriter::OnBeginAttributes() { BeginCollection(BeginAttributesToken); } - + void TYsonWriter::OnEndAttributes() { EndCollection(EndAttributesToken); if (Format == NYson::EYsonFormat::Pretty) { Stream->Write(' '); } } - + void TYsonWriter::OnRaw(TStringBuf yson, EYsonType type) { if (EnableRaw) { Stream->Write(yson); @@ -325,19 +325,19 @@ namespace NYson { TYsonConsumerBase::OnRaw(yson, type); } } - + TYsonWriter::TState TYsonWriter::State() const { TState state; state.Depth = Depth; state.BeforeFirstItem = BeforeFirstItem; return state; - } - + } + void TYsonWriter::Reset(const TState& state) { Depth = state.Depth; BeforeFirstItem = state.BeforeFirstItem; - } - + } + //////////////////////////////////////////////////////////////////////////////// void ReformatYsonStream( @@ -351,5 +351,5 @@ namespace NYson { } //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/writer.h b/library/cpp/yson/writer.h index 0ae04be6e8..40f5d7d501 100644 --- a/library/cpp/yson/writer.h +++ b/library/cpp/yson/writer.h @@ -1,17 +1,17 @@ -#pragma once - -#include "public.h" -#include "token.h" -#include "consumer.h" - -#include <util/generic/noncopyable.h> - +#pragma once + +#include "public.h" +#include "token.h" +#include "consumer.h" + +#include <util/generic/noncopyable.h> + class IOutputStream; class IZeroCopyInput; - + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TYsonWriter : public TYsonConsumerBase, private TNonCopyable { @@ -20,7 +20,7 @@ namespace NYson { private: int Depth; bool BeforeFirstItem; - + friend class TYsonWriter; }; @@ -37,23 +37,23 @@ namespace NYson { void OnDoubleScalar(double value) override; void OnBooleanScalar(bool value) override; void OnEntity() override; - + void OnBeginList() override; void OnListItem() override; void OnEndList() override; - + void OnBeginMap() override; void OnKeyedItem(TStringBuf key) override; void OnEndMap() override; - + void OnBeginAttributes() override; void OnEndAttributes() override; - + void OnRaw(TStringBuf yson, EYsonType type = ::NYson::EYsonType::Node) override; - + TState State() const; void Reset(const TState& state); - + protected: IOutputStream* Stream; EYsonFormat Format; @@ -62,28 +62,28 @@ namespace NYson { int Depth; bool BeforeFirstItem; - + static const int IndentSize = 4; - + void WriteIndent(); void WriteStringScalar(const TStringBuf& value); - + void BeginCollection(ETokenType beginToken); void CollectionItem(ETokenType separatorToken); void EndCollection(ETokenType endToken); - + bool IsTopLevelFragmentContext() const; void EndNode(); }; - + //////////////////////////////////////////////////////////////////////////////// - + void ReformatYsonStream( IInputStream* input, IOutputStream* output, EYsonFormat format = EYsonFormat::Binary, EYsonType type = ::NYson::EYsonType::Node); - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/ya.make b/library/cpp/yson/ya.make index b30109a31f..c55a189b10 100644 --- a/library/cpp/yson/ya.make +++ b/library/cpp/yson/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - +LIBRARY() + OWNER( ermolovd g:yt ) - + PEERDIR( library/cpp/yt/misc library/cpp/yt/yson ) -SRCS( +SRCS( consumer.cpp - lexer.cpp + lexer.cpp parser.cpp - token.cpp - tokenizer.cpp + token.cpp + tokenizer.cpp varint.cpp - writer.cpp -) - -END() + writer.cpp +) + +END() diff --git a/library/cpp/yson/zigzag.h b/library/cpp/yson/zigzag.h index dc130a7410..2f1190508f 100644 --- a/library/cpp/yson/zigzag.h +++ b/library/cpp/yson/zigzag.h @@ -1,31 +1,31 @@ -#pragma once - -#include <util/system/defaults.h> - +#pragma once + +#include <util/system/defaults.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + //! Functions that provide coding of integers with property: 0 <= f(x) <= 2 * |x| //! Actually taken 'as is' from protobuf/wire_format_lite.h - + inline ui32 ZigZagEncode32(i32 n) { // Note: the right-shift must be arithmetic return (ui32(n) << 1) ^ (n >> 31); } - + inline i32 ZigZagDecode32(ui32 n) { return (n >> 1) ^ -static_cast<i32>(n & 1); } - + inline ui64 ZigZagEncode64(i64 n) { // Note: the right-shift must be arithmetic return (ui64(n) << 1) ^ (n >> 63); } - + inline i64 ZigZagDecode64(ui64 n) { return (n >> 1) ^ -static_cast<i64>(n & 1); } - + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/util/generic/guid.cpp b/util/generic/guid.cpp index bda7a82244..8b907457bc 100644 --- a/util/generic/guid.cpp +++ b/util/generic/guid.cpp @@ -125,7 +125,7 @@ bool GetGuid(const TStringBuf s, TGUID& result) { } result.dw[partId] = static_cast<ui32>(partValue); return true; -} +} // Parses GUID from s and checks that it's valid. // In case of error returns TGUID(). diff --git a/ydb/core/base/appdata.cpp b/ydb/core/base/appdata.cpp index 757cc87b11..f9e517fc42 100644 --- a/ydb/core/base/appdata.cpp +++ b/ydb/core/base/appdata.cpp @@ -42,7 +42,7 @@ TAppData::TDefaultTabletTypes::TDefaultTabletTypes() , Mediator(TTabletTypes::TX_MEDIATOR) , Kesus(TTabletTypes::KESUS) , Hive(TTabletTypes::Hive) - , SysViewProcessor(TTabletTypes::SysViewProcessor) + , SysViewProcessor(TTabletTypes::SysViewProcessor) , ColumnShard(TTabletTypes::COLUMNSHARD) , TestShard(TTabletTypes::TestShard) , SequenceShard(TTabletTypes::SequenceShard) diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index bb066bb173..c666f7468c 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -98,7 +98,7 @@ struct TAppData { TTabletTypes::EType Mediator; TTabletTypes::EType Kesus; TTabletTypes::EType Hive; - TTabletTypes::EType SysViewProcessor; + TTabletTypes::EType SysViewProcessor; TTabletTypes::EType ColumnShard; TTabletTypes::EType TestShard; TTabletTypes::EType SequenceShard; diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h index 4b42f2ee86..a2faee326e 100644 --- a/ydb/core/base/blobstorage.h +++ b/ydb/core/base/blobstorage.h @@ -799,8 +799,8 @@ struct TEvBlobStorage { EvControllerMigrationBatchRequest, EvControllerMigrationDone, - EvControllerUpdateSystemViews, - + EvControllerUpdateSystemViews, + // proxy - node controller interface EvConfigureProxy = EvPut + 13 * 512, EvProxyConfigurationRequest, // DEPRECATED diff --git a/ydb/core/base/counters.cpp b/ydb/core/base/counters.cpp index 16215f38fc..b192fa36a1 100644 --- a/ydb/core/base/counters.cpp +++ b/ydb/core/base/counters.cpp @@ -47,8 +47,8 @@ static const THashSet<TString> DATABASE_ATTRIBUTE_SERVICES static const THashSet<TString> DATABASE_ATTRIBUTE_LABELS = {{ TString("cloud_id"), - TString("folder_id"), - TString("database_id") + TString("folder_id"), + TString("database_id") }}; using NMonitoring::TDynamicCounters; diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 5b00d9d974..f5fedfe19b 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -111,7 +111,7 @@ struct TKikimrEvents : TEvents { ES_TX_ALLOCATOR_CLIENT, ES_PQ_CLUSTER_TRACKER, ES_NET_CLASSIFIER, - ES_SYSTEM_VIEW, + ES_SYSTEM_VIEW, ES_TENANT_NODE_ENUMERATOR, ES_SERVICE_ACCOUNT_SERVICE, ES_INDEX_BUILD, diff --git a/ydb/core/base/pool_stats_collector.cpp b/ydb/core/base/pool_stats_collector.cpp index 6db2127887..7dcbf28954 100644 --- a/ydb/core/base/pool_stats_collector.cpp +++ b/ydb/core/base/pool_stats_collector.cpp @@ -58,9 +58,9 @@ private: IActor *CreateStatsCollector(ui32 intervalSec, const TActorSystemSetup& setup, - NMonitoring::TDynamicCounterPtr counters) + NMonitoring::TDynamicCounterPtr counters) { - return new TStatsCollectingActor(intervalSec, setup, counters); + return new TStatsCollectingActor(intervalSec, setup, counters); } } // namespace NKikimr diff --git a/ydb/core/base/pool_stats_collector.h b/ydb/core/base/pool_stats_collector.h index 045314ec66..cbf06f9f51 100644 --- a/ydb/core/base/pool_stats_collector.h +++ b/ydb/core/base/pool_stats_collector.h @@ -12,6 +12,6 @@ namespace NKikimr { IActor* CreateStatsCollector(ui32 intervalSec, const TActorSystemSetup& setup, - NMonitoring::TDynamicCounterPtr counters); + NMonitoring::TDynamicCounterPtr counters); } diff --git a/ydb/core/base/shared_data.cpp b/ydb/core/base/shared_data.cpp index 632469fc98..c375acc515 100644 --- a/ydb/core/base/shared_data.cpp +++ b/ydb/core/base/shared_data.cpp @@ -1,32 +1,32 @@ #include "shared_data.h" -#include <library/cpp/actors/core/memory_tracker.h> - +#include <library/cpp/actors/core/memory_tracker.h> + #include <util/system/sys_alloc.h> #include <util/system/sanitizers.h> namespace NKikimr { - static constexpr char MemoryLabelSharedData[] = "Tablet/TSharedData/Buffers"; - + static constexpr char MemoryLabelSharedData[] = "Tablet/TSharedData/Buffers"; + char* TSharedData::Allocate(size_t size) { char* data = nullptr; if (size > 0) { if (size >= MaxDataSize) { throw std::length_error("Allocate size overflow"); } - auto allocSize = OverheadSize + size; - char* raw = reinterpret_cast<char*>(y_allocate(allocSize)); - - auto* privateHeader = reinterpret_cast<TPrivateHeader*>(raw); - privateHeader->AllocSize = allocSize; - NActors::NMemory::TLabel<MemoryLabelSharedData>::Add(allocSize); - - auto* header = reinterpret_cast<THeader*>(raw + PrivateHeaderSize); + auto allocSize = OverheadSize + size; + char* raw = reinterpret_cast<char*>(y_allocate(allocSize)); + + auto* privateHeader = reinterpret_cast<TPrivateHeader*>(raw); + privateHeader->AllocSize = allocSize; + NActors::NMemory::TLabel<MemoryLabelSharedData>::Add(allocSize); + + auto* header = reinterpret_cast<THeader*>(raw + PrivateHeaderSize); header->RefCount = 1; header->Owner = nullptr; - - data = raw + OverheadSize; + + data = raw + OverheadSize; NSan::Poison(data, size); } return data; @@ -34,14 +34,14 @@ namespace NKikimr { void TSharedData::Deallocate(char* data) noexcept { if (data) { - char* raw = data - OverheadSize; - - auto* privateHeader = reinterpret_cast<TPrivateHeader*>(raw); - NActors::NMemory::TLabel<MemoryLabelSharedData>::Sub(privateHeader->AllocSize); - - auto* header = reinterpret_cast<THeader*>(raw + PrivateHeaderSize); + char* raw = data - OverheadSize; + + auto* privateHeader = reinterpret_cast<TPrivateHeader*>(raw); + NActors::NMemory::TLabel<MemoryLabelSharedData>::Sub(privateHeader->AllocSize); + + auto* header = reinterpret_cast<THeader*>(raw + PrivateHeaderSize); Y_VERIFY_DEBUG(header->Owner == nullptr); - + y_deallocate(raw); } } diff --git a/ydb/core/base/shared_data.h b/ydb/core/base/shared_data.h index 7899ca6c8d..2ccfc0e187 100644 --- a/ydb/core/base/shared_data.h +++ b/ydb/core/base/shared_data.h @@ -15,24 +15,24 @@ namespace NKikimr { virtual void Deallocate(char*) noexcept = 0; }; - struct TPrivateHeader { - size_t AllocSize; - size_t Pad; - }; - - static_assert(sizeof(TPrivateHeader) == 16, "TPrivateHeader has an unexpected size"); - + struct TPrivateHeader { + size_t AllocSize; + size_t Pad; + }; + + static_assert(sizeof(TPrivateHeader) == 16, "TPrivateHeader has an unexpected size"); + struct THeader { TAtomic RefCount; IOwner* Owner; }; - static_assert(sizeof(THeader) == 16, "THeader has an unexpected size"); - + static_assert(sizeof(THeader) == 16, "THeader has an unexpected size"); + enum : size_t { - PrivateHeaderSize = sizeof(TPrivateHeader), - HeaderSize = sizeof(THeader), - OverheadSize = PrivateHeaderSize + HeaderSize, + PrivateHeaderSize = sizeof(TPrivateHeader), + HeaderSize = sizeof(THeader), + OverheadSize = PrivateHeaderSize + HeaderSize, MaxDataSize = (std::numeric_limits<size_t>::max() - OverheadSize) }; diff --git a/ydb/core/base/tablet_types.h b/ydb/core/base/tablet_types.h index bde322dbd9..f585819299 100644 --- a/ydb/core/base/tablet_types.h +++ b/ydb/core/base/tablet_types.h @@ -36,7 +36,7 @@ public: static constexpr EType TENANT_SLOT_BROKER = TenantSlotBroker; static constexpr EType CONSOLE = Console; static constexpr EType KESUS = Kesus; - static constexpr EType SYS_VIEW_PROCESSOR = SysViewProcessor; + static constexpr EType SYS_VIEW_PROCESSOR = SysViewProcessor; static constexpr EType FILESTORE = FileStore; static constexpr EType TESTSHARD = TestShard; static constexpr EType SEQUENCESHARD = SequenceShard; diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp index 3d974726da..addca8c533 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp @@ -62,9 +62,9 @@ TDsProxyNodeMon::TDsProxyNodeMon(TIntrusivePtr<NMonitoring::TDynamicCounters> &c { auto group = Group->GetSubgroup("subsystem", "restart_histo"); - auto histoGroup = group->GetSubgroup("sensor", "restart_histo"); + auto histoGroup = group->GetSubgroup("sensor", "restart_histo"); for (size_t i = 0; i < RestartHisto.size(); ++i) { - RestartHisto[i] = histoGroup->GetNamedCounter("restartCount", ToString(i), true); + RestartHisto[i] = histoGroup->GetNamedCounter("restartCount", ToString(i), true); } } // Accelerate counters @@ -189,7 +189,7 @@ void TDsProxyNodeMon::CheckNodeMonCountersForDeviceType(TPDiskCategory::EDeviceT auto getNamedHisto = [&subGroup, &type] (const TString& name) { auto buckets = NMonitoring::ExplicitHistogram(GetCommonLatencyHistBounds(type)); - return subGroup->GetHistogram(name, std::move(buckets)); + return subGroup->GetHistogram(name, std::move(buckets)); }; PutTabletLogResponseTimeHist256Ki[idx] = getNamedHisto("putTabletLog256KiMs"); diff --git a/ydb/core/blobstorage/incrhuge/incrhuge.h b/ydb/core/blobstorage/incrhuge/incrhuge.h index 4b9f8190b5..c2f5848e72 100644 --- a/ydb/core/blobstorage/incrhuge/incrhuge.h +++ b/ydb/core/blobstorage/incrhuge/incrhuge.h @@ -54,7 +54,7 @@ namespace NKikimr { // TWritePayload structure contains user fields needed by VDisk when receiving write confirmation. It is // simply moved from request to response. - struct TWritePayload { + struct TWritePayload { virtual ~TWritePayload() = default; }; diff --git a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_common.h b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_common.h index 48d248e0d9..bc9f706f26 100644 --- a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_common.h +++ b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_common.h @@ -201,7 +201,7 @@ namespace NKikimr { // event callback base; it is a class that holds some data and has an apply function which is called when // log entry is written or write is failed or for other event; uses NALF incremental allocator, because it // is short-term object - class IEventCallback { + class IEventCallback { public: virtual ~IEventCallback() = default; virtual void Apply(NKikimrProto::EReplyStatus status, IEventBase *result, const TActorContext& ctx) = 0; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h index 0511bdd611..8e722ce42a 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h @@ -243,7 +243,7 @@ struct TEvLogResult; // LOG //////////////////////////////////////////////////////////////////////////// struct TEvLog : public TEventLocal<TEvLog, TEvBlobStorage::EvLog> { - struct ICallback { + struct ICallback { virtual ~ICallback() = default; virtual void operator ()(TActorSystem *actorSystem, const TEvLogResult &ev) = 0; }; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_countedqueuemanyone.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_countedqueuemanyone.h index 3351f17cbf..a0ac88b250 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_countedqueuemanyone.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_countedqueuemanyone.h @@ -13,7 +13,7 @@ namespace NPDisk { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template<typename T, ui32 TSize> class TCountedQueueManyOne { - NThreading::TObstructiveConsumerAuxQueue<T> Queue; + NThreading::TObstructiveConsumerAuxQueue<T> Queue; TAtomic SizeLowerEstimate; TMutex ProducedMutex; TCondVar ProducedCondVar; diff --git a/ydb/core/blobstorage/testload/test_load_actor.cpp b/ydb/core/blobstorage/testload/test_load_actor.cpp index a173372112..c8788b1310 100644 --- a/ydb/core/blobstorage/testload/test_load_actor.cpp +++ b/ydb/core/blobstorage/testload/test_load_actor.cpp @@ -207,19 +207,19 @@ public: break; } - case NKikimrBlobStorage::TEvTestLoadRequest::CommandCase::kMemoryLoadStart: { - const auto& cmd = record.GetMemoryLoadStart(); - const ui64 tag = GetOrGenerateTag(cmd); - if (LoadActors.count(tag) != 0) { - ythrow TLoadActorException() << Sprintf("duplicate load actor with Tag# %" PRIu64, tag); - } - - LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Create new memory load actor with tag# " << tag); - LoadActors.emplace(tag, ctx.Register(CreateMemoryTestLoad( - cmd, ctx.SelfID, GetServiceCounters(Counters, "load_actor"), 0, tag))); - break; - } - + case NKikimrBlobStorage::TEvTestLoadRequest::CommandCase::kMemoryLoadStart: { + const auto& cmd = record.GetMemoryLoadStart(); + const ui64 tag = GetOrGenerateTag(cmd); + if (LoadActors.count(tag) != 0) { + ythrow TLoadActorException() << Sprintf("duplicate load actor with Tag# %" PRIu64, tag); + } + + LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Create new memory load actor with tag# " << tag); + LoadActors.emplace(tag, ctx.Register(CreateMemoryTestLoad( + cmd, ctx.SelfID, GetServiceCounters(Counters, "load_actor"), 0, tag))); + break; + } + default: { TString protoTxt; google::protobuf::TextFormat::PrintToString(record, &protoTxt); diff --git a/ydb/core/blobstorage/testload/test_load_actor.h b/ydb/core/blobstorage/testload/test_load_actor.h index f67a851861..38b7fb3d30 100644 --- a/ydb/core/blobstorage/testload/test_load_actor.h +++ b/ydb/core/blobstorage/testload/test_load_actor.h @@ -51,15 +51,15 @@ namespace NKikimr { NActors::IActor *CreateKeyValueWriterTestLoad(const NKikimrBlobStorage::TEvTestLoadRequest::TKeyValueLoadStart& cmd, const NActors::TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, ui64 index, ui64 tag); - + NActors::IActor *CreateKqpWriterTestLoad(const NKikimrBlobStorage::TEvTestLoadRequest::TKqpLoadStart& cmd, const NActors::TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, ui64 index, ui64 tag); - NActors::IActor *CreateMemoryTestLoad(const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd, - const NActors::TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, - ui64 index, ui64 tag); - + NActors::IActor *CreateMemoryTestLoad(const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd, + const NActors::TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + ui64 index, ui64 tag); + struct TLoadReport : public TThrRefBase { enum ELoadType { LOAD_READ, diff --git a/ydb/core/blobstorage/testload/test_load_memory.cpp b/ydb/core/blobstorage/testload/test_load_memory.cpp index 27db469055..9da565186a 100644 --- a/ydb/core/blobstorage/testload/test_load_memory.cpp +++ b/ydb/core/blobstorage/testload/test_load_memory.cpp @@ -1,141 +1,141 @@ -#include "test_load_actor.h" - +#include "test_load_actor.h" + #include <ydb/core/base/appdata.h> - -#include <library/cpp/monlib/service/pages/templates.h> - -namespace NKikimr { - -class TMemoryTestLoadActor : public TActorBootstrapped<TMemoryTestLoadActor> { - enum { - EvAllocateBlock = EventSpaceBegin(TEvents::ES_PRIVATE), - EvEnd - }; - - struct TEvAllocateBlock : public TEventLocal<TEvAllocateBlock, EvAllocateBlock> {}; - - const TActorId Parent; - const ui64 Tag; - - TDuration Duration; - ui64 BlockSize; - TDuration Interval; - - TInstant TestStartTime; - TVector<TVector<char>> Blocks; - ui64 AllocatedSize = 0; - -public: + +#include <library/cpp/monlib/service/pages/templates.h> + +namespace NKikimr { + +class TMemoryTestLoadActor : public TActorBootstrapped<TMemoryTestLoadActor> { + enum { + EvAllocateBlock = EventSpaceBegin(TEvents::ES_PRIVATE), + EvEnd + }; + + struct TEvAllocateBlock : public TEventLocal<TEvAllocateBlock, EvAllocateBlock> {}; + + const TActorId Parent; + const ui64 Tag; + + TDuration Duration; + ui64 BlockSize; + TDuration Interval; + + TInstant TestStartTime; + TVector<TVector<char>> Blocks; + ui64 AllocatedSize = 0; + +public: static constexpr auto ActorActivityType() { - return NKikimrServices::TActivity::BS_LOAD_PDISK_LOG_WRITE; - } - - TMemoryTestLoadActor(const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd, - const TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, ui64 index, ui64 tag) - : Parent(parent) - , Tag(tag) - { - Y_UNUSED(counters); - Y_UNUSED(index); - - VERIFY_PARAM(DurationSeconds); - Duration = TDuration::Seconds(cmd.GetDurationSeconds()); - - VERIFY_PARAM(BlockSize); - BlockSize = cmd.GetBlockSize(); - - VERIFY_PARAM(IntervalUs); - Interval = TDuration::MicroSeconds(cmd.GetIntervalUs()); - - Blocks.reserve(Duration.MicroSeconds() / Interval.MicroSeconds() + 1); - } - - void Bootstrap(const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag - << " TMemoryTestLoadActor Bootstrap called"); - - Become(&TMemoryTestLoadActor::StateFunc); - - LOG_INFO_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag - << " Schedule PoisonPill"); - - ctx.Schedule(Duration, new TEvents::TEvPoisonPill); - ctx.Schedule(Interval, new TEvAllocateBlock); - TestStartTime = TAppData::TimeProvider->Now(); - } - - void HandlePoisonPill(const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag - << " Handle PoisonPill"); - - TIntrusivePtr<TLoadReport> report(new TLoadReport()); - report->Duration = Duration; - ctx.Send(Parent, new TEvTestLoadFinished(Tag, report, "OK")); - Die(ctx); - } - - void Handle(TEvAllocateBlock::TPtr&, const TActorContext& ctx) { - auto size = RandomNumber<ui64>(BlockSize * 2 + 1); - - Blocks.push_back({}); - auto& block = Blocks.back(); - block.resize(size); - for (size_t i = 0; i < size; ++i) { - block[i] = 0; - } - AllocatedSize += size; - - LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag - << " Handle AllocateBlock"); - - ctx.Schedule(Interval, new TEvAllocateBlock); - } - - void Handle(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) { -#define PARAM(NAME, VALUE) \ - TABLER() { \ - TABLED() { str << NAME; } \ - TABLED() { str << VALUE; } \ - } - TStringStream str; - HTML(str) { - TABLE_CLASS("table table-condensed") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "Parameter"; } - TABLEH() { str << "Value"; } - } - } - TABLEBODY() { - PARAM("Elapsed time / Duration", - (TAppData::TimeProvider->Now() - TestStartTime).Seconds() << "s / " - << Duration.Seconds() << "s"); - PARAM("Interval", Interval.MicroSeconds() << "us"); - PARAM("Block size", BlockSize); - PARAM("Allocated bytes", AllocatedSize); - PARAM("Allocated blocks", Blocks.size()); - } - } - } - - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str(), ev->Get()->SubRequestId)); - } - - STRICT_STFUNC(StateFunc, - CFunc(TEvents::TSystem::PoisonPill, HandlePoisonPill) - HFunc(TEvAllocateBlock, Handle) - HFunc(NMon::TEvHttpInfo, Handle) - ) -}; - -IActor* CreateMemoryTestLoad( - const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd, - const TActorId& parent, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, - ui64 index, - ui64 tag) -{ - return new TMemoryTestLoadActor(cmd, parent, counters, index, tag); -} - -} // NKikimr + return NKikimrServices::TActivity::BS_LOAD_PDISK_LOG_WRITE; + } + + TMemoryTestLoadActor(const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd, + const TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, ui64 index, ui64 tag) + : Parent(parent) + , Tag(tag) + { + Y_UNUSED(counters); + Y_UNUSED(index); + + VERIFY_PARAM(DurationSeconds); + Duration = TDuration::Seconds(cmd.GetDurationSeconds()); + + VERIFY_PARAM(BlockSize); + BlockSize = cmd.GetBlockSize(); + + VERIFY_PARAM(IntervalUs); + Interval = TDuration::MicroSeconds(cmd.GetIntervalUs()); + + Blocks.reserve(Duration.MicroSeconds() / Interval.MicroSeconds() + 1); + } + + void Bootstrap(const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag + << " TMemoryTestLoadActor Bootstrap called"); + + Become(&TMemoryTestLoadActor::StateFunc); + + LOG_INFO_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag + << " Schedule PoisonPill"); + + ctx.Schedule(Duration, new TEvents::TEvPoisonPill); + ctx.Schedule(Interval, new TEvAllocateBlock); + TestStartTime = TAppData::TimeProvider->Now(); + } + + void HandlePoisonPill(const TActorContext& ctx) { + LOG_INFO_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag + << " Handle PoisonPill"); + + TIntrusivePtr<TLoadReport> report(new TLoadReport()); + report->Duration = Duration; + ctx.Send(Parent, new TEvTestLoadFinished(Tag, report, "OK")); + Die(ctx); + } + + void Handle(TEvAllocateBlock::TPtr&, const TActorContext& ctx) { + auto size = RandomNumber<ui64>(BlockSize * 2 + 1); + + Blocks.push_back({}); + auto& block = Blocks.back(); + block.resize(size); + for (size_t i = 0; i < size; ++i) { + block[i] = 0; + } + AllocatedSize += size; + + LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag + << " Handle AllocateBlock"); + + ctx.Schedule(Interval, new TEvAllocateBlock); + } + + void Handle(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) { +#define PARAM(NAME, VALUE) \ + TABLER() { \ + TABLED() { str << NAME; } \ + TABLED() { str << VALUE; } \ + } + TStringStream str; + HTML(str) { + TABLE_CLASS("table table-condensed") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "Parameter"; } + TABLEH() { str << "Value"; } + } + } + TABLEBODY() { + PARAM("Elapsed time / Duration", + (TAppData::TimeProvider->Now() - TestStartTime).Seconds() << "s / " + << Duration.Seconds() << "s"); + PARAM("Interval", Interval.MicroSeconds() << "us"); + PARAM("Block size", BlockSize); + PARAM("Allocated bytes", AllocatedSize); + PARAM("Allocated blocks", Blocks.size()); + } + } + } + + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str(), ev->Get()->SubRequestId)); + } + + STRICT_STFUNC(StateFunc, + CFunc(TEvents::TSystem::PoisonPill, HandlePoisonPill) + HFunc(TEvAllocateBlock, Handle) + HFunc(NMon::TEvHttpInfo, Handle) + ) +}; + +IActor* CreateMemoryTestLoad( + const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd, + const TActorId& parent, + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + ui64 index, + ui64 tag) +{ + return new TMemoryTestLoadActor(cmd, parent, counters, index, tag); +} + +} // NKikimr diff --git a/ydb/core/blobstorage/testload/ya.make b/ydb/core/blobstorage/testload/ya.make index 8015a87b23..f1448fe240 100644 --- a/ydb/core/blobstorage/testload/ya.make +++ b/ydb/core/blobstorage/testload/ya.make @@ -25,7 +25,7 @@ SRCS( test_load_gen.h test_load_interval_gen.h test_load_keyvalue_write.cpp - test_load_memory.cpp + test_load_memory.cpp test_load_pdisk_read.cpp test_load_pdisk_write.cpp test_load_pdisk_log.cpp diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_histograms.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_histograms.cpp index 6a4d39f1f6..d9cce39e9e 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_histograms.cpp +++ b/ydb/core/blobstorage/vdisk/common/vdisk_histograms.cpp @@ -16,7 +16,7 @@ namespace NKikimr { std::make_pair(&VPutUserDataLatencyHistogram, "PutUserData" ), std::make_pair(&VPutAsyncBlobLatencyHistogram, "PutAsyncBlob") }) { - *item.first = std::make_shared<NVDiskMon::TLtcHisto>(counters, "sensor", item.second, type); + *item.first = std::make_shared<NVDiskMon::TLtcHisto>(counters, "sensor", item.second, type); } } diff --git a/ydb/core/client/flat_ut.cpp b/ydb/core/client/flat_ut.cpp index 2aa3d5eb0a..362c052aa1 100644 --- a/ydb/core/client/flat_ut.cpp +++ b/ydb/core/client/flat_ut.cpp @@ -1114,7 +1114,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) { Y_UNIT_TEST(Ls) { TPortManager pm; ui16 port = pm.GetPort(2134); - TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false)); + TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false)); TFlatMsgBusClient annoyingClient(port); annoyingClient.InitRoot(); @@ -1207,7 +1207,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) { Y_UNIT_TEST(PathSorting) { TPortManager pm; ui16 port = pm.GetPort(2134); - TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false)); + TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false)); TFlatMsgBusClient annoyingClient(port); @@ -1253,7 +1253,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) { Y_UNIT_TEST(LsPathId) { TPortManager pm; ui16 port = pm.GetPort(2134); - TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false)); + TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false)); TFlatMsgBusClient annoyingClient(port); diff --git a/ydb/core/client/server/msgbus_server_s3_listing.cpp b/ydb/core/client/server/msgbus_server_s3_listing.cpp index 4c0abf38c1..3626866322 100644 --- a/ydb/core/client/server/msgbus_server_s3_listing.cpp +++ b/ydb/core/client/server/msgbus_server_s3_listing.cpp @@ -396,7 +396,7 @@ private: ui64 shardId = KeyRange->Partitions[idx].ShardId; THolder<TEvDataShard::TEvS3ListingRequest> ev(new TEvDataShard::TEvS3ListingRequest()); - ev->Record.SetTableId(KeyRange->TableId.PathId.LocalPathId); + ev->Record.SetTableId(KeyRange->TableId.PathId.LocalPathId); ev->Record.SetSerializedKeyPrefix(PrefixColumns.GetBuffer()); ev->Record.SetPathColumnPrefix(Request->GetPathColumnPrefix()); ev->Record.SetPathColumnDelimiter(Request->GetPathColumnDelimiter()); diff --git a/ydb/core/cms/console/console__create_tenant.cpp b/ydb/core/cms/console/console__create_tenant.cpp index e113cb3047..77d8166d54 100644 --- a/ydb/core/cms/console/console__create_tenant.cpp +++ b/ydb/core/cms/console/console__create_tenant.cpp @@ -126,7 +126,7 @@ public: Tenant->IsExternalSubdomain = Self->FeatureFlags.GetEnableExternalSubdomains(); Tenant->IsExternalHive = Self->FeatureFlags.GetEnableExternalHive(); - Tenant->IsExternalSysViewProcessor = Self->FeatureFlags.GetEnablePersistentQueryStats(); + Tenant->IsExternalSysViewProcessor = Self->FeatureFlags.GetEnablePersistentQueryStats(); if (rec.options().disable_external_subdomain()) { Tenant->IsExternalSubdomain = false; @@ -143,11 +143,11 @@ public: Tenant->TimeCastBucketsPerMediator = 0; Tenant->IsExternalSubdomain = false; Tenant->IsExternalHive = false; - Tenant->IsExternalSysViewProcessor = false; + Tenant->IsExternalSysViewProcessor = false; } Tenant->IsExternalHive &= Tenant->IsExternalSubdomain; // external hive without external sub domain is pointless - Tenant->IsExternalSysViewProcessor &= Tenant->IsExternalSubdomain; + Tenant->IsExternalSysViewProcessor &= Tenant->IsExternalSubdomain; Tenant->StorageUnitsQuota = Self->Config.DefaultStorageUnitsQuota; Tenant->ComputationalUnitsQuota = Self->Config.DefaultComputationalUnitsQuota; diff --git a/ydb/core/cms/console/console__scheme.h b/ydb/core/cms/console/console__scheme.h index 1f61019db2..29eb6c3035 100644 --- a/ydb/core/cms/console/console__scheme.h +++ b/ydb/core/cms/console/console__scheme.h @@ -42,7 +42,7 @@ struct Schema : NIceDb::Schema { // SharedDomainId { struct SharedDomainSchemeShardId : Column<21, NScheme::NTypeIds::Uint64> {}; struct SharedDomainPathId : Column<22, NScheme::NTypeIds::Uint64> {}; - struct IsExternalSysViewProcessor : Column<23, NScheme::NTypeIds::Bool> {}; + struct IsExternalSysViewProcessor : Column<23, NScheme::NTypeIds::Bool> {}; // } // SharedDomainId struct SchemaOperationQuotas : Column<24, NScheme::NTypeIds::String> {}; struct CreateIdempotencyKey : Column<25, NScheme::NTypeIds::Utf8> {}; diff --git a/ydb/core/cms/console/console_tenants_manager.cpp b/ydb/core/cms/console/console_tenants_manager.cpp index b40b1eed02..0db3f5dced 100644 --- a/ydb/core/cms/console/console_tenants_manager.cpp +++ b/ydb/core/cms/console/console_tenants_manager.cpp @@ -455,9 +455,9 @@ public: if (Tenant->IsExternalHive) { subdomain.SetExternalHive(true); } - if (Tenant->IsExternalSysViewProcessor) { - subdomain.SetExternalSysViewProcessor(true); - } + if (Tenant->IsExternalSysViewProcessor) { + subdomain.SetExternalSysViewProcessor(true); + } } if (SharedTenant) { @@ -483,9 +483,9 @@ public: if (Tenant->IsExternalHive) { subdomain.SetExternalHive(true); } - if (Tenant->IsExternalSysViewProcessor) { - subdomain.SetExternalSysViewProcessor(true); - } + if (Tenant->IsExternalSysViewProcessor) { + subdomain.SetExternalSysViewProcessor(true); + } } for (auto &pr : (SharedTenant ? SharedTenant->StoragePools : Tenant->StoragePools)) { @@ -1183,7 +1183,7 @@ TTenantsManager::TTenant::TTenant(const TString &path, , Generation(0) , IsExternalSubdomain(false) , IsExternalHive(false) - , IsExternalSysViewProcessor(false) + , IsExternalSysViewProcessor(false) , AreResourcesShared(false) { } @@ -2248,7 +2248,7 @@ void TTenantsManager::DbAddTenant(TTenant::TPtr tenant, << " errorcode=" << tenant->ErrorCode << " isExternalSubDomain=" << tenant->IsExternalSubdomain << " isExternalHive=" << tenant->IsExternalHive - << " isExternalSysViewProcessor=" << tenant->IsExternalSysViewProcessor + << " isExternalSysViewProcessor=" << tenant->IsExternalSysViewProcessor << " areResourcesShared=" << tenant->AreResourcesShared << " sharedDomainId=" << tenant->SharedDomainId); @@ -2269,7 +2269,7 @@ void TTenantsManager::DbAddTenant(TTenant::TPtr tenant, NIceDb::TUpdate<Schema::Tenants::ErrorCode>(tenant->ErrorCode), NIceDb::TUpdate<Schema::Tenants::IsExternalSubDomain>(tenant->IsExternalSubdomain), NIceDb::TUpdate<Schema::Tenants::IsExternalHive>(tenant->IsExternalHive), - NIceDb::TUpdate<Schema::Tenants::IsExternalSysViewProcessor>(tenant->IsExternalSysViewProcessor), + NIceDb::TUpdate<Schema::Tenants::IsExternalSysViewProcessor>(tenant->IsExternalSysViewProcessor), NIceDb::TUpdate<Schema::Tenants::AreResourcesShared>(tenant->AreResourcesShared), NIceDb::TUpdate<Schema::Tenants::CreateIdempotencyKey>(tenant->CreateIdempotencyKey)); @@ -2373,7 +2373,7 @@ bool TTenantsManager::DbLoadState(TTransactionContext &txc, const TActorContext = static_cast<Ydb::StatusIds::StatusCode>(tenantRowset.GetValueOrDefault<Schema::Tenants::ErrorCode>(0)); bool isExternalSubDomain = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalSubDomain>(false); bool isExternalHive = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalHive>(false); - bool isExternalSysViewProcessor = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalSysViewProcessor>(false); + bool isExternalSysViewProcessor = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalSysViewProcessor>(false); const bool areResourcesShared = tenantRowset.GetValueOrDefault<Schema::Tenants::AreResourcesShared>(false); TTenant::TPtr tenant = new TTenant(path, state, userToken); @@ -2392,7 +2392,7 @@ bool TTenantsManager::DbLoadState(TTransactionContext &txc, const TActorContext tenant->Issue = issue; tenant->IsExternalSubdomain = isExternalSubDomain; tenant->IsExternalHive = isExternalHive; - tenant->IsExternalSysViewProcessor = isExternalSysViewProcessor; + tenant->IsExternalSysViewProcessor = isExternalSysViewProcessor; tenant->AreResourcesShared = areResourcesShared; if (tenantRowset.HaveValue<Schema::Tenants::SchemaOperationQuotas>()) { diff --git a/ydb/core/cms/console/console_tenants_manager.h b/ydb/core/cms/console/console_tenants_manager.h index 9e6d481e23..a21615dd62 100644 --- a/ydb/core/cms/console/console_tenants_manager.h +++ b/ydb/core/cms/console/console_tenants_manager.h @@ -518,7 +518,7 @@ public: TDomainId SharedDomainId; bool IsExternalSubdomain; bool IsExternalHive; - bool IsExternalSysViewProcessor; + bool IsExternalSysViewProcessor; bool AreResourcesShared; THashSet<TTenant::TPtr> HostedTenants; diff --git a/ydb/core/control/immediate_control_board_impl.cpp b/ydb/core/control/immediate_control_board_impl.cpp index 48bcf2c773..fa26926fae 100644 --- a/ydb/core/control/immediate_control_board_impl.cpp +++ b/ydb/core/control/immediate_control_board_impl.cpp @@ -7,49 +7,49 @@ namespace NKikimr { bool TControlBoard::RegisterLocalControl(TControlWrapper control, TString name) { - bool result = true; - if (Board.Has(name)) { - result = false; + bool result = true; + if (Board.Has(name)) { + result = false; } - Board.Insert(name, control.Control); - return result; + Board.Insert(name, control.Control); + return result; } void TControlBoard::RegisterSharedControl(TControlWrapper& control, TString name) { - control.Control = Board.InsertIfAbsent(name, control.Control); + control.Control = Board.InsertIfAbsent(name, control.Control); } void TControlBoard::RestoreDefaults() { - for (auto& bucket : Board.Buckets) { - TReadGuard guard(bucket.GetLock()); - for (auto &control : bucket.GetMap()) { - control.second->RestoreDefault(); - } + for (auto& bucket : Board.Buckets) { + TReadGuard guard(bucket.GetLock()); + for (auto &control : bucket.GetMap()) { + control.second->RestoreDefault(); + } } } void TControlBoard::RestoreDefault(TString name) { - TIntrusivePtr<TControl> control; - if (Board.Get(name, control)) { - control->RestoreDefault(); - } + TIntrusivePtr<TControl> control; + if (Board.Get(name, control)) { + control->RestoreDefault(); + } } bool TControlBoard::SetValue(TString name, TAtomic value, TAtomic &outPrevValue) { - TIntrusivePtr<TControl> control; - if (Board.Get(name, control)) { - outPrevValue = control->SetFromHtmlRequest(value); - return control->IsDefault(); + TIntrusivePtr<TControl> control; + if (Board.Get(name, control)) { + outPrevValue = control->SetFromHtmlRequest(value); + return control->IsDefault(); } return true; } // Only for tests void TControlBoard::GetValue(TString name, TAtomic &outValue, bool &outIsControlExists) const { - TIntrusivePtr<TControl> control; - outIsControlExists = Board.Get(name, control); + TIntrusivePtr<TControl> control; + outIsControlExists = Board.Get(name, control); if (outIsControlExists) { - outValue = control->Get(); + outValue = control->Get(); } } @@ -68,34 +68,34 @@ TString TControlBoard::RenderAsHtml() const { } } TABLEBODY() { - for (const auto& bucket : Board.Buckets) { - TReadGuard guard(bucket.GetLock()); - for (const auto &item : bucket.GetMap()) { - TABLER() { - TABLED() { str << item.first; } - TABLED() { str << item.second->RangeAsString(); } - TABLED() { - if (item.second->IsDefault()) { - str << "<p>" << item.second->Get() << "</p>"; - } else { - str << "<p style='color:red;'><b>" << item.second->Get() << " </b></p>"; - } + for (const auto& bucket : Board.Buckets) { + TReadGuard guard(bucket.GetLock()); + for (const auto &item : bucket.GetMap()) { + TABLER() { + TABLED() { str << item.first; } + TABLED() { str << item.second->RangeAsString(); } + TABLED() { + if (item.second->IsDefault()) { + str << "<p>" << item.second->Get() << "</p>"; + } else { + str << "<p style='color:red;'><b>" << item.second->Get() << " </b></p>"; + } } - TABLED() { - if (item.second->IsDefault()) { - str << "<p>" << item.second->GetDefault() << "</p>"; - } else { - str << "<p style='color:red;'><b>" << item.second->GetDefault() << " </b></p>"; - } + TABLED() { + if (item.second->IsDefault()) { + str << "<p>" << item.second->GetDefault() << "</p>"; + } else { + str << "<p style='color:red;'><b>" << item.second->GetDefault() << " </b></p>"; + } } - TABLED() { - str << "<form class='form_horizontal' method='post'>"; - str << "<input name='" << item.first << "' type='text' value='" - << item.second->Get() << "'/>"; - str << "<button type='submit' style='color:red;'><b>Change</b></button>"; - str << "</form>"; - } - TABLED() { str << !item.second->IsDefault(); } + TABLED() { + str << "<form class='form_horizontal' method='post'>"; + str << "<input name='" << item.first << "' type='text' value='" + << item.second->Get() << "'/>"; + str << "<button type='submit' style='color:red;'><b>Change</b></button>"; + str << "</form>"; + } + TABLED() { str << !item.second->IsDefault(); } } } } @@ -108,4 +108,4 @@ TString TControlBoard::RenderAsHtml() const { return str.Str(); } -} +} diff --git a/ydb/core/control/immediate_control_board_impl.h b/ydb/core/control/immediate_control_board_impl.h index dc79f15a93..a01e09f4d6 100644 --- a/ydb/core/control/immediate_control_board_impl.h +++ b/ydb/core/control/immediate_control_board_impl.h @@ -1,5 +1,5 @@ #pragma once - + #include "defs.h" #include "immediate_control_board_wrapper.h" @@ -8,8 +8,8 @@ namespace NKikimr { class TControlBoard : public TThrRefBase { -private: - TConcurrentRWHashMap<TString, TIntrusivePtr<TControl>, 16> Board; +private: + TConcurrentRWHashMap<TString, TIntrusivePtr<TControl>, 16> Board; public: bool RegisterLocalControl(TControlWrapper control, TString name); diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index df8cb17048..450d4135c6 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -1127,17 +1127,17 @@ public: } const TString& key = items.at(0); - if (key.size() > TUserAttributesLimits::MaxNameLen) { + if (key.size() > TUserAttributesLimits::MaxNameLen) { Cerr << "Key '" << key << "' too long" - << ", max: " << TUserAttributesLimits::MaxNameLen + << ", max: " << TUserAttributesLimits::MaxNameLen << ", actual: " << key.size() << Endl; exit(1); } const TString& value = items.at(1); - if (value.size() > TUserAttributesLimits::MaxValueLen) { + if (value.size() > TUserAttributesLimits::MaxValueLen) { Cerr << "Value '" << value << "' too long" - << ", max: " << TUserAttributesLimits::MaxValueLen + << ", max: " << TUserAttributesLimits::MaxValueLen << ", actual: " << value.size() << Endl; exit(1); } diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index 2d6fb9391b..faf1797413 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -56,7 +56,7 @@ union TBasicKikimrServicesMask { bool EnablePersQueueClusterDiscovery:1; bool EnableNetClassifier:1; bool EnablePersQueueClusterTracker:1; - bool EnableSysViewService:1; + bool EnableSysViewService:1; bool EnableMeteringWriter:1; bool EnableSchemeBoardMonitoring:1; bool EnableConfigsCache:1; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 23e70a19df..819c1478d1 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -918,8 +918,8 @@ void TLocalServiceInitializer::InitializeServices( new TTabletSetupInfo(&NKesus::CreateKesusTablet, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.Hive] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&CreateDefaultHive, TMailboxType::ReadAsFilled, importantPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); - localConfig->TabletClassInfo[appData->DefaultTabletTypes.SysViewProcessor] = TLocalConfig::TTabletClassInfo( - new TTabletSetupInfo(&NSysView::CreateSysViewProcessor, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); + localConfig->TabletClassInfo[appData->DefaultTabletTypes.SysViewProcessor] = TLocalConfig::TTabletClassInfo( + new TTabletSetupInfo(&NSysView::CreateSysViewProcessor, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.TestShard] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&NTestShard::CreateTestShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.ColumnShard] = TLocalConfig::TTabletClassInfo( @@ -1637,7 +1637,7 @@ void TStatsCollectorInitializer::InitializeServices( IActor* statsCollector = CreateStatsCollector( 1, // seconds *setup, - appData->Counters); + appData->Counters); setup->LocalServices.emplace_back( TActorId(), TActorSetupCmd( @@ -1657,7 +1657,7 @@ void TStatsCollectorInitializer::InitializeServices( IActor* procStatCollector = CreateProcStatCollector( 5, // seconds - appData->Counters); + appData->Counters); setup->LocalServices.emplace_back( TActorId(), TActorSetupCmd( @@ -1904,16 +1904,16 @@ TMemoryTrackerInitializer::TMemoryTrackerInitializer(const TKikimrRunConfig& run : IKikimrServicesInitializer(runConfig) {} -void TMemoryTrackerInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) -{ - auto* actor = NMemory::CreateMemoryTrackerActor(TDuration::MilliSeconds(20), appData->Counters); - setup->LocalServices.emplace_back( - TActorId(), - TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId) - ); -} +void TMemoryTrackerInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) +{ + auto* actor = NMemory::CreateMemoryTrackerActor(TDuration::MilliSeconds(20), appData->Counters); + setup->LocalServices.emplace_back( + TActorId(), + TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId) + ); +} TQuoterServiceInitializer::TQuoterServiceInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) @@ -2165,29 +2165,29 @@ void TConfigValidatorsInitializer::InitializeServices(NActors::TActorSystemSetup NConsole::RegisterCoreValidators(); } -TSysViewServiceInitializer::TSysViewServiceInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{ -} +TSysViewServiceInitializer::TSysViewServiceInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{ +} -void TSysViewServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - NSysView::TExtCountersConfig config; +void TSysViewServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + NSysView::TExtCountersConfig config; for (ui32 i = 0; i < setup->GetExecutorsCount(); ++i) { - config.Pools.push_back(NSysView::TExtCountersConfig::TPool{ + config.Pools.push_back(NSysView::TExtCountersConfig::TPool{ setup->GetPoolName(i), setup->GetThreads(i)}); - } - - // external counters only for dynamic nodes - bool hasExternalCounters = Config.GetDynamicNodeConfig().HasNodeInfo(); - - auto actor = NSysView::CreateSysViewService(std::move(config), hasExternalCounters); - + } + + // external counters only for dynamic nodes + bool hasExternalCounters = Config.GetDynamicNodeConfig().HasNodeInfo(); + + auto actor = NSysView::CreateSysViewService(std::move(config), hasExternalCounters); + setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( - NSysView::MakeSysViewServiceID(NodeId), - TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->UserPoolId))); -} - + NSysView::MakeSysViewServiceID(NodeId), + TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->UserPoolId))); +} + TMeteringWriterInitializer::TMeteringWriterInitializer(const TKikimrRunConfig &runConfig) : IKikimrServicesInitializer(runConfig) { diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 40aa6e47d5..407ce1bb7b 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -364,13 +364,13 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TMemoryTrackerInitializer : public IKikimrServicesInitializer { -public: +class TMemoryTrackerInitializer : public IKikimrServicesInitializer { +public: TMemoryTrackerInitializer(const TKikimrRunConfig& runConfig); void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - +}; + class TQuoterServiceInitializer : public IKikimrServicesInitializer { public: TQuoterServiceInitializer(const TKikimrRunConfig& runConfig); @@ -451,13 +451,13 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TSysViewServiceInitializer : public IKikimrServicesInitializer { -public: - TSysViewServiceInitializer(const TKikimrRunConfig& runConfig); +class TSysViewServiceInitializer : public IKikimrServicesInitializer { +public: + TSysViewServiceInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - class TMeteringWriterInitializer : public IKikimrServicesInitializer { public: TMeteringWriterInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index ef331f4782..a4f74aa4e0 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -114,7 +114,7 @@ #include <ydb/core/node_whiteboard/node_whiteboard.h> #include <ydb/core/tablet/node_tablet_monitor.h> -#include <library/cpp/actors/core/memory_track.h> +#include <library/cpp/actors/core/memory_track.h> #include <library/cpp/actors/prof/tag.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> @@ -1264,10 +1264,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TMemProfMonitorInitializer(runConfig)); -#if defined(ENABLE_MEMORY_TRACKING) - sil->AddServiceInitializer(new TMemoryTrackerInitializer(runConfig)); -#endif - +#if defined(ENABLE_MEMORY_TRACKING) + sil->AddServiceInitializer(new TMemoryTrackerInitializer(runConfig)); +#endif + if (serviceMask.EnableKqp) { sil->AddServiceInitializer(new TKqpServiceInitializer(runConfig, ModuleFactories)); } @@ -1299,10 +1299,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TQuoterServiceInitializer(runConfig)); } - if (serviceMask.EnableSysViewService) { - sil->AddServiceInitializer(new TSysViewServiceInitializer(runConfig)); - } - + if (serviceMask.EnableSysViewService) { + sil->AddServiceInitializer(new TSysViewServiceInitializer(runConfig)); + } + if (serviceMask.EnableMeteringWriter) { sil->AddServiceInitializer(new TMeteringWriterInitializer(runConfig)); } diff --git a/ydb/core/engine/minikql/flat_local_minikql_host.h b/ydb/core/engine/minikql/flat_local_minikql_host.h index da3ea2c44f..edfd182286 100644 --- a/ydb/core/engine/minikql/flat_local_minikql_host.h +++ b/ydb/core/engine/minikql/flat_local_minikql_host.h @@ -21,7 +21,7 @@ private: bool IsMyKey(const TTableId& tableId, const TArrayRef<const TCell>& row) const override { Y_UNUSED(row); - return (tableId.PathId.OwnerId == GetShardId()); + return (tableId.PathId.OwnerId == GetShardId()); } TRowVersion GetWriteVersion(const TTableId& tableId) const override diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp index b6802a64a1..36c35f32cc 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.cpp +++ b/ydb/core/engine/minikql/minikql_engine_host.cpp @@ -933,7 +933,7 @@ ui64 TEngineHost::GetTableSchemaVersion(const TTableId&) const { } ui64 TEngineHost::LocalTableId(const TTableId& tableId) const { - return tableId.PathId.LocalPathId; + return tableId.PathId.LocalPathId; } void TEngineHost::ConvertKeys(const TScheme::TTableInfo* tableInfo, const TArrayRef<const TCell>& row, diff --git a/ydb/core/engine/mkql_engine_flat_ut.cpp b/ydb/core/engine/mkql_engine_flat_ut.cpp index 733b016009..e2b12e2d7a 100644 --- a/ydb/core/engine/mkql_engine_flat_ut.cpp +++ b/ydb/core/engine/mkql_engine_flat_ut.cpp @@ -99,7 +99,7 @@ namespace { void TwoShardResolver(TKeyDesc& key) { key.Status = TKeyDesc::EStatus::Ok; - key.Partitions.push_back(TKeyDesc::TPartitionInfo(key.TableId.PathId.LocalPathId == Table1Id ? Shard1 : Shard2)); + key.Partitions.push_back(TKeyDesc::TPartitionInfo(key.TableId.PathId.LocalPathId == Table1Id ? Shard1 : Shard2)); } struct TDriver { diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index 8db29589ec..44b25c4a5f 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -330,7 +330,7 @@ public: virtual void ReplyUnavaliable() = 0; virtual bool Validate(TString& error) = 0; - virtual void UseDatabase(const TString& database) = 0; + virtual void UseDatabase(const TString& database) = 0; // This method allows to set hook for unary call. // The hook will be called at the reply time @@ -479,10 +479,10 @@ public: return true; } - void UseDatabase(const TString& database) override { - Y_UNUSED(database); - } - + void UseDatabase(const TString& database) override { + Y_UNUSED(database); + } + TActorId GetFromId() const { return From_; } @@ -638,10 +638,10 @@ public: return true; } - void UseDatabase(const TString& database) override { - Ctx_->UseDatabase(database); - } - + void UseDatabase(const TString& database) override { + Ctx_->UseDatabase(database); + } + IStreamCtx* GetStreamCtx() { return Ctx_.Get(); } @@ -863,10 +863,10 @@ public: return true; } - void UseDatabase(const TString& database) override { - Ctx_->UseDatabase(database); - } - + void UseDatabase(const TString& database) override { + Ctx_->UseDatabase(database); + } + void ReplyUnavaliable() override { TResponse* resp = CreateResponseMessage(); TCommonResponseFiller<TResp, TDerived::IsOp>::Fill(*resp, IssueManager.GetIssues(), CostInfo, Ydb::StatusIds::UNAVAILABLE); diff --git a/ydb/core/grpc_services/counters/counters.cpp b/ydb/core/grpc_services/counters/counters.cpp index eea7474631..8fc8ede917 100644 --- a/ydb/core/grpc_services/counters/counters.cpp +++ b/ydb/core/grpc_services/counters/counters.cpp @@ -1,645 +1,645 @@ -#include "counters.h" - +#include "counters.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> #include <ydb/core/sys_view/service/db_counters.h> #include <ydb/core/sys_view/service/sysview_service.h> #include <ydb/core/util/concurrent_rw_hash.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -#include <library/cpp/deprecated/enum_codegen/enum_codegen.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; - -struct TYdbRpcCounters { - TYdbRpcCounters(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName, - const char* requestName, bool forDatabase); - - NMonitoring::TDynamicCounters::TCounterPtr RequestCount; - NMonitoring::TDynamicCounters::TCounterPtr RequestInflight; - NMonitoring::TDynamicCounters::TCounterPtr RequestBytes; - NMonitoring::TDynamicCounters::TCounterPtr RequestInflightBytes; - NMonitoring::TDynamicCounters::TCounterPtr RequestRpcError; - - NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes; - NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcError; - NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcNotAuthenticated; - NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcResourceExhausted; - THashMap<ui32, NMonitoring::TDynamicCounters::TCounterPtr> ResponseByStatus; -}; - + +#include <library/cpp/deprecated/enum_codegen/enum_codegen.h> + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; + +struct TYdbRpcCounters { + TYdbRpcCounters(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName, + const char* requestName, bool forDatabase); + + NMonitoring::TDynamicCounters::TCounterPtr RequestCount; + NMonitoring::TDynamicCounters::TCounterPtr RequestInflight; + NMonitoring::TDynamicCounters::TCounterPtr RequestBytes; + NMonitoring::TDynamicCounters::TCounterPtr RequestInflightBytes; + NMonitoring::TDynamicCounters::TCounterPtr RequestRpcError; + + NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes; + NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcError; + NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcNotAuthenticated; + NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcResourceExhausted; + THashMap<ui32, NMonitoring::TDynamicCounters::TCounterPtr> ResponseByStatus; +}; + class TYdbCounterBlock : public NGrpc::ICounterBlock { -protected: - bool Streaming = false; - bool Percentile = false; - - // "Internal" counters - // TODO: Switch to public YDB counters - NMonitoring::TDynamicCounters::TCounterPtr TotalCounter; - NMonitoring::TDynamicCounters::TCounterPtr InflyCounter; - NMonitoring::TDynamicCounters::TCounterPtr NotOkRequestCounter; - NMonitoring::TDynamicCounters::TCounterPtr NotOkResponseCounter; - NMonitoring::TDynamicCounters::TCounterPtr RequestBytes; - NMonitoring::TDynamicCounters::TCounterPtr InflyRequestBytes; - NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes; - NMonitoring::TDynamicCounters::TCounterPtr NotAuthenticated; - NMonitoring::TDynamicCounters::TCounterPtr ResourceExhausted; - NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutDatabase; - NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutToken; - NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutTls; - NMonitoring::TPercentileTracker<4, 512, 15> RequestHistMs; - std::array<NMonitoring::TDynamicCounters::TCounterPtr, 2> GRpcStatusCounters; - - TYdbRpcCounters YdbCounters; - -public: - TYdbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName, - const char* requestName, bool percentile, bool streaming, - bool forDatabase = false, NMonitoring::TDynamicCounterPtr internalGroup = {}); - - void CountNotOkRequest() override { - NotOkRequestCounter->Inc(); - YdbCounters.RequestRpcError->Inc(); - } - - void CountNotOkResponse() override { - NotOkResponseCounter->Inc(); - YdbCounters.ResponseRpcError->Inc(); - } - - void CountNotAuthenticated() override { - NotAuthenticated->Inc(); - YdbCounters.ResponseRpcNotAuthenticated->Inc(); - } - - void CountResourceExhausted() override { - ResourceExhausted->Inc(); - YdbCounters.ResponseRpcResourceExhausted->Inc(); - } - - void CountRequestsWithoutDatabase() override { - RequestsWithoutDatabase->Inc(); - } - - void CountRequestsWithoutToken() override { - RequestsWithoutToken->Inc(); - } - - void CountRequestWithoutTls() override { - RequestsWithoutTls->Inc(); - } - - void CountRequestBytes(ui32 requestSize) override { - *RequestBytes += requestSize; - *YdbCounters.RequestBytes += requestSize; - } - - void CountResponseBytes(ui32 responseSize) override { - *ResponseBytes += responseSize; - *YdbCounters.ResponseBytes += responseSize; - } - - void StartProcessing(ui32 requestSize) override { - TotalCounter->Inc(); - InflyCounter->Inc(); - *RequestBytes += requestSize; - *InflyRequestBytes += requestSize; - - YdbCounters.RequestCount->Inc(); - YdbCounters.RequestInflight->Inc(); - *YdbCounters.RequestBytes += requestSize; - *YdbCounters.RequestInflightBytes += requestSize; - } - - void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status, - TDuration requestDuration) override - { - InflyCounter->Dec(); - *InflyRequestBytes -= requestSize; - *ResponseBytes += responseSize; - - YdbCounters.RequestInflight->Dec(); - *YdbCounters.RequestInflightBytes -= requestSize; - *YdbCounters.ResponseBytes += responseSize; - - if (!ok) { - NotOkResponseCounter->Inc(); - YdbCounters.ResponseRpcError->Inc(); - } else if (!Streaming) { - auto statusCounter = YdbCounters.ResponseByStatus.FindPtr(status); - if (statusCounter) { - (*statusCounter)->Inc(); - } else { - YdbCounters.ResponseByStatus[0]->Inc(); - } - } - - if (Percentile) { - RequestHistMs.Increment(requestDuration.MilliSeconds()); - } - } - +protected: + bool Streaming = false; + bool Percentile = false; + + // "Internal" counters + // TODO: Switch to public YDB counters + NMonitoring::TDynamicCounters::TCounterPtr TotalCounter; + NMonitoring::TDynamicCounters::TCounterPtr InflyCounter; + NMonitoring::TDynamicCounters::TCounterPtr NotOkRequestCounter; + NMonitoring::TDynamicCounters::TCounterPtr NotOkResponseCounter; + NMonitoring::TDynamicCounters::TCounterPtr RequestBytes; + NMonitoring::TDynamicCounters::TCounterPtr InflyRequestBytes; + NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes; + NMonitoring::TDynamicCounters::TCounterPtr NotAuthenticated; + NMonitoring::TDynamicCounters::TCounterPtr ResourceExhausted; + NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutDatabase; + NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutToken; + NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutTls; + NMonitoring::TPercentileTracker<4, 512, 15> RequestHistMs; + std::array<NMonitoring::TDynamicCounters::TCounterPtr, 2> GRpcStatusCounters; + + TYdbRpcCounters YdbCounters; + +public: + TYdbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName, + const char* requestName, bool percentile, bool streaming, + bool forDatabase = false, NMonitoring::TDynamicCounterPtr internalGroup = {}); + + void CountNotOkRequest() override { + NotOkRequestCounter->Inc(); + YdbCounters.RequestRpcError->Inc(); + } + + void CountNotOkResponse() override { + NotOkResponseCounter->Inc(); + YdbCounters.ResponseRpcError->Inc(); + } + + void CountNotAuthenticated() override { + NotAuthenticated->Inc(); + YdbCounters.ResponseRpcNotAuthenticated->Inc(); + } + + void CountResourceExhausted() override { + ResourceExhausted->Inc(); + YdbCounters.ResponseRpcResourceExhausted->Inc(); + } + + void CountRequestsWithoutDatabase() override { + RequestsWithoutDatabase->Inc(); + } + + void CountRequestsWithoutToken() override { + RequestsWithoutToken->Inc(); + } + + void CountRequestWithoutTls() override { + RequestsWithoutTls->Inc(); + } + + void CountRequestBytes(ui32 requestSize) override { + *RequestBytes += requestSize; + *YdbCounters.RequestBytes += requestSize; + } + + void CountResponseBytes(ui32 responseSize) override { + *ResponseBytes += responseSize; + *YdbCounters.ResponseBytes += responseSize; + } + + void StartProcessing(ui32 requestSize) override { + TotalCounter->Inc(); + InflyCounter->Inc(); + *RequestBytes += requestSize; + *InflyRequestBytes += requestSize; + + YdbCounters.RequestCount->Inc(); + YdbCounters.RequestInflight->Inc(); + *YdbCounters.RequestBytes += requestSize; + *YdbCounters.RequestInflightBytes += requestSize; + } + + void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status, + TDuration requestDuration) override + { + InflyCounter->Dec(); + *InflyRequestBytes -= requestSize; + *ResponseBytes += responseSize; + + YdbCounters.RequestInflight->Dec(); + *YdbCounters.RequestInflightBytes -= requestSize; + *YdbCounters.ResponseBytes += responseSize; + + if (!ok) { + NotOkResponseCounter->Inc(); + YdbCounters.ResponseRpcError->Inc(); + } else if (!Streaming) { + auto statusCounter = YdbCounters.ResponseByStatus.FindPtr(status); + if (statusCounter) { + (*statusCounter)->Inc(); + } else { + YdbCounters.ResponseByStatus[0]->Inc(); + } + } + + if (Percentile) { + RequestHistMs.Increment(requestDuration.MilliSeconds()); + } + } + NGrpc::ICounterBlockPtr Clone() override { - return this; - } - - void Update() { - if (Percentile) { - RequestHistMs.Update(); - } - } -}; - -TYdbRpcCounters::TYdbRpcCounters(const NMonitoring::TDynamicCounterPtr& counters, - const char* serviceName, const char* requestName, bool forDatabase) -{ - NMonitoring::TDynamicCounterPtr ydbGroup; - if (forDatabase) { - ydbGroup = counters; - } else { - ydbGroup = GetServiceCounters(counters, "ydb"); - } - - auto serviceGroup = ydbGroup->GetSubgroup("api_service", serviceName); - auto typeGroup = serviceGroup->GetSubgroup("method", requestName); - - RequestCount = typeGroup->GetNamedCounter("name", "api.grpc.request.count", true); - RequestInflight = typeGroup->GetNamedCounter("name", "api.grpc.request.inflight_count", false); - RequestBytes = typeGroup->GetNamedCounter("name", "api.grpc.request.bytes", true); - RequestInflightBytes = typeGroup->GetNamedCounter("name", "api.grpc.request.inflight_bytes", false); - RequestRpcError = typeGroup->GetNamedCounter("name", "api.grpc.request.dropped_count", true); - - ResponseBytes = typeGroup->GetNamedCounter("name", "api.grpc.response.bytes", true); - ResponseRpcError = typeGroup->GetNamedCounter("name", "api.grpc.response.dropped_count", true); - - auto countName = "api.grpc.response.count"; - - ResponseRpcNotAuthenticated = - typeGroup->GetSubgroup("status", "UNAUTHENTICATED")->GetNamedCounter("name", countName, true); - ResponseRpcResourceExhausted = - typeGroup->GetSubgroup("status", "RESOURCE_EXHAUSTED")->GetNamedCounter("name", countName, true); - - ResponseByStatus[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED] = - typeGroup->GetSubgroup("status", "UNSPECIFIED")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::SUCCESS] = - typeGroup->GetSubgroup("status", "SUCCESS")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::BAD_REQUEST] = - typeGroup->GetSubgroup("status", "BAD_REQUEST")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::UNAUTHORIZED] = - typeGroup->GetSubgroup("status", "UNAUTHORIZED")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::INTERNAL_ERROR] = - typeGroup->GetSubgroup("status", "INTERNAL_ERROR")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::ABORTED] = - typeGroup->GetSubgroup("status", "ABORTED")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::UNAVAILABLE] = - typeGroup->GetSubgroup("status", "UNAVAILABLE")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::OVERLOADED] = - typeGroup->GetSubgroup("status", "OVERLOADED")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::SCHEME_ERROR] = - typeGroup->GetSubgroup("status", "SCHEME_ERROR")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::GENERIC_ERROR] = - typeGroup->GetSubgroup("status", "GENERIC_ERROR")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::TIMEOUT] = - typeGroup->GetSubgroup("status", "TIMEOUT")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::BAD_SESSION] = - typeGroup->GetSubgroup("status", "BAD_SESSION")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::PRECONDITION_FAILED] = - typeGroup->GetSubgroup("status", "PRECONDITION_FAILED")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::ALREADY_EXISTS] = - typeGroup->GetSubgroup("status", "ALREADY_EXISTS")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::NOT_FOUND] = - typeGroup->GetSubgroup("status", "NOT_FOUND")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::SESSION_EXPIRED] = - typeGroup->GetSubgroup("status", "SESSION_EXPIRED")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::CANCELLED] = - typeGroup->GetSubgroup("status", "CANCELLED")->GetNamedCounter("name", countName, true); - ResponseByStatus[Ydb::StatusIds::SESSION_BUSY] = - typeGroup->GetSubgroup("status", "SESSION_BUSY")->GetNamedCounter("name", countName, true); -} - -TYdbCounterBlock::TYdbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName, - const char* requestName, bool percentile, bool streaming, - bool forDatabase, NMonitoring::TDynamicCounterPtr internalGroup) - : Streaming(streaming) - , Percentile(percentile) - , YdbCounters(counters, serviceName, requestName, forDatabase) -{ - // group for all counters - NMonitoring::TDynamicCounterPtr group; - if (forDatabase) { - group = internalGroup->GetSubgroup("group", "grpc"); - } else { - group = GetServiceCounters(counters, "grpc")->GetSubgroup("subsystem", "serverStats"); - } - - // aggregated (non-request-specific counters) - NotOkRequestCounter = group->GetCounter("notOkRequest", true); - NotOkResponseCounter = group->GetCounter("notOkResponse", true); - RequestBytes = group->GetCounter("requestBytes", true); - InflyRequestBytes = group->GetCounter("inflyRequestBytes", false); - ResponseBytes = group->GetCounter("responseBytes", true); - NotAuthenticated = group->GetCounter("notAuthenticated", true); - ResourceExhausted = group->GetCounter("resourceExhausted", true); - RequestsWithoutDatabase = group->GetCounter("requestsWithoutDatabase", true); - RequestsWithoutToken = group->GetCounter("requestsWithoutToken", true); - RequestsWithoutTls = group->GetCounter("requestsWithoutTls", true); - - // subgroup for request-specific counters - auto subgroup = group->GetSubgroup(streaming ? "stream" : "request", requestName); - TotalCounter = subgroup->GetCounter("total", true); - InflyCounter = subgroup->GetCounter("infly", false); - - if (Percentile) { - RequestHistMs.Initialize(group, "event", "request", "ms", {0.5f, 0.9f, 0.99f, 0.999f, 1.0f}); - } -} - -using TYdbCounterBlockPtr = TIntrusivePtr<TYdbCounterBlock>; - - -#define DB_GRPC_SIMPLE_COUNTERS_MAP(XX) \ - XX(DB_GRPC_REQ_INFLIGHT_COUNT, YdbCounters.RequestInflight) \ - XX(DB_GRPC_REQ_INFLIGHT_BYTES, YdbCounters.RequestInflightBytes) - -#define DB_GRPC_CUMULATIVE_COUNTERS_MAP(XX) \ - XX(DB_GRPC_REQ_COUNT, YdbCounters.RequestCount) \ - XX(DB_GRPC_REQ_BYTES, YdbCounters.RequestBytes) \ - XX(DB_GRPC_REQ_RPC_ERROR, YdbCounters.RequestRpcError) \ - XX(DB_GRPC_RSP_BYTES, YdbCounters.ResponseBytes) \ - XX(DB_GRPC_RSP_RPC_ERROR, YdbCounters.ResponseRpcError) \ - XX(DB_GRPC_RSP_RPC_NOT_AUTH, YdbCounters.ResponseRpcNotAuthenticated) \ - XX(DB_GRPC_RSP_RPC_RESOURCE_EXHAUSTED, YdbCounters.ResponseRpcResourceExhausted) \ - XX(DB_GRPC_RSP_UNSPECIFIED, YdbCounters.ResponseByStatus[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED]) \ - XX(DB_GRPC_RSP_SUCCESS, YdbCounters.ResponseByStatus[Ydb::StatusIds::SUCCESS]) \ - XX(DB_GRPC_RSP_BAD_REQUEST, YdbCounters.ResponseByStatus[Ydb::StatusIds::BAD_REQUEST]) \ - XX(DB_GRPC_RSP_UNAUTHORIZED, YdbCounters.ResponseByStatus[Ydb::StatusIds::UNAUTHORIZED]) \ - XX(DB_GRPC_RSP_INTERNAL_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::INTERNAL_ERROR]) \ - XX(DB_GRPC_RSP_ABORTED, YdbCounters.ResponseByStatus[Ydb::StatusIds::ABORTED]) \ - XX(DB_GRPC_RSP_UNAVAILABLE, YdbCounters.ResponseByStatus[Ydb::StatusIds::UNAVAILABLE]) \ - XX(DB_GRPC_RSP_OVERLOADED, YdbCounters.ResponseByStatus[Ydb::StatusIds::OVERLOADED]) \ - XX(DB_GRPC_RSP_SCHEME_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::SCHEME_ERROR]) \ - XX(DB_GRPC_RSP_GENERIC_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::GENERIC_ERROR]) \ - XX(DB_GRPC_RSP_TIMEOUT, YdbCounters.ResponseByStatus[Ydb::StatusIds::TIMEOUT]) \ - XX(DB_GRPC_RSP_BAD_SESSION, YdbCounters.ResponseByStatus[Ydb::StatusIds::BAD_SESSION]) \ - XX(DB_GRPC_RSP_PRECONDITION_FAILED, YdbCounters.ResponseByStatus[Ydb::StatusIds::PRECONDITION_FAILED]) \ - XX(DB_GRPC_RSP_ALREADY_EXISTS, YdbCounters.ResponseByStatus[Ydb::StatusIds::ALREADY_EXISTS]) \ - XX(DB_GRPC_RSP_NOT_FOUND, YdbCounters.ResponseByStatus[Ydb::StatusIds::NOT_FOUND]) \ - XX(DB_GRPC_RSP_SESSION_EXPIRED, YdbCounters.ResponseByStatus[Ydb::StatusIds::SESSION_EXPIRED]) \ - XX(DB_GRPC_RSP_CANCELLED, YdbCounters.ResponseByStatus[Ydb::StatusIds::CANCELLED]) \ - XX(DB_GRPC_RSP_SESSION_BUSY, YdbCounters.ResponseByStatus[Ydb::StatusIds::SESSION_BUSY]) - -class TYdbDbCounterBlock : public TYdbCounterBlock { -public: - TYdbDbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName, - const char* requestName, bool percentile, bool streaming, - NMonitoring::TDynamicCounterPtr internalGroup = {}) - : TYdbCounterBlock(counters, serviceName, requestName, percentile, streaming, true, internalGroup) - {} - - void ToProto(NKikimrSysView::TDbGRpcCounters& counters) { - auto* main = counters.MutableRequestCounters(); - auto* simple = main->MutableSimple(); - auto* cumulative = main->MutableCumulative(); - - simple->Resize(DB_GRPC_SIMPLE_COUNTER_SIZE, 0); - cumulative->Resize(DB_GRPC_CUMULATIVE_COUNTER_SIZE, 0); - -#define SAVE_SIMPLE_COUNTER(INDEX, TARGET) { (*simple)[INDEX] = (TARGET)->Val(); } -#define SAVE_CUMULATIVE_COUNTER(INDEX, TARGET) { (*cumulative)[INDEX] = (TARGET)->Val(); } - - DB_GRPC_SIMPLE_COUNTERS_MAP(SAVE_SIMPLE_COUNTER) - DB_GRPC_CUMULATIVE_COUNTERS_MAP(SAVE_CUMULATIVE_COUNTER) - } - - void FromProto(NKikimrSysView::TDbGRpcCounters& counters) { - auto* main = counters.MutableRequestCounters(); - auto* simple = main->MutableSimple(); - auto* cumulative = main->MutableCumulative(); - - simple->Resize(DB_GRPC_SIMPLE_COUNTER_SIZE, 0); - cumulative->Resize(DB_GRPC_CUMULATIVE_COUNTER_SIZE, 0); - -#define LOAD_SIMPLE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*simple)[INDEX]); } -#define LOAD_CUMULATIVE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*cumulative)[INDEX]); } - - DB_GRPC_SIMPLE_COUNTERS_MAP(LOAD_SIMPLE_COUNTER) - DB_GRPC_CUMULATIVE_COUNTERS_MAP(LOAD_CUMULATIVE_COUNTER) - } - - void AggregateFrom(TYdbDbCounterBlock& other) { -#define COPY_SIMPLE_COUNTER(INDEX, TARGET) { *TARGET += other.TARGET->Val(); } -#define COPY_CUMULATIVE_COUNTER(INDEX, TARGET) { *TARGET += other.TARGET->Val(); } - - DB_GRPC_SIMPLE_COUNTERS_MAP(COPY_SIMPLE_COUNTER) - DB_GRPC_CUMULATIVE_COUNTERS_MAP(COPY_CUMULATIVE_COUNTER) - } - -private: - enum ESimpleCounter { - DB_GRPC_SIMPLE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) - DB_GRPC_SIMPLE_COUNTER_SIZE - }; - enum ECumulativeCounter { - DB_GRPC_CUMULATIVE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) - DB_GRPC_CUMULATIVE_COUNTER_SIZE - }; -}; - -using TYdbDbCounterBlockPtr = TIntrusivePtr<TYdbDbCounterBlock>; - - -class TGRpcDbCounters : public NSysView::IDbCounters { -public: - TGRpcDbCounters() - : Counters(new NMonitoring::TDynamicCounters) - , InternalGroup(new NMonitoring::TDynamicCounters) - {} - - TGRpcDbCounters(NMonitoring::TDynamicCounterPtr counters, NMonitoring::TDynamicCounterPtr internalGroup) - : Counters(counters) - , InternalGroup(internalGroup) - {} - - void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override { - for (auto& bucket : CounterBlocks.Buckets) { - TReadGuard guard(bucket.GetLock()); - for (auto& [key, block] : bucket.GetMap()) { - auto* proto = counters.FindOrAddGRpcCounters(key.first, key.second); - block->ToProto(*proto); - } - } - } - - void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override { - for (auto& proto : *counters.Proto().MutableGRpcCounters()) { - auto block = GetCounterBlock(proto.GetGRpcService(), proto.GetGRpcRequest()); - block->FromProto(proto); - } - } - - TYdbDbCounterBlockPtr GetCounterBlock(const TString& serviceName, const TString& requestName) { - auto key = std::make_pair(serviceName, requestName); - - TYdbDbCounterBlockPtr dbCounters; - if (CounterBlocks.Get(key, dbCounters)) { - return dbCounters; - } - - return CounterBlocks.InsertIfAbsentWithInit(key, [&] { - return new TYdbDbCounterBlock(Counters, serviceName.c_str(), requestName.c_str(), false, false, InternalGroup); - }); - } - -private: - NMonitoring::TDynamicCounterPtr Counters; - NMonitoring::TDynamicCounterPtr InternalGroup; - - using TKey = std::pair<TString, TString>; - TConcurrentRWHashMap<TKey, TYdbDbCounterBlockPtr, 16> CounterBlocks; -}; - -using TGRpcDbCountersPtr = TIntrusivePtr<TGRpcDbCounters>; - - -class TGRpcDbCountersRegistry { - class TGRpcDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback { - public: - void OnDatabaseRemoved(const TString& database, TPathId) override { - Singleton<TGRpcDbCountersRegistry>()->RemoveDbCounters(database); - } - }; - -public: - void Initialize(TActorSystem* actorSystem) { - if (Y_LIKELY(ActorSystem)) { - return; - } - ActorSystem = actorSystem; - - auto callback = MakeIntrusive<TGRpcDbWatcherCallback>(); - DbWatcherActorId = ActorSystem->Register(NSysView::CreateDbWatcherActor(callback)); - } - - TYdbDbCounterBlockPtr GetCounterBlock( - const TString& database, const TString& serviceName, const TString& requestName) - { - TGRpcDbCountersPtr dbCounters; - if (DbCounters.Get(database, dbCounters)) { - return dbCounters->GetCounterBlock(serviceName, requestName); - } - + return this; + } + + void Update() { + if (Percentile) { + RequestHistMs.Update(); + } + } +}; + +TYdbRpcCounters::TYdbRpcCounters(const NMonitoring::TDynamicCounterPtr& counters, + const char* serviceName, const char* requestName, bool forDatabase) +{ + NMonitoring::TDynamicCounterPtr ydbGroup; + if (forDatabase) { + ydbGroup = counters; + } else { + ydbGroup = GetServiceCounters(counters, "ydb"); + } + + auto serviceGroup = ydbGroup->GetSubgroup("api_service", serviceName); + auto typeGroup = serviceGroup->GetSubgroup("method", requestName); + + RequestCount = typeGroup->GetNamedCounter("name", "api.grpc.request.count", true); + RequestInflight = typeGroup->GetNamedCounter("name", "api.grpc.request.inflight_count", false); + RequestBytes = typeGroup->GetNamedCounter("name", "api.grpc.request.bytes", true); + RequestInflightBytes = typeGroup->GetNamedCounter("name", "api.grpc.request.inflight_bytes", false); + RequestRpcError = typeGroup->GetNamedCounter("name", "api.grpc.request.dropped_count", true); + + ResponseBytes = typeGroup->GetNamedCounter("name", "api.grpc.response.bytes", true); + ResponseRpcError = typeGroup->GetNamedCounter("name", "api.grpc.response.dropped_count", true); + + auto countName = "api.grpc.response.count"; + + ResponseRpcNotAuthenticated = + typeGroup->GetSubgroup("status", "UNAUTHENTICATED")->GetNamedCounter("name", countName, true); + ResponseRpcResourceExhausted = + typeGroup->GetSubgroup("status", "RESOURCE_EXHAUSTED")->GetNamedCounter("name", countName, true); + + ResponseByStatus[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED] = + typeGroup->GetSubgroup("status", "UNSPECIFIED")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::SUCCESS] = + typeGroup->GetSubgroup("status", "SUCCESS")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::BAD_REQUEST] = + typeGroup->GetSubgroup("status", "BAD_REQUEST")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::UNAUTHORIZED] = + typeGroup->GetSubgroup("status", "UNAUTHORIZED")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::INTERNAL_ERROR] = + typeGroup->GetSubgroup("status", "INTERNAL_ERROR")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::ABORTED] = + typeGroup->GetSubgroup("status", "ABORTED")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::UNAVAILABLE] = + typeGroup->GetSubgroup("status", "UNAVAILABLE")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::OVERLOADED] = + typeGroup->GetSubgroup("status", "OVERLOADED")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::SCHEME_ERROR] = + typeGroup->GetSubgroup("status", "SCHEME_ERROR")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::GENERIC_ERROR] = + typeGroup->GetSubgroup("status", "GENERIC_ERROR")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::TIMEOUT] = + typeGroup->GetSubgroup("status", "TIMEOUT")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::BAD_SESSION] = + typeGroup->GetSubgroup("status", "BAD_SESSION")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::PRECONDITION_FAILED] = + typeGroup->GetSubgroup("status", "PRECONDITION_FAILED")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::ALREADY_EXISTS] = + typeGroup->GetSubgroup("status", "ALREADY_EXISTS")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::NOT_FOUND] = + typeGroup->GetSubgroup("status", "NOT_FOUND")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::SESSION_EXPIRED] = + typeGroup->GetSubgroup("status", "SESSION_EXPIRED")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::CANCELLED] = + typeGroup->GetSubgroup("status", "CANCELLED")->GetNamedCounter("name", countName, true); + ResponseByStatus[Ydb::StatusIds::SESSION_BUSY] = + typeGroup->GetSubgroup("status", "SESSION_BUSY")->GetNamedCounter("name", countName, true); +} + +TYdbCounterBlock::TYdbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName, + const char* requestName, bool percentile, bool streaming, + bool forDatabase, NMonitoring::TDynamicCounterPtr internalGroup) + : Streaming(streaming) + , Percentile(percentile) + , YdbCounters(counters, serviceName, requestName, forDatabase) +{ + // group for all counters + NMonitoring::TDynamicCounterPtr group; + if (forDatabase) { + group = internalGroup->GetSubgroup("group", "grpc"); + } else { + group = GetServiceCounters(counters, "grpc")->GetSubgroup("subsystem", "serverStats"); + } + + // aggregated (non-request-specific counters) + NotOkRequestCounter = group->GetCounter("notOkRequest", true); + NotOkResponseCounter = group->GetCounter("notOkResponse", true); + RequestBytes = group->GetCounter("requestBytes", true); + InflyRequestBytes = group->GetCounter("inflyRequestBytes", false); + ResponseBytes = group->GetCounter("responseBytes", true); + NotAuthenticated = group->GetCounter("notAuthenticated", true); + ResourceExhausted = group->GetCounter("resourceExhausted", true); + RequestsWithoutDatabase = group->GetCounter("requestsWithoutDatabase", true); + RequestsWithoutToken = group->GetCounter("requestsWithoutToken", true); + RequestsWithoutTls = group->GetCounter("requestsWithoutTls", true); + + // subgroup for request-specific counters + auto subgroup = group->GetSubgroup(streaming ? "stream" : "request", requestName); + TotalCounter = subgroup->GetCounter("total", true); + InflyCounter = subgroup->GetCounter("infly", false); + + if (Percentile) { + RequestHistMs.Initialize(group, "event", "request", "ms", {0.5f, 0.9f, 0.99f, 0.999f, 1.0f}); + } +} + +using TYdbCounterBlockPtr = TIntrusivePtr<TYdbCounterBlock>; + + +#define DB_GRPC_SIMPLE_COUNTERS_MAP(XX) \ + XX(DB_GRPC_REQ_INFLIGHT_COUNT, YdbCounters.RequestInflight) \ + XX(DB_GRPC_REQ_INFLIGHT_BYTES, YdbCounters.RequestInflightBytes) + +#define DB_GRPC_CUMULATIVE_COUNTERS_MAP(XX) \ + XX(DB_GRPC_REQ_COUNT, YdbCounters.RequestCount) \ + XX(DB_GRPC_REQ_BYTES, YdbCounters.RequestBytes) \ + XX(DB_GRPC_REQ_RPC_ERROR, YdbCounters.RequestRpcError) \ + XX(DB_GRPC_RSP_BYTES, YdbCounters.ResponseBytes) \ + XX(DB_GRPC_RSP_RPC_ERROR, YdbCounters.ResponseRpcError) \ + XX(DB_GRPC_RSP_RPC_NOT_AUTH, YdbCounters.ResponseRpcNotAuthenticated) \ + XX(DB_GRPC_RSP_RPC_RESOURCE_EXHAUSTED, YdbCounters.ResponseRpcResourceExhausted) \ + XX(DB_GRPC_RSP_UNSPECIFIED, YdbCounters.ResponseByStatus[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED]) \ + XX(DB_GRPC_RSP_SUCCESS, YdbCounters.ResponseByStatus[Ydb::StatusIds::SUCCESS]) \ + XX(DB_GRPC_RSP_BAD_REQUEST, YdbCounters.ResponseByStatus[Ydb::StatusIds::BAD_REQUEST]) \ + XX(DB_GRPC_RSP_UNAUTHORIZED, YdbCounters.ResponseByStatus[Ydb::StatusIds::UNAUTHORIZED]) \ + XX(DB_GRPC_RSP_INTERNAL_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::INTERNAL_ERROR]) \ + XX(DB_GRPC_RSP_ABORTED, YdbCounters.ResponseByStatus[Ydb::StatusIds::ABORTED]) \ + XX(DB_GRPC_RSP_UNAVAILABLE, YdbCounters.ResponseByStatus[Ydb::StatusIds::UNAVAILABLE]) \ + XX(DB_GRPC_RSP_OVERLOADED, YdbCounters.ResponseByStatus[Ydb::StatusIds::OVERLOADED]) \ + XX(DB_GRPC_RSP_SCHEME_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::SCHEME_ERROR]) \ + XX(DB_GRPC_RSP_GENERIC_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::GENERIC_ERROR]) \ + XX(DB_GRPC_RSP_TIMEOUT, YdbCounters.ResponseByStatus[Ydb::StatusIds::TIMEOUT]) \ + XX(DB_GRPC_RSP_BAD_SESSION, YdbCounters.ResponseByStatus[Ydb::StatusIds::BAD_SESSION]) \ + XX(DB_GRPC_RSP_PRECONDITION_FAILED, YdbCounters.ResponseByStatus[Ydb::StatusIds::PRECONDITION_FAILED]) \ + XX(DB_GRPC_RSP_ALREADY_EXISTS, YdbCounters.ResponseByStatus[Ydb::StatusIds::ALREADY_EXISTS]) \ + XX(DB_GRPC_RSP_NOT_FOUND, YdbCounters.ResponseByStatus[Ydb::StatusIds::NOT_FOUND]) \ + XX(DB_GRPC_RSP_SESSION_EXPIRED, YdbCounters.ResponseByStatus[Ydb::StatusIds::SESSION_EXPIRED]) \ + XX(DB_GRPC_RSP_CANCELLED, YdbCounters.ResponseByStatus[Ydb::StatusIds::CANCELLED]) \ + XX(DB_GRPC_RSP_SESSION_BUSY, YdbCounters.ResponseByStatus[Ydb::StatusIds::SESSION_BUSY]) + +class TYdbDbCounterBlock : public TYdbCounterBlock { +public: + TYdbDbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName, + const char* requestName, bool percentile, bool streaming, + NMonitoring::TDynamicCounterPtr internalGroup = {}) + : TYdbCounterBlock(counters, serviceName, requestName, percentile, streaming, true, internalGroup) + {} + + void ToProto(NKikimrSysView::TDbGRpcCounters& counters) { + auto* main = counters.MutableRequestCounters(); + auto* simple = main->MutableSimple(); + auto* cumulative = main->MutableCumulative(); + + simple->Resize(DB_GRPC_SIMPLE_COUNTER_SIZE, 0); + cumulative->Resize(DB_GRPC_CUMULATIVE_COUNTER_SIZE, 0); + +#define SAVE_SIMPLE_COUNTER(INDEX, TARGET) { (*simple)[INDEX] = (TARGET)->Val(); } +#define SAVE_CUMULATIVE_COUNTER(INDEX, TARGET) { (*cumulative)[INDEX] = (TARGET)->Val(); } + + DB_GRPC_SIMPLE_COUNTERS_MAP(SAVE_SIMPLE_COUNTER) + DB_GRPC_CUMULATIVE_COUNTERS_MAP(SAVE_CUMULATIVE_COUNTER) + } + + void FromProto(NKikimrSysView::TDbGRpcCounters& counters) { + auto* main = counters.MutableRequestCounters(); + auto* simple = main->MutableSimple(); + auto* cumulative = main->MutableCumulative(); + + simple->Resize(DB_GRPC_SIMPLE_COUNTER_SIZE, 0); + cumulative->Resize(DB_GRPC_CUMULATIVE_COUNTER_SIZE, 0); + +#define LOAD_SIMPLE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*simple)[INDEX]); } +#define LOAD_CUMULATIVE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*cumulative)[INDEX]); } + + DB_GRPC_SIMPLE_COUNTERS_MAP(LOAD_SIMPLE_COUNTER) + DB_GRPC_CUMULATIVE_COUNTERS_MAP(LOAD_CUMULATIVE_COUNTER) + } + + void AggregateFrom(TYdbDbCounterBlock& other) { +#define COPY_SIMPLE_COUNTER(INDEX, TARGET) { *TARGET += other.TARGET->Val(); } +#define COPY_CUMULATIVE_COUNTER(INDEX, TARGET) { *TARGET += other.TARGET->Val(); } + + DB_GRPC_SIMPLE_COUNTERS_MAP(COPY_SIMPLE_COUNTER) + DB_GRPC_CUMULATIVE_COUNTERS_MAP(COPY_CUMULATIVE_COUNTER) + } + +private: + enum ESimpleCounter { + DB_GRPC_SIMPLE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) + DB_GRPC_SIMPLE_COUNTER_SIZE + }; + enum ECumulativeCounter { + DB_GRPC_CUMULATIVE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) + DB_GRPC_CUMULATIVE_COUNTER_SIZE + }; +}; + +using TYdbDbCounterBlockPtr = TIntrusivePtr<TYdbDbCounterBlock>; + + +class TGRpcDbCounters : public NSysView::IDbCounters { +public: + TGRpcDbCounters() + : Counters(new NMonitoring::TDynamicCounters) + , InternalGroup(new NMonitoring::TDynamicCounters) + {} + + TGRpcDbCounters(NMonitoring::TDynamicCounterPtr counters, NMonitoring::TDynamicCounterPtr internalGroup) + : Counters(counters) + , InternalGroup(internalGroup) + {} + + void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override { + for (auto& bucket : CounterBlocks.Buckets) { + TReadGuard guard(bucket.GetLock()); + for (auto& [key, block] : bucket.GetMap()) { + auto* proto = counters.FindOrAddGRpcCounters(key.first, key.second); + block->ToProto(*proto); + } + } + } + + void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override { + for (auto& proto : *counters.Proto().MutableGRpcCounters()) { + auto block = GetCounterBlock(proto.GetGRpcService(), proto.GetGRpcRequest()); + block->FromProto(proto); + } + } + + TYdbDbCounterBlockPtr GetCounterBlock(const TString& serviceName, const TString& requestName) { + auto key = std::make_pair(serviceName, requestName); + + TYdbDbCounterBlockPtr dbCounters; + if (CounterBlocks.Get(key, dbCounters)) { + return dbCounters; + } + + return CounterBlocks.InsertIfAbsentWithInit(key, [&] { + return new TYdbDbCounterBlock(Counters, serviceName.c_str(), requestName.c_str(), false, false, InternalGroup); + }); + } + +private: + NMonitoring::TDynamicCounterPtr Counters; + NMonitoring::TDynamicCounterPtr InternalGroup; + + using TKey = std::pair<TString, TString>; + TConcurrentRWHashMap<TKey, TYdbDbCounterBlockPtr, 16> CounterBlocks; +}; + +using TGRpcDbCountersPtr = TIntrusivePtr<TGRpcDbCounters>; + + +class TGRpcDbCountersRegistry { + class TGRpcDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback { + public: + void OnDatabaseRemoved(const TString& database, TPathId) override { + Singleton<TGRpcDbCountersRegistry>()->RemoveDbCounters(database); + } + }; + +public: + void Initialize(TActorSystem* actorSystem) { + if (Y_LIKELY(ActorSystem)) { + return; + } + ActorSystem = actorSystem; + + auto callback = MakeIntrusive<TGRpcDbWatcherCallback>(); + DbWatcherActorId = ActorSystem->Register(NSysView::CreateDbWatcherActor(callback)); + } + + TYdbDbCounterBlockPtr GetCounterBlock( + const TString& database, const TString& serviceName, const TString& requestName) + { + TGRpcDbCountersPtr dbCounters; + if (DbCounters.Get(database, dbCounters)) { + return dbCounters->GetCounterBlock(serviceName, requestName); + } + dbCounters = DbCounters.InsertIfAbsentWithInit(database, [&database, this] { - auto counters = MakeIntrusive<TGRpcDbCounters>(); - - if (ActorSystem) { - auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>( - NKikimrSysView::GRPC, database, counters); - - ActorSystem->Send(NSysView::MakeSysViewServiceID(ActorSystem->NodeId), evRegister.Release()); - - if (DbWatcherActorId) { - auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database); - ActorSystem->Send(DbWatcherActorId, evWatch.Release()); - } - } - - return counters; - }); - - return dbCounters->GetCounterBlock(serviceName, requestName); - } - - void RemoveDbCounters(const TString& database) { - DbCounters.Erase(database); - } - -private: - TConcurrentRWHashMap<TString, TIntrusivePtr<TGRpcDbCounters>, 256> DbCounters; - TActorSystem* ActorSystem = {}; - TActorId DbWatcherActorId; -}; - - + auto counters = MakeIntrusive<TGRpcDbCounters>(); + + if (ActorSystem) { + auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>( + NKikimrSysView::GRPC, database, counters); + + ActorSystem->Send(NSysView::MakeSysViewServiceID(ActorSystem->NodeId), evRegister.Release()); + + if (DbWatcherActorId) { + auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database); + ActorSystem->Send(DbWatcherActorId, evWatch.Release()); + } + } + + return counters; + }); + + return dbCounters->GetCounterBlock(serviceName, requestName); + } + + void RemoveDbCounters(const TString& database) { + DbCounters.Erase(database); + } + +private: + TConcurrentRWHashMap<TString, TIntrusivePtr<TGRpcDbCounters>, 256> DbCounters; + TActorSystem* ActorSystem = {}; + TActorId DbWatcherActorId; +}; + + class TYdbCounterBlockWrapper : public NGrpc::ICounterBlock { - TYdbCounterBlockPtr Common; - TString ServiceName; - TString RequestName; - bool Percentile = false; - bool Streaming = false; - - NMonitoring::TDynamicCounterPtr Root; - TYdbDbCounterBlockPtr Db; - -public: - TYdbCounterBlockWrapper(TYdbCounterBlockPtr common, const TString& serviceName, const TString& requestName, - bool percentile, bool streaming) - : Common(common) - , ServiceName(serviceName) - , RequestName(requestName) - , Percentile(percentile) - , Streaming(streaming) - , Root(new NMonitoring::TDynamicCounters) - , Db(new TYdbDbCounterBlock(Root, serviceName.c_str(), requestName.c_str(), percentile, streaming, Root)) - {} - - void CountNotOkRequest() override { - Common->CountNotOkRequest(); - Db->CountNotOkRequest(); - } - - void CountNotOkResponse() override { - Common->CountNotOkResponse(); - Db->CountNotOkResponse(); - } - - void CountNotAuthenticated() override { - Common->CountNotAuthenticated(); - Db->CountNotAuthenticated(); - } - - void CountResourceExhausted() override { - Common->CountResourceExhausted(); - Db->CountResourceExhausted(); - } - - void CountRequestsWithoutDatabase() override { - Common->CountRequestsWithoutDatabase(); - Db->CountRequestsWithoutDatabase(); - } - - void CountRequestsWithoutToken() override { - Common->CountRequestsWithoutToken(); - Db->CountRequestsWithoutToken(); - } - - void CountRequestWithoutTls() override { - Common->CountRequestWithoutTls(); - Db->CountRequestWithoutTls(); - } - - void CountRequestBytes(ui32 requestSize) override { - Common->CountRequestBytes(requestSize); - Db->CountRequestBytes(requestSize); - } - - void CountResponseBytes(ui32 responseSize) override { - Common->CountResponseBytes(responseSize); - Db->CountResponseBytes(responseSize); - } - - void StartProcessing(ui32 requestSize) override { - Common->StartProcessing(requestSize); - Db->StartProcessing(requestSize); - } - - void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status, - TDuration requestDuration) override - { - Common->FinishProcessing(requestSize, responseSize, ok, status, requestDuration); - Db->FinishProcessing(requestSize, responseSize, ok, status, requestDuration); - } - + TYdbCounterBlockPtr Common; + TString ServiceName; + TString RequestName; + bool Percentile = false; + bool Streaming = false; + + NMonitoring::TDynamicCounterPtr Root; + TYdbDbCounterBlockPtr Db; + +public: + TYdbCounterBlockWrapper(TYdbCounterBlockPtr common, const TString& serviceName, const TString& requestName, + bool percentile, bool streaming) + : Common(common) + , ServiceName(serviceName) + , RequestName(requestName) + , Percentile(percentile) + , Streaming(streaming) + , Root(new NMonitoring::TDynamicCounters) + , Db(new TYdbDbCounterBlock(Root, serviceName.c_str(), requestName.c_str(), percentile, streaming, Root)) + {} + + void CountNotOkRequest() override { + Common->CountNotOkRequest(); + Db->CountNotOkRequest(); + } + + void CountNotOkResponse() override { + Common->CountNotOkResponse(); + Db->CountNotOkResponse(); + } + + void CountNotAuthenticated() override { + Common->CountNotAuthenticated(); + Db->CountNotAuthenticated(); + } + + void CountResourceExhausted() override { + Common->CountResourceExhausted(); + Db->CountResourceExhausted(); + } + + void CountRequestsWithoutDatabase() override { + Common->CountRequestsWithoutDatabase(); + Db->CountRequestsWithoutDatabase(); + } + + void CountRequestsWithoutToken() override { + Common->CountRequestsWithoutToken(); + Db->CountRequestsWithoutToken(); + } + + void CountRequestWithoutTls() override { + Common->CountRequestWithoutTls(); + Db->CountRequestWithoutTls(); + } + + void CountRequestBytes(ui32 requestSize) override { + Common->CountRequestBytes(requestSize); + Db->CountRequestBytes(requestSize); + } + + void CountResponseBytes(ui32 responseSize) override { + Common->CountResponseBytes(responseSize); + Db->CountResponseBytes(responseSize); + } + + void StartProcessing(ui32 requestSize) override { + Common->StartProcessing(requestSize); + Db->StartProcessing(requestSize); + } + + void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status, + TDuration requestDuration) override + { + Common->FinishProcessing(requestSize, responseSize, ok, status, requestDuration); + Db->FinishProcessing(requestSize, responseSize, ok, status, requestDuration); + } + NGrpc::ICounterBlockPtr Clone() override { - return new TYdbCounterBlockWrapper(Common, ServiceName, RequestName, Percentile, Streaming); - } - - void UseDatabase(const TString& database) override { - if (database.empty()) { - return; - } - - auto block = Singleton<TGRpcDbCountersRegistry>()->GetCounterBlock( - database, ServiceName, RequestName); - block->AggregateFrom(*Db); - Db = block; - } -}; - -class TUpdaterActor - : public TActor<TUpdaterActor> -{ - TVector<TYdbCounterBlockPtr> Counters; - -public: - enum : ui32 { - EvRegisterItem = EventSpaceBegin(TEvents::ES_PRIVATE), - }; - - struct TEvRegisterItem - : TEventLocal<TEvRegisterItem, EvRegisterItem> - { - TYdbCounterBlockPtr Counters; - - TEvRegisterItem(TYdbCounterBlockPtr counters) - : Counters(std::move(counters)) - {} - }; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::GRPC_UPDATER; - } - - TUpdaterActor() - : TActor(&TThis::StateFunc) - {} - - void HandleWakeup(const TActorContext& ctx) { - ctx.Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup); - for (const auto& counter : Counters) { - counter->Update(); - } - } - - void Handle(TEvRegisterItem::TPtr& ev, const TActorContext& ctx) { - Counters.push_back(std::move(ev->Get()->Counters)); - if (Counters.size() == 1) { - HandleWakeup(ctx); - } - } - - STRICT_STFUNC(StateFunc, { - CFunc(TEvents::TSystem::Wakeup, HandleWakeup) - HFunc(TEvRegisterItem, Handle) - }) -}; - -TServiceCounterCB::TServiceCounterCB(NMonitoring::TDynamicCounterPtr counters, TActorSystem *actorSystem) - : Counters(std::move(counters)) - , ActorSystem(actorSystem) -{ - if (ActorSystem) { - ActorId = ActorSystem->Register(new TUpdaterActor); - Singleton<TGRpcDbCountersRegistry>()->Initialize(ActorSystem); - } -} - + return new TYdbCounterBlockWrapper(Common, ServiceName, RequestName, Percentile, Streaming); + } + + void UseDatabase(const TString& database) override { + if (database.empty()) { + return; + } + + auto block = Singleton<TGRpcDbCountersRegistry>()->GetCounterBlock( + database, ServiceName, RequestName); + block->AggregateFrom(*Db); + Db = block; + } +}; + +class TUpdaterActor + : public TActor<TUpdaterActor> +{ + TVector<TYdbCounterBlockPtr> Counters; + +public: + enum : ui32 { + EvRegisterItem = EventSpaceBegin(TEvents::ES_PRIVATE), + }; + + struct TEvRegisterItem + : TEventLocal<TEvRegisterItem, EvRegisterItem> + { + TYdbCounterBlockPtr Counters; + + TEvRegisterItem(TYdbCounterBlockPtr counters) + : Counters(std::move(counters)) + {} + }; + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::GRPC_UPDATER; + } + + TUpdaterActor() + : TActor(&TThis::StateFunc) + {} + + void HandleWakeup(const TActorContext& ctx) { + ctx.Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup); + for (const auto& counter : Counters) { + counter->Update(); + } + } + + void Handle(TEvRegisterItem::TPtr& ev, const TActorContext& ctx) { + Counters.push_back(std::move(ev->Get()->Counters)); + if (Counters.size() == 1) { + HandleWakeup(ctx); + } + } + + STRICT_STFUNC(StateFunc, { + CFunc(TEvents::TSystem::Wakeup, HandleWakeup) + HFunc(TEvRegisterItem, Handle) + }) +}; + +TServiceCounterCB::TServiceCounterCB(NMonitoring::TDynamicCounterPtr counters, TActorSystem *actorSystem) + : Counters(std::move(counters)) + , ActorSystem(actorSystem) +{ + if (ActorSystem) { + ActorId = ActorSystem->Register(new TUpdaterActor); + Singleton<TGRpcDbCountersRegistry>()->Initialize(ActorSystem); + } +} + NGrpc::ICounterBlockPtr TServiceCounterCB::operator()(const char* serviceName, - const char* requestName, bool percentile, bool streaming) const -{ - auto block = MakeIntrusive<TYdbCounterBlock>(Counters, serviceName, requestName, percentile, streaming); - - if (ActorSystem) { - ActorSystem->Send(ActorId, new TUpdaterActor::TEvRegisterItem(block)); - } - + const char* requestName, bool percentile, bool streaming) const +{ + auto block = MakeIntrusive<TYdbCounterBlock>(Counters, serviceName, requestName, percentile, streaming); + + if (ActorSystem) { + ActorSystem->Send(ActorId, new TUpdaterActor::TEvRegisterItem(block)); + } + NGrpc::ICounterBlockPtr res(block); - if (ActorSystem && AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters()) { - res = MakeIntrusive<TYdbCounterBlockWrapper>(block, serviceName, requestName, percentile, streaming); - } - - return res; -} - -TIntrusivePtr<NSysView::IDbCounters> CreateGRpcDbCounters( - NMonitoring::TDynamicCounterPtr externalGroup, - NMonitoring::TDynamicCounterPtr internalGroup) -{ - return new TGRpcDbCounters(externalGroup, internalGroup); -} - -} // namespace NGRpcService -} // namespace NKikimr + if (ActorSystem && AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters()) { + res = MakeIntrusive<TYdbCounterBlockWrapper>(block, serviceName, requestName, percentile, streaming); + } + + return res; +} + +TIntrusivePtr<NSysView::IDbCounters> CreateGRpcDbCounters( + NMonitoring::TDynamicCounterPtr externalGroup, + NMonitoring::TDynamicCounterPtr internalGroup) +{ + return new TGRpcDbCounters(externalGroup, internalGroup); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/counters/counters.h b/ydb/core/grpc_services/counters/counters.h index 1f01cd94c0..da499dac9e 100644 --- a/ydb/core/grpc_services/counters/counters.h +++ b/ydb/core/grpc_services/counters/counters.h @@ -1,34 +1,34 @@ -#pragma once - +#pragma once + #include <ydb/core/sys_view/common/events.h> - + #include <library/cpp/grpc/server/grpc_counters.h> - -namespace NKikimr { -namespace NGRpcService { - -class TServiceCounterCB { -public: - TServiceCounterCB(NMonitoring::TDynamicCounterPtr counters, TActorSystem *actorSystem); - + +namespace NKikimr { +namespace NGRpcService { + +class TServiceCounterCB { +public: + TServiceCounterCB(NMonitoring::TDynamicCounterPtr counters, TActorSystem *actorSystem); + NGrpc::ICounterBlockPtr operator()(const char* serviceName, const char* requestName, - bool percentile = true, bool streaming = false) const; - -private: - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; - TActorSystem* ActorSystem; - TActorId ActorId; -}; - -inline TServiceCounterCB CreateCounterCb(NMonitoring::TDynamicCounterPtr counters, - TActorSystem *actorSystem) -{ - return TServiceCounterCB(std::move(counters), actorSystem); -} - -TIntrusivePtr<NSysView::IDbCounters> CreateGRpcDbCounters( - NMonitoring::TDynamicCounterPtr externalGroup, - NMonitoring::TDynamicCounterPtr internalGroup); - -} // namespace NGRpcService -} // namespace NKikimr + bool percentile = true, bool streaming = false) const; + +private: + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + TActorSystem* ActorSystem; + TActorId ActorId; +}; + +inline TServiceCounterCB CreateCounterCb(NMonitoring::TDynamicCounterPtr counters, + TActorSystem *actorSystem) +{ + return TServiceCounterCB(std::move(counters), actorSystem); +} + +TIntrusivePtr<NSysView::IDbCounters> CreateGRpcDbCounters( + NMonitoring::TDynamicCounterPtr externalGroup, + NMonitoring::TDynamicCounterPtr internalGroup); + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/counters/ya.make b/ydb/core/grpc_services/counters/ya.make index 9247d66487..01c329a887 100644 --- a/ydb/core/grpc_services/counters/ya.make +++ b/ydb/core/grpc_services/counters/ya.make @@ -1,21 +1,21 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - counters.cpp - counters.h -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + counters.cpp + counters.h +) + +PEERDIR( ydb/core/base ydb/core/protos ydb/core/sys_view/service -) - +) + YQL_LAST_ABI_VERSION() - -END() + +END() diff --git a/ydb/core/grpc_services/grpc_helper.cpp b/ydb/core/grpc_services/grpc_helper.cpp index 8f0a59d78b..b1ef484cc7 100644 --- a/ydb/core/grpc_services/grpc_helper.cpp +++ b/ydb/core/grpc_services/grpc_helper.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NGRpcService { -//using namespace NActors; +//using namespace NActors; NGrpc::IGRpcRequestLimiterPtr TCreateLimiterCB::operator()(const char* serviceName, const char* requestName, i64 limit) const { TString fullName = TString(serviceName) + "_" + requestName; diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp index aa847ae47a..4baae606a9 100644 --- a/ydb/core/grpc_services/rpc_read_columns.cpp +++ b/ydb/core/grpc_services/rpc_read_columns.cpp @@ -86,7 +86,7 @@ public: , ShardRequestCount(0) , ShardReplyCount(0) , SysViewMaxRows(100000) - , SysViewMaxBytes(10*1024*1024) + , SysViewMaxBytes(10*1024*1024) , SysViewRowsReceived(0) {} @@ -237,11 +237,11 @@ private: "Snapshots are ignored when scanning system views")); } - if (auto maxRows = Request->GetProtoRequest()->max_rows(); maxRows && maxRows <= 100) - SysViewMaxRows = maxRows; + if (auto maxRows = Request->GetProtoRequest()->max_rows(); maxRows && maxRows <= 100) + SysViewMaxRows = maxRows; - if (auto maxBytes = Request->GetProtoRequest()->max_bytes(); maxBytes && maxBytes <= 10000) - SysViewMaxBytes = maxBytes; + if (auto maxBytes = Request->GetProtoRequest()->max_bytes(); maxBytes && maxBytes <= 10000) + SysViewMaxBytes = maxBytes; // List of columns requested by user TVector<std::pair<TString, NScheme::TTypeId>> valueColumnNamesAndTypes; @@ -315,7 +315,7 @@ private: ctx); } - SysViewScanActor = ctx.Register(tableScanActor.Release()); + SysViewScanActor = ctx.Register(tableScanActor.Release()); auto ackEv = MakeHolder<NKqp::TEvKqpCompute::TEvScanDataAck>(0); ctx.Send(SysViewScanActor, ackEv.Release()); @@ -428,18 +428,18 @@ private: } } - void Handle(NKqp::TEvKqpCompute::TEvScanError::TPtr& ev, const TActorContext& ctx) { + void Handle(NKqp::TEvKqpCompute::TEvScanError::TPtr& ev, const TActorContext& ctx) { NYql::TIssues issues; Ydb::StatusIds::StatusCode status = ev->Get()->Record.GetStatus(); NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues); - ReplyWithError(status, issues, ctx); - } - + ReplyWithError(status, issues, ctx); + } + STFUNC(StateSysViewScan) { switch (ev->GetTypeRewrite()) { HFunc(NKqp::TEvKqpCompute::TEvScanData, Handle); - HFunc(NKqp::TEvKqpCompute::TEvScanError, Handle); + HFunc(NKqp::TEvKqpCompute::TEvScanError, Handle); CFunc(TEvents::TSystem::Wakeup, HandleTimeout); default: @@ -616,7 +616,7 @@ private: // Send request to the first shard std::unique_ptr<TEvDataShard::TEvReadColumnsRequest> ev = std::make_unique<TEvDataShard::TEvReadColumnsRequest>(); - ev->Record.SetTableId(KeyRange->TableId.PathId.LocalPathId); + ev->Record.SetTableId(KeyRange->TableId.PathId.LocalPathId); for (TString col : Request->GetProtoRequest()->columns()) { ev->Record.AddColumns(col); } @@ -754,16 +754,16 @@ private: } } - void ReplyWithError(StatusIds::StatusCode status, const NYql::TIssues& issues, const TActorContext& ctx) { - Finished = true; - Request->RaiseIssues(issues); + void ReplyWithError(StatusIds::StatusCode status, const NYql::TIssues& issues, const TActorContext& ctx) { + Finished = true; + Request->RaiseIssues(issues); Request->ReplyWithYdbStatus(status); - - if (!WaitingResolveReply) { - Die(ctx); - } - } - + + if (!WaitingResolveReply) { + Die(ctx); + } + } + void ReplyWithResult(StatusIds::StatusCode status, const Ydb::ClickhouseInternal::ScanResult& result, const TActorContext& ctx) { diff --git a/ydb/core/grpc_streaming/grpc_streaming.h b/ydb/core/grpc_streaming/grpc_streaming.h index dd80817422..ecac6fcf17 100644 --- a/ydb/core/grpc_streaming/grpc_streaming.h +++ b/ydb/core/grpc_streaming/grpc_streaming.h @@ -116,7 +116,7 @@ public: virtual TString GetPeerName() const = 0; virtual TVector<TStringBuf> GetPeerMetaValues(TStringBuf key) const = 0; virtual grpc_compression_level GetCompressionLevel() const = 0; - virtual void UseDatabase(const TString& database) = 0; + virtual void UseDatabase(const TString& database) = 0; }; template<class TIn, class TOut, class TServer, int LoggerServiceId> @@ -196,7 +196,7 @@ private: void MaybeClone() { if (!Server->IsShuttingDown()) { - Start(Server, this->Service, this->CQ, AcceptRequest, AcceptCallback, ActorSystem, Name, Counters->Clone(), Limiter); + Start(Server, this->Service, this->CQ, AcceptRequest, AcceptCallback, ActorSystem, Name, Counters->Clone(), Limiter); } } @@ -605,10 +605,10 @@ private: } } - void UseDatabase(const TString& database) { - Counters->UseDatabase(database); - } - + void UseDatabase(const TString& database) { + Counters->UseDatabase(database); + } + private: bool IncRequest() { if (Limiter) { @@ -678,10 +678,10 @@ private: return Self->GetCompressionLevel(); } - void UseDatabase(const TString& database) override { - Self->UseDatabase(database); - } - + void UseDatabase(const TString& database) override { + Self->UseDatabase(database); + } + private: TIntrusivePtr<TSelf> Self; }; diff --git a/ydb/core/kqp/common/kqp_tx_info.h b/ydb/core/kqp/common/kqp_tx_info.h index fac4ac844b..a3e57414ae 100644 --- a/ydb/core/kqp/common/kqp_tx_info.h +++ b/ydb/core/kqp/common/kqp_tx_info.h @@ -1,47 +1,47 @@ -#pragma once - -#include <util/datetime/base.h> - +#pragma once + +#include <util/datetime/base.h> + #include <optional> -namespace NKikimr { -namespace NKqp { - +namespace NKikimr { +namespace NKqp { + struct TKqpForceNewEngineState { ui32 ForceNewEnginePercent = 0; ui32 ForceNewEngineLevel = 0; std::optional<bool> ForcedNewEngine; }; -struct TKqpTransactionInfo { - enum class EKind { - Pure, - ReadOnly, - WriteOnly, - ReadWrite - }; - - enum class EStatus { - Active, - Committed, - Aborted - }; - +struct TKqpTransactionInfo { + enum class EKind { + Pure, + ReadOnly, + WriteOnly, + ReadWrite + }; + + enum class EStatus { + Active, + Committed, + Aborted + }; + enum class EEngine { OldEngine, NewEngine }; -public: - EStatus Status; - EKind Kind; +public: + EStatus Status; + EKind Kind; std::optional<EEngine> TxEngine; - TDuration TotalDuration; - TDuration ServerDuration; - ui32 QueriesCount = 0; + TDuration TotalDuration; + TDuration ServerDuration; + ui32 QueriesCount = 0; TKqpForceNewEngineState ForceNewEngineState; -}; - -} // namespace NKqp -} // namespace NKikimr - +}; + +} // namespace NKqp +} // namespace NKikimr + diff --git a/ydb/core/kqp/counters/kqp_counters.cpp b/ydb/core/kqp/counters/kqp_counters.cpp index 41eeba73d5..26a28a0249 100644 --- a/ydb/core/kqp/counters/kqp_counters.cpp +++ b/ydb/core/kqp/counters/kqp_counters.cpp @@ -7,11 +7,11 @@ #include <ydb/core/sys_view/service/sysview_service.h> #include <library/cpp/actors/core/log.h> - + #include <util/generic/size_literals.h> #include <ydb/library/yql/core/issue/protos/issue_id.pb.h> - + namespace NKikimr { namespace NKqp { @@ -22,33 +22,33 @@ NMonitoring::TDynamicCounterPtr TKqpCountersBase::GetQueryReplayCounters() const return QueryReplayGroup; } -void TKqpCountersBase::CreateYdbTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name) { - auto txKindGroup = YdbGroup->GetSubgroup("tx_kind", name); - - auto& ydbTxCounters = YdbTxByKind[kind]; - ydbTxCounters.TotalDuration = txKindGroup->GetNamedHistogram("name", - "table.transaction.total_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); - ydbTxCounters.ServerDuration = txKindGroup->GetNamedHistogram("name", - "table.transaction.server_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); - ydbTxCounters.ClientDuration = txKindGroup->GetNamedHistogram("name", - "table.transaction.client_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); -} - -void TKqpCountersBase::UpdateYdbTxCounters(const TKqpTransactionInfo& txInfo, - THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters>& txCounters) -{ - auto byKind = txCounters.FindPtr(txInfo.Kind); - if (!byKind) { - return; - } - - byKind->TotalDuration->Collect(txInfo.TotalDuration.MilliSeconds()); - byKind->ServerDuration->Collect(txInfo.ServerDuration.MilliSeconds()); - auto clientDuration = txInfo.TotalDuration - txInfo.ServerDuration; - byKind->ClientDuration->Collect(clientDuration.MilliSeconds()); -} - -void TKqpCountersBase::Init() { +void TKqpCountersBase::CreateYdbTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name) { + auto txKindGroup = YdbGroup->GetSubgroup("tx_kind", name); + + auto& ydbTxCounters = YdbTxByKind[kind]; + ydbTxCounters.TotalDuration = txKindGroup->GetNamedHistogram("name", + "table.transaction.total_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); + ydbTxCounters.ServerDuration = txKindGroup->GetNamedHistogram("name", + "table.transaction.server_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); + ydbTxCounters.ClientDuration = txKindGroup->GetNamedHistogram("name", + "table.transaction.client_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); +} + +void TKqpCountersBase::UpdateYdbTxCounters(const TKqpTransactionInfo& txInfo, + THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters>& txCounters) +{ + auto byKind = txCounters.FindPtr(txInfo.Kind); + if (!byKind) { + return; + } + + byKind->TotalDuration->Collect(txInfo.TotalDuration.MilliSeconds()); + byKind->ServerDuration->Collect(txInfo.ServerDuration.MilliSeconds()); + auto clientDuration = txInfo.TotalDuration - txInfo.ServerDuration; + byKind->ClientDuration->Collect(clientDuration.MilliSeconds()); +} + +void TKqpCountersBase::Init() { /* Requests */ QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetCounter("Requests/QueryExecute", true); @@ -75,10 +75,10 @@ void TKqpCountersBase::Init() { PingSessionRequests = KqpGroup->GetCounter("Requests/PingSession", true); RequestBytes = KqpGroup->GetCounter("Requests/Bytes", true); - YdbRequestBytes = YdbGroup->GetNamedCounter("name", "table.query.request.bytes", true); + YdbRequestBytes = YdbGroup->GetNamedCounter("name", "table.query.request.bytes", true); QueryBytes = KqpGroup->GetCounter("Requests/QueryBytes", true); ParametersBytes = KqpGroup->GetCounter("Requests/ParametersBytes", true); - YdbParametersBytes = YdbGroup->GetNamedCounter("name", "table.query.request.parameters_bytes", true); + YdbParametersBytes = YdbGroup->GetNamedCounter("name", "table.query.request.parameters_bytes", true); SqlV0Translations = KqpGroup->GetCounter("Requests/Sql/V0", true); SqlV1Translations = KqpGroup->GetCounter("Requests/Sql/V1", true); @@ -121,30 +121,30 @@ void TKqpCountersBase::Init() { NMonitoring::ExplicitHistogram({1_MB, 9_MB, 29_MB})); /* Request latency */ - QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram( - "Query/ExecuteLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1)); - QueryLatencies[NKikimrKqp::QUERY_ACTION_EXPLAIN] = KqpGroup->GetHistogram( - "Query/ExplainLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); - QueryLatencies[NKikimrKqp::QUERY_ACTION_VALIDATE] = KqpGroup->GetHistogram( - "Query/ValidateLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); - QueryLatencies[NKikimrKqp::QUERY_ACTION_PREPARE] = KqpGroup->GetHistogram( - "Query/PrepareLatencyMs", NMonitoring::ExponentialHistogram(16, 2, 1)); - QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram( - "Query/ExecPreparedLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1)); - QueryLatencies[NKikimrKqp::QUERY_ACTION_BEGIN_TX] = KqpGroup->GetHistogram( - "Query/BeginTxLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); - QueryLatencies[NKikimrKqp::QUERY_ACTION_COMMIT_TX] = KqpGroup->GetHistogram( - "Query/CommitTxLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1)); - QueryLatencies[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX] = KqpGroup->GetHistogram( - "Query/RollbackTxLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); - - YdbQueryExecuteLatency = YdbGroup->GetNamedHistogram("name", - "table.query.execution.latency_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); - - // TODO move to grpc - YdbResponsesLocksInvalidated = YdbGroup->GetSubgroup("issue_type", "optimistic_locks_invalidation") - ->GetNamedCounter("name", "api.grpc.response.issues", true); - + QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram( + "Query/ExecuteLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1)); + QueryLatencies[NKikimrKqp::QUERY_ACTION_EXPLAIN] = KqpGroup->GetHistogram( + "Query/ExplainLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); + QueryLatencies[NKikimrKqp::QUERY_ACTION_VALIDATE] = KqpGroup->GetHistogram( + "Query/ValidateLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); + QueryLatencies[NKikimrKqp::QUERY_ACTION_PREPARE] = KqpGroup->GetHistogram( + "Query/PrepareLatencyMs", NMonitoring::ExponentialHistogram(16, 2, 1)); + QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram( + "Query/ExecPreparedLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1)); + QueryLatencies[NKikimrKqp::QUERY_ACTION_BEGIN_TX] = KqpGroup->GetHistogram( + "Query/BeginTxLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); + QueryLatencies[NKikimrKqp::QUERY_ACTION_COMMIT_TX] = KqpGroup->GetHistogram( + "Query/CommitTxLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1)); + QueryLatencies[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX] = KqpGroup->GetHistogram( + "Query/RollbackTxLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); + + YdbQueryExecuteLatency = YdbGroup->GetNamedHistogram("name", + "table.query.execution.latency_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); + + // TODO move to grpc + YdbResponsesLocksInvalidated = YdbGroup->GetSubgroup("issue_type", "optimistic_locks_invalidation") + ->GetNamedCounter("name", "api.grpc.response.issues", true); + /* Response statuses */ YdbResponses[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED] = KqpGroup->GetCounter("YdbResponses/Unspecified", true); YdbResponses[Ydb::StatusIds::SUCCESS] = KqpGroup->GetCounter("YdbResponses/Success", true); @@ -169,18 +169,18 @@ void TKqpCountersBase::Init() { OtherYdbResponses = KqpGroup->GetCounter("YdbResponses/Other", true); ResponseBytes = KqpGroup->GetCounter("Responses/Bytes", true); - YdbResponseBytes = YdbGroup->GetNamedCounter("name", "table.query.response.bytes", true); + YdbResponseBytes = YdbGroup->GetNamedCounter("name", "table.query.response.bytes", true); QueryResultsBytes = KqpGroup->GetCounter("Responses/QueryResultBytes", true); /* Workers */ - WorkerLifeSpan = KqpGroup->GetHistogram( - "Workers/LifeSpanMs", NMonitoring::ExponentialHistogram(20, 2, 1)); - QueriesPerWorker = KqpGroup->GetHistogram( - "Workers/QueriesPerWorkerQ", NMonitoring::ExponentialHistogram(16, 2, 1)); - + WorkerLifeSpan = KqpGroup->GetHistogram( + "Workers/LifeSpanMs", NMonitoring::ExponentialHistogram(20, 2, 1)); + QueriesPerWorker = KqpGroup->GetHistogram( + "Workers/QueriesPerWorkerQ", NMonitoring::ExponentialHistogram(16, 2, 1)); + WorkersCreated = KqpGroup->GetCounter("Workers/Created", true); WorkersClosedIdle = KqpGroup->GetCounter("Workers/ClosedIdle", true); - YdbWorkersClosedIdle = YdbGroup->GetNamedCounter("name", "table.session.closed_by_idle_count", true); + YdbWorkersClosedIdle = YdbGroup->GetNamedCounter("name", "table.session.closed_by_idle_count", true); WorkersClosedError = KqpGroup->GetCounter("Workers/ClosedError", true); WorkersClosedRequest = KqpGroup->GetCounter("Workers/ClosedRequest", true); ActiveWorkers = KqpGroup->GetCounter("Workers/Active", false); @@ -189,49 +189,49 @@ void TKqpCountersBase::Init() { SessionBalancerCV = KqpGroup->GetCounter("SessionBalancer/CV", false); SessionBalancerShutdowns = KqpGroup->GetCounter("SessionBalancer/Shutdown", true); - YdbActiveWorkers = YdbGroup->GetNamedCounter("name", "table.session.active_count", false); + YdbActiveWorkers = YdbGroup->GetNamedCounter("name", "table.session.active_count", false); + + WorkerCleanupLatency = KqpGroup->GetHistogram( + "Workers/CleanupLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); - WorkerCleanupLatency = KqpGroup->GetHistogram( - "Workers/CleanupLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1)); - /* Transactions */ TxCreated = KqpGroup->GetCounter("Transactions/Created", true); TxAborted = KqpGroup->GetCounter("Transactions/Aborted", true); TxCommited = KqpGroup->GetCounter("Transactions/Commited", true); TxEvicted = KqpGroup->GetCounter("Transactions/Evicted", true); - TxActivePerSession = KqpGroup->GetHistogram( - "Transactions/TxActivePerSession", NMonitoring::ExponentialHistogram(16, 2, 1)); - TxAbortedPerSession = KqpGroup->GetHistogram( - "Transactions/TxAbortedPerSession", NMonitoring::ExponentialHistogram(16, 2, 1)); - - CompileRequestsCompile = KqpGroup->GetCounter("Compilation/Requests/Compile", true); - CompileRequestsGet = KqpGroup->GetCounter("Compilation/Requests/Get", true); - CompileRequestsInvalidate = KqpGroup->GetCounter("Compilation/Requests/Invalidate", true); - CompileRequestsRejected = KqpGroup->GetCounter("Compilation/Requests/Rejected", true); - CompileRequestsTimeout = KqpGroup->GetCounter("Compilation/Requests/Timeout", true); + TxActivePerSession = KqpGroup->GetHistogram( + "Transactions/TxActivePerSession", NMonitoring::ExponentialHistogram(16, 2, 1)); + TxAbortedPerSession = KqpGroup->GetHistogram( + "Transactions/TxAbortedPerSession", NMonitoring::ExponentialHistogram(16, 2, 1)); + + CompileRequestsCompile = KqpGroup->GetCounter("Compilation/Requests/Compile", true); + CompileRequestsGet = KqpGroup->GetCounter("Compilation/Requests/Get", true); + CompileRequestsInvalidate = KqpGroup->GetCounter("Compilation/Requests/Invalidate", true); + CompileRequestsRejected = KqpGroup->GetCounter("Compilation/Requests/Rejected", true); + CompileRequestsTimeout = KqpGroup->GetCounter("Compilation/Requests/Timeout", true); CompileRequestsRecompile = KqpGroup->GetCounter("Compilation/Requests/Recompile", true); - - CompileCpuTime = KqpGroup->GetHistogram( - "Compilation/CPUTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1)); - - CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::Pure, "pure"); - CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::ReadOnly, "read_only"); - CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::WriteOnly, "write_only"); - CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::ReadWrite, "read_write"); - - CompileQueryCacheHits = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_hits", true); - CompileQueryCacheMisses = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_misses", true); - - CompileTotal = YdbGroup->GetNamedCounter("name", "table.query.compilation.count", true); - CompileErrors = YdbGroup->GetNamedCounter("name", "table.query.compilation.error_count", true); - CompileActive = YdbGroup->GetNamedCounter("name", "table.query.compilation.active_count", false); - - YdbCompileDuration = YdbGroup->GetNamedHistogram("name", - "table.query.compilation.latency_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); -} - -void TKqpCountersBase::ReportQueryAction(NKikimrKqp::EQueryAction action) { + + CompileCpuTime = KqpGroup->GetHistogram( + "Compilation/CPUTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1)); + + CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::Pure, "pure"); + CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::ReadOnly, "read_only"); + CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::WriteOnly, "write_only"); + CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::ReadWrite, "read_write"); + + CompileQueryCacheHits = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_hits", true); + CompileQueryCacheMisses = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_misses", true); + + CompileTotal = YdbGroup->GetNamedCounter("name", "table.query.compilation.count", true); + CompileErrors = YdbGroup->GetNamedCounter("name", "table.query.compilation.error_count", true); + CompileActive = YdbGroup->GetNamedCounter("name", "table.query.compilation.active_count", false); + + YdbCompileDuration = YdbGroup->GetNamedHistogram("name", + "table.query.compilation.latency_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1)); +} + +void TKqpCountersBase::ReportQueryAction(NKikimrKqp::EQueryAction action) { auto counter = QueryActionRequests.FindPtr(action); if (counter) { (*counter)->Inc(); @@ -241,7 +241,7 @@ void TKqpCountersBase::ReportQueryAction(NKikimrKqp::EQueryAction action) { OtherQueryRequests->Inc(); } -void TKqpCountersBase::ReportQueryType(NKikimrKqp::EQueryType type) { +void TKqpCountersBase::ReportQueryType(NKikimrKqp::EQueryType type) { auto counter = QueryTypes.FindPtr(type); if (counter) { (*counter)->Inc(); @@ -255,40 +255,40 @@ void TKqpCountersBase::ReportSessionShutdownRequest() { SessionBalancerShutdowns->Inc(); } -void TKqpCountersBase::ReportCreateSession(ui64 requestSize) { - CreateSessionRequests->Inc(); - *RequestBytes += requestSize; - *YdbRequestBytes += requestSize; -} - -void TKqpCountersBase::ReportPingSession(ui64 requestSize) { - PingSessionRequests->Inc(); - *RequestBytes += requestSize; - *YdbRequestBytes += requestSize; -} - -void TKqpCountersBase::ReportCloseSession(ui64 requestSize) { - CloseSessionRequests->Inc(); - *RequestBytes += requestSize; - *YdbRequestBytes += requestSize; -} - -void TKqpCountersBase::ReportQueryRequest(const NKikimrKqp::TQueryRequest& request) { - ReportQueryAction(request.GetAction()); - ReportQueryType(request.GetType()); - - auto requestBytes = request.ByteSize(); - auto parametersBytes = request.GetParameters().ByteSize(); - - *RequestBytes += requestBytes; - *YdbRequestBytes += requestBytes; - - *QueryBytes += request.GetQuery().size(); - - *ParametersBytes += parametersBytes; - *YdbParametersBytes += parametersBytes; -} - +void TKqpCountersBase::ReportCreateSession(ui64 requestSize) { + CreateSessionRequests->Inc(); + *RequestBytes += requestSize; + *YdbRequestBytes += requestSize; +} + +void TKqpCountersBase::ReportPingSession(ui64 requestSize) { + PingSessionRequests->Inc(); + *RequestBytes += requestSize; + *YdbRequestBytes += requestSize; +} + +void TKqpCountersBase::ReportCloseSession(ui64 requestSize) { + CloseSessionRequests->Inc(); + *RequestBytes += requestSize; + *YdbRequestBytes += requestSize; +} + +void TKqpCountersBase::ReportQueryRequest(const NKikimrKqp::TQueryRequest& request) { + ReportQueryAction(request.GetAction()); + ReportQueryType(request.GetType()); + + auto requestBytes = request.ByteSize(); + auto parametersBytes = request.GetParameters().ByteSize(); + + *RequestBytes += requestBytes; + *YdbRequestBytes += requestBytes; + + *QueryBytes += request.GetQuery().size(); + + *ParametersBytes += parametersBytes; + *YdbParametersBytes += parametersBytes; +} + void TKqpCountersBase::ReportQueryWithRangeScan() { QueriesWithRangeScan->Inc(); } @@ -322,9 +322,9 @@ void TKqpCountersBase::ReportQueryMaxShardProgramSize(ui64 programSize) { } void TKqpCountersBase::ReportResponseStatus(ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus) { - *ResponseBytes += responseSize; - *YdbResponseBytes += responseSize; - + *ResponseBytes += responseSize; + *YdbResponseBytes += responseSize; + auto ydbCounter = YdbResponses.FindPtr(ydbStatus); if (ydbCounter) { (*ydbCounter)->Inc(); @@ -333,11 +333,11 @@ void TKqpCountersBase::ReportResponseStatus(ui64 responseSize, Ydb::StatusIds::S } } -void TKqpCountersBase::ReportResultsBytes(ui64 resultsBytes) { - *QueryResultsBytes += resultsBytes; -} - -TString TKqpCountersBase::GetIssueName(ui32 issueCode) { +void TKqpCountersBase::ReportResultsBytes(ui64 resultsBytes) { + *QueryResultsBytes += resultsBytes; +} + +TString TKqpCountersBase::GetIssueName(ui32 issueCode) { auto kikimrIssueDescriptor = NKikimrIssues::TIssuesIds::EIssueCode_descriptor(); if (kikimrIssueDescriptor) { auto valueDescriptor = kikimrIssueDescriptor->FindValueByNumber(issueCode); @@ -357,7 +357,7 @@ TString TKqpCountersBase::GetIssueName(ui32 issueCode) { return TStringBuilder() << "CODE:" << ToString(issueCode); } -void TKqpCountersBase::ReportIssues(const Ydb::Issue::IssueMessage& issue) { +void TKqpCountersBase::ReportIssues(const Ydb::Issue::IssueMessage& issue) { auto issueCounter = IssueCounters.FindPtr(issue.issue_code()); if (!issueCounter) { auto counterName = TStringBuilder() << "Issues/" << GetIssueName(issue.issue_code()); @@ -378,23 +378,23 @@ void TKqpCountersBase::ReportIssues(const Ydb::Issue::IssueMessage& issue) { } } -void TKqpCountersBase::ReportQueryLatency(NKikimrKqp::EQueryAction action, const TDuration& duration) { +void TKqpCountersBase::ReportQueryLatency(NKikimrKqp::EQueryAction action, const TDuration& duration) { if (action == NKikimrKqp::QUERY_ACTION_EXECUTE || action == NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED || action == NKikimrKqp::QUERY_ACTION_COMMIT_TX || action == NKikimrKqp::QUERY_ACTION_ROLLBACK_TX) { - YdbQueryExecuteLatency->Collect(duration.MilliSeconds()); - } - + YdbQueryExecuteLatency->Collect(duration.MilliSeconds()); + } + auto counter = QueryLatencies.FindPtr(action); - if (counter) { - (*counter)->Collect(duration.MilliSeconds()); + if (counter) { + (*counter)->Collect(duration.MilliSeconds()); } } -void TKqpCountersBase::ReportTransaction(const TKqpTransactionInfo& txInfo) { +void TKqpCountersBase::ReportTransaction(const TKqpTransactionInfo& txInfo) { switch (txInfo.Status) { case TKqpTransactionInfo::EStatus::Active: return; @@ -405,10 +405,10 @@ void TKqpCountersBase::ReportTransaction(const TKqpTransactionInfo& txInfo) { TxCommited->Inc(); break; } - UpdateYdbTxCounters(txInfo, YdbTxByKind); + UpdateYdbTxCounters(txInfo, YdbTxByKind); } -void TKqpCountersBase::ReportSqlVersion(ui16 sqlVersion) { +void TKqpCountersBase::ReportSqlVersion(ui16 sqlVersion) { switch (sqlVersion) { case 0: SqlV0Translations->Inc(); @@ -422,12 +422,12 @@ void TKqpCountersBase::ReportSqlVersion(ui16 sqlVersion) { } } -void TKqpCountersBase::ReportWorkerCreated() { - WorkersCreated->Inc(); - ActiveWorkers->Inc(); - YdbActiveWorkers->Inc(); -} - +void TKqpCountersBase::ReportWorkerCreated() { + WorkersCreated->Inc(); + ActiveWorkers->Inc(); + YdbActiveWorkers->Inc(); +} + void TKqpCountersBase::ReportSessionBalancerCV(ui32 value) { SessionBalancerCV->Set(value); @@ -437,324 +437,324 @@ void TKqpCountersBase::ReportProxyForwardedRequest() { ProxyForwardedRequests->Inc(); } -void TKqpCountersBase::ReportWorkerFinished(TDuration lifeSpan) { - WorkerLifeSpan->Collect(lifeSpan.MilliSeconds()); - ActiveWorkers->Dec(); - YdbActiveWorkers->Dec(); -} - -void TKqpCountersBase::ReportWorkerCleanupLatency(TDuration cleanupTime) { - WorkerCleanupLatency->Collect(cleanupTime.MilliSeconds()); -} - -void TKqpCountersBase::ReportWorkerClosedIdle() { - WorkersClosedIdle->Inc(); - YdbWorkersClosedIdle->Inc(); -} - -void TKqpCountersBase::ReportWorkerClosedError() { - WorkersClosedError->Inc(); -} - -void TKqpCountersBase::ReportWorkerClosedRequest() { - WorkersClosedRequest->Inc(); -} - -void TKqpCountersBase::ReportQueriesPerWorker(ui32 queryId) { - QueriesPerWorker->Collect(queryId); -} - -void TKqpCountersBase::ReportBeginTransaction(ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx) { - TxEvicted->Add(evictedTx); - TxAborted->Add(evictedTx); - TxActivePerSession->Collect(currentActiveTx); - TxAbortedPerSession->Collect(currentAbortedTx); -} - -void TKqpCountersBase::ReportTxCreated() { - TxCreated->Inc(); -} - -void TKqpCountersBase::ReportTxAborted(ui32 abortedCount) { - TxAborted->Add(abortedCount); -} - -void TKqpCountersBase::ReportQueryCacheHit(bool hit) { - if (hit) { - CompileQueryCacheHits->Inc(); - } else { - CompileQueryCacheMisses->Inc(); - } -} - -void TKqpCountersBase::ReportCompileStart() { - CompileTotal->Inc(); - CompileActive->Inc(); -} - -void TKqpCountersBase::ReportCompileFinish() { - CompileActive->Dec(); -} - -void TKqpCountersBase::ReportCompileError() { - CompileErrors->Inc(); -} - -void TKqpCountersBase::ReportCompileRequestCompile() { - CompileRequestsCompile->Inc(); -} - -void TKqpCountersBase::ReportCompileRequestGet() { - CompileRequestsGet->Inc(); -} - -void TKqpCountersBase::ReportCompileRequestInvalidate() { - CompileRequestsInvalidate->Inc(); -} - -void TKqpCountersBase::ReportCompileRequestRejected() { - CompileRequestsRejected->Inc(); -} - -void TKqpCountersBase::ReportCompileRequestTimeout() { - CompileRequestsTimeout->Inc(); -} - -void TKqpCountersBase::ReportCompileDurations(TDuration duration, TDuration cpuTime) { - YdbCompileDuration->Collect(duration.MilliSeconds()); - CompileCpuTime->Collect(cpuTime.MilliSeconds()); -} - +void TKqpCountersBase::ReportWorkerFinished(TDuration lifeSpan) { + WorkerLifeSpan->Collect(lifeSpan.MilliSeconds()); + ActiveWorkers->Dec(); + YdbActiveWorkers->Dec(); +} + +void TKqpCountersBase::ReportWorkerCleanupLatency(TDuration cleanupTime) { + WorkerCleanupLatency->Collect(cleanupTime.MilliSeconds()); +} + +void TKqpCountersBase::ReportWorkerClosedIdle() { + WorkersClosedIdle->Inc(); + YdbWorkersClosedIdle->Inc(); +} + +void TKqpCountersBase::ReportWorkerClosedError() { + WorkersClosedError->Inc(); +} + +void TKqpCountersBase::ReportWorkerClosedRequest() { + WorkersClosedRequest->Inc(); +} + +void TKqpCountersBase::ReportQueriesPerWorker(ui32 queryId) { + QueriesPerWorker->Collect(queryId); +} + +void TKqpCountersBase::ReportBeginTransaction(ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx) { + TxEvicted->Add(evictedTx); + TxAborted->Add(evictedTx); + TxActivePerSession->Collect(currentActiveTx); + TxAbortedPerSession->Collect(currentAbortedTx); +} + +void TKqpCountersBase::ReportTxCreated() { + TxCreated->Inc(); +} + +void TKqpCountersBase::ReportTxAborted(ui32 abortedCount) { + TxAborted->Add(abortedCount); +} + +void TKqpCountersBase::ReportQueryCacheHit(bool hit) { + if (hit) { + CompileQueryCacheHits->Inc(); + } else { + CompileQueryCacheMisses->Inc(); + } +} + +void TKqpCountersBase::ReportCompileStart() { + CompileTotal->Inc(); + CompileActive->Inc(); +} + +void TKqpCountersBase::ReportCompileFinish() { + CompileActive->Dec(); +} + +void TKqpCountersBase::ReportCompileError() { + CompileErrors->Inc(); +} + +void TKqpCountersBase::ReportCompileRequestCompile() { + CompileRequestsCompile->Inc(); +} + +void TKqpCountersBase::ReportCompileRequestGet() { + CompileRequestsGet->Inc(); +} + +void TKqpCountersBase::ReportCompileRequestInvalidate() { + CompileRequestsInvalidate->Inc(); +} + +void TKqpCountersBase::ReportCompileRequestRejected() { + CompileRequestsRejected->Inc(); +} + +void TKqpCountersBase::ReportCompileRequestTimeout() { + CompileRequestsTimeout->Inc(); +} + +void TKqpCountersBase::ReportCompileDurations(TDuration duration, TDuration cpuTime) { + YdbCompileDuration->Collect(duration.MilliSeconds()); + CompileCpuTime->Collect(cpuTime.MilliSeconds()); +} + void TKqpCountersBase::ReportRecompileRequestGet() { CompileRequestsRecompile->Inc(); } - - -TKqpDbCounters::TKqpDbCounters() { - Counters = new NMonitoring::TDynamicCounters(); - KqpGroup = Counters->GetSubgroup("group", "kqp"); - YdbGroup = Counters->GetSubgroup("group", "ydb"); - - Init(); -} - -TKqpDbCounters::TKqpDbCounters(const NMonitoring::TDynamicCounterPtr& externalGroup, - const NMonitoring::TDynamicCounterPtr& internalGroup) -{ - Counters = internalGroup; - KqpGroup = Counters->GetSubgroup("group", "kqp"); - YdbGroup = externalGroup; + + +TKqpDbCounters::TKqpDbCounters() { + Counters = new NMonitoring::TDynamicCounters(); + KqpGroup = Counters->GetSubgroup("group", "kqp"); + YdbGroup = Counters->GetSubgroup("group", "ydb"); + + Init(); +} + +TKqpDbCounters::TKqpDbCounters(const NMonitoring::TDynamicCounterPtr& externalGroup, + const NMonitoring::TDynamicCounterPtr& internalGroup) +{ + Counters = internalGroup; + KqpGroup = Counters->GetSubgroup("group", "kqp"); + YdbGroup = externalGroup; QueryReplayGroup = KqpGroup->GetSubgroup("subsystem", "unified_agent_query_replay"); - - Init(); -} - -template <typename T> -void SaveHistogram(T& histogram, int index, const NMon::THistogramCounterHelper& helper) { - auto* buckets = histogram[index].MutableBuckets(); - auto count = helper.GetBucketCount(); - buckets->Resize(count, 0); - for (size_t i = 0; i < count; ++i) { - (*buckets)[i] = helper.GetBucketValue(i); - } -} - -template <typename T> -void SaveHistogram(T& histogram, int index, const NMonitoring::THistogramPtr& hgram) { - auto* buckets = histogram[index].MutableBuckets(); - auto snapshot = hgram->Snapshot(); - auto count = snapshot->Count(); - buckets->Resize(count, 0); - for (size_t i = 0; i < count; ++i) { - (*buckets)[i] = snapshot->Value(i); - } -} - -void TKqpDbCounters::ToProto(NKikimr::NSysView::TDbServiceCounters& counters) { - auto* main = counters.Proto().MutableMain(); - auto* simple = main->MutableSimple(); - auto* cumulative = main->MutableCumulative(); - auto* histogram = main->MutableHistogram(); - - simple->Resize(DB_KQP_SIMPLE_COUNTER_SIZE, 0); - cumulative->Resize(DB_KQP_CUMULATIVE_COUNTER_SIZE, 0); - if (main->HistogramSize() < DB_KQP_HISTOGRAM_COUNTER_SIZE) { - auto missing = DB_KQP_HISTOGRAM_COUNTER_SIZE - main->HistogramSize(); - for (; missing > 0; --missing) { - main->AddHistogram(); - } - } - -#define SAVE_SIMPLE_COUNTER(INDEX, TARGET) { (*simple)[INDEX] = (TARGET)->Val(); } -#define SAVE_CUMULATIVE_COUNTER(INDEX, TARGET) { (*cumulative)[INDEX] = (TARGET)->Val(); } -#define SAVE_HISTOGRAM_COUNTER(INDEX, TARGET) { SaveHistogram(*histogram, INDEX, (TARGET)); } - - DB_KQP_SIMPLE_COUNTERS_MAP(SAVE_SIMPLE_COUNTER) - DB_KQP_CUMULATIVE_COUNTERS_MAP(SAVE_CUMULATIVE_COUNTER) - DB_KQP_HISTOGRAM_COUNTERS_MAP(SAVE_HISTOGRAM_COUNTER) -} - -template <typename T> -void LoadHistogram(T& histogram, int index, NMon::THistogramCounterHelper& helper) { - auto* buckets = histogram[index].MutableBuckets(); - auto count = helper.GetBucketCount(); - buckets->Resize(count, 0); - for (size_t i = 0; i < count; ++i) { - helper.SetBucketValue(i, (*buckets)[i]); - } -} - -template <typename T> -void LoadHistogram(T& histogram, int index, NMonitoring::THistogramPtr& hgram) { - auto* buckets = histogram[index].MutableBuckets(); - auto snapshot = hgram->Snapshot(); - auto count = snapshot->Count(); - buckets->Resize(count, 0); - hgram->Reset(); - for (ui32 i = 0; i < count; ++i) { - hgram->Collect(snapshot->UpperBound(i), (*buckets)[i]); - } -} - -void TKqpDbCounters::FromProto(NKikimr::NSysView::TDbServiceCounters& counters) { - auto* main = counters.Proto().MutableMain(); - auto* simple = main->MutableSimple(); - auto* cumulative = main->MutableCumulative(); - auto* histogram = main->MutableHistogram(); - - simple->Resize(DB_KQP_SIMPLE_COUNTER_SIZE, 0); - cumulative->Resize(DB_KQP_CUMULATIVE_COUNTER_SIZE, 0); - if (main->HistogramSize() < DB_KQP_HISTOGRAM_COUNTER_SIZE) { - auto missing = DB_KQP_HISTOGRAM_COUNTER_SIZE - main->HistogramSize(); - for (; missing > 0; --missing) { - main->AddHistogram(); - } - } - -#define LOAD_SIMPLE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*simple)[INDEX]); } -#define LOAD_CUMULATIVE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*cumulative)[INDEX]); } -#define LOAD_HISTOGRAM_COUNTER(INDEX, TARGET) { LoadHistogram(*histogram, INDEX, (TARGET)); } - - DB_KQP_SIMPLE_COUNTERS_MAP(LOAD_SIMPLE_COUNTER) - DB_KQP_CUMULATIVE_COUNTERS_MAP(LOAD_CUMULATIVE_COUNTER) - DB_KQP_HISTOGRAM_COUNTERS_MAP(LOAD_HISTOGRAM_COUNTER) -} - - -class TKqpDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback { - TIntrusivePtr<TKqpCounters> Counters; - -public: - explicit TKqpDbWatcherCallback(TIntrusivePtr<TKqpCounters> counters) - : Counters(counters) - {} - - void OnDatabaseRemoved(const TString& database, TPathId) override { - Counters->RemoveDbCounters(database); - } -}; - - -void TKqpCounters::CreateTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name) { - auto& txKindCounters = TxByKind[kind]; - - txKindCounters.TotalDuration = KqpGroup->GetHistogram("Transactions/" + name + "/TotalDurationMs", - NMonitoring::ExponentialHistogram(20, 2, 1)); - - txKindCounters.ServerDuration = KqpGroup->GetHistogram("Transactions/" + name + "/ServerDurationMs", - NMonitoring::ExponentialHistogram(20, 2, 1)); - - txKindCounters.ClientDuration = KqpGroup->GetHistogram("Transactions/" + name + "/ClientDurationMs", - NMonitoring::ExponentialHistogram(20, 2, 1)); - - txKindCounters.Queries = KqpGroup->GetHistogram("Transactions/" + name + "/Queries", - NMonitoring::ExponentialHistogram(16, 2, 1)); -} - -void TKqpCounters::UpdateTxCounters(const TKqpTransactionInfo& txInfo, - THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters>& txCounters) -{ - auto byKind = txCounters.FindPtr(txInfo.Kind); - if (!byKind) { - return; - } - - byKind->TotalDuration->Collect(txInfo.TotalDuration.MilliSeconds()); - byKind->ServerDuration->Collect(txInfo.ServerDuration.MilliSeconds()); + + Init(); +} + +template <typename T> +void SaveHistogram(T& histogram, int index, const NMon::THistogramCounterHelper& helper) { + auto* buckets = histogram[index].MutableBuckets(); + auto count = helper.GetBucketCount(); + buckets->Resize(count, 0); + for (size_t i = 0; i < count; ++i) { + (*buckets)[i] = helper.GetBucketValue(i); + } +} + +template <typename T> +void SaveHistogram(T& histogram, int index, const NMonitoring::THistogramPtr& hgram) { + auto* buckets = histogram[index].MutableBuckets(); + auto snapshot = hgram->Snapshot(); + auto count = snapshot->Count(); + buckets->Resize(count, 0); + for (size_t i = 0; i < count; ++i) { + (*buckets)[i] = snapshot->Value(i); + } +} + +void TKqpDbCounters::ToProto(NKikimr::NSysView::TDbServiceCounters& counters) { + auto* main = counters.Proto().MutableMain(); + auto* simple = main->MutableSimple(); + auto* cumulative = main->MutableCumulative(); + auto* histogram = main->MutableHistogram(); + + simple->Resize(DB_KQP_SIMPLE_COUNTER_SIZE, 0); + cumulative->Resize(DB_KQP_CUMULATIVE_COUNTER_SIZE, 0); + if (main->HistogramSize() < DB_KQP_HISTOGRAM_COUNTER_SIZE) { + auto missing = DB_KQP_HISTOGRAM_COUNTER_SIZE - main->HistogramSize(); + for (; missing > 0; --missing) { + main->AddHistogram(); + } + } + +#define SAVE_SIMPLE_COUNTER(INDEX, TARGET) { (*simple)[INDEX] = (TARGET)->Val(); } +#define SAVE_CUMULATIVE_COUNTER(INDEX, TARGET) { (*cumulative)[INDEX] = (TARGET)->Val(); } +#define SAVE_HISTOGRAM_COUNTER(INDEX, TARGET) { SaveHistogram(*histogram, INDEX, (TARGET)); } + + DB_KQP_SIMPLE_COUNTERS_MAP(SAVE_SIMPLE_COUNTER) + DB_KQP_CUMULATIVE_COUNTERS_MAP(SAVE_CUMULATIVE_COUNTER) + DB_KQP_HISTOGRAM_COUNTERS_MAP(SAVE_HISTOGRAM_COUNTER) +} + +template <typename T> +void LoadHistogram(T& histogram, int index, NMon::THistogramCounterHelper& helper) { + auto* buckets = histogram[index].MutableBuckets(); + auto count = helper.GetBucketCount(); + buckets->Resize(count, 0); + for (size_t i = 0; i < count; ++i) { + helper.SetBucketValue(i, (*buckets)[i]); + } +} + +template <typename T> +void LoadHistogram(T& histogram, int index, NMonitoring::THistogramPtr& hgram) { + auto* buckets = histogram[index].MutableBuckets(); + auto snapshot = hgram->Snapshot(); + auto count = snapshot->Count(); + buckets->Resize(count, 0); + hgram->Reset(); + for (ui32 i = 0; i < count; ++i) { + hgram->Collect(snapshot->UpperBound(i), (*buckets)[i]); + } +} + +void TKqpDbCounters::FromProto(NKikimr::NSysView::TDbServiceCounters& counters) { + auto* main = counters.Proto().MutableMain(); + auto* simple = main->MutableSimple(); + auto* cumulative = main->MutableCumulative(); + auto* histogram = main->MutableHistogram(); + + simple->Resize(DB_KQP_SIMPLE_COUNTER_SIZE, 0); + cumulative->Resize(DB_KQP_CUMULATIVE_COUNTER_SIZE, 0); + if (main->HistogramSize() < DB_KQP_HISTOGRAM_COUNTER_SIZE) { + auto missing = DB_KQP_HISTOGRAM_COUNTER_SIZE - main->HistogramSize(); + for (; missing > 0; --missing) { + main->AddHistogram(); + } + } + +#define LOAD_SIMPLE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*simple)[INDEX]); } +#define LOAD_CUMULATIVE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*cumulative)[INDEX]); } +#define LOAD_HISTOGRAM_COUNTER(INDEX, TARGET) { LoadHistogram(*histogram, INDEX, (TARGET)); } + + DB_KQP_SIMPLE_COUNTERS_MAP(LOAD_SIMPLE_COUNTER) + DB_KQP_CUMULATIVE_COUNTERS_MAP(LOAD_CUMULATIVE_COUNTER) + DB_KQP_HISTOGRAM_COUNTERS_MAP(LOAD_HISTOGRAM_COUNTER) +} + + +class TKqpDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback { + TIntrusivePtr<TKqpCounters> Counters; + +public: + explicit TKqpDbWatcherCallback(TIntrusivePtr<TKqpCounters> counters) + : Counters(counters) + {} + + void OnDatabaseRemoved(const TString& database, TPathId) override { + Counters->RemoveDbCounters(database); + } +}; + + +void TKqpCounters::CreateTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name) { + auto& txKindCounters = TxByKind[kind]; + + txKindCounters.TotalDuration = KqpGroup->GetHistogram("Transactions/" + name + "/TotalDurationMs", + NMonitoring::ExponentialHistogram(20, 2, 1)); + + txKindCounters.ServerDuration = KqpGroup->GetHistogram("Transactions/" + name + "/ServerDurationMs", + NMonitoring::ExponentialHistogram(20, 2, 1)); + + txKindCounters.ClientDuration = KqpGroup->GetHistogram("Transactions/" + name + "/ClientDurationMs", + NMonitoring::ExponentialHistogram(20, 2, 1)); + + txKindCounters.Queries = KqpGroup->GetHistogram("Transactions/" + name + "/Queries", + NMonitoring::ExponentialHistogram(16, 2, 1)); +} + +void TKqpCounters::UpdateTxCounters(const TKqpTransactionInfo& txInfo, + THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters>& txCounters) +{ + auto byKind = txCounters.FindPtr(txInfo.Kind); + if (!byKind) { + return; + } + + byKind->TotalDuration->Collect(txInfo.TotalDuration.MilliSeconds()); + byKind->ServerDuration->Collect(txInfo.ServerDuration.MilliSeconds()); byKind->ClientDuration->Collect((txInfo.TotalDuration - txInfo.ServerDuration).MilliSeconds()); - byKind->Queries->Collect(txInfo.QueriesCount); -} - -TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, const TActorContext* ctx) - : AllocCounters(counters, "kqp") -{ - Counters = counters; - KqpGroup = GetServiceCounters(counters, "kqp"); - YdbGroup = GetServiceCounters(counters, "ydb"); + byKind->Queries->Collect(txInfo.QueriesCount); +} + +TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, const TActorContext* ctx) + : AllocCounters(counters, "kqp") +{ + Counters = counters; + KqpGroup = GetServiceCounters(counters, "kqp"); + YdbGroup = GetServiceCounters(counters, "ydb"); QueryReplayGroup = KqpGroup->GetSubgroup("subsystem", "unified_agent_query_replay"); - - Init(); - - if (ctx) { - ActorSystem = ctx->ActorSystem(); - if (AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters()) { - auto callback = MakeIntrusive<TKqpDbWatcherCallback>(this); - DbWatcherActorId = ActorSystem->Register(NSysView::CreateDbWatcherActor(callback)); - } - } - - /* Transactions */ - CreateTxKindCounters(TKqpTransactionInfo::EKind::Pure, "Pure"); - CreateTxKindCounters(TKqpTransactionInfo::EKind::ReadOnly, "ReadOnly"); - CreateTxKindCounters(TKqpTransactionInfo::EKind::WriteOnly, "WriteOnly"); - CreateTxKindCounters(TKqpTransactionInfo::EKind::ReadWrite, "ReadWrite"); - - /* Compile service */ - CompileQueryCacheSize = YdbGroup->GetNamedCounter("name", "table.query.compilation.cached_query_count", false); - CompileQueryCacheBytes = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_size_bytes", false); - CompileQueryCacheEvicted = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_evictions", true); - - CompileQueueSize = KqpGroup->GetCounter("Compilation/QueueSize", false); - ForceNewEngineCompileErrors = KqpGroup->GetCounter("Compilation/ForceNEErrors", true); - - /* Resource Manager */ - RmComputeActors = KqpGroup->GetCounter("RM/ComputeActors", false); - RmMemory = KqpGroup->GetCounter("RM/Memory", false); + + Init(); + + if (ctx) { + ActorSystem = ctx->ActorSystem(); + if (AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters()) { + auto callback = MakeIntrusive<TKqpDbWatcherCallback>(this); + DbWatcherActorId = ActorSystem->Register(NSysView::CreateDbWatcherActor(callback)); + } + } + + /* Transactions */ + CreateTxKindCounters(TKqpTransactionInfo::EKind::Pure, "Pure"); + CreateTxKindCounters(TKqpTransactionInfo::EKind::ReadOnly, "ReadOnly"); + CreateTxKindCounters(TKqpTransactionInfo::EKind::WriteOnly, "WriteOnly"); + CreateTxKindCounters(TKqpTransactionInfo::EKind::ReadWrite, "ReadWrite"); + + /* Compile service */ + CompileQueryCacheSize = YdbGroup->GetNamedCounter("name", "table.query.compilation.cached_query_count", false); + CompileQueryCacheBytes = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_size_bytes", false); + CompileQueryCacheEvicted = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_evictions", true); + + CompileQueueSize = KqpGroup->GetCounter("Compilation/QueueSize", false); + ForceNewEngineCompileErrors = KqpGroup->GetCounter("Compilation/ForceNEErrors", true); + + /* Resource Manager */ + RmComputeActors = KqpGroup->GetCounter("RM/ComputeActors", false); + RmMemory = KqpGroup->GetCounter("RM/Memory", false); RmExternalMemory = KqpGroup->GetCounter("RM/ExternalMemory", false); - RmNotEnoughMemory = KqpGroup->GetCounter("RM/NotEnoughMemory", true); - RmNotEnoughComputeActors = KqpGroup->GetCounter("RM/NotEnoughComputeActors", true); - RmExtraMemAllocs = KqpGroup->GetCounter("RM/ExtraMemAllocs", true); - RmInternalError = KqpGroup->GetCounter("RM/InternalError", true); - - /* Spilling */ - SpillingWriteBlobs = KqpGroup->GetCounter("Spilling/WriteBlobs", true); - SpillingReadBlobs = KqpGroup->GetCounter("Spilling/ReadBlobs", true); - SpillingStoredBlobs = KqpGroup->GetCounter("Spilling/StoredBlobs", false); - SpillingTotalSpaceUsed = KqpGroup->GetCounter("Spilling/TotalSpaceUsed", false); - SpillingTooBigFileErrors = KqpGroup->GetCounter("Spilling/TooBigFileErrors", true); - SpillingNoSpaceErrors = KqpGroup->GetCounter("Spilling/NoSpaceErrors", true); - SpillingIoErrors = KqpGroup->GetCounter("Spilling/IoErrors", true); + RmNotEnoughMemory = KqpGroup->GetCounter("RM/NotEnoughMemory", true); + RmNotEnoughComputeActors = KqpGroup->GetCounter("RM/NotEnoughComputeActors", true); + RmExtraMemAllocs = KqpGroup->GetCounter("RM/ExtraMemAllocs", true); + RmInternalError = KqpGroup->GetCounter("RM/InternalError", true); + + /* Spilling */ + SpillingWriteBlobs = KqpGroup->GetCounter("Spilling/WriteBlobs", true); + SpillingReadBlobs = KqpGroup->GetCounter("Spilling/ReadBlobs", true); + SpillingStoredBlobs = KqpGroup->GetCounter("Spilling/StoredBlobs", false); + SpillingTotalSpaceUsed = KqpGroup->GetCounter("Spilling/TotalSpaceUsed", false); + SpillingTooBigFileErrors = KqpGroup->GetCounter("Spilling/TooBigFileErrors", true); + SpillingNoSpaceErrors = KqpGroup->GetCounter("Spilling/NoSpaceErrors", true); + SpillingIoErrors = KqpGroup->GetCounter("Spilling/IoErrors", true); /* Scan queries */ ScanQueryShardDisconnect = KqpGroup->GetCounter("ScanQuery/ShardDisconnect", true); ScanQueryShardResolve = KqpGroup->GetCounter("ScanQuery/ShardResolve", true); ScanQueryRateLimitLatency = KqpGroup->GetHistogram( "ScanQuery/RateLimitLatency", NMonitoring::ExponentialHistogram(20, 2, 1)); - - - // NewEngine - NewEngineForcedQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram( - "Query/ExecuteLatency_NEForced", NMonitoring::ExponentialHistogram(20, 2, 1)); - NewEngineForcedQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram( - "Query/ExecPreparedLatency_NEForced", NMonitoring::ExponentialHistogram(20, 2, 1)); - NewEngineCompatibleQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram( - "Query/ExecuteLatency_NECompatible", NMonitoring::ExponentialHistogram(20, 2, 1)); - NewEngineCompatibleQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram( - "Query/ExecPreparedLatency_NECompatible", NMonitoring::ExponentialHistogram(20, 2, 1)); - NewEngineForcedComputeCpuTime = KqpGroup->GetCounter("Query/ComputeCpuTime_NEForced", true); - NewEngineCompatibleComputeCpuTime = KqpGroup->GetCounter("Query/ComputeCpuTime_NECompatible", true); - NewEngineForcedQueryCount = KqpGroup->GetCounter("Query/Count_NEForced", true); - NewEngineCompatibleQueryCount = KqpGroup->GetCounter("Query/Count_NECompatible", true); + + + // NewEngine + NewEngineForcedQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram( + "Query/ExecuteLatency_NEForced", NMonitoring::ExponentialHistogram(20, 2, 1)); + NewEngineForcedQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram( + "Query/ExecPreparedLatency_NEForced", NMonitoring::ExponentialHistogram(20, 2, 1)); + NewEngineCompatibleQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram( + "Query/ExecuteLatency_NECompatible", NMonitoring::ExponentialHistogram(20, 2, 1)); + NewEngineCompatibleQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram( + "Query/ExecPreparedLatency_NECompatible", NMonitoring::ExponentialHistogram(20, 2, 1)); + NewEngineForcedComputeCpuTime = KqpGroup->GetCounter("Query/ComputeCpuTime_NEForced", true); + NewEngineCompatibleComputeCpuTime = KqpGroup->GetCounter("Query/ComputeCpuTime_NECompatible", true); + NewEngineForcedQueryCount = KqpGroup->GetCounter("Query/Count_NEForced", true); + NewEngineCompatibleQueryCount = KqpGroup->GetCounter("Query/Count_NECompatible", true); LiteralTxTotalTimeHistogram = KqpGroup->GetHistogram( "NE/LiteralTxTotalTimeMs", NMonitoring::ExponentialHistogram(10, 2, 1)); @@ -762,8 +762,8 @@ TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, cons "NE/DataTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1)); ScanTxTotalTimeHistogram = KqpGroup->GetHistogram( "NE/ScanTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1)); -} - +} + NMonitoring::TDynamicCounterPtr TKqpCounters::GetQueryReplayCounters() const { return QueryReplayGroup; } @@ -784,355 +784,355 @@ void TKqpCounters::ReportProxyForwardedRequest(TKqpDbCountersPtr dbCounters) { } } -void TKqpCounters::ReportSessionShutdownRequest(TKqpDbCountersPtr dbCounters) { +void TKqpCounters::ReportSessionShutdownRequest(TKqpDbCountersPtr dbCounters) { TKqpCountersBase::ReportSessionShutdownRequest(); - if (dbCounters) { - dbCounters->ReportSessionShutdownRequest(); - } -} - -void TKqpCounters::ReportCreateSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) { - TKqpCountersBase::ReportCreateSession(requestSize); - if (dbCounters) { - dbCounters->ReportCreateSession(requestSize); - } -} - -void TKqpCounters::ReportPingSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) { - TKqpCountersBase::ReportPingSession(requestSize); - if (dbCounters) { - dbCounters->ReportPingSession(requestSize); - } -} - -void TKqpCounters::ReportCloseSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) { - TKqpCountersBase::ReportCloseSession(requestSize); - if (dbCounters) { - dbCounters->ReportCloseSession(requestSize); - } -} - -void TKqpCounters::ReportQueryRequest(TKqpDbCountersPtr dbCounters, const NKikimrKqp::TQueryRequest& request) { - TKqpCountersBase::ReportQueryRequest(request); - if (dbCounters) { - dbCounters->ReportQueryRequest(request); - } -} - -void TKqpCounters::ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters) { + if (dbCounters) { + dbCounters->ReportSessionShutdownRequest(); + } +} + +void TKqpCounters::ReportCreateSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) { + TKqpCountersBase::ReportCreateSession(requestSize); + if (dbCounters) { + dbCounters->ReportCreateSession(requestSize); + } +} + +void TKqpCounters::ReportPingSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) { + TKqpCountersBase::ReportPingSession(requestSize); + if (dbCounters) { + dbCounters->ReportPingSession(requestSize); + } +} + +void TKqpCounters::ReportCloseSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) { + TKqpCountersBase::ReportCloseSession(requestSize); + if (dbCounters) { + dbCounters->ReportCloseSession(requestSize); + } +} + +void TKqpCounters::ReportQueryRequest(TKqpDbCountersPtr dbCounters, const NKikimrKqp::TQueryRequest& request) { + TKqpCountersBase::ReportQueryRequest(request); + if (dbCounters) { + dbCounters->ReportQueryRequest(request); + } +} + +void TKqpCounters::ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters) { TKqpCountersBase::ReportQueryWithRangeScan(); - if (dbCounters) { - dbCounters->ReportQueryWithRangeScan(); + if (dbCounters) { + dbCounters->ReportQueryWithRangeScan(); } } -void TKqpCounters::ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters) { +void TKqpCounters::ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters) { TKqpCountersBase::ReportQueryWithFullScan(); - if (dbCounters) { - dbCounters->ReportQueryWithFullScan(); + if (dbCounters) { + dbCounters->ReportQueryWithFullScan(); } } -void TKqpCounters::ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount) { +void TKqpCounters::ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount) { TKqpCountersBase::ReportQueryAffectedShards(shardsCount); - if (dbCounters) { - dbCounters->ReportQueryAffectedShards(shardsCount); + if (dbCounters) { + dbCounters->ReportQueryAffectedShards(shardsCount); } } -void TKqpCounters::ReportQueryReadSets(TKqpDbCountersPtr dbCounters, ui64 readSetsCount) { +void TKqpCounters::ReportQueryReadSets(TKqpDbCountersPtr dbCounters, ui64 readSetsCount) { TKqpCountersBase::ReportQueryReadSets(readSetsCount); - if (dbCounters) { - dbCounters->ReportQueryReadSets(readSetsCount); + if (dbCounters) { + dbCounters->ReportQueryReadSets(readSetsCount); } } -void TKqpCounters::ReportQueryReadBytes(TKqpDbCountersPtr dbCounters, ui64 bytesCount) { +void TKqpCounters::ReportQueryReadBytes(TKqpDbCountersPtr dbCounters, ui64 bytesCount) { TKqpCountersBase::ReportQueryReadBytes(bytesCount); - if (dbCounters) { - dbCounters->ReportQueryReadBytes(bytesCount); + if (dbCounters) { + dbCounters->ReportQueryReadBytes(bytesCount); } } -void TKqpCounters::ReportQueryReadRows(TKqpDbCountersPtr dbCounters, ui64 rowsCount) { +void TKqpCounters::ReportQueryReadRows(TKqpDbCountersPtr dbCounters, ui64 rowsCount) { TKqpCountersBase::ReportQueryReadRows(rowsCount); - if (dbCounters) { - dbCounters->ReportQueryReadRows(rowsCount); + if (dbCounters) { + dbCounters->ReportQueryReadRows(rowsCount); } } -void TKqpCounters::ReportQueryMaxShardReplySize(TKqpDbCountersPtr dbCounters, ui64 replySize) { +void TKqpCounters::ReportQueryMaxShardReplySize(TKqpDbCountersPtr dbCounters, ui64 replySize) { TKqpCountersBase::ReportQueryMaxShardReplySize(replySize); - if (dbCounters) { - dbCounters->ReportQueryMaxShardReplySize(replySize); + if (dbCounters) { + dbCounters->ReportQueryMaxShardReplySize(replySize); } } -void TKqpCounters::ReportQueryMaxShardProgramSize(TKqpDbCountersPtr dbCounters, ui64 programSize) { +void TKqpCounters::ReportQueryMaxShardProgramSize(TKqpDbCountersPtr dbCounters, ui64 programSize) { TKqpCountersBase::ReportQueryMaxShardProgramSize(programSize); - if (dbCounters) { - dbCounters->ReportQueryMaxShardProgramSize(programSize); + if (dbCounters) { + dbCounters->ReportQueryMaxShardProgramSize(programSize); } } -void TKqpCounters::ReportResponseStatus(TKqpDbCountersPtr dbCounters, ui64 responseSize, +void TKqpCounters::ReportResponseStatus(TKqpDbCountersPtr dbCounters, ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus) -{ +{ TKqpCountersBase::ReportResponseStatus(responseSize, ydbStatus); - if (dbCounters) { + if (dbCounters) { dbCounters->ReportResponseStatus(responseSize, ydbStatus); - } -} - -void TKqpCounters::ReportResultsBytes(TKqpDbCountersPtr dbCounters, ui64 resultsSize) { - TKqpCountersBase::ReportResultsBytes(resultsSize); - if (dbCounters) { - dbCounters->ReportResultsBytes(resultsSize); - } -} - -void TKqpCounters::ReportIssues(TKqpDbCountersPtr dbCounters, const Ydb::Issue::IssueMessage& issue) { - TKqpCountersBase::ReportIssues(issue); - if (dbCounters) { - dbCounters->ReportIssues(issue); - } -} - -void TKqpCounters::ReportQueryLatency(TKqpDbCountersPtr dbCounters, - NKikimrKqp::EQueryAction action, const TDuration& duration) -{ - TKqpCountersBase::ReportQueryLatency(action, duration); - if (dbCounters) { - dbCounters->ReportQueryLatency(action, duration); - } -} - -void TKqpCounters::ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTransactionInfo& txInfo) { - TKqpCountersBase::ReportTransaction(txInfo); - if (dbCounters) { - dbCounters->ReportTransaction(txInfo); - } - if (txInfo.Status == TKqpTransactionInfo::EStatus::Committed) { - UpdateTxCounters(txInfo, TxByKind); - } -} - -void TKqpCounters::ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion) { - TKqpCountersBase::ReportSqlVersion(sqlVersion); - if (dbCounters) { - dbCounters->ReportSqlVersion(sqlVersion); - } -} - -void TKqpCounters::ReportWorkerCreated(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportWorkerCreated(); - if (dbCounters) { - dbCounters->ReportWorkerCreated(); - } -} - -void TKqpCounters::ReportWorkerFinished(TKqpDbCountersPtr dbCounters, TDuration lifeSpan) { - TKqpCountersBase::ReportWorkerFinished(lifeSpan); - if (dbCounters) { - dbCounters->ReportWorkerFinished(lifeSpan); - } -} - -void TKqpCounters::ReportWorkerCleanupLatency(TKqpDbCountersPtr dbCounters, TDuration cleanupTime) { - TKqpCountersBase::ReportWorkerCleanupLatency(cleanupTime); - if (dbCounters) { - dbCounters->ReportWorkerCleanupLatency(cleanupTime); - } -} - -void TKqpCounters::ReportWorkerClosedIdle(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportWorkerClosedIdle(); - if (dbCounters) { - dbCounters->ReportWorkerClosedIdle(); - } -} - -void TKqpCounters::ReportWorkerClosedError(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportWorkerClosedError(); - if (dbCounters) { - dbCounters->ReportWorkerClosedError(); - } -} - -void TKqpCounters::ReportWorkerClosedRequest(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportWorkerClosedRequest(); - if (dbCounters) { - dbCounters->ReportWorkerClosedRequest(); - } -} - -void TKqpCounters::ReportQueriesPerWorker(TKqpDbCountersPtr dbCounters, ui32 queryId) { - TKqpCountersBase::ReportQueriesPerWorker(queryId); - if (dbCounters) { - dbCounters->ReportQueriesPerWorker(queryId); - } -} - -void TKqpCounters::ReportBeginTransaction(TKqpDbCountersPtr dbCounters, - ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx) -{ - TKqpCountersBase::ReportBeginTransaction(evictedTx, currentActiveTx, currentAbortedTx); - if (dbCounters) { - dbCounters->ReportBeginTransaction(evictedTx, currentActiveTx, currentAbortedTx); - } -} - -void TKqpCounters::ReportTxCreated(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportTxCreated(); - if (dbCounters) { - dbCounters->ReportTxCreated(); - } -} - -void TKqpCounters::ReportTxAborted(TKqpDbCountersPtr dbCounters, ui32 abortedCount) { - TKqpCountersBase::ReportTxAborted(abortedCount); - if (dbCounters) { - dbCounters->ReportTxAborted(abortedCount); - } -} - -void TKqpCounters::ReportQueryCacheHit(TKqpDbCountersPtr dbCounters, bool hit) { - TKqpCountersBase::ReportQueryCacheHit(hit); - if (dbCounters) { - dbCounters->ReportQueryCacheHit(hit); - } -} - -void TKqpCounters::ReportCompileStart(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportCompileStart(); - if (dbCounters) { - dbCounters->ReportCompileStart(); - } -} - -void TKqpCounters::ReportCompileFinish(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportCompileFinish(); - if (dbCounters) { - dbCounters->ReportCompileFinish(); - } -} - -void TKqpCounters::ReportCompileError(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportCompileError(); - if (dbCounters) { - dbCounters->ReportCompileError(); - } -} - -void TKqpCounters::ReportCompileRequestCompile(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportCompileRequestCompile(); - if (dbCounters) { - dbCounters->ReportCompileRequestCompile(); - } -} - -void TKqpCounters::ReportCompileRequestGet(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportCompileRequestGet(); - if (dbCounters) { - dbCounters->ReportCompileRequestGet(); - } -} - -void TKqpCounters::ReportCompileRequestInvalidate(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportCompileRequestInvalidate(); - if (dbCounters) { - dbCounters->ReportCompileRequestInvalidate(); - } -} - -void TKqpCounters::ReportCompileRequestRejected(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportCompileRequestRejected(); - if (dbCounters) { - dbCounters->ReportCompileRequestRejected(); - } -} - -void TKqpCounters::ReportCompileRequestTimeout(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportCompileRequestTimeout(); - if (dbCounters) { - dbCounters->ReportCompileRequestTimeout(); - } -} - -void TKqpCounters::ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuration duration, TDuration cpuTime) { - TKqpCountersBase::ReportCompileDurations(duration, cpuTime); - if (dbCounters) { - dbCounters->ReportCompileDurations(duration, cpuTime); - } -} - -void TKqpCounters::ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters) { + } +} + +void TKqpCounters::ReportResultsBytes(TKqpDbCountersPtr dbCounters, ui64 resultsSize) { + TKqpCountersBase::ReportResultsBytes(resultsSize); + if (dbCounters) { + dbCounters->ReportResultsBytes(resultsSize); + } +} + +void TKqpCounters::ReportIssues(TKqpDbCountersPtr dbCounters, const Ydb::Issue::IssueMessage& issue) { + TKqpCountersBase::ReportIssues(issue); + if (dbCounters) { + dbCounters->ReportIssues(issue); + } +} + +void TKqpCounters::ReportQueryLatency(TKqpDbCountersPtr dbCounters, + NKikimrKqp::EQueryAction action, const TDuration& duration) +{ + TKqpCountersBase::ReportQueryLatency(action, duration); + if (dbCounters) { + dbCounters->ReportQueryLatency(action, duration); + } +} + +void TKqpCounters::ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTransactionInfo& txInfo) { + TKqpCountersBase::ReportTransaction(txInfo); + if (dbCounters) { + dbCounters->ReportTransaction(txInfo); + } + if (txInfo.Status == TKqpTransactionInfo::EStatus::Committed) { + UpdateTxCounters(txInfo, TxByKind); + } +} + +void TKqpCounters::ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion) { + TKqpCountersBase::ReportSqlVersion(sqlVersion); + if (dbCounters) { + dbCounters->ReportSqlVersion(sqlVersion); + } +} + +void TKqpCounters::ReportWorkerCreated(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportWorkerCreated(); + if (dbCounters) { + dbCounters->ReportWorkerCreated(); + } +} + +void TKqpCounters::ReportWorkerFinished(TKqpDbCountersPtr dbCounters, TDuration lifeSpan) { + TKqpCountersBase::ReportWorkerFinished(lifeSpan); + if (dbCounters) { + dbCounters->ReportWorkerFinished(lifeSpan); + } +} + +void TKqpCounters::ReportWorkerCleanupLatency(TKqpDbCountersPtr dbCounters, TDuration cleanupTime) { + TKqpCountersBase::ReportWorkerCleanupLatency(cleanupTime); + if (dbCounters) { + dbCounters->ReportWorkerCleanupLatency(cleanupTime); + } +} + +void TKqpCounters::ReportWorkerClosedIdle(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportWorkerClosedIdle(); + if (dbCounters) { + dbCounters->ReportWorkerClosedIdle(); + } +} + +void TKqpCounters::ReportWorkerClosedError(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportWorkerClosedError(); + if (dbCounters) { + dbCounters->ReportWorkerClosedError(); + } +} + +void TKqpCounters::ReportWorkerClosedRequest(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportWorkerClosedRequest(); + if (dbCounters) { + dbCounters->ReportWorkerClosedRequest(); + } +} + +void TKqpCounters::ReportQueriesPerWorker(TKqpDbCountersPtr dbCounters, ui32 queryId) { + TKqpCountersBase::ReportQueriesPerWorker(queryId); + if (dbCounters) { + dbCounters->ReportQueriesPerWorker(queryId); + } +} + +void TKqpCounters::ReportBeginTransaction(TKqpDbCountersPtr dbCounters, + ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx) +{ + TKqpCountersBase::ReportBeginTransaction(evictedTx, currentActiveTx, currentAbortedTx); + if (dbCounters) { + dbCounters->ReportBeginTransaction(evictedTx, currentActiveTx, currentAbortedTx); + } +} + +void TKqpCounters::ReportTxCreated(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportTxCreated(); + if (dbCounters) { + dbCounters->ReportTxCreated(); + } +} + +void TKqpCounters::ReportTxAborted(TKqpDbCountersPtr dbCounters, ui32 abortedCount) { + TKqpCountersBase::ReportTxAborted(abortedCount); + if (dbCounters) { + dbCounters->ReportTxAborted(abortedCount); + } +} + +void TKqpCounters::ReportQueryCacheHit(TKqpDbCountersPtr dbCounters, bool hit) { + TKqpCountersBase::ReportQueryCacheHit(hit); + if (dbCounters) { + dbCounters->ReportQueryCacheHit(hit); + } +} + +void TKqpCounters::ReportCompileStart(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportCompileStart(); + if (dbCounters) { + dbCounters->ReportCompileStart(); + } +} + +void TKqpCounters::ReportCompileFinish(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportCompileFinish(); + if (dbCounters) { + dbCounters->ReportCompileFinish(); + } +} + +void TKqpCounters::ReportCompileError(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportCompileError(); + if (dbCounters) { + dbCounters->ReportCompileError(); + } +} + +void TKqpCounters::ReportCompileRequestCompile(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportCompileRequestCompile(); + if (dbCounters) { + dbCounters->ReportCompileRequestCompile(); + } +} + +void TKqpCounters::ReportCompileRequestGet(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportCompileRequestGet(); + if (dbCounters) { + dbCounters->ReportCompileRequestGet(); + } +} + +void TKqpCounters::ReportCompileRequestInvalidate(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportCompileRequestInvalidate(); + if (dbCounters) { + dbCounters->ReportCompileRequestInvalidate(); + } +} + +void TKqpCounters::ReportCompileRequestRejected(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportCompileRequestRejected(); + if (dbCounters) { + dbCounters->ReportCompileRequestRejected(); + } +} + +void TKqpCounters::ReportCompileRequestTimeout(TKqpDbCountersPtr dbCounters) { + TKqpCountersBase::ReportCompileRequestTimeout(); + if (dbCounters) { + dbCounters->ReportCompileRequestTimeout(); + } +} + +void TKqpCounters::ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuration duration, TDuration cpuTime) { + TKqpCountersBase::ReportCompileDurations(duration, cpuTime); + if (dbCounters) { + dbCounters->ReportCompileDurations(duration, cpuTime); + } +} + +void TKqpCounters::ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters) { TKqpCountersBase::ReportRecompileRequestGet(); - if (dbCounters) { - dbCounters->ReportRecompileRequestGet(); - } -} - -void TKqpCounters::ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action, TDuration duration, - ui64 computeCpuTime) -{ - NewEngineForcedComputeCpuTime->Add(computeCpuTime); - NewEngineForcedQueryCount->Inc(); - - if (auto counter = NewEngineForcedQueryLatencies.FindPtr(action)) { - (*counter)->Collect(duration.MilliSeconds()); - } -} - -void TKqpCounters::ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction action, TDuration duration, - ui64 computeCpuTime) -{ - NewEngineCompatibleComputeCpuTime->Add(computeCpuTime); - NewEngineCompatibleQueryCount->Inc(); - - if (auto counter = NewEngineCompatibleQueryLatencies.FindPtr(action)) { - (*counter)->Collect(duration.MilliSeconds()); - } -} - + if (dbCounters) { + dbCounters->ReportRecompileRequestGet(); + } +} + +void TKqpCounters::ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action, TDuration duration, + ui64 computeCpuTime) +{ + NewEngineForcedComputeCpuTime->Add(computeCpuTime); + NewEngineForcedQueryCount->Inc(); + + if (auto counter = NewEngineForcedQueryLatencies.FindPtr(action)) { + (*counter)->Collect(duration.MilliSeconds()); + } +} + +void TKqpCounters::ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction action, TDuration duration, + ui64 computeCpuTime) +{ + NewEngineCompatibleComputeCpuTime->Add(computeCpuTime); + NewEngineCompatibleQueryCount->Inc(); + + if (auto counter = NewEngineCompatibleQueryLatencies.FindPtr(action)) { + (*counter)->Collect(duration.MilliSeconds()); + } +} + const NMonitoring::TDynamicCounters::TCounterPtr TKqpCounters::RecompileRequestGet() const { return TKqpCountersBase::CompileRequestsRecompile; } -NMonitoring::TDynamicCounters::TCounterPtr TKqpCounters::GetQueryTypeCounter( - NKikimrKqp::EQueryType queryType) -{ - return QueryTypes[queryType]; -} - -TKqpDbCountersPtr TKqpCounters::GetDbCounters(const TString& database) { - if (!ActorSystem || !AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters() || database.empty()) { - return {}; - } - - TKqpDbCountersPtr dbCounters; - if (DbCounters.Get(database, dbCounters)) { - return dbCounters; - } - - return DbCounters.InsertIfAbsentWithInit(database, [&database, this] { - auto counters = MakeIntrusive<TKqpDbCounters>(); - - auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>( - NKikimrSysView::KQP, database, counters); - ActorSystem->Send(NSysView::MakeSysViewServiceID(ActorSystem->NodeId), evRegister.Release()); - - if (DbWatcherActorId) { - auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database); - ActorSystem->Send(DbWatcherActorId, evWatch.Release()); - } - return counters; - }); -} - -void TKqpCounters::RemoveDbCounters(const TString& database) { - DbCounters.Erase(database); -} - +NMonitoring::TDynamicCounters::TCounterPtr TKqpCounters::GetQueryTypeCounter( + NKikimrKqp::EQueryType queryType) +{ + return QueryTypes[queryType]; +} + +TKqpDbCountersPtr TKqpCounters::GetDbCounters(const TString& database) { + if (!ActorSystem || !AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters() || database.empty()) { + return {}; + } + + TKqpDbCountersPtr dbCounters; + if (DbCounters.Get(database, dbCounters)) { + return dbCounters; + } + + return DbCounters.InsertIfAbsentWithInit(database, [&database, this] { + auto counters = MakeIntrusive<TKqpDbCounters>(); + + auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>( + NKikimrSysView::KQP, database, counters); + ActorSystem->Send(NSysView::MakeSysViewServiceID(ActorSystem->NodeId), evRegister.Release()); + + if (DbWatcherActorId) { + auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database); + ActorSystem->Send(DbWatcherActorId, evWatch.Release()); + } + return counters; + }); +} + +void TKqpCounters::RemoveDbCounters(const TString& database) { + DbCounters.Erase(database); +} + } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/counters/kqp_counters.h b/ydb/core/kqp/counters/kqp_counters.h index 8a24403618..53a8f7d2c2 100644 --- a/ydb/core/kqp/counters/kqp_counters.h +++ b/ydb/core/kqp/counters/kqp_counters.h @@ -1,10 +1,10 @@ #pragma once -#include "kqp_db_counters.h" - -#include <library/cpp/actors/helpers/mon_histogram_helper.h> -#include <library/cpp/monlib/metrics/histogram_collector.h> - +#include "kqp_db_counters.h" + +#include <library/cpp/actors/helpers/mon_histogram_helper.h> +#include <library/cpp/monlib/metrics/histogram_collector.h> + #include <ydb/core/util/concurrent_rw_hash.h> #include <ydb/core/kqp/common/kqp_tx_info.h> #include <ydb/core/kqp/common/kqp_tx_info.h> @@ -14,35 +14,35 @@ #include <ydb/library/yql/minikql/aligned_page_pool.h> -#include <util/system/spinlock.h> +#include <util/system/spinlock.h> namespace NKikimr { namespace NKqp { -class TKqpCountersBase { -protected: - struct TYdbTxByKindCounters { - NMonitoring::THistogramPtr TotalDuration; - NMonitoring::THistogramPtr ServerDuration; - NMonitoring::THistogramPtr ClientDuration; +class TKqpCountersBase { +protected: + struct TYdbTxByKindCounters { + NMonitoring::THistogramPtr TotalDuration; + NMonitoring::THistogramPtr ServerDuration; + NMonitoring::THistogramPtr ClientDuration; }; -protected: - void CreateYdbTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name); - static void UpdateYdbTxCounters(const TKqpTransactionInfo& txInfo, - THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters>& txCounters); - - void Init(); - +protected: + void CreateYdbTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name); + static void UpdateYdbTxCounters(const TKqpTransactionInfo& txInfo, + THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters>& txCounters); + + void Init(); + void ReportQueryAction(NKikimrKqp::EQueryAction action); void ReportQueryType(NKikimrKqp::EQueryType type); - + void ReportSessionShutdownRequest(); - void ReportCreateSession(ui64 requestSize); - void ReportPingSession(ui64 requestSize); - void ReportCloseSession(ui64 requestSize); - void ReportQueryRequest(const NKikimrKqp::TQueryRequest& request); - + void ReportCreateSession(ui64 requestSize); + void ReportPingSession(ui64 requestSize); + void ReportCloseSession(ui64 requestSize); + void ReportQueryRequest(const NKikimrKqp::TQueryRequest& request); + void ReportQueryWithRangeScan(); void ReportQueryWithFullScan(); void ReportQueryAffectedShards(ui64 shardsCount); @@ -53,50 +53,50 @@ protected: void ReportQueryMaxShardProgramSize(ui64 programSize); void ReportResponseStatus(ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus); - void ReportResultsBytes(ui64 resultsSize); - - static TString GetIssueName(ui32 issueCode); - void ReportIssues(const Ydb::Issue::IssueMessage& issue); - + void ReportResultsBytes(ui64 resultsSize); + + static TString GetIssueName(ui32 issueCode); + void ReportIssues(const Ydb::Issue::IssueMessage& issue); + void ReportQueryLatency(NKikimrKqp::EQueryAction action, const TDuration& duration); - - void ReportTransaction(const TKqpTransactionInfo& txInfo); - + + void ReportTransaction(const TKqpTransactionInfo& txInfo); + void ReportSqlVersion(ui16 sqlVersion); - void ReportWorkerCreated(); - void ReportWorkerFinished(TDuration lifeSpan); - void ReportWorkerCleanupLatency(TDuration cleanupTime); - void ReportWorkerClosedIdle(); - void ReportWorkerClosedError(); - void ReportWorkerClosedRequest(); - void ReportQueriesPerWorker(ui32 queryId); + void ReportWorkerCreated(); + void ReportWorkerFinished(TDuration lifeSpan); + void ReportWorkerCleanupLatency(TDuration cleanupTime); + void ReportWorkerClosedIdle(); + void ReportWorkerClosedError(); + void ReportWorkerClosedRequest(); + void ReportQueriesPerWorker(ui32 queryId); void ReportProxyForwardedRequest(); void ReportSessionBalancerCV(ui32 value); - void ReportBeginTransaction(ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx); - - void ReportTxCreated(); - void ReportTxAborted(ui32 abortedCount); - - void ReportQueryCacheHit(bool hit); - void ReportCompileStart(); - void ReportCompileFinish(); - void ReportCompileError(); - void ReportCompileRequestCompile(); - void ReportCompileRequestGet(); - void ReportCompileRequestInvalidate(); - void ReportCompileRequestRejected(); - void ReportCompileRequestTimeout(); - void ReportCompileDurations(TDuration duration, TDuration cpuTime); + void ReportBeginTransaction(ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx); + + void ReportTxCreated(); + void ReportTxAborted(ui32 abortedCount); + + void ReportQueryCacheHit(bool hit); + void ReportCompileStart(); + void ReportCompileFinish(); + void ReportCompileError(); + void ReportCompileRequestCompile(); + void ReportCompileRequestGet(); + void ReportCompileRequestInvalidate(); + void ReportCompileRequestRejected(); + void ReportCompileRequestTimeout(); + void ReportCompileDurations(TDuration duration, TDuration cpuTime); void ReportRecompileRequestGet(); NMonitoring::TDynamicCounterPtr GetQueryReplayCounters() const; - -protected: - NMonitoring::TDynamicCounterPtr Counters; - NMonitoring::TDynamicCounterPtr KqpGroup; - NMonitoring::TDynamicCounterPtr YdbGroup; + +protected: + NMonitoring::TDynamicCounterPtr Counters; + NMonitoring::TDynamicCounterPtr KqpGroup; + NMonitoring::TDynamicCounterPtr YdbGroup; NMonitoring::TDynamicCounterPtr QueryReplayGroup; // Requests @@ -107,10 +107,10 @@ protected: NMonitoring::TDynamicCounters::TCounterPtr PingSessionRequests; NMonitoring::TDynamicCounters::TCounterPtr RequestBytes; - NMonitoring::TDynamicCounters::TCounterPtr YdbRequestBytes; + NMonitoring::TDynamicCounters::TCounterPtr YdbRequestBytes; NMonitoring::TDynamicCounters::TCounterPtr QueryBytes; NMonitoring::TDynamicCounters::TCounterPtr ParametersBytes; - NMonitoring::TDynamicCounters::TCounterPtr YdbParametersBytes; + NMonitoring::TDynamicCounters::TCounterPtr YdbParametersBytes; NMonitoring::TDynamicCounters::TCounterPtr SqlV0Translations; NMonitoring::TDynamicCounters::TCounterPtr SqlV1Translations; @@ -129,189 +129,189 @@ protected: NMonitoring::THistogramPtr QueryMaxShardProgramSize; // Request latency - THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> QueryLatencies; - NMonitoring::THistogramPtr YdbQueryExecuteLatency; + THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> QueryLatencies; + NMonitoring::THistogramPtr YdbQueryExecuteLatency; // Responses NMonitoring::TDynamicCounters::TCounterPtr OtherResponses; - NMonitoring::TDynamicCounters::TCounterPtr YdbResponsesLocksInvalidated; - + NMonitoring::TDynamicCounters::TCounterPtr YdbResponsesLocksInvalidated; + THashMap<Ydb::StatusIds::StatusCode, NMonitoring::TDynamicCounters::TCounterPtr> YdbResponses; NMonitoring::TDynamicCounters::TCounterPtr OtherYdbResponses; NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes; - NMonitoring::TDynamicCounters::TCounterPtr YdbResponseBytes; + NMonitoring::TDynamicCounters::TCounterPtr YdbResponseBytes; NMonitoring::TDynamicCounters::TCounterPtr QueryResultsBytes; THashMap<ui32, NMonitoring::TDynamicCounters::TCounterPtr> IssueCounters; // Workers - NMonitoring::THistogramPtr WorkerLifeSpan; - NMonitoring::THistogramPtr QueriesPerWorker; + NMonitoring::THistogramPtr WorkerLifeSpan; + NMonitoring::THistogramPtr QueriesPerWorker; NMonitoring::TDynamicCounters::TCounterPtr WorkersCreated; NMonitoring::TDynamicCounters::TCounterPtr WorkersClosedIdle; - NMonitoring::TDynamicCounters::TCounterPtr YdbWorkersClosedIdle; + NMonitoring::TDynamicCounters::TCounterPtr YdbWorkersClosedIdle; NMonitoring::TDynamicCounters::TCounterPtr WorkersClosedError; NMonitoring::TDynamicCounters::TCounterPtr WorkersClosedRequest; NMonitoring::TDynamicCounters::TCounterPtr ActiveWorkers; - NMonitoring::TDynamicCounters::TCounterPtr YdbActiveWorkers; + NMonitoring::TDynamicCounters::TCounterPtr YdbActiveWorkers; NMonitoring::TDynamicCounters::TCounterPtr SessionBalancerCV; NMonitoring::TDynamicCounters::TCounterPtr SessionBalancerShutdowns; NMonitoring::TDynamicCounters::TCounterPtr ProxyForwardedRequests; - NMonitoring::THistogramPtr WorkerCleanupLatency; - - // Transactions - NMonitoring::TDynamicCounters::TCounterPtr TxCreated; - NMonitoring::TDynamicCounters::TCounterPtr TxAborted; - NMonitoring::TDynamicCounters::TCounterPtr TxCommited; - NMonitoring::TDynamicCounters::TCounterPtr TxEvicted; - NMonitoring::THistogramPtr TxActivePerSession; - NMonitoring::THistogramPtr TxAbortedPerSession; - THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters> YdbTxByKind; - - // Compile service - NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheHits; - NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheMisses; - NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsCompile; - NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsGet; - NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsInvalidate; - NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsRejected; - NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsTimeout; + NMonitoring::THistogramPtr WorkerCleanupLatency; + + // Transactions + NMonitoring::TDynamicCounters::TCounterPtr TxCreated; + NMonitoring::TDynamicCounters::TCounterPtr TxAborted; + NMonitoring::TDynamicCounters::TCounterPtr TxCommited; + NMonitoring::TDynamicCounters::TCounterPtr TxEvicted; + NMonitoring::THistogramPtr TxActivePerSession; + NMonitoring::THistogramPtr TxAbortedPerSession; + THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters> YdbTxByKind; + + // Compile service + NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheHits; + NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheMisses; + NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsCompile; + NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsGet; + NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsInvalidate; + NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsRejected; + NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsTimeout; NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsRecompile; - NMonitoring::TDynamicCounters::TCounterPtr CompileTotal; - NMonitoring::TDynamicCounters::TCounterPtr CompileErrors; - NMonitoring::TDynamicCounters::TCounterPtr CompileActive; - NMonitoring::THistogramPtr CompileCpuTime; - NMonitoring::THistogramPtr YdbCompileDuration; -}; - - -class TKqpDbCounters : public NSysView::IDbCounters, public TKqpCountersBase { - friend class TKqpCounters; - -public: - // per database internal counters, not exposed - TKqpDbCounters(); - - // created in SVP, exposed - explicit TKqpDbCounters(const NMonitoring::TDynamicCounterPtr& externalGroup, - const NMonitoring::TDynamicCounterPtr& internalGroup); - - void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override; - void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override; - -private: - enum ESimpleCounter { - DB_KQP_SIMPLE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) - DB_KQP_SIMPLE_COUNTER_SIZE - }; - enum ECumulativeCounter { - DB_KQP_CUMULATIVE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) - DB_KQP_CUMULATIVE_COUNTER_SIZE - }; - enum EHistogramCounter { - DB_KQP_HISTOGRAM_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) - DB_KQP_HISTOGRAM_COUNTER_SIZE - }; - - struct TDeprecatedCounter { - void Set(ui64) {} - ui64 Val() { return 0; } - }; - TDeprecatedCounter DeprecatedCounter; -}; - -using TKqpDbCountersPtr = TIntrusivePtr<TKqpDbCounters>; - - -class TKqpCounters : public TThrRefBase, public TKqpCountersBase { -private: - struct TTxByKindCounters { - NMonitoring::THistogramPtr TotalDuration; - NMonitoring::THistogramPtr ServerDuration; - NMonitoring::THistogramPtr ClientDuration; - NMonitoring::THistogramPtr Queries; - }; - -private: - void CreateTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name); - static void UpdateTxCounters(const TKqpTransactionInfo& txInfo, - THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters>& txCounters); - -public: - explicit TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, const TActorContext* ctx = nullptr); - + NMonitoring::TDynamicCounters::TCounterPtr CompileTotal; + NMonitoring::TDynamicCounters::TCounterPtr CompileErrors; + NMonitoring::TDynamicCounters::TCounterPtr CompileActive; + NMonitoring::THistogramPtr CompileCpuTime; + NMonitoring::THistogramPtr YdbCompileDuration; +}; + + +class TKqpDbCounters : public NSysView::IDbCounters, public TKqpCountersBase { + friend class TKqpCounters; + +public: + // per database internal counters, not exposed + TKqpDbCounters(); + + // created in SVP, exposed + explicit TKqpDbCounters(const NMonitoring::TDynamicCounterPtr& externalGroup, + const NMonitoring::TDynamicCounterPtr& internalGroup); + + void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override; + void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override; + +private: + enum ESimpleCounter { + DB_KQP_SIMPLE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) + DB_KQP_SIMPLE_COUNTER_SIZE + }; + enum ECumulativeCounter { + DB_KQP_CUMULATIVE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) + DB_KQP_CUMULATIVE_COUNTER_SIZE + }; + enum EHistogramCounter { + DB_KQP_HISTOGRAM_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE) + DB_KQP_HISTOGRAM_COUNTER_SIZE + }; + + struct TDeprecatedCounter { + void Set(ui64) {} + ui64 Val() { return 0; } + }; + TDeprecatedCounter DeprecatedCounter; +}; + +using TKqpDbCountersPtr = TIntrusivePtr<TKqpDbCounters>; + + +class TKqpCounters : public TThrRefBase, public TKqpCountersBase { +private: + struct TTxByKindCounters { + NMonitoring::THistogramPtr TotalDuration; + NMonitoring::THistogramPtr ServerDuration; + NMonitoring::THistogramPtr ClientDuration; + NMonitoring::THistogramPtr Queries; + }; + +private: + void CreateTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name); + static void UpdateTxCounters(const TKqpTransactionInfo& txInfo, + THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters>& txCounters); + +public: + explicit TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, const TActorContext* ctx = nullptr); + void ReportProxyForwardedRequest(TKqpDbCountersPtr dbCounters); void ReportSessionBalancerCV(TKqpDbCountersPtr dbCounters, ui32 val); - void ReportSessionShutdownRequest(TKqpDbCountersPtr dbCounters); - void ReportCreateSession(TKqpDbCountersPtr dbCounters, ui64 requestSize); - void ReportPingSession(TKqpDbCountersPtr dbCounters, ui64 requestSize); - void ReportCloseSession(TKqpDbCountersPtr dbCounters, ui64 requestSize); - void ReportQueryRequest(TKqpDbCountersPtr dbCounters, const NKikimrKqp::TQueryRequest& request); - + void ReportSessionShutdownRequest(TKqpDbCountersPtr dbCounters); + void ReportCreateSession(TKqpDbCountersPtr dbCounters, ui64 requestSize); + void ReportPingSession(TKqpDbCountersPtr dbCounters, ui64 requestSize); + void ReportCloseSession(TKqpDbCountersPtr dbCounters, ui64 requestSize); + void ReportQueryRequest(TKqpDbCountersPtr dbCounters, const NKikimrKqp::TQueryRequest& request); + void ReportResponseStatus(TKqpDbCountersPtr dbCounters, ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus); - void ReportResultsBytes(TKqpDbCountersPtr dbCounters, ui64 resultsSize); - void ReportIssues(TKqpDbCountersPtr dbCounters, const Ydb::Issue::IssueMessage& issue); - - void ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters); - void ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters); - void ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount); - void ReportQueryReadSets(TKqpDbCountersPtr dbCounters, ui64 readSetsCount); - void ReportQueryReadBytes(TKqpDbCountersPtr dbCounters, ui64 bytesCount); - void ReportQueryReadRows(TKqpDbCountersPtr dbCounters, ui64 rowsCount); - void ReportQueryMaxShardReplySize(TKqpDbCountersPtr dbCounters, ui64 replySize); - void ReportQueryMaxShardProgramSize(TKqpDbCountersPtr dbCounters, ui64 programSize); - - void ReportQueryLatency(TKqpDbCountersPtr dbCounters, - NKikimrKqp::EQueryAction action, const TDuration& duration); - void ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion); - void ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTransactionInfo& txInfo); - + void ReportResultsBytes(TKqpDbCountersPtr dbCounters, ui64 resultsSize); + void ReportIssues(TKqpDbCountersPtr dbCounters, const Ydb::Issue::IssueMessage& issue); + + void ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters); + void ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters); + void ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount); + void ReportQueryReadSets(TKqpDbCountersPtr dbCounters, ui64 readSetsCount); + void ReportQueryReadBytes(TKqpDbCountersPtr dbCounters, ui64 bytesCount); + void ReportQueryReadRows(TKqpDbCountersPtr dbCounters, ui64 rowsCount); + void ReportQueryMaxShardReplySize(TKqpDbCountersPtr dbCounters, ui64 replySize); + void ReportQueryMaxShardProgramSize(TKqpDbCountersPtr dbCounters, ui64 programSize); + + void ReportQueryLatency(TKqpDbCountersPtr dbCounters, + NKikimrKqp::EQueryAction action, const TDuration& duration); + void ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion); + void ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTransactionInfo& txInfo); + // NewEngine - void ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action, - TDuration duration, ui64 computeCpuTime); - void ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction action, - TDuration duration, ui64 computeCpuTime); - - void ReportWorkerCreated(TKqpDbCountersPtr dbCounters); - void ReportWorkerFinished(TKqpDbCountersPtr dbCounters, TDuration lifeSpan); - void ReportWorkerCleanupLatency(TKqpDbCountersPtr dbCounters, TDuration cleanupTime); - void ReportWorkerClosedIdle(TKqpDbCountersPtr dbCounters); - void ReportWorkerClosedError(TKqpDbCountersPtr dbCounters); - void ReportWorkerClosedRequest(TKqpDbCountersPtr dbCounters); - void ReportQueriesPerWorker(TKqpDbCountersPtr dbCounters, ui32 queryId); - - void ReportBeginTransaction(TKqpDbCountersPtr dbCounters, - ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx); - - void ReportTxCreated(TKqpDbCountersPtr dbCounters); - void ReportTxAborted(TKqpDbCountersPtr dbCounters, ui32 abortedCount); - - void ReportQueryCacheHit(TKqpDbCountersPtr dbCounters, bool hit); - void ReportCompileStart(TKqpDbCountersPtr dbCounters); - void ReportCompileFinish(TKqpDbCountersPtr dbCounters); - void ReportCompileError(TKqpDbCountersPtr dbCounters); - void ReportCompileRequestCompile(TKqpDbCountersPtr dbCounters); - void ReportCompileRequestGet(TKqpDbCountersPtr dbCounters); - void ReportCompileRequestInvalidate(TKqpDbCountersPtr dbCounters); - void ReportCompileRequestRejected(TKqpDbCountersPtr dbCounters); - void ReportCompileRequestTimeout(TKqpDbCountersPtr dbCounters); - void ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuration duration, TDuration cpuTime); - void ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters); - + void ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action, + TDuration duration, ui64 computeCpuTime); + void ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction action, + TDuration duration, ui64 computeCpuTime); + + void ReportWorkerCreated(TKqpDbCountersPtr dbCounters); + void ReportWorkerFinished(TKqpDbCountersPtr dbCounters, TDuration lifeSpan); + void ReportWorkerCleanupLatency(TKqpDbCountersPtr dbCounters, TDuration cleanupTime); + void ReportWorkerClosedIdle(TKqpDbCountersPtr dbCounters); + void ReportWorkerClosedError(TKqpDbCountersPtr dbCounters); + void ReportWorkerClosedRequest(TKqpDbCountersPtr dbCounters); + void ReportQueriesPerWorker(TKqpDbCountersPtr dbCounters, ui32 queryId); + + void ReportBeginTransaction(TKqpDbCountersPtr dbCounters, + ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx); + + void ReportTxCreated(TKqpDbCountersPtr dbCounters); + void ReportTxAborted(TKqpDbCountersPtr dbCounters, ui32 abortedCount); + + void ReportQueryCacheHit(TKqpDbCountersPtr dbCounters, bool hit); + void ReportCompileStart(TKqpDbCountersPtr dbCounters); + void ReportCompileFinish(TKqpDbCountersPtr dbCounters); + void ReportCompileError(TKqpDbCountersPtr dbCounters); + void ReportCompileRequestCompile(TKqpDbCountersPtr dbCounters); + void ReportCompileRequestGet(TKqpDbCountersPtr dbCounters); + void ReportCompileRequestInvalidate(TKqpDbCountersPtr dbCounters); + void ReportCompileRequestRejected(TKqpDbCountersPtr dbCounters); + void ReportCompileRequestTimeout(TKqpDbCountersPtr dbCounters); + void ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuration duration, TDuration cpuTime); + void ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters); + const NMonitoring::TDynamicCounters::TCounterPtr RecompileRequestGet() const; NMonitoring::TDynamicCounterPtr GetQueryReplayCounters() const; - NMonitoring::TDynamicCounters::TCounterPtr GetQueryTypeCounter(NKikimrKqp::EQueryType queryType); - - TKqpDbCountersPtr GetDbCounters(const TString& database); - void RemoveDbCounters(const TString& database); - -public: - // Transactions - THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters> TxByKind; - + NMonitoring::TDynamicCounters::TCounterPtr GetQueryTypeCounter(NKikimrKqp::EQueryType queryType); + + TKqpDbCountersPtr GetDbCounters(const TString& database); + void RemoveDbCounters(const TString& database); + +public: + // Transactions + THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters> TxByKind; + // Compile service NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheSize; NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheBytes; @@ -343,24 +343,24 @@ public: NMonitoring::THistogramPtr ScanQueryRateLimitLatency; // NewEngine vs OldEngine - THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineForcedQueryLatencies; - THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineCompatibleQueryLatencies; - NMonitoring::TDynamicCounters::TCounterPtr NewEngineForcedComputeCpuTime; - NMonitoring::TDynamicCounters::TCounterPtr NewEngineForcedQueryCount; - NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleComputeCpuTime; - NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleQueryCount; - + THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineForcedQueryLatencies; + THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineCompatibleQueryLatencies; + NMonitoring::TDynamicCounters::TCounterPtr NewEngineForcedComputeCpuTime; + NMonitoring::TDynamicCounters::TCounterPtr NewEngineForcedQueryCount; + NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleComputeCpuTime; + NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleQueryCount; + // NewEngine tx duration NMonitoring::THistogramPtr LiteralTxTotalTimeHistogram; NMonitoring::THistogramPtr DataTxTotalTimeHistogram; NMonitoring::THistogramPtr ScanTxTotalTimeHistogram; TAlignedPagePoolCounters AllocCounters; - - // db counters - TConcurrentRWHashMap<TString, TKqpDbCountersPtr, 256> DbCounters; - TActorSystem* ActorSystem = nullptr; - TActorId DbWatcherActorId; + + // db counters + TConcurrentRWHashMap<TString, TKqpDbCountersPtr, 256> DbCounters; + TActorSystem* ActorSystem = nullptr; + TActorId DbWatcherActorId; }; struct TKqpRequestCounters : public TThrRefBase { diff --git a/ydb/core/kqp/counters/kqp_db_counters.h b/ydb/core/kqp/counters/kqp_db_counters.h index 6aba15df70..814c382d4c 100644 --- a/ydb/core/kqp/counters/kqp_db_counters.h +++ b/ydb/core/kqp/counters/kqp_db_counters.h @@ -1,120 +1,120 @@ -#pragma once - -#include <library/cpp/deprecated/enum_codegen/enum_codegen.h> - -namespace NKikimr { -namespace NKqp { - -#define DB_KQP_SIMPLE_COUNTERS_MAP(XX) \ - XX(DB_KQP_WORKERS_ACTIVE, ActiveWorkers) \ - XX(DB_KQP_YDB_WORKERS_ACTIVE, YdbActiveWorkers) \ - XX(DB_KQP_COMPILE_ACTIVE, CompileActive) - -#define DB_KQP_CUMULATIVE_COUNTERS_MAP(XX) \ - XX(DB_KQP_REQ_QUERY_EXECUTE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE]) \ - XX(DB_KQP_REQ_QUERY_EXPLAIN, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXPLAIN]) \ - XX(DB_KQP_REQ_QUERY_VALIDATE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_VALIDATE]) \ - XX(DB_KQP_REQ_QUERY_PREPARE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_PREPARE]) \ - XX(DB_KQP_REQ_QUERY_EXECUTE_PREPARED, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED]) \ - XX(DB_KQP_REQ_QUERY_BEGIN_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_BEGIN_TX]) \ - XX(DB_KQP_REQ_QUERY_COMMIT_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_COMMIT_TX]) \ - XX(DB_KQP_REQ_QUERY_ROLLBACK_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX]) \ - XX(DB_KQP_REQ_QUERY_PARSE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_PARSE]) \ - XX(DB_KQP_REQ_QUERY_OTHER, OtherQueryRequests) \ - XX(DB_KQP_CLOSE_SESSION_REQ, CloseSessionRequests) \ - XX(DB_KQP_CREATE_SESSION_REQ, PingSessionRequests) \ - XX(DB_KQP_REQUEST_BYTES, RequestBytes) \ - XX(DB_KQP_QUERY_BYTES, QueryBytes) \ - XX(DB_KQP_PARAMS_BYTES, ParametersBytes) \ - XX(DB_KQP_SQL_V0_TRANSLATIONS, SqlV0Translations) \ - XX(DB_KQP_SQL_V1_TRANSLATIONS, SqlV1Translations) \ - XX(DB_KQP_SQL_UNKNOWN_TRANSLATIONS, SqlUnknownTranslations) \ - XX(DB_KQP_QUERY_TYPE_UNDEFINED, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_UNDEFINED]) \ - XX(DB_KQP_QUERY_TYPE_SQL_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DML]) \ - XX(DB_KQP_QUERY_TYPE_SQL_DDL, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DDL]) \ - XX(DB_KQP_QUERY_TYPE_PREPARED_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_PREPARED_DML]) \ - XX(DB_KQP_QUERY_TYPE_AST_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_DML]) \ - XX(DB_KQP_QUERY_TYPE_SQL_SCRIPT, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT]) \ +#pragma once + +#include <library/cpp/deprecated/enum_codegen/enum_codegen.h> + +namespace NKikimr { +namespace NKqp { + +#define DB_KQP_SIMPLE_COUNTERS_MAP(XX) \ + XX(DB_KQP_WORKERS_ACTIVE, ActiveWorkers) \ + XX(DB_KQP_YDB_WORKERS_ACTIVE, YdbActiveWorkers) \ + XX(DB_KQP_COMPILE_ACTIVE, CompileActive) + +#define DB_KQP_CUMULATIVE_COUNTERS_MAP(XX) \ + XX(DB_KQP_REQ_QUERY_EXECUTE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE]) \ + XX(DB_KQP_REQ_QUERY_EXPLAIN, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXPLAIN]) \ + XX(DB_KQP_REQ_QUERY_VALIDATE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_VALIDATE]) \ + XX(DB_KQP_REQ_QUERY_PREPARE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_PREPARE]) \ + XX(DB_KQP_REQ_QUERY_EXECUTE_PREPARED, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED]) \ + XX(DB_KQP_REQ_QUERY_BEGIN_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_BEGIN_TX]) \ + XX(DB_KQP_REQ_QUERY_COMMIT_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_COMMIT_TX]) \ + XX(DB_KQP_REQ_QUERY_ROLLBACK_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX]) \ + XX(DB_KQP_REQ_QUERY_PARSE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_PARSE]) \ + XX(DB_KQP_REQ_QUERY_OTHER, OtherQueryRequests) \ + XX(DB_KQP_CLOSE_SESSION_REQ, CloseSessionRequests) \ + XX(DB_KQP_CREATE_SESSION_REQ, PingSessionRequests) \ + XX(DB_KQP_REQUEST_BYTES, RequestBytes) \ + XX(DB_KQP_QUERY_BYTES, QueryBytes) \ + XX(DB_KQP_PARAMS_BYTES, ParametersBytes) \ + XX(DB_KQP_SQL_V0_TRANSLATIONS, SqlV0Translations) \ + XX(DB_KQP_SQL_V1_TRANSLATIONS, SqlV1Translations) \ + XX(DB_KQP_SQL_UNKNOWN_TRANSLATIONS, SqlUnknownTranslations) \ + XX(DB_KQP_QUERY_TYPE_UNDEFINED, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_UNDEFINED]) \ + XX(DB_KQP_QUERY_TYPE_SQL_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DML]) \ + XX(DB_KQP_QUERY_TYPE_SQL_DDL, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DDL]) \ + XX(DB_KQP_QUERY_TYPE_PREPARED_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_PREPARED_DML]) \ + XX(DB_KQP_QUERY_TYPE_AST_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_DML]) \ + XX(DB_KQP_QUERY_TYPE_SQL_SCRIPT, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT]) \ XX(DB_KQP_QUERY_TYPE_SQL_SCRIPT_STREAMING, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT_STREAMING]) \ - XX(DB_KQP_QUERY_TYPE_SQL_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN]) \ - XX(DB_KQP_QUERY_TYPE_AST_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN]) \ - XX(DB_KQP_RSP_STATUS_CODE_UNSPECIFIED, YdbResponses[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED]) \ - XX(DB_KQP_RSP_SUCCESS, YdbResponses[Ydb::StatusIds::SUCCESS]) \ - XX(DB_KQP_RSP_BAD_REQUEST, YdbResponses[Ydb::StatusIds::BAD_REQUEST]) \ - XX(DB_KQP_RSP_UNAUTHORIZED, YdbResponses[Ydb::StatusIds::UNAUTHORIZED]) \ - XX(DB_KQP_RSP_INTERNAL_ERROR, YdbResponses[Ydb::StatusIds::INTERNAL_ERROR]) \ - XX(DB_KQP_RSP_ABORTED, YdbResponses[Ydb::StatusIds::ABORTED]) \ - XX(DB_KQP_RSP_UNAVAILABLE, YdbResponses[Ydb::StatusIds::UNAVAILABLE]) \ - XX(DB_KQP_RSP_OVERLOADED, YdbResponses[Ydb::StatusIds::OVERLOADED]) \ - XX(DB_KQP_RSP_SCHEME_ERROR, YdbResponses[Ydb::StatusIds::SCHEME_ERROR]) \ - XX(DB_KQP_RSP_GENERIC_ERROR, YdbResponses[Ydb::StatusIds::GENERIC_ERROR]) \ - XX(DB_KQP_RSP_TIMEOUT, YdbResponses[Ydb::StatusIds::TIMEOUT]) \ - XX(DB_KQP_RSP_BAD_SESSION, YdbResponses[Ydb::StatusIds::BAD_SESSION]) \ - XX(DB_KQP_RSP_PRECONDITION_FAILED, YdbResponses[Ydb::StatusIds::PRECONDITION_FAILED]) \ - XX(DB_KQP_RSP_ALREADY_EXISTS, YdbResponses[Ydb::StatusIds::ALREADY_EXISTS]) \ - XX(DB_KQP_RSP_NOT_FOUND, YdbResponses[Ydb::StatusIds::NOT_FOUND]) \ - XX(DB_KQP_RSP_SESSION_EXPIRED, YdbResponses[Ydb::StatusIds::SESSION_EXPIRED]) \ - XX(DB_KQP_RSP_CANCELLED, YdbResponses[Ydb::StatusIds::CANCELLED]) \ - XX(DB_KQP_RSP_UNDETERMINED, YdbResponses[Ydb::StatusIds::UNDETERMINED]) \ - XX(DB_KQP_RSP_UNSUPPORTED, YdbResponses[Ydb::StatusIds::UNSUPPORTED]) \ - XX(DB_KQP_RSP_SESSION_BUSY, YdbResponses[Ydb::StatusIds::SESSION_BUSY]) \ - XX(DB_KQP_RSP_OTHER, OtherYdbResponses) \ - XX(DB_KQP_RESPONSE_BYTES, ResponseBytes) \ - XX(DB_KQP_QUERY_RESULTS_BYTES, QueryResultsBytes) \ - XX(DB_KQP_WORKERS_CREATED, WorkersCreated) \ - XX(DB_KQP_WORKERS_CLOSED_IDLE, WorkersClosedIdle) \ - XX(DB_KQP_WORKERS_CLOSED_ERROR, WorkersClosedError) \ - XX(DB_KQP_WORKERS_CLOSED_REQUEST, WorkersClosedRequest) \ - XX(DB_KQP_TX_CREATED, TxCreated) \ - XX(DB_KQP_TX_ABORTED, TxAborted) \ - XX(DB_KQP_TX_COMMITED, TxCommited) \ - XX(DB_KQP_TX_EVICTED, TxEvicted) \ - XX(DB_KQP_COMPILE_QUERY_CACHE_HITS, CompileQueryCacheHits) \ - XX(DB_KQP_COMPILE_QUERY_CACHE_MISSES, CompileQueryCacheMisses) \ - XX(DB_KQP_COMPILE_REQ_COMPILE, CompileRequestsCompile) \ - XX(DB_KQP_COMPILE_REQ_GET, CompileRequestsGet) \ - XX(DB_KQP_COMPILE_REQ_INVALIDATE, CompileRequestsInvalidate) \ - XX(DB_KQP_COMPILE_REQ_REJECTED, CompileRequestsRejected) \ - XX(DB_KQP_COMPILE_REQ_TIMEOUT, CompileRequestsTimeout) \ - XX(DB_KQP_COMPILE_TOTAL, CompileTotal) \ - XX(DB_KQP_COMPILE_ERRORS, CompileErrors) \ - XX(DB_KQP_DEPRECATED1, &DeprecatedCounter) \ - XX(DB_KQP_YDB_WORKERS_CLOSED_IDLE, YdbWorkersClosedIdle) \ - XX(DB_KQP_YDB_REQUEST_BYTES, YdbRequestBytes) \ - XX(DB_KQP_YDB_PARAMS_BYTES, YdbParametersBytes) \ + XX(DB_KQP_QUERY_TYPE_SQL_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN]) \ + XX(DB_KQP_QUERY_TYPE_AST_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN]) \ + XX(DB_KQP_RSP_STATUS_CODE_UNSPECIFIED, YdbResponses[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED]) \ + XX(DB_KQP_RSP_SUCCESS, YdbResponses[Ydb::StatusIds::SUCCESS]) \ + XX(DB_KQP_RSP_BAD_REQUEST, YdbResponses[Ydb::StatusIds::BAD_REQUEST]) \ + XX(DB_KQP_RSP_UNAUTHORIZED, YdbResponses[Ydb::StatusIds::UNAUTHORIZED]) \ + XX(DB_KQP_RSP_INTERNAL_ERROR, YdbResponses[Ydb::StatusIds::INTERNAL_ERROR]) \ + XX(DB_KQP_RSP_ABORTED, YdbResponses[Ydb::StatusIds::ABORTED]) \ + XX(DB_KQP_RSP_UNAVAILABLE, YdbResponses[Ydb::StatusIds::UNAVAILABLE]) \ + XX(DB_KQP_RSP_OVERLOADED, YdbResponses[Ydb::StatusIds::OVERLOADED]) \ + XX(DB_KQP_RSP_SCHEME_ERROR, YdbResponses[Ydb::StatusIds::SCHEME_ERROR]) \ + XX(DB_KQP_RSP_GENERIC_ERROR, YdbResponses[Ydb::StatusIds::GENERIC_ERROR]) \ + XX(DB_KQP_RSP_TIMEOUT, YdbResponses[Ydb::StatusIds::TIMEOUT]) \ + XX(DB_KQP_RSP_BAD_SESSION, YdbResponses[Ydb::StatusIds::BAD_SESSION]) \ + XX(DB_KQP_RSP_PRECONDITION_FAILED, YdbResponses[Ydb::StatusIds::PRECONDITION_FAILED]) \ + XX(DB_KQP_RSP_ALREADY_EXISTS, YdbResponses[Ydb::StatusIds::ALREADY_EXISTS]) \ + XX(DB_KQP_RSP_NOT_FOUND, YdbResponses[Ydb::StatusIds::NOT_FOUND]) \ + XX(DB_KQP_RSP_SESSION_EXPIRED, YdbResponses[Ydb::StatusIds::SESSION_EXPIRED]) \ + XX(DB_KQP_RSP_CANCELLED, YdbResponses[Ydb::StatusIds::CANCELLED]) \ + XX(DB_KQP_RSP_UNDETERMINED, YdbResponses[Ydb::StatusIds::UNDETERMINED]) \ + XX(DB_KQP_RSP_UNSUPPORTED, YdbResponses[Ydb::StatusIds::UNSUPPORTED]) \ + XX(DB_KQP_RSP_SESSION_BUSY, YdbResponses[Ydb::StatusIds::SESSION_BUSY]) \ + XX(DB_KQP_RSP_OTHER, OtherYdbResponses) \ + XX(DB_KQP_RESPONSE_BYTES, ResponseBytes) \ + XX(DB_KQP_QUERY_RESULTS_BYTES, QueryResultsBytes) \ + XX(DB_KQP_WORKERS_CREATED, WorkersCreated) \ + XX(DB_KQP_WORKERS_CLOSED_IDLE, WorkersClosedIdle) \ + XX(DB_KQP_WORKERS_CLOSED_ERROR, WorkersClosedError) \ + XX(DB_KQP_WORKERS_CLOSED_REQUEST, WorkersClosedRequest) \ + XX(DB_KQP_TX_CREATED, TxCreated) \ + XX(DB_KQP_TX_ABORTED, TxAborted) \ + XX(DB_KQP_TX_COMMITED, TxCommited) \ + XX(DB_KQP_TX_EVICTED, TxEvicted) \ + XX(DB_KQP_COMPILE_QUERY_CACHE_HITS, CompileQueryCacheHits) \ + XX(DB_KQP_COMPILE_QUERY_CACHE_MISSES, CompileQueryCacheMisses) \ + XX(DB_KQP_COMPILE_REQ_COMPILE, CompileRequestsCompile) \ + XX(DB_KQP_COMPILE_REQ_GET, CompileRequestsGet) \ + XX(DB_KQP_COMPILE_REQ_INVALIDATE, CompileRequestsInvalidate) \ + XX(DB_KQP_COMPILE_REQ_REJECTED, CompileRequestsRejected) \ + XX(DB_KQP_COMPILE_REQ_TIMEOUT, CompileRequestsTimeout) \ + XX(DB_KQP_COMPILE_TOTAL, CompileTotal) \ + XX(DB_KQP_COMPILE_ERRORS, CompileErrors) \ + XX(DB_KQP_DEPRECATED1, &DeprecatedCounter) \ + XX(DB_KQP_YDB_WORKERS_CLOSED_IDLE, YdbWorkersClosedIdle) \ + XX(DB_KQP_YDB_REQUEST_BYTES, YdbRequestBytes) \ + XX(DB_KQP_YDB_PARAMS_BYTES, YdbParametersBytes) \ XX(DB_KQP_YDB_RSP_LOCKS_INVALIDATED, YdbResponsesLocksInvalidated) \ XX(DB_KQP_QUERIES_WITH_RANGE_SCAN, QueriesWithRangeScan) \ - XX(DB_KQP_QUERIES_WITH_FULL_SCAN, QueriesWithFullScan) \ - XX(DB_KQP_YDB_RESPONSE_BYTES, YdbResponseBytes) - -#define DB_KQP_HISTOGRAM_COUNTERS_MAP(XX) \ - XX(DB_KQP_QUERY_LATENCY_EXECUTE, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE]) \ - XX(DB_KQP_QUERY_LATENCY_EXPLAIN, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXPLAIN]) \ - XX(DB_KQP_QUERY_LATENCY_VALIDATE, QueryLatencies[NKikimrKqp::QUERY_ACTION_VALIDATE]) \ - XX(DB_KQP_QUERY_LATENCY_PREPARE, QueryLatencies[NKikimrKqp::QUERY_ACTION_PREPARE]) \ - XX(DB_KQP_QUERY_LATENCY_EXECUTE_PREPARED, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED]) \ - XX(DB_KQP_QUERY_LATENCY_BEGIN_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_BEGIN_TX]) \ - XX(DB_KQP_QUERY_LATENCY_COMMIT_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_COMMIT_TX]) \ - XX(DB_KQP_QUERY_LATENCY_ROLLBACK_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX]) \ - XX(DB_KQP_WORKER_LIFE_SPAN, WorkerLifeSpan) \ - XX(DB_KQP_QUERIES_PER_WORKER, QueriesPerWorker) \ - XX(DB_KQP_WORKER_CLEANUP_LATENCY, WorkerCleanupLatency) \ - XX(DB_KQP_TX_ACTIVE_PER_SESSION, TxActivePerSession) \ - XX(DB_KQP_TX_ABORTED_PER_SESSION, TxAbortedPerSession) \ - XX(DB_KQP_PURE_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].TotalDuration) \ - XX(DB_KQP_PURE_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].ServerDuration) \ - XX(DB_KQP_PURE_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].ClientDuration) \ - XX(DB_KQP_RO_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].TotalDuration) \ - XX(DB_KQP_RO_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].ServerDuration) \ - XX(DB_KQP_RO_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].ClientDuration) \ - XX(DB_KQP_WO_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].TotalDuration) \ - XX(DB_KQP_WO_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].ServerDuration) \ - XX(DB_KQP_WO_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].ClientDuration) \ - XX(DB_KQP_RW_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].TotalDuration) \ - XX(DB_KQP_RW_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].ServerDuration) \ - XX(DB_KQP_RW_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].ClientDuration) \ - XX(DB_KQP_YDB_COMPILE_DURATION, YdbCompileDuration) \ - XX(DB_KQP_COMPILE_CPU_TIME, CompileCpuTime) \ + XX(DB_KQP_QUERIES_WITH_FULL_SCAN, QueriesWithFullScan) \ + XX(DB_KQP_YDB_RESPONSE_BYTES, YdbResponseBytes) + +#define DB_KQP_HISTOGRAM_COUNTERS_MAP(XX) \ + XX(DB_KQP_QUERY_LATENCY_EXECUTE, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE]) \ + XX(DB_KQP_QUERY_LATENCY_EXPLAIN, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXPLAIN]) \ + XX(DB_KQP_QUERY_LATENCY_VALIDATE, QueryLatencies[NKikimrKqp::QUERY_ACTION_VALIDATE]) \ + XX(DB_KQP_QUERY_LATENCY_PREPARE, QueryLatencies[NKikimrKqp::QUERY_ACTION_PREPARE]) \ + XX(DB_KQP_QUERY_LATENCY_EXECUTE_PREPARED, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED]) \ + XX(DB_KQP_QUERY_LATENCY_BEGIN_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_BEGIN_TX]) \ + XX(DB_KQP_QUERY_LATENCY_COMMIT_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_COMMIT_TX]) \ + XX(DB_KQP_QUERY_LATENCY_ROLLBACK_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX]) \ + XX(DB_KQP_WORKER_LIFE_SPAN, WorkerLifeSpan) \ + XX(DB_KQP_QUERIES_PER_WORKER, QueriesPerWorker) \ + XX(DB_KQP_WORKER_CLEANUP_LATENCY, WorkerCleanupLatency) \ + XX(DB_KQP_TX_ACTIVE_PER_SESSION, TxActivePerSession) \ + XX(DB_KQP_TX_ABORTED_PER_SESSION, TxAbortedPerSession) \ + XX(DB_KQP_PURE_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].TotalDuration) \ + XX(DB_KQP_PURE_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].ServerDuration) \ + XX(DB_KQP_PURE_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].ClientDuration) \ + XX(DB_KQP_RO_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].TotalDuration) \ + XX(DB_KQP_RO_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].ServerDuration) \ + XX(DB_KQP_RO_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].ClientDuration) \ + XX(DB_KQP_WO_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].TotalDuration) \ + XX(DB_KQP_WO_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].ServerDuration) \ + XX(DB_KQP_WO_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].ClientDuration) \ + XX(DB_KQP_RW_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].TotalDuration) \ + XX(DB_KQP_RW_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].ServerDuration) \ + XX(DB_KQP_RW_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].ClientDuration) \ + XX(DB_KQP_YDB_COMPILE_DURATION, YdbCompileDuration) \ + XX(DB_KQP_COMPILE_CPU_TIME, CompileCpuTime) \ XX(DB_KQP_YDB_QUERY_LATENCY_EXECUTE, YdbQueryExecuteLatency) \ XX(DB_KQP_QUERY_AFFECTED_SHARDS, QueryAffectedShardsCount) \ XX(DB_KQP_QUERY_READ_SETS, QueryReadSetsCount) \ @@ -122,5 +122,5 @@ namespace NKqp { XX(DB_KQP_QUERY_READ_ROWS, QueryReadRows) \ XX(DB_KQP_QUERY_MAX_SHARD_REPLY, QueryMaxShardReplySize) \ XX(DB_KQP_QUERY_MAX_SHARD_PROGRAM, QueryMaxShardProgramSize) -} -} +} +} diff --git a/ydb/core/kqp/counters/ya.make b/ydb/core/kqp/counters/ya.make index c4d9f4fe1e..45f9d3f3c9 100644 --- a/ydb/core/kqp/counters/ya.make +++ b/ydb/core/kqp/counters/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - -OWNER( - spuchin - g:kikimr -) - -SRCS( - kqp_counters.cpp - kqp_counters.h - kqp_db_counters.h -) - -PEERDIR( +LIBRARY() + +OWNER( + spuchin + g:kikimr +) + +SRCS( + kqp_counters.cpp + kqp_counters.h + kqp_db_counters.h +) + +PEERDIR( ydb/core/base ydb/core/protos ydb/core/sys_view/service -) - -END() +) + +END() diff --git a/ydb/core/kqp/executer/kqp_scan_executer.cpp b/ydb/core/kqp/executer/kqp_scan_executer.cpp index 8bbdbaea24..07c29f64e2 100644 --- a/ydb/core/kqp/executer/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer/kqp_scan_executer.cpp @@ -173,7 +173,7 @@ private: void HandleExecute(TEvDqCompute::TEvState::TPtr& ev) { TActorId computeActor = ev->Sender; - auto& state = ev->Get()->Record; + auto& state = ev->Get()->Record; ui64 taskId = state.GetTaskId(); LOG_D("Got execution state from compute actor: " << computeActor @@ -978,8 +978,8 @@ private: } TBase::ReplyErrorAndDie(status, issues); - } - + } + void PassAway() override { for (auto channelPair: ResultChannelProxies) { LOG_D("terminate result channel " << channelPair.first << " proxy at " << channelPair.second->SelfId()); diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index 6cf3495bb8..44af6d1168 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -217,13 +217,13 @@ public: TVector<const TString> queryPlans; for (auto id : SessionCtx->Query().ExecutionOrder) { - auto result = SessionCtx->Query().Results.FindPtr(id); + auto result = SessionCtx->Query().Results.FindPtr(id); if (result) { queryPlans.push_back(SerializeAnalyzePlan(result->QueryStats)); AddQueryStats(queryResult.QueryStats, std::move(result->QueryStats)); - } - } - + } + } + FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder); queryResult.SqlVersion = SqlVersion; queryResult.QueryPlan = SerializeScriptPlan(queryPlans); @@ -291,8 +291,8 @@ public: auto& execResult = ExecuteCtx->QueryResults[0]; queryResult.QueryStats.Swap(&execResult.QueryStats); queryResult.QueryPlan = SerializeAnalyzePlan(queryResult.QueryStats); - } - + } + queryResult.SqlVersion = SqlVersion; } diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp index cb1347ed6f..42b4eca7c6 100644 --- a/ydb/core/kqp/host/kqp_runner.cpp +++ b/ydb/core/kqp/host/kqp_runner.cpp @@ -409,16 +409,16 @@ private: // OldEngine only YQL_ENSURE(!engine.has_value() || *engine == TKqpTransactionInfo::EEngine::OldEngine); - for (const auto& [name, table] : TransformCtx->Tables->GetTables()) { - if (!table.Metadata->SysView.empty()) { + for (const auto& [name, table] : TransformCtx->Tables->GetTables()) { + if (!table.Metadata->SysView.empty()) { ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), TStringBuilder() - << "Table " << table.Metadata->Name << " is a system view. " - << "System views are not supported by data queries." - )); + << "Table " << table.Metadata->Name << " is a system view. " + << "System views are not supported by data queries." + )); return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues())); - } - } - + } + } + auto program = BuildKiProgram(dataQuery, *TransformCtx->Tables, ctx, sysColumnsEnabled); KqlOptimizeTransformer->Rewind(); diff --git a/ydb/core/kqp/kqp.h b/ydb/core/kqp/kqp.h index 4f183d2f33..09f794591d 100644 --- a/ydb/core/kqp/kqp.h +++ b/ydb/core/kqp/kqp.h @@ -368,9 +368,9 @@ struct TEvKqp { : UserToken(userToken) , Uid(uid) , Query(std::move(query)) - , KeepInCache(keepInCache) + , KeepInCache(keepInCache) , Deadline(deadline) - , DbCounters(dbCounters) + , DbCounters(dbCounters) { Y_ENSURE(Uid.Defined() != Query.Defined()); } @@ -381,7 +381,7 @@ struct TEvKqp { bool KeepInCache = false; // it is allowed for local event to use absolute time (TInstant) instead of time interval (TDuration) TInstant Deadline; - TKqpDbCountersPtr DbCounters; + TKqpDbCountersPtr DbCounters; TMaybe<bool> DocumentApiRestricted; }; @@ -392,14 +392,14 @@ struct TEvKqp { , Uid(uid) , Query(query) , Deadline(deadline) - , DbCounters(dbCounters) {} + , DbCounters(dbCounters) {} TString UserToken; TString Uid; TMaybe<TKqpQueryId> Query; TInstant Deadline; - TKqpDbCountersPtr DbCounters; + TKqpDbCountersPtr DbCounters; }; struct TEvCompileResponse : public TEventLocal<TEvCompileResponse, TKqpEvents::EvCompileResponse> { @@ -417,12 +417,12 @@ struct TEvKqp { struct TEvCompileInvalidateRequest : public TEventLocal<TEvCompileInvalidateRequest, TKqpEvents::EvCompileInvalidateRequest> { - TEvCompileInvalidateRequest(const TString& uid, TKqpDbCountersPtr dbCounters) - : Uid(uid) - , DbCounters(dbCounters) {} + TEvCompileInvalidateRequest(const TString& uid, TKqpDbCountersPtr dbCounters) + : Uid(uid) + , DbCounters(dbCounters) {} TString Uid; - TKqpDbCountersPtr DbCounters; + TKqpDbCountersPtr DbCounters; }; struct TEvInitiateShutdownRequest : public TEventLocal<TEvInitiateShutdownRequest, TKqpEvents::EvInitiateShutdownRequest> { diff --git a/ydb/core/kqp/kqp_compile_actor.cpp b/ydb/core/kqp/kqp_compile_actor.cpp index ecb2ad49e5..27cf296a03 100644 --- a/ydb/core/kqp/kqp_compile_actor.cpp +++ b/ydb/core/kqp/kqp_compile_actor.cpp @@ -45,7 +45,7 @@ public: TKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings, const TTableServiceConfig& serviceConfig, TIntrusivePtr<TModuleResolverState> moduleResolverState, - TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken, + TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken, TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine) : Owner(owner) , ModuleResolverState(moduleResolverState) @@ -53,7 +53,7 @@ public: , Uid(uid) , Query(query) , UserToken(userToken) - , DbCounters(dbCounters) + , DbCounters(dbCounters) , Config(MakeIntrusive<TKikimrConfiguration>()) , CompilationTimeout(TDuration::MilliSeconds(serviceConfig.GetCompileTimeoutMs())) , RecompileWithNewEngine(recompileWithNewEngine) @@ -72,8 +72,8 @@ public: void Bootstrap(const TActorContext& ctx) { StartTime = TInstant::Now(); - Counters->ReportCompileStart(DbCounters); - + Counters->ReportCompileStart(DbCounters); + LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Start compilation" << ", self: " << ctx.SelfID << ", cluster: " << Query.Cluster @@ -211,7 +211,7 @@ private: stats.SetCpuTimeUs(CompileCpuTime.MicroSeconds()); ctx.Send(Owner, responseEv.Release()); - Counters->ReportCompileFinish(DbCounters); + Counters->ReportCompileFinish(DbCounters); Die(ctx); } @@ -251,9 +251,9 @@ private: auto kqpResult = std::move(AsyncCompileResult->GetResult()); auto status = GetYdbStatus(kqpResult); - auto database = Query.Database; + auto database = Query.Database; if (kqpResult.SqlVersion) { - Counters->ReportSqlVersion(DbCounters, *kqpResult.SqlVersion); + Counters->ReportSqlVersion(DbCounters, *kqpResult.SqlVersion); } if (status == Ydb::StatusIds::SUCCESS) { @@ -269,7 +269,7 @@ private: auto now = TInstant::Now(); auto duration = now - StartTime; - Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime); + Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime); LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Compilation successful" << ", self: " << ctx.SelfID @@ -301,7 +301,7 @@ private: << ", self: " << ctx.SelfID << ", status: " << Ydb::StatusIds_StatusCode_Name(status) << ", issues: " << kqpResult.Issues().ToString()); - Counters->ReportCompileError(DbCounters); + Counters->ReportCompileError(DbCounters); } Reply(KqpCompileResult, ctx); @@ -338,13 +338,13 @@ private: KqpCompileResult->PreparedQueryNewEngine.reset(kqpResult.PreparingQuery.release()); auto duration = TInstant::Now() - RecompileStartTime; - Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime); + Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime); LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation successful" << ", self: " << ctx.SelfID << ", duration: " << duration); } else { - Counters->ReportCompileError(DbCounters); + Counters->ReportCompileError(DbCounters); Counters->ForceNewEngineCompileErrors->Inc(); LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation failed" @@ -375,7 +375,7 @@ private: TString Uid; TKqpQueryId Query; TString UserToken; - TKqpDbCountersPtr DbCounters; + TKqpDbCountersPtr DbCounters; TKikimrConfiguration::TPtr Config; TDuration CompilationTimeout; bool RecompileWithNewEngine; @@ -402,7 +402,7 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings, const TTableServiceConfig& serviceConfig, TIntrusivePtr<TModuleResolverState> moduleResolverState, - TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken, + TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken, TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine) { return new TKqpCompileActor(owner, kqpSettings, serviceConfig, moduleResolverState, counters, uid, diff --git a/ydb/core/kqp/kqp_compile_request.cpp b/ydb/core/kqp/kqp_compile_request.cpp index e82d5b890c..2c18af159b 100644 --- a/ydb/core/kqp/kqp_compile_request.cpp +++ b/ydb/core/kqp/kqp_compile_request.cpp @@ -24,14 +24,14 @@ public: } TKqpCompileRequestActor(const TActorId& owner, const TString& userToken, const TMaybe<TString>& uid, - TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters) + TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters) : Owner(owner) , UserToken(userToken) , Uid(uid) , Query(std::move(query)) , KeepInCache(keepInCache) - , Deadline(deadline) - , DbCounters(dbCounters) {} + , Deadline(deadline) + , DbCounters(dbCounters) {} void Bootstrap(const TActorContext& ctx) { TimeoutTimerId = CreateLongTimer(ctx, Deadline - TInstant::Now(), @@ -41,7 +41,7 @@ public: std::swap(Query, query); auto compileEv = MakeHolder<TEvKqp::TEvCompileRequest>(UserToken, Uid, std::move(query), - KeepInCache, Deadline, DbCounters); + KeepInCache, Deadline, DbCounters); ctx.Send(MakeKqpCompileServiceID(ctx.SelfID.NodeId()), compileEv.Release()); Become(&TKqpCompileRequestActor::MainState); @@ -80,7 +80,7 @@ public: << ", queryUid: " << compileResult.Uid); auto recompileEv = MakeHolder<TEvKqp::TEvRecompileRequest>(UserToken, compileResult.Uid, compileResult.Query, - Deadline, DbCounters); + Deadline, DbCounters); ctx.Send(MakeKqpCompileServiceID(ctx.SelfID.NodeId()), recompileEv.Release()); DeferredResponse.Reset(); @@ -287,7 +287,7 @@ private: TMaybe<TKqpQueryId> Query; bool KeepInCache = false; TInstant Deadline; - TKqpDbCountersPtr DbCounters; + TKqpDbCountersPtr DbCounters; TActorId TimeoutTimerId; THashMap<TTableId, ui64> TableVersions; THolder<TEvKqp::TEvCompileResponse> DeferredResponse; @@ -295,9 +295,9 @@ private: IActor* CreateKqpCompileRequestActor(const TActorId& owner, const TString& userToken, const TMaybe<TString>& uid, - TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters) + TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters) { - return new TKqpCompileRequestActor(owner, userToken, uid, std::move(query), keepInCache, deadline, dbCounters); + return new TKqpCompileRequestActor(owner, userToken, uid, std::move(query), keepInCache, deadline, dbCounters); } } // namespace NKqp diff --git a/ydb/core/kqp/kqp_compile_service.cpp b/ydb/core/kqp/kqp_compile_service.cpp index 49114bf8f8..da886224a2 100644 --- a/ydb/core/kqp/kqp_compile_service.cpp +++ b/ydb/core/kqp/kqp_compile_service.cpp @@ -10,7 +10,7 @@ #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/cache/cache.h> +#include <library/cpp/cache/cache.h> #include <util/string/escape.h> @@ -180,14 +180,14 @@ private: struct TKqpCompileRequest { TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache, - const TString& userToken, const TInstant& deadline, TKqpDbCountersPtr dbCounters) + const TString& userToken, const TInstant& deadline, TKqpDbCountersPtr dbCounters) : Sender(sender) , Query(std::move(query)) , Uid(uid) , KeepInCache(keepInCache) , UserToken(userToken) - , Deadline(deadline) - , DbCounters(dbCounters) {} + , Deadline(deadline) + , DbCounters(dbCounters) {} TActorId Sender; TKqpQueryId Query; @@ -195,7 +195,7 @@ struct TKqpCompileRequest { bool KeepInCache = false; TString UserToken; TInstant Deadline; - TKqpDbCountersPtr DbCounters; + TKqpDbCountersPtr DbCounters; TActorId CompileActor; }; @@ -417,16 +417,16 @@ private: *Counters->CompileQueryCacheBytes = QueryCache.Bytes(); auto userSid = NACLib::TUserToken(request.UserToken).GetUserSID(); - auto dbCounters = request.DbCounters; + auto dbCounters = request.DbCounters; if (request.Uid) { - Counters->ReportCompileRequestGet(dbCounters); + Counters->ReportCompileRequestGet(dbCounters); auto compileResult = QueryCache.FindByUid(*request.Uid, request.KeepInCache); if (compileResult) { Y_ENSURE(compileResult->Query); if (compileResult->Query->UserSid == userSid) { - Counters->ReportQueryCacheHit(dbCounters, true); + Counters->ReportQueryCacheHit(dbCounters, true); LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Served query from cache by uid" << ", sender: " << ev->Sender @@ -444,7 +444,7 @@ private: } } - Counters->ReportQueryCacheHit(dbCounters, false); + Counters->ReportQueryCacheHit(dbCounters, false); LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Query not found" << ", sender: " << ev->Sender @@ -455,7 +455,7 @@ private: return; } - Counters->ReportCompileRequestCompile(dbCounters); + Counters->ReportCompileRequestCompile(dbCounters); Y_ENSURE(request.Query); auto& query = *request.Query; @@ -468,7 +468,7 @@ private: auto compileResult = QueryCache.FindByQuery(query, request.KeepInCache); if (compileResult) { - Counters->ReportQueryCacheHit(dbCounters, true); + Counters->ReportQueryCacheHit(dbCounters, true); LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Served query from cache" << ", sender: " << ev->Sender @@ -478,13 +478,13 @@ private: return; } - Counters->ReportQueryCacheHit(dbCounters, false); + Counters->ReportQueryCacheHit(dbCounters, false); TKqpCompileRequest compileRequest(ev->Sender, CreateGuidAsString(), std::move(*request.Query), - request.KeepInCache, request.UserToken, request.Deadline, dbCounters); + request.KeepInCache, request.UserToken, request.Deadline, dbCounters); if (!RequestsQueue.Enqueue(std::move(compileRequest))) { - Counters->ReportCompileRequestRejected(dbCounters); + Counters->ReportCompileRequestRejected(dbCounters); LOG_WARN_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Requests queue size limit exceeded" << ", sender: " << ev->Sender @@ -519,18 +519,18 @@ private: LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Received recompile request" << ", sender: " << ev->Sender); - auto dbCounters = request.DbCounters; - Counters->ReportRecompileRequestGet(dbCounters); + auto dbCounters = request.DbCounters; + Counters->ReportRecompileRequestGet(dbCounters); auto compileResult = QueryCache.FindByUid(request.Uid, false); if (compileResult || request.Query) { - Counters->ReportCompileRequestCompile(dbCounters); + Counters->ReportCompileRequestCompile(dbCounters); TKqpCompileRequest compileRequest(ev->Sender, request.Uid, compileResult ? *compileResult->Query : *request.Query, - true, request.UserToken, request.Deadline, dbCounters); + true, request.UserToken, request.Deadline, dbCounters); if (!RequestsQueue.Enqueue(std::move(compileRequest))) { - Counters->ReportCompileRequestRejected(dbCounters); + Counters->ReportCompileRequestRejected(dbCounters); LOG_WARN_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Requests queue size limit exceeded" << ", sender: " << ev->Sender @@ -623,10 +623,10 @@ private: LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Received invalidate request" << ", sender: " << ev->Sender - << ", queryUid: " << request.Uid); + << ", queryUid: " << request.Uid); - auto dbCounters = request.DbCounters; - Counters->ReportCompileRequestInvalidate(dbCounters); + auto dbCounters = request.DbCounters; + Counters->ReportCompileRequestInvalidate(dbCounters); QueryCache.EraseByUid(request.Uid); } @@ -657,7 +657,7 @@ private: << ", sender: " << request->Sender << ", deadline: " << request->Deadline); - Counters->ReportCompileRequestTimeout(request->DbCounters); + Counters->ReportCompileRequestTimeout(request->DbCounters); NYql::TIssue issue(NYql::TPosition(), "Compilation timed out."); ReplyError(request->Sender, "", Ydb::StatusIds::TIMEOUT, {issue}, ctx); diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp index 8013638d33..47489a9bd5 100644 --- a/ydb/core/kqp/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/kqp_ic_gateway.cpp @@ -1007,10 +1007,10 @@ public: } TMaybe<NYql::TKikimrClusterConfig> GetClusterConfig(const TString& cluster) override { - Y_UNUSED(cluster); - return {}; - } - + Y_UNUSED(cluster); + return {}; + } + TMaybe<TString> GetSetting(const TString& cluster, const TString& name) override { Y_UNUSED(cluster); Y_UNUSED(name); diff --git a/ydb/core/kqp/kqp_impl.h b/ydb/core/kqp/kqp_impl.h index d9f81fe65e..7d6f172899 100644 --- a/ydb/core/kqp/kqp_impl.h +++ b/ydb/core/kqp/kqp_impl.h @@ -51,11 +51,11 @@ IActor* CreateKqpCompileService(const NKikimrConfig::TTableServiceConfig& servic IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings, const NKikimrConfig::TTableServiceConfig& serviceConfig, TIntrusivePtr<TModuleResolverState> moduleResolverState, - TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken, + TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken, TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine); IActor* CreateKqpCompileRequestActor(const TActorId& owner, const TString& userToken, const TMaybe<TString>& uid, - TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters); + TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters); struct TKqpWorkerSettings { TString Cluster; @@ -64,14 +64,14 @@ struct TKqpWorkerSettings { NKikimrConfig::TTableServiceConfig Service; - TKqpDbCountersPtr DbCounters; - + TKqpDbCountersPtr DbCounters; + TKqpWorkerSettings(const TString& cluster, const TString& database, - const NKikimrConfig::TTableServiceConfig& serviceConfig, TKqpDbCountersPtr dbCounters) + const NKikimrConfig::TTableServiceConfig& serviceConfig, TKqpDbCountersPtr dbCounters) : Cluster(cluster) , Database(database) - , Service(serviceConfig) - , DbCounters(dbCounters) {} + , Service(serviceConfig) + , DbCounters(dbCounters) {} }; IActor* CreateKqpWorkerActor(const TActorId& owner, const TString& sessionId, diff --git a/ydb/core/kqp/kqp_worker_actor.cpp b/ydb/core/kqp/kqp_worker_actor.cpp index efdc2ebb1b..cccb8a51e8 100644 --- a/ydb/core/kqp/kqp_worker_actor.cpp +++ b/ydb/core/kqp/kqp_worker_actor.cpp @@ -179,7 +179,7 @@ public: void Bootstrap(const TActorContext& ctx) { LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, "Worker bootstrapped, workerId: " << ctx.SelfID); - Counters->ReportWorkerCreated(Settings.DbCounters); + Counters->ReportWorkerCreated(Settings.DbCounters); std::shared_ptr<NYql::IKikimrGateway::IKqpTableMetadataLoader> loader = std::make_shared<TKqpTableMetadataLoader>(TlsActivationContext->ActorSystem(), false); Gateway = CreateKikimrIcGateway(Settings.Cluster, Settings.Database, std::move(loader), @@ -200,7 +200,7 @@ public: auto requestInfo = TKqpRequestInfo(event.GetTraceId(), event.GetRequest().GetSessionId()); if (CheckRequest(requestInfo, ev->Sender, proxyRequestId, ctx)) { LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, requestInfo << "Session closed due to explicit close event"); - Counters->ReportWorkerClosedRequest(Settings.DbCounters); + Counters->ReportWorkerClosedRequest(Settings.DbCounters); FinalCleanup(ctx); } } @@ -448,7 +448,7 @@ public: if (timerId == IdleTimerId) { LOG_NOTICE_S(ctx, NKikimrServices::KQP_WORKER, TKqpRequestInfo("", SessionId) << "Worker idle timeout, worker destroyed"); - Counters->ReportWorkerClosedIdle(Settings.DbCounters); + Counters->ReportWorkerClosedIdle(Settings.DbCounters); FinalCleanup(ctx); } } @@ -527,7 +527,7 @@ public: TKqpRequestInfo(QueryState->TraceId, SessionId), Ydb::StatusIds::BAD_SESSION, "Session is being closed", ctx); - Counters->ReportWorkerClosedError(Settings.DbCounters); + Counters->ReportWorkerClosedError(Settings.DbCounters); FinalCleanup(ctx); } @@ -876,7 +876,7 @@ private: } auto compileRequestActor = CreateKqpCompileRequestActor(ctx.SelfID, QueryState->UserToken, uid, - std::move(query), keepInCache, compileDeadline, Settings.DbCounters); + std::move(query), keepInCache, compileDeadline, Settings.DbCounters); ctx.ExecutorThread.RegisterActor(compileRequestActor); Become(&TKqpWorkerActor::CompileQueryState); @@ -907,7 +907,7 @@ private: if (Settings.LongSession) { QueryState.Reset(); } else { - Counters->ReportWorkerClosedError(Settings.DbCounters); + Counters->ReportWorkerClosedError(Settings.DbCounters); FinalCleanup(ctx); } }; @@ -931,11 +931,11 @@ private: case Ydb::Table::TransactionControl::kBeginTx: { beginTxResult = BeginTransaction(txControl.begin_tx()); - Counters->ReportBeginTransaction(Settings.DbCounters, - beginTxResult.EvictedTx, - beginTxResult.CurrentActiveTx, - beginTxResult.CurrentAbortedTx); - + Counters->ReportBeginTransaction(Settings.DbCounters, + beginTxResult.EvictedTx, + beginTxResult.CurrentActiveTx, + beginTxResult.CurrentAbortedTx); + if (!beginTxResult.Success()) { QueryState->AsyncQueryResult = MakeKikimrResultHolder( NCommon::ResultFromErrors<TQueryResult>(beginTxResult.Issues())); @@ -944,7 +944,7 @@ private: return; } - Counters->ReportTxCreated(Settings.DbCounters); + Counters->ReportTxCreated(Settings.DbCounters); QueryState->TxId = beginTxResult.TxId; if (commit) { @@ -1206,7 +1206,7 @@ private: if (isFinal) { StopIdleTimer(ctx); - Counters->ReportQueriesPerWorker(Settings.DbCounters, QueryId); + Counters->ReportQueriesPerWorker(Settings.DbCounters, QueryId); MakeNewQueryState(); } @@ -1214,7 +1214,7 @@ private: if (Settings.LongSession) { if (isFinal) { auto abortedCount = KqpHost->AbortAll(); - Counters->ReportTxAborted(Settings.DbCounters, abortedCount); + Counters->ReportTxAborted(Settings.DbCounters, abortedCount); } CleanupState->AsyncResult = KqpHost->RollbackAborted(); } else { @@ -1234,16 +1234,16 @@ private: Y_VERIFY(CleanupState); if (CleanupState->AsyncResult) { - auto cleanupTime = TInstant::Now() - CleanupState->Start; - Counters->ReportWorkerCleanupLatency(Settings.DbCounters, cleanupTime); + auto cleanupTime = TInstant::Now() - CleanupState->Start; + Counters->ReportWorkerCleanupLatency(Settings.DbCounters, cleanupTime); } bool isFinal = CleanupState->Final; CleanupState.Reset(); if (isFinal) { - auto lifeSpan = TInstant::Now() - CreationTime; - Counters->ReportWorkerFinished(Settings.DbCounters, lifeSpan); + auto lifeSpan = TInstant::Now() - CreationTime; + Counters->ReportWorkerFinished(Settings.DbCounters, lifeSpan); auto closeEv = MakeHolder<TEvKqp::TEvCloseSessionResponse>(); closeEv->Record.SetStatus(Ydb::StatusIds::SUCCESS); @@ -1523,20 +1523,20 @@ private: if (status == Ydb::StatusIds::INTERNAL_ERROR) { LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, requestInfo << "Worker destroyed due to internal error"); - Counters->ReportWorkerClosedError(Settings.DbCounters); + Counters->ReportWorkerClosedError(Settings.DbCounters); return false; } if (status == Ydb::StatusIds::BAD_SESSION) { LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, requestInfo << "Worker destroyed due to session error"); - Counters->ReportWorkerClosedError(Settings.DbCounters); + Counters->ReportWorkerClosedError(Settings.DbCounters); return false; } } else { if (status != Ydb::StatusIds::SUCCESS) { LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, requestInfo << "Worker destroyed due to query error"); - Counters->ReportWorkerClosedError(Settings.DbCounters); + Counters->ReportWorkerClosedError(Settings.DbCounters); return false; } } @@ -1544,7 +1544,7 @@ private: if (!keepSession) { LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, requestInfo << "Worker destroyed due to negative keep session flag"); - Counters->ReportWorkerClosedRequest(Settings.DbCounters); + Counters->ReportWorkerClosedRequest(Settings.DbCounters); return false; } @@ -1616,17 +1616,17 @@ private: auto& record = responseEv->Record.GetRef(); auto status = record.GetYdbStatus(); - auto now = TInstant::Now(); - auto queryDuration = now - QueryState->StartTime; + auto now = TInstant::Now(); + auto queryDuration = now - QueryState->StartTime; if (status == Ydb::StatusIds::SUCCESS) { - Counters->ReportQueryLatency(Settings.DbCounters, queryRequest.GetAction(), queryDuration); + Counters->ReportQueryLatency(Settings.DbCounters, queryRequest.GetAction(), queryDuration); auto maxReadType = ExtractMostHeavyReadType(queryResult.QueryPlan); if (maxReadType == ETableReadType::FullScan) { - Counters->ReportQueryWithFullScan(Settings.DbCounters); + Counters->ReportQueryWithFullScan(Settings.DbCounters); } else if (maxReadType == ETableReadType::Scan) { - Counters->ReportQueryWithRangeScan(Settings.DbCounters); + Counters->ReportQueryWithRangeScan(Settings.DbCounters); } ui32 affectedShardsCount = 0; @@ -1640,12 +1640,12 @@ private: } } - Counters->ReportQueryAffectedShards(Settings.DbCounters, affectedShardsCount); - Counters->ReportQueryReadRows(Settings.DbCounters, readRowsCount); - Counters->ReportQueryReadBytes(Settings.DbCounters, readBytesCount); - Counters->ReportQueryReadSets(Settings.DbCounters, queryResult.QueryStats.GetReadSetsCount()); - Counters->ReportQueryMaxShardReplySize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardReplySize()); - Counters->ReportQueryMaxShardProgramSize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardProgramSize()); + Counters->ReportQueryAffectedShards(Settings.DbCounters, affectedShardsCount); + Counters->ReportQueryReadRows(Settings.DbCounters, readRowsCount); + Counters->ReportQueryReadBytes(Settings.DbCounters, readBytesCount); + Counters->ReportQueryReadSets(Settings.DbCounters, queryResult.QueryStats.GetReadSetsCount()); + Counters->ReportQueryMaxShardReplySize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardReplySize()); + Counters->ReportQueryMaxShardProgramSize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardProgramSize()); if (QueryState->QueryCompileResult && QueryState->QueryCompileResult->PreparedQueryNewEngine && QueryState->NewEngineCompatibleQuery) @@ -1665,7 +1665,7 @@ private: } if (queryResult.SqlVersion) { - Counters->ReportSqlVersion(Settings.DbCounters, *queryResult.SqlVersion); + Counters->ReportSqlVersion(Settings.DbCounters, *queryResult.SqlVersion); } if (Settings.LongSession && status != Ydb::StatusIds::SUCCESS) { @@ -1712,7 +1712,7 @@ private: if (IsExecuteAction(queryRequest.GetAction())) { auto ru = CalcRequestUnit(stats); record.SetConsumedRu(ru); - CollectSystemViewQueryStats(ctx, &stats, queryDuration, queryRequest.GetDatabase(), ru); + CollectSystemViewQueryStats(ctx, &stats, queryDuration, queryRequest.GetDatabase(), ru); SlowLogQuery(ctx, requestInfo, queryDuration, status, [&record](){ ui64 resultsSize = 0; for (auto& result : record.GetResponse().GetResults()) { @@ -1796,38 +1796,38 @@ private: "Pending previous query completion", ctx); } - void CollectSystemViewQueryStats(const TActorContext& ctx, - const NKqpProto::TKqpStatsQuery* stats, TDuration queryDuration, - const TString& database, ui64 requestUnits) - { - auto type = QueryState->Request.GetType(); - switch (type) { - case NKikimrKqp::QUERY_TYPE_SQL_DML: - case NKikimrKqp::QUERY_TYPE_PREPARED_DML: - case NKikimrKqp::QUERY_TYPE_SQL_SCAN: + void CollectSystemViewQueryStats(const TActorContext& ctx, + const NKqpProto::TKqpStatsQuery* stats, TDuration queryDuration, + const TString& database, ui64 requestUnits) + { + auto type = QueryState->Request.GetType(); + switch (type) { + case NKikimrKqp::QUERY_TYPE_SQL_DML: + case NKikimrKqp::QUERY_TYPE_PREPARED_DML: + case NKikimrKqp::QUERY_TYPE_SQL_SCAN: case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { - auto userSID = NACLib::TUserToken(QueryState->UserToken).GetUserSID(); - NSysView::CollectQueryStats(ctx, stats, queryDuration, ExtractQueryText(), - userSID, QueryState->ParametersSize, database, type, requestUnits); + auto userSID = NACLib::TUserToken(QueryState->UserToken).GetUserSID(); + NSysView::CollectQueryStats(ctx, stats, queryDuration, ExtractQueryText(), + userSID, QueryState->ParametersSize, database, type, requestUnits); break; - } - default: - break; - } - } - - TString ExtractQueryText() const { - auto compileResult = QueryState->QueryCompileResult; - if (compileResult) { - if (compileResult->Query) { - return compileResult->Query->Text; - } - return {}; - } - return QueryState->Request.GetQuery(); - } - + } + default: + break; + } + } + + TString ExtractQueryText() const { + auto compileResult = QueryState->QueryCompileResult; + if (compileResult) { + if (compileResult->Query) { + return compileResult->Query->Text; + } + return {}; + } + return QueryState->Request.GetQuery(); + } + void SlowLogQuery(const TActorContext &ctx, const TKqpRequestInfo& requestInfo, const TDuration& duration, Ydb::StatusIds::StatusCode status, const std::function<ui64()>& resultsSizeFunc) { @@ -1858,7 +1858,7 @@ private: username = "UNAUTHENTICATED"; } - auto queryText = ExtractQueryText(); + auto queryText = ExtractQueryText(); auto paramsText = TStringBuilder() << ToString(QueryState->ParametersSize) @@ -2037,7 +2037,7 @@ private: TString replyTxId; if (txInfo) { - Counters->ReportTransaction(Settings.DbCounters, *txInfo); + Counters->ReportTransaction(Settings.DbCounters, *txInfo); switch (txInfo->Status) { case TKqpTransactionInfo::EStatus::Active: @@ -2057,7 +2057,7 @@ private: } TString InvalidateQuery(const TKqpCompileResult& compileResult, const TActorContext& ctx) { - auto invalidateEv = MakeHolder<TEvKqp::TEvCompileInvalidateRequest>(compileResult.Uid, Settings.DbCounters); + auto invalidateEv = MakeHolder<TEvKqp::TEvCompileInvalidateRequest>(compileResult.Uid, Settings.DbCounters); ctx.Send(MakeKqpCompileServiceID(ctx.SelfID.NodeId()), invalidateEv.Release()); return compileResult.Uid; diff --git a/ydb/core/kqp/node/kqp_node.cpp b/ydb/core/kqp/node/kqp_node.cpp index 7acf87f65f..24f8f2ac79 100644 --- a/ydb/core/kqp/node/kqp_node.cpp +++ b/ydb/core/kqp/node/kqp_node.cpp @@ -16,7 +16,7 @@ #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/monlib/service/pages/templates.h> -#include <util/string/join.h> +#include <util/string/join.h> namespace NKikimr { namespace NKqp { diff --git a/ydb/core/kqp/prepare/kqp_prepare.h b/ydb/core/kqp/prepare/kqp_prepare.h index 295f55ecbb..d114a44945 100644 --- a/ydb/core/kqp/prepare/kqp_prepare.h +++ b/ydb/core/kqp/prepare/kqp_prepare.h @@ -127,7 +127,7 @@ struct TKqlTransformContext : TThrRefBase { ReplyTarget = {}; AnalyzeResults = {}; MkqlResults.clear(); - QueryStats = {}; + QueryStats = {}; PreparingKql = nullptr; PreparedKql = nullptr; PhysicalQuery = nullptr; diff --git a/ydb/core/kqp/prepare/kqp_query_exec.cpp b/ydb/core/kqp/prepare/kqp_query_exec.cpp index f07f557b36..bc2d4e94ce 100644 --- a/ydb/core/kqp/prepare/kqp_query_exec.cpp +++ b/ydb/core/kqp/prepare/kqp_query_exec.cpp @@ -387,7 +387,7 @@ private: void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats::TTxStats& txStats) { auto& dstExec = *dst.AddExecutions(); NKqpProto::TKqpExecutionExtraStats executionExtraStats; - + dstExec.SetDurationUs(txStats.GetDurationUs()); dstExec.SetCpuTimeUs(txStats.GetComputeCpuTimeUsec()); @@ -444,18 +444,18 @@ void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats:: // NOTE: This might be incorrect in case when single shard has several // tables, i.e. collocated tables. affectedShards += table.GetShardCount(); - } - + } + executionExtraStats.SetAffectedShards(affectedShards); dstExec.MutableExtra()->PackFrom(executionExtraStats); -} - +} + } // namespace void TKqlTransformContext::AddMkqlStats(const TString& program, NKikimrQueryStats::TTxStats&& txStats) { Y_UNUSED(program); - + ExtractQueryStats(QueryStats, txStats); } diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index f0ac0ccd54..035d6b7701 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -17,13 +17,13 @@ public: : SessionCtx(sessionCtx) {} private: - TStatus HandleClusterConfig(TKiClusterConfig node, TExprContext& ctx) override { - Y_UNUSED(node); - Y_UNUSED(ctx); - - return TStatus::Ok; - } - + TStatus HandleClusterConfig(TKiClusterConfig node, TExprContext& ctx) override { + Y_UNUSED(node); + Y_UNUSED(ctx); + + return TStatus::Ok; + } + TStatus HandleWriteTable(TKiWriteTable node, TExprContext& ctx) override { Y_UNUSED(ctx); @@ -332,35 +332,35 @@ public: return KikimrProviderName; } - TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override { - auto config = Gateway->GetClusterConfig(cluster); - if (!config) { - return {}; - } - + TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override { + auto config = Gateway->GetClusterConfig(cluster); + if (!config) { + return {}; + } + TPositionHandle pos; - - TVector<TExprBase> locators; - auto& grpcData = config->GetGrpc(); - for (size_t index = 0; index < grpcData.LocatorsSize(); ++index) { - locators.push_back(Build<TCoAtom>(ctx, pos) - .Value(grpcData.GetLocators(index)) - .Done()); - } - - return Build<TKiClusterConfig>(ctx, pos) - .GrpcData<TKiGrpcData>() - .Locators<TCoAtomList>() - .Add(locators) - .Build() - .TimeoutMs<TCoAtom>().Build(ToString(grpcData.GetTimeoutMs())) - .MaxMessageSizeBytes<TCoAtom>().Build(ToString(grpcData.GetMaxMessageSizeBytes())) - .MaxInFlight<TCoAtom>().Build(ToString(grpcData.GetMaxInFlight())) - .Build() - .TvmId<TCoAtom>().Build(ToString(config->GetTvmId())) - .Done().Ptr(); - } - + + TVector<TExprBase> locators; + auto& grpcData = config->GetGrpc(); + for (size_t index = 0; index < grpcData.LocatorsSize(); ++index) { + locators.push_back(Build<TCoAtom>(ctx, pos) + .Value(grpcData.GetLocators(index)) + .Done()); + } + + return Build<TKiClusterConfig>(ctx, pos) + .GrpcData<TKiGrpcData>() + .Locators<TCoAtomList>() + .Add(locators) + .Build() + .TimeoutMs<TCoAtom>().Build(ToString(grpcData.GetTimeoutMs())) + .MaxMessageSizeBytes<TCoAtom>().Build(ToString(grpcData.GetMaxMessageSizeBytes())) + .MaxInFlight<TCoAtom>().Build(ToString(grpcData.GetMaxInFlight())) + .Build() + .TvmId<TCoAtom>().Build(ToString(config->GetTvmId())) + .Done().Ptr(); + } + IGraphTransformer& GetIntentDeterminationTransformer() override { return *IntentDeterminationTransformer; } @@ -698,10 +698,10 @@ IGraphTransformer::TStatus TKiSinkVisitorTransformer::DoTransform(TExprNode::TPt auto callable = TCallable(input); - if (auto node = callable.Maybe<TKiClusterConfig>()) { - return HandleClusterConfig(node.Cast(), ctx); - } - + if (auto node = callable.Maybe<TKiClusterConfig>()) { + return HandleClusterConfig(node.Cast(), ctx); + } + if (auto node = callable.Maybe<TKiWriteTable>()) { return HandleWriteTable(node.Cast(), ctx); } diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json index 2b4225c182..c54d82a2ca 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json @@ -426,24 +426,24 @@ ] }, { - "Name": "TKiGrpcData", - "Base": "TExprBase", - "Match": {"Type": "Tuple"}, - "Children": [ - {"Index": 0, "Name": "Locators", "Type": "TCoAtomList"}, - {"Index": 1, "Name": "TimeoutMs", "Type": "TCoAtom"}, - {"Index": 2, "Name": "MaxMessageSizeBytes", "Type": "TCoAtom"}, - {"Index": 3, "Name": "MaxInFlight", "Type": "TCoAtom"} - ] - }, - { - "Name": "TKiClusterConfig", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "KiClusterConfig"}, - "Children": [ - {"Index": 0, "Name": "GrpcData", "Type": "TKiGrpcData"}, - {"Index": 1, "Name": "TvmId", "Type": "TCoAtom"} - ] + "Name": "TKiGrpcData", + "Base": "TExprBase", + "Match": {"Type": "Tuple"}, + "Children": [ + {"Index": 0, "Name": "Locators", "Type": "TCoAtomList"}, + {"Index": 1, "Name": "TimeoutMs", "Type": "TCoAtom"}, + {"Index": 2, "Name": "MaxMessageSizeBytes", "Type": "TCoAtom"}, + {"Index": 3, "Name": "MaxInFlight", "Type": "TCoAtom"} + ] + }, + { + "Name": "TKiClusterConfig", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KiClusterConfig"}, + "Children": [ + {"Index": 0, "Name": "GrpcData", "Type": "TKiGrpcData"}, + {"Index": 1, "Name": "TvmId", "Type": "TCoAtom"} + ] } ] } diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index d944b1aebd..7598494a2b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -585,7 +585,7 @@ public: virtual bool HasCluster(const TString& cluster) = 0; virtual TVector<TString> GetClusters() = 0; virtual TString GetDefaultCluster() = 0; - virtual TMaybe<TKikimrClusterConfig> GetClusterConfig(const TString& cluster) = 0; + virtual TMaybe<TKikimrClusterConfig> GetClusterConfig(const TString& cluster) = 0; virtual TMaybe<TString> GetSetting(const TString& cluster, const TString& name) = 0; virtual void SetToken(const TString& cluster, const TString& token) = 0; diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp index 3ffa3a6114..0ba0b34952 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp @@ -475,7 +475,7 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, const TMaybe<bool>& useNewEngine, T TKiExploreTxResults txExplore; if (!ExploreTx(commit.World(), ctx, kiDataSink, txExplore)) { - return node.Ptr(); + return node.Ptr(); } if (txExplore.HasExecute) { @@ -484,7 +484,7 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, const TMaybe<bool>& useNewEngine, T auto txSyncSet = txExplore.GetSyncSet(); if (!CheckTx(commit.World(), kiDataSink, txExplore.Ops, txSyncSet)) { - return node.Ptr(); + return node.Ptr(); } bool hasScheme; @@ -578,7 +578,7 @@ TExprNode::TPtr KiBuildResult(TExprBase node, const TString& cluster, TExprCont auto resFill = node.Cast<TResFill>(); if (resFill.DelegatedSource().Value() != KikimrProviderName) { - return node.Ptr(); + return node.Ptr(); } if (resFill.Data().Maybe<TCoNth>().Tuple().Maybe<TCoRight>().Input().Maybe<TKiExecDataQuery>()) { diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index 8ee0475fe9..635d164827 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -38,7 +38,7 @@ struct TKikimrData { DataSourceNames.insert(TKiReadTableScheme::CallableName()); DataSourceNames.insert(TKiReadTableList::CallableName()); - DataSinkNames.insert(TKiClusterConfig::CallableName()); + DataSinkNames.insert(TKiClusterConfig::CallableName()); DataSinkNames.insert(TKiWriteTable::CallableName()); DataSinkNames.insert(TKiUpdateTable::CallableName()); DataSinkNames.insert(TKiDeleteTable::CallableName()); diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h index 31e0cdf1d8..e1aa3cda2b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h @@ -27,7 +27,7 @@ public: TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final; private: - virtual TStatus HandleClusterConfig(NNodes::TKiClusterConfig, TExprContext& ctx) = 0; + virtual TStatus HandleClusterConfig(NNodes::TKiClusterConfig, TExprContext& ctx) = 0; virtual TStatus HandleWriteTable(NNodes::TKiWriteTable node, TExprContext& ctx) = 0; virtual TStatus HandleUpdateTable(NNodes::TKiUpdateTable node, TExprContext& ctx) = 0; virtual TStatus HandleDeleteTable(NNodes::TKiDeleteTable node, TExprContext& ctx) = 0; diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index 1dccb61e09..692165204a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -269,19 +269,19 @@ public: , SessionCtx(sessionCtx) {} private: - virtual TStatus HandleClusterConfig(TKiClusterConfig node, TExprContext& ctx) override { - if (!EnsureTuple(node.GrpcData().Ref(), ctx)) { - return TStatus::Error; - } - - if (!EnsureAtom(node.TvmId().Ref(), ctx)) { - return TStatus::Error; - } - + virtual TStatus HandleClusterConfig(TKiClusterConfig node, TExprContext& ctx) override { + if (!EnsureTuple(node.GrpcData().Ref(), ctx)) { + return TStatus::Error; + } + + if (!EnsureAtom(node.TvmId().Ref(), ctx)) { + return TStatus::Error; + } + node.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>()); - return TStatus::Ok; - } - + return TStatus::Ok; + } + virtual TStatus HandleWriteTable(TKiWriteTable node, TExprContext& ctx) override { if (!EnsureWorldType(node.World().Ref(), ctx)) { return TStatus::Error; diff --git a/ydb/core/kqp/proxy/kqp_proxy_service.cpp b/ydb/core/kqp/proxy/kqp_proxy_service.cpp index 24a764ed41..decd2cb301 100644 --- a/ydb/core/kqp/proxy/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy/kqp_proxy_service.cpp @@ -122,11 +122,11 @@ class TLocalSessionsRegistry { public: TLocalSessionsRegistry() = default; - TKqpSessionInfo* Create(const TString& sessionId, const TActorId& workerId, - const TString& database, TKqpDbCountersPtr dbCounters) - { + TKqpSessionInfo* Create(const TString& sessionId, const TActorId& workerId, + const TString& database, TKqpDbCountersPtr dbCounters) + { RecentUsedSessions.push_front(sessionId); - auto result = LocalSessions.emplace(sessionId, TKqpSessionInfo(sessionId, workerId, database, dbCounters)); + auto result = LocalSessions.emplace(sessionId, TKqpSessionInfo(sessionId, workerId, database, dbCounters)); Links.emplace(sessionId, RecentUsedSessions.begin()); Y_VERIFY(result.second, "Duplicate session id!"); TargetIdIndex.emplace(workerId, sessionId); @@ -434,7 +434,7 @@ public: KQP_PROXY_LOG_I("Received tenant pool status, serving tenants: " << JoinRange(", ", Tenants.begin(), Tenants.end())); for (auto& [_, sessionInfo] : LocalSessions) { - if (!sessionInfo.Database.empty() && !Tenants.contains(sessionInfo.Database)) { + if (!sessionInfo.Database.empty() && !Tenants.contains(sessionInfo.Database)) { auto closeSessionEv = MakeHolder<TEvKqp::TEvCloseSessionRequest>(); closeSessionEv->Record.MutableRequest()->SetSessionId(sessionInfo.SessionId); Send(sessionInfo.WorkerId, closeSessionEv.Release()); @@ -514,7 +514,7 @@ public: auto responseEv = MakeHolder<TEvKqp::TEvCreateSessionResponse>(); TProcessResult<TKqpSessionInfo*> result; - TKqpDbCountersPtr dbCounters; + TKqpDbCountersPtr dbCounters; const auto deadline = TInstant::MicroSeconds(event.GetDeadlineUs()); @@ -523,18 +523,18 @@ public: { auto& response = *responseEv->Record.MutableResponse(); response.SetSessionId(result.Value->SessionId); - dbCounters = result.Value->DbCounters; - } else { - dbCounters = Counters->GetDbCounters(request.GetDatabase()); + dbCounters = result.Value->DbCounters; + } else { + dbCounters = Counters->GetDbCounters(request.GetDatabase()); } - LogRequest(request, requestInfo, ev->Sender, dbCounters); - + LogRequest(request, requestInfo, ev->Sender, dbCounters); + responseEv->Record.SetResourceExhausted(result.ResourceExhausted); responseEv->Record.SetYdbStatus(result.YdbStatus); responseEv->Record.SetError(result.Error); - LogResponse(event.GetTraceId(), responseEv->Record, dbCounters); + LogResponse(event.GetTraceId(), responseEv->Record, dbCounters); Send(ev->Sender, responseEv.Release(), 0, ev->Cookie); } @@ -543,20 +543,20 @@ public: auto& request = *event.MutableRequest(); TString traceId = event.GetTraceId(); TKqpRequestInfo requestInfo(traceId); - ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvQueryRequest); + ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvQueryRequest); + + auto dbCounters = GetDbCountersForSession(request.GetSessionId()); - auto dbCounters = GetDbCountersForSession(request.GetSessionId()); - auto queryLimitBytes = TableServiceConfig.GetQueryLimitBytes(); if (queryLimitBytes && IsSqlQuery(request.GetType())) { auto querySizeBytes = request.GetQuery().size(); if (querySizeBytes > queryLimitBytes) { TString error = TStringBuilder() << "Query text size exceeds limit (" << querySizeBytes << "b > " << queryLimitBytes << "b)"; ReplyProcessError(Ydb::StatusIds::BAD_REQUEST, error, requestId); - if (!dbCounters) { - dbCounters = Counters->GetDbCounters(request.GetDatabase()); - } - LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); + if (!dbCounters) { + dbCounters = Counters->GetDbCounters(request.GetDatabase()); + } + LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); return; } } @@ -565,11 +565,11 @@ public: if (paramsLimitBytes) { auto paramsBytes = request.GetParameters().ByteSizeLong(); if (paramsBytes > paramsLimitBytes) { - if (!dbCounters) { - dbCounters = Counters->GetDbCounters(request.GetDatabase()); - } - LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); - + if (!dbCounters) { + dbCounters = Counters->GetDbCounters(request.GetDatabase()); + } + LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); + TString error = TStringBuilder() << "Parameters size exceeds limit (" << paramsBytes << "b > " << paramsLimitBytes << "b)"; ReplyProcessError(Ydb::StatusIds::BAD_REQUEST, error, requestId); return; @@ -580,17 +580,17 @@ public: if (!request.GetSessionId().empty()) { TProcessResult<TActorId> result; if (!TryGetSessionTargetActor(request.GetSessionId(), requestInfo, result)) { - if (!dbCounters) { - dbCounters = Counters->GetDbCounters(request.GetDatabase()); - } - LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); + if (!dbCounters) { + dbCounters = Counters->GetDbCounters(request.GetDatabase()); + } + LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); ReplyProcessError(result.YdbStatus, result.Error, requestId); return; } targetId = result.Value; - - LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); + + LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); } else { TString cluster = request.GetCluster(); if (cluster.empty()) { @@ -599,23 +599,23 @@ public: TProcessResult<TKqpSessionInfo*> result; if (!CreateNewSessionWorker(requestInfo, cluster, false, request.GetDatabase(), result)) { - if (!dbCounters) { - dbCounters = Counters->GetDbCounters(request.GetDatabase()); - } - LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); + if (!dbCounters) { + dbCounters = Counters->GetDbCounters(request.GetDatabase()); + } + LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); ReplyProcessError(result.YdbStatus, result.Error, requestId); return; } targetId = result.Value->WorkerId; request.SetSessionId(result.Value->SessionId); - dbCounters = result.Value->DbCounters; - - LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); + dbCounters = result.Value->DbCounters; + + LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); } TString sessionId = request.GetSessionId(); - PendingRequests.SetSessionId(requestId, sessionId, dbCounters); + PendingRequests.SetSessionId(requestId, sessionId, dbCounters); // We add extra milliseconds to the user-specified timeout, so it means we give additional priority for worker replies, // because it is much better to give detailed error message rather than generic timeout. // For example, it helps to avoid race in event order when worker and proxy recieve timeout at the same moment. @@ -633,10 +633,10 @@ public: TKqpRequestInfo requestInfo(event.GetTraceId()); TString sessionId = request.GetSessionId(); - auto dbCounters = GetDbCountersForSession(sessionId); - - LogRequest(request, requestInfo, ev->Sender, dbCounters); - + auto dbCounters = GetDbCountersForSession(sessionId); + + LogRequest(request, requestInfo, ev->Sender, dbCounters); + if (!sessionId.empty()) { TProcessResult<TActorId> result; if (TryGetSessionTargetActor(sessionId, requestInfo, result)) { @@ -651,15 +651,15 @@ public: auto traceId = event.GetTraceId(); TKqpRequestInfo requestInfo(traceId); - auto sessionId = request.GetSessionId(); - ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvPingSessionRequest); + auto sessionId = request.GetSessionId(); + ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvPingSessionRequest); + + auto dbCounters = GetDbCountersForSession(sessionId); + + LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); - auto dbCounters = GetDbCountersForSession(sessionId); - - LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters); - TProcessResult<TActorId> result; - if (!TryGetSessionTargetActor(sessionId, requestInfo, result)) { + if (!TryGetSessionTargetActor(sessionId, requestInfo, result)) { ReplyProcessError(result.YdbStatus, result.Error, requestId); return; } @@ -669,7 +669,7 @@ public: timeout = TDuration::MilliSeconds(Min(timeout.MilliSeconds(), (ui64)request.GetTimeoutMs())); } - PendingRequests.SetSessionId(requestId, sessionId, dbCounters); + PendingRequests.SetSessionId(requestId, sessionId, dbCounters); StartQueryTimeout(requestId, timeout); Send(result.Value, ev->Release().Release(), IEventHandle::FlagTrackDelivery, requestId); } @@ -685,7 +685,7 @@ public: return; } - LogResponse(proxyRequest->TraceId, ev->Get()->Record, proxyRequest->DbCounters); + LogResponse(proxyRequest->TraceId, ev->Get()->Record, proxyRequest->DbCounters); Send(proxyRequest->Sender, ev->Release().Release(), 0, proxyRequest->SenderCookie); TKqpRequestInfo requestInfo(proxyRequest->TraceId); @@ -993,7 +993,7 @@ public: private: void LogResponse(const TKqpRequestInfo& requestInfo, - const NKikimrKqp::TEvProcessResponse& event, TKqpDbCountersPtr dbCounters) + const NKikimrKqp::TEvProcessResponse& event, TKqpDbCountersPtr dbCounters) { auto status = event.GetYdbStatus(); if (status != Ydb::StatusIds::SUCCESS) { @@ -1004,74 +1004,74 @@ private: } void LogResponse(const TKqpRequestInfo& requestInfo, - const TEvKqp::TProtoArenaHolder<NKikimrKqp::TEvQueryResponse>& holder, - TKqpDbCountersPtr dbCounters) + const TEvKqp::TProtoArenaHolder<NKikimrKqp::TEvQueryResponse>& holder, + TKqpDbCountersPtr dbCounters) { Y_UNUSED(requestInfo); const auto& event = holder.GetRef(); - Counters->ReportResponseStatus(dbCounters, event.ByteSize(), + Counters->ReportResponseStatus(dbCounters, event.ByteSize(), event.GetYdbStatus()); - + for (auto& issue : event.GetResponse().GetQueryIssues()) { - Counters->ReportIssues(dbCounters, issue); + Counters->ReportIssues(dbCounters, issue); } - ui64 resultsBytes = 0; + ui64 resultsBytes = 0; for (auto& result : event.GetResponse().GetResults()) { - resultsBytes += result.ByteSize(); + resultsBytes += result.ByteSize(); } - Counters->ReportResultsBytes(dbCounters, resultsBytes); + Counters->ReportResultsBytes(dbCounters, resultsBytes); } void LogResponse(const TKqpRequestInfo& requestInfo, - const NKikimrKqp::TEvCreateSessionResponse& event, TKqpDbCountersPtr dbCounters) + const NKikimrKqp::TEvCreateSessionResponse& event, TKqpDbCountersPtr dbCounters) { Y_UNUSED(requestInfo); - - Counters->ReportResponseStatus(dbCounters, event.ByteSize(), + + Counters->ReportResponseStatus(dbCounters, event.ByteSize(), event.GetYdbStatus()); } void LogResponse(const TKqpRequestInfo& requestInfo, - const NKikimrKqp::TEvPingSessionResponse& event, TKqpDbCountersPtr dbCounters) + const NKikimrKqp::TEvPingSessionResponse& event, TKqpDbCountersPtr dbCounters) { Y_UNUSED(requestInfo); - - Counters->ReportResponseStatus(dbCounters, event.ByteSize(), event.GetStatus()); + + Counters->ReportResponseStatus(dbCounters, event.ByteSize(), event.GetStatus()); } void LogRequest(const NKikimrKqp::TCloseSessionRequest& request, - const TKqpRequestInfo& requestInfo, const TActorId& sender, - TKqpDbCountersPtr dbCounters) + const TKqpRequestInfo& requestInfo, const TActorId& sender, + TKqpDbCountersPtr dbCounters) { KQP_PROXY_LOG_D(requestInfo << "Received close session request, sender: " << sender << ", SessionId: " << request.GetSessionId()); - Counters->ReportCloseSession(dbCounters, request.ByteSize()); + Counters->ReportCloseSession(dbCounters, request.ByteSize()); } void LogRequest(const NKikimrKqp::TQueryRequest& request, - const TKqpRequestInfo& requestInfo, const TActorId& sender, ui64 requestId, - TKqpDbCountersPtr dbCounters) + const TKqpRequestInfo& requestInfo, const TActorId& sender, ui64 requestId, + TKqpDbCountersPtr dbCounters) { KQP_PROXY_LOG_D(requestInfo << "Received new query request, sender: " << sender << ", RequestId: " << requestId << ", Query: \"" << request.GetQuery().substr(0, 10000) << "\""); - Counters->ReportQueryRequest(dbCounters, request); + Counters->ReportQueryRequest(dbCounters, request); } void LogRequest(const NKikimrKqp::TCreateSessionRequest& request, - const TKqpRequestInfo& requestInfo, const TActorId& sender, - TKqpDbCountersPtr dbCounters) + const TKqpRequestInfo& requestInfo, const TActorId& sender, + TKqpDbCountersPtr dbCounters) { KQP_PROXY_LOG_D(requestInfo << "Received create session request, sender: " << sender); - Counters->ReportCreateSession(dbCounters, request.ByteSize()); + Counters->ReportCreateSession(dbCounters, request.ByteSize()); } void LogRequest(const NKikimrKqp::TPingSessionRequest& request, - const TKqpRequestInfo& requestInfo, const TActorId& sender, ui64 requestId, - TKqpDbCountersPtr dbCounters) + const TKqpRequestInfo& requestInfo, const TActorId& sender, ui64 requestId, + TKqpDbCountersPtr dbCounters) { KQP_PROXY_LOG_D(requestInfo << "Received ping session request, sender: " << sender << " selfID: " << SelfId() << ", RequestId: " << requestId); - Counters->ReportPingSession(dbCounters, request.ByteSize()); + Counters->ReportPingSession(dbCounters, request.ByteSize()); } bool ReplyProcessError(Ydb::StatusIds::StatusCode ydbStatus, const TString& message, ui64 requestId) @@ -1155,16 +1155,16 @@ private: auto sessionId = EncodeSessionId(SelfId().NodeId(), CreateGuidAsString()); - auto dbCounters = Counters->GetDbCounters(database); - - TKqpWorkerSettings workerSettings(cluster, database, TableServiceConfig, dbCounters); + auto dbCounters = Counters->GetDbCounters(database); + + TKqpWorkerSettings workerSettings(cluster, database, TableServiceConfig, dbCounters); workerSettings.LongSession = longSession; IActor* workerActor = AppData()->FeatureFlags.GetEnableKqpSessionActor() ? CreateKqpSessionActor(SelfId(), sessionId, KqpSettings, workerSettings, ModuleResolverState, Counters) : CreateKqpWorkerActor(SelfId(), sessionId, KqpSettings, workerSettings, ModuleResolverState, Counters); auto workerId = TlsActivationContext->ExecutorThread.RegisterActor(workerActor, TMailboxType::HTSwap, AppData()->UserPoolId); - TKqpSessionInfo* sessionInfo = LocalSessions.Create(sessionId, workerId, database, dbCounters); + TKqpSessionInfo* sessionInfo = LocalSessions.Create(sessionId, workerId, database, dbCounters); KQP_PROXY_LOG_D(requestInfo << "Created new session" << ", sessionId: " << sessionInfo->SessionId @@ -1256,11 +1256,11 @@ private: NYql::NDq::SetYqlLogLevels(yqlPriority); } - TKqpDbCountersPtr GetDbCountersForSession(const TString& sessionId) const { - auto localSession = LocalSessions.FindPtr(sessionId); - return localSession ? localSession->DbCounters : nullptr; - } - + TKqpDbCountersPtr GetDbCountersForSession(const TString& sessionId) const { + auto localSession = LocalSessions.FindPtr(sessionId); + return localSession ? localSession->DbCounters : nullptr; + } + private: NYql::NLog::YqlLoggerScope YqlLoggerScope; NKikimrConfig::TLogConfig LogConfig; @@ -1281,7 +1281,7 @@ private: TIntrusivePtr<TKqpCounters> Counters; TLocalSessionsRegistry LocalSessions; - + bool ServerWorkerBalancerComplete = false; std::optional<TString> SelfDataCenterId; TVector<NKikimrKqp::TKqpProxyNodeResources> PeerProxyNodeResources; diff --git a/ydb/core/kqp/proxy/kqp_proxy_service.h b/ydb/core/kqp/proxy/kqp_proxy_service.h index 4c1c92bdbe..f5059d3f36 100644 --- a/ydb/core/kqp/proxy/kqp_proxy_service.h +++ b/ydb/core/kqp/proxy/kqp_proxy_service.h @@ -16,10 +16,10 @@ struct TKqpProxyRequest { TString TraceId; ui32 EventType; TString SessionId; - TKqpDbCountersPtr DbCounters; + TKqpDbCountersPtr DbCounters; TKqpProxyRequest(const TActorId& sender, ui64 senderCookie, const TString& traceId, - ui32 eventType) + ui32 eventType) : Sender(sender) , SenderCookie(senderCookie) , TraceId(traceId) @@ -27,9 +27,9 @@ struct TKqpProxyRequest { , SessionId() {} - void SetSessionId(const TString& sessionId, TKqpDbCountersPtr dbCounters) { + void SetSessionId(const TString& sessionId, TKqpDbCountersPtr dbCounters) { SessionId = sessionId; - DbCounters = dbCounters; + DbCounters = dbCounters; } }; @@ -43,9 +43,9 @@ public: : RequestId(0) {} - ui64 RegisterRequest(const TActorId& sender, ui64 senderCookie, const TString& traceId, ui32 eventType) { + ui64 RegisterRequest(const TActorId& sender, ui64 senderCookie, const TString& traceId, ui32 eventType) { ui64 NewRequestId = ++RequestId; - PendingRequests.emplace(NewRequestId, TKqpProxyRequest(sender, senderCookie, traceId, eventType)); + PendingRequests.emplace(NewRequestId, TKqpProxyRequest(sender, senderCookie, traceId, eventType)); return NewRequestId; } @@ -53,9 +53,9 @@ public: return PendingRequests.FindPtr(requestId); } - void SetSessionId(ui64 requestId, const TString& sessionId, TKqpDbCountersPtr dbCounters) { + void SetSessionId(ui64 requestId, const TString& sessionId, TKqpDbCountersPtr dbCounters) { TKqpProxyRequest* ptr = PendingRequests.FindPtr(requestId); - ptr->SetSessionId(sessionId, dbCounters); + ptr->SetSessionId(sessionId, dbCounters); } void Erase(ui64 requestId) { @@ -77,18 +77,18 @@ struct TKqpSessionInfo { TString SessionId; TActorId WorkerId; TString Database; - TKqpDbCountersPtr DbCounters; + TKqpDbCountersPtr DbCounters; ui32 UseFrequency; TInstant LastRequestAt; TInstant CreatedAt; TInstant ShutdownStartedAt; - TKqpSessionInfo(const TString& sessionId, const TActorId& workerId, - const TString& database, TKqpDbCountersPtr dbCounters) + TKqpSessionInfo(const TString& sessionId, const TActorId& workerId, + const TString& database, TKqpDbCountersPtr dbCounters) : SessionId(sessionId) , WorkerId(workerId) , Database(database) - , DbCounters(dbCounters) + , DbCounters(dbCounters) , UseFrequency(0) , ShutdownStartedAt() { diff --git a/ydb/core/kqp/rm/kqp_rm.cpp b/ydb/core/kqp/rm/kqp_rm.cpp index d187e4c641..30fc08ff5f 100644 --- a/ydb/core/kqp/rm/kqp_rm.cpp +++ b/ydb/core/kqp/rm/kqp_rm.cpp @@ -231,7 +231,7 @@ public: void Bootstrap() { ActorSystem = TlsActivationContext->ActorSystem(); if (!Counters) { - Counters = MakeIntrusive<TKqpCounters>(AppData()->Counters); + Counters = MakeIntrusive<TKqpCounters>(AppData()->Counters); } LOG_D("Start KqpResourceManagerActor at " << SelfId() << " with ResourceBroker at " << ResourceBrokerId); diff --git a/ydb/core/kqp/runtime/kqp_compute.h b/ydb/core/kqp/runtime/kqp_compute.h index e6f8492b41..c55a4e2a21 100644 --- a/ydb/core/kqp/runtime/kqp_compute.h +++ b/ydb/core/kqp/runtime/kqp_compute.h @@ -4,18 +4,18 @@ #include <ydb/core/scheme/scheme_tabledefs.h> #include <ydb/core/tablet_flat/flat_row_eggs.h> - + // TODO rename file to runtime_compute_context.h namespace NKikimr { namespace NMiniKQL { class TKqpComputeContextBase : public NYql::NDq::TDqComputeContextBase { -public: - struct TColumn { - NTable::TTag Tag; - NScheme::TTypeId Type; - }; +public: + struct TColumn { + NTable::TTag Tag; + NScheme::TTypeId Type; + }; // used only at then building of a computation graph, to inject taskId in runtime nodes void SetCurrentTaskId(ui64 taskId) { CurrentTaskId = taskId; } @@ -23,7 +23,7 @@ public: private: ui64 CurrentTaskId = 0; -}; +}; TComputationNodeFactory GetKqpBaseComputeFactory(const TKqpComputeContextBase* computeCtx); diff --git a/ydb/core/kqp/runtime/kqp_program_builder.cpp b/ydb/core/kqp/runtime/kqp_program_builder.cpp index ff5de8411f..b5169c19d8 100644 --- a/ydb/core/kqp/runtime/kqp_program_builder.cpp +++ b/ydb/core/kqp/runtime/kqp_program_builder.cpp @@ -73,8 +73,8 @@ TRuntimeNode BuildKeyPrefixIndicesList(const TProgramBuilder& builder, const TSt TRuntimeNode BuildTableIdLiteral(const TTableId& tableId, TProgramBuilder& builder) { TVector<TRuntimeNode> tupleItems { - builder.NewDataLiteral<ui64>(tableId.PathId.OwnerId), - builder.NewDataLiteral<ui64>(tableId.PathId.LocalPathId), + builder.NewDataLiteral<ui64>(tableId.PathId.OwnerId), + builder.NewDataLiteral<ui64>(tableId.PathId.LocalPathId), builder.NewDataLiteral<NUdf::EDataSlot::String>(tableId.SysViewInfo), builder.NewDataLiteral<ui64>(tableId.SchemaVersion), }; diff --git a/ydb/core/kqp/runtime/kqp_read_table.cpp b/ydb/core/kqp/runtime/kqp_read_table.cpp index e566dc0bea..358ebd0a3e 100644 --- a/ydb/core/kqp/runtime/kqp_read_table.cpp +++ b/ydb/core/kqp/runtime/kqp_read_table.cpp @@ -1,11 +1,11 @@ -#include "kqp_read_table.h" -#include "kqp_scan_data.h" - +#include "kqp_read_table.h" +#include "kqp_scan_data.h" + #include <ydb/core/engine/mkql_keys.h> - + #include <ydb/library/yql/minikql/mkql_node_cast.h> - -namespace NKikimr { + +namespace NKikimr { namespace NKqp { @@ -32,43 +32,43 @@ NUdf::TDataTypeId UnwrapDataTypeFromStruct(const TStructType& structType, ui32 i } // namespace NKqp -namespace NMiniKQL { - -using namespace NTable; -using namespace NUdf; - -static TCell BuildKeyTupleCell(const TType* type, const TUnboxedValue& value, const TTypeEnvironment& env) { - MKQL_ENSURE_S(type); - - auto keyType = type; - auto keyValue = value; - - if (type->IsOptional()) { - if (!value) { - return TCell(); - } - - keyType = AS_TYPE(TOptionalType, type)->GetItemType(); - keyValue = value.GetOptionalValue(); - } - - return MakeCell(AS_TYPE(TDataType, keyType)->GetSchemeType(), keyValue, env, true); -} - -void BuildKeyTupleCells(const TTupleType* tupleType, const TUnboxedValue& tupleValue, TVector<TCell>& cells, - const TTypeEnvironment& env) -{ - MKQL_ENSURE_S(tupleType); - - cells.resize(tupleType->GetElementsCount()); - - for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { - auto keyType = tupleType->GetElementType(i); - auto keyValue = tupleValue.GetElement(i); - cells[i] = BuildKeyTupleCell(keyType, keyValue, env); - } -} - +namespace NMiniKQL { + +using namespace NTable; +using namespace NUdf; + +static TCell BuildKeyTupleCell(const TType* type, const TUnboxedValue& value, const TTypeEnvironment& env) { + MKQL_ENSURE_S(type); + + auto keyType = type; + auto keyValue = value; + + if (type->IsOptional()) { + if (!value) { + return TCell(); + } + + keyType = AS_TYPE(TOptionalType, type)->GetItemType(); + keyValue = value.GetOptionalValue(); + } + + return MakeCell(AS_TYPE(TDataType, keyType)->GetSchemeType(), keyValue, env, true); +} + +void BuildKeyTupleCells(const TTupleType* tupleType, const TUnboxedValue& tupleValue, TVector<TCell>& cells, + const TTypeEnvironment& env) +{ + MKQL_ENSURE_S(tupleType); + + cells.resize(tupleType->GetElementsCount()); + + for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { + auto keyType = tupleType->GetElementType(i); + auto keyValue = tupleValue.GetElement(i); + cells[i] = BuildKeyTupleCell(keyType, keyValue, env); + } +} + void ParseReadColumns(const TType* readType, const TRuntimeNode& tagsNode, TSmallVec<TKqpComputeContextBase::TColumn>& columns, TSmallVec<TKqpComputeContextBase::TColumn>& systemColumns) { @@ -106,37 +106,37 @@ void ParseWideReadColumns(const TCallable& callable, const TRuntimeNode& tagsNod { auto tags = AS_VALUE(TStructLiteral, tagsNode); MKQL_ENSURE_S(tags); - + TType* returnType = callable.GetType()->GetReturnType(); MKQL_ENSURE_S(returnType->GetKind() == TType::EKind::Flow); - + auto itemType = AS_TYPE(TFlowType, returnType)->GetItemType(); MKQL_ENSURE_S(itemType->GetKind() == TType::EKind::Tuple); - + auto tupleType = AS_TYPE(TTupleType, itemType); MKQL_ENSURE_S(tags->GetValuesCount() == tupleType->GetElementsCount()); - + columns.reserve(tupleType->GetElementsCount()); - + for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { auto memberType = tupleType->GetElementType(i); - + if (memberType->GetKind() == TType::EKind::Optional) { memberType = AS_TYPE(TOptionalType, memberType)->GetItemType(); } - + MKQL_ENSURE_S(memberType->GetKind() == TType::EKind::Data); - + NTable::TTag columnId = AS_VALUE(TDataLiteral, tags->GetValue(i))->AsValue().Get<ui32>(); - + if (IsSystemColumn(columnId)) { systemColumns.push_back({columnId, AS_TYPE(TDataType, memberType)->GetSchemeType()}); } else { columns.push_back({columnId, AS_TYPE(TDataType, memberType)->GetSchemeType()}); } } -} - +} + TParseReadTableResult ParseWideReadTable(TCallable& callable) { MKQL_ENSURE_S(callable.GetInputsCount() >= 4); @@ -230,7 +230,7 @@ TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable) { } namespace { - + class TKqpScanWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase> { using TBase = TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase>; public: diff --git a/ydb/core/kqp/runtime/kqp_read_table.h b/ydb/core/kqp/runtime/kqp_read_table.h index 67133361b9..99c9cc3f05 100644 --- a/ydb/core/kqp/runtime/kqp_read_table.h +++ b/ydb/core/kqp/runtime/kqp_read_table.h @@ -1,29 +1,29 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_scan_data.h> #include <ydb/core/tablet_flat/flat_database.h> #include <ydb/core/scheme/scheme_tabledefs.h> - + #include <ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> - -namespace NKikimr { -namespace NMiniKQL { - -void BuildKeyTupleCells(const TTupleType* tupleType, const NUdf::TUnboxedValue& tupleValue, - TVector<TCell>& cells, const TTypeEnvironment& env); - + +namespace NKikimr { +namespace NMiniKQL { + +void BuildKeyTupleCells(const TTupleType* tupleType, const NUdf::TUnboxedValue& tupleValue, + TVector<TCell>& cells, const TTypeEnvironment& env); + struct TParseReadTableResultBase { - ui32 CallableId = 0; - TTableId TableId; - - TSmallVec<TKqpComputeContextBase::TColumn> Columns; + ui32 CallableId = 0; + TTableId TableId; + + TSmallVec<TKqpComputeContextBase::TColumn> Columns; TSmallVec<TKqpComputeContextBase::TColumn> SystemColumns; - TSmallVec<bool> SkipNullKeys; + TSmallVec<bool> SkipNullKeys; TNode* ItemsLimit = nullptr; bool Reverse = false; -}; - +}; + struct TParseReadTableResult : TParseReadTableResultBase { TTupleLiteral* FromTuple = nullptr; bool FromInclusive = false; @@ -40,12 +40,12 @@ void ParseReadColumns(const TType* readType, const TRuntimeNode& tagsNode, TParseReadTableResult ParseWideReadTable(TCallable& callable); TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable); - + IComputationNode* WrapKqpScanWideReadTableRanges(TCallable& callable, const TComputationNodeFactoryContext& ctx, TKqpScanComputeContext& computeCtx); IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, TKqpScanComputeContext& computeCtx); - + } // namespace NMiniKQL } // namespace NKikimr diff --git a/ydb/core/kqp/runtime/kqp_scan_data.cpp b/ydb/core/kqp/runtime/kqp_scan_data.cpp index fb09ed4be2..c4d8e24196 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data.cpp +++ b/ydb/core/kqp/runtime/kqp_scan_data.cpp @@ -1,14 +1,14 @@ -#include "kqp_scan_data.h" - +#include "kqp_scan_data.h" + #include <ydb/core/engine/minikql/minikql_engine_host.h> #include <ydb/core/protos/tx_datashard.pb.h> - + #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/utils/yql_panic.h> -namespace NKikimr { -namespace NMiniKQL { - +namespace NKikimr { +namespace NMiniKQL { + namespace { struct TBytesStatistics { @@ -248,15 +248,15 @@ std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& val return {sizes.AllocatedBytes, sizes.DataBytes}; } -TKqpScanComputeContext::TScanData::TScanData(const TTableId& tableId, const TTableRange& range, +TKqpScanComputeContext::TScanData::TScanData(const TTableId& tableId, const TTableRange& range, const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys) - : TableId(tableId) - , Range(range) + : TableId(tableId) + , Range(range) , SkipNullKeys(skipNullKeys) - , Columns(columns) + , Columns(columns) , SystemColumns(systemColumns) -{} - +{} + TKqpScanComputeContext::TScanData::TScanData(const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta, NYql::NDqProto::EDqStatsMode statsMode) { @@ -378,8 +378,8 @@ ui64 TKqpScanComputeContext::TScanData::AddRows(const arrow::RecordBatch& batch, } return stats.AllocatedBytes; -} - +} + NUdf::TUnboxedValue TKqpScanComputeContext::TScanData::TakeRow() { YQL_ENSURE(!RowBatches.empty()); auto& batch = RowBatches.front(); @@ -396,7 +396,7 @@ NUdf::TUnboxedValue TKqpScanComputeContext::TScanData::TakeRow() { void TKqpScanComputeContext::AddTableScan(ui32, const TTableId& tableId, const TTableRange& range, const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys) -{ +{ auto scanData = TKqpScanComputeContext::TScanData(tableId, range, columns, systemColumns, skipNullKeys); if (Y_UNLIKELY(StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC)) { @@ -408,9 +408,9 @@ void TKqpScanComputeContext::AddTableScan(ui32, const TTableId& tableId, const T } auto result = Scans.emplace(0, std::move(scanData)); - Y_ENSURE(result.second); -} - + Y_ENSURE(result.second); +} + void TKqpScanComputeContext::AddTableScan(ui32, const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta, NYql::NDqProto::EDqStatsMode statsMode) { @@ -422,15 +422,15 @@ void TKqpScanComputeContext::AddTableScan(ui32, const NKikimrTxDataShard::TKqpTr TKqpScanComputeContext::TScanData& TKqpScanComputeContext::GetTableScan(ui32) { auto scanData = Scans.FindPtr(0); - Y_ENSURE(scanData); - - return *scanData; -} - -TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::GetTableScans() { - return Scans; -} - + Y_ENSURE(scanData); + + return *scanData; +} + +TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::GetTableScans() { + return Scans; +} + const TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::GetTableScans() const { return Scans; } @@ -438,29 +438,29 @@ const TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::Get TIntrusivePtr<IKqpTableReader> TKqpScanComputeContext::ReadTable(ui32) const { auto scanData = Scans.FindPtr(0); Y_ENSURE(scanData); - Y_ENSURE(scanData->TableReader); - - return scanData->TableReader; -} - -class TKqpTableReader : public IKqpTableReader { -public: + Y_ENSURE(scanData->TableReader); + + return scanData->TableReader; +} + +class TKqpTableReader : public IKqpTableReader { +public: TKqpTableReader(TKqpScanComputeContext::TScanData& scanData) : ScanData(scanData) {} - + NUdf::EFetchStatus Next(NUdf::TUnboxedValue& result) override { if (ScanData.IsEmpty()) { if (ScanData.IsFinished()) { - return NUdf::EFetchStatus::Finish; - } - return NUdf::EFetchStatus::Yield; - } - + return NUdf::EFetchStatus::Finish; + } + return NUdf::EFetchStatus::Yield; + } + result = std::move(ScanData.TakeRow()); - return NUdf::EFetchStatus::Ok; - } - + return NUdf::EFetchStatus::Ok; + } + EFetchResult Next(NUdf::TUnboxedValue* const* result) override { if (ScanData.IsEmpty()) { if (ScanData.IsFinished()) { @@ -479,13 +479,13 @@ public: return EFetchResult::One; } -private: - TKqpScanComputeContext::TScanData& ScanData; -}; - +private: + TKqpScanComputeContext::TScanData& ScanData; +}; + TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData) { return MakeIntrusive<TKqpTableReader>(scanData); -} - +} + } // namespace NMiniKQL -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/kqp/runtime/kqp_scan_data.h b/ydb/core/kqp/runtime/kqp_scan_data.h index 9dfe941e49..c8e7d82b32 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data.h +++ b/ydb/core/kqp/runtime/kqp_scan_data.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/services.pb.h> -#include "kqp_compute.h" - +#include "kqp_compute.h" + #include <ydb/core/engine/minikql/minikql_engine_host.h> #include <ydb/core/formats/arrow_helpers.h> #include <ydb/core/scheme/scheme_tabledefs.h> #include <ydb/core/tablet_flat/flat_database.h> - + #include <ydb/library/yql/dq/actors/protos/dq_stats.pb.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> @@ -19,9 +19,9 @@ namespace NKikimrTxDataShard { class TKqpTransaction_TScanTaskMeta; } -namespace NKikimr { -namespace NMiniKQL { - +namespace NKikimr { +namespace NMiniKQL { + std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& value, NScheme::TTypeId type); class IKqpTableReader : public TSimpleRefCount<IKqpTableReader> { @@ -87,13 +87,13 @@ public: public: ui64 TaskId = 0; - TTableId TableId; + TTableId TableId; TString TablePath; - TSerializedTableRange Range; + TSerializedTableRange Range; TSmallVec<bool> SkipNullKeys; // shared with actor via TableReader - TIntrusivePtr<IKqpTableReader> TableReader; + TIntrusivePtr<IKqpTableReader> TableReader; struct TBasicStats { size_t Rows = 0; @@ -126,37 +126,37 @@ public: TQueue<RowBatch> RowBatches; ui64 StoredBytes = 0; bool Finished = false; - }; - -public: + }; + +public: explicit TKqpScanComputeContext(NYql::NDqProto::EDqStatsMode statsMode) : StatsMode(statsMode) {} - - TIntrusivePtr<IKqpTableReader> ReadTable(ui32 callableId) const; - - void AddTableScan(ui32 callableId, const TTableId& tableId, const TTableRange& range, + + TIntrusivePtr<IKqpTableReader> ReadTable(ui32 callableId) const; + + void AddTableScan(ui32 callableId, const TTableId& tableId, const TTableRange& range, const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys); - + void AddTableScan(ui32 callableId, const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta, NYql::NDqProto::EDqStatsMode statsMode); - TScanData& GetTableScan(ui32 callableId); - TMap<ui32, TScanData>& GetTableScans(); + TScanData& GetTableScan(ui32 callableId); + TMap<ui32, TScanData>& GetTableScans(); const TMap<ui32, TScanData>& GetTableScans() const; - + void Clear() { for (auto& scan: Scans) { scan.second.Clear(); } Scans.clear(); } - -private: + +private: const NYql::NDqProto::EDqStatsMode StatsMode; - TMap<ui32, TScanData> Scans; -}; - + TMap<ui32, TScanData> Scans; +}; + TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData); - + } // namespace NMiniKQL -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/kqp/runtime/ya.make b/ydb/core/kqp/runtime/ya.make index 9543fea453..1143b6bc89 100644 --- a/ydb/core/kqp/runtime/ya.make +++ b/ydb/core/kqp/runtime/ya.make @@ -13,11 +13,11 @@ SRCS( kqp_output_stream.cpp kqp_program_builder.cpp kqp_program_builder.h - kqp_read_table.cpp - kqp_read_table.h + kqp_read_table.cpp + kqp_read_table.h kqp_runtime_impl.h - kqp_scan_data.cpp - kqp_scan_data.h + kqp_scan_data.cpp + kqp_scan_data.h kqp_spilling_file.cpp kqp_tasks_runner.cpp kqp_tasks_runner.h diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index 403ff67a85..9bc7889f92 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -131,15 +131,15 @@ TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) { } TKikimrRunner::TKikimrRunner(const TVector<NKikimrKqp::TKqpSetting>& kqpSettings, const TString& authToken, - const TString& domainRoot, ui32 nodeCount) + const TString& domainRoot, ui32 nodeCount) : TKikimrRunner(TKikimrSettings() .SetKqpSettings(kqpSettings) .SetAuthToken(authToken) .SetDomainRoot(domainRoot) .SetNodeCount(nodeCount)) {} -TKikimrRunner::TKikimrRunner(const NKikimrConfig::TAppConfig& appConfig, const TString& authToken, - const TString& domainRoot, ui32 nodeCount) +TKikimrRunner::TKikimrRunner(const NKikimrConfig::TAppConfig& appConfig, const TString& authToken, + const TString& domainRoot, ui32 nodeCount) : TKikimrRunner(TKikimrSettings() .SetAppConfig(appConfig) .SetAuthToken(authToken) diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h index b8033b495f..c1d64b65c3 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.h +++ b/ydb/core/kqp/ut/common/kqp_ut_common.h @@ -99,10 +99,10 @@ public: TKikimrRunner(const TKikimrSettings& settings); TKikimrRunner(const TVector<NKikimrKqp::TKqpSetting>& kqpSettings, const TString& authToken = "", - const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1); + const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1); TKikimrRunner(const NKikimrConfig::TAppConfig& appConfig, const TString& authToken = "", - const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1); + const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1); TKikimrRunner(const NKikimrConfig::TAppConfig& appConfig, const TVector<NKikimrKqp::TKqpSetting>& kqpSettings, const TString& authToken = "", const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1); @@ -110,7 +110,7 @@ public: TKikimrRunner(const NKikimrConfig::TFeatureFlags& featureFlags, const TString& authToken = "", const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1); - TKikimrRunner(const TString& authToken = "", const TString& domainRoot = KikimrDefaultUtDomainRoot, + TKikimrRunner(const TString& authToken = "", const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1); ~TKikimrRunner() { @@ -119,7 +119,7 @@ public: Client.Reset(); } - const TString& GetEndpoint() const { return Endpoint; } + const TString& GetEndpoint() const { return Endpoint; } const NYdb::TDriver& GetDriver() const { return *Driver; } NYdb::NScheme::TSchemeClient GetSchemeClient() const { return NYdb::NScheme::TSchemeClient(*Driver); } Tests::TClient& GetTestClient() const { return *Client; } diff --git a/ydb/core/kqp/ut/kqp_acl_ut.cpp b/ydb/core/kqp/ut/kqp_acl_ut.cpp index 0764e66f4d..203c8ba618 100644 --- a/ydb/core/kqp/ut/kqp_acl_ut.cpp +++ b/ydb/core/kqp/ut/kqp_acl_ut.cpp @@ -1,126 +1,126 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> - + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -namespace NKikimr { -namespace NKqp { - -using namespace NYdb; -using namespace NYdb::NTable; - -Y_UNIT_TEST_SUITE(KqpNewEngineAcl) { - Y_UNIT_TEST(FailNavigate) { + +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; + +Y_UNIT_TEST_SUITE(KqpNewEngineAcl) { + Y_UNIT_TEST(FailNavigate) { TKikimrRunner kikimr("user0@builtin"); - - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; SELECT * FROM `/Root/TwoShard`; - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } - - Y_UNIT_TEST(FailResolve) { + )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + } + + Y_UNIT_TEST(FailResolve) { TKikimrRunner kikimr; - { - NYdb::NScheme::TPermissions permissions("user0@builtin", - {"ydb.deprecated.describe_schema"} - ); - auto schemeClient = kikimr.GetSchemeClient(); - auto result = schemeClient.ModifyPermissions("/Root/TwoShard", - NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) - ).ExtractValueSync(); - AssertSuccessResult(result); - } - - auto driverConfig = TDriverConfig() - .SetEndpoint(kikimr.GetEndpoint()) - .SetAuthToken("user0@builtin"); - auto driver = TDriver(driverConfig); + { + NYdb::NScheme::TPermissions permissions("user0@builtin", + {"ydb.deprecated.describe_schema"} + ); + auto schemeClient = kikimr.GetSchemeClient(); + auto result = schemeClient.ModifyPermissions("/Root/TwoShard", + NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) + ).ExtractValueSync(); + AssertSuccessResult(result); + } + + auto driverConfig = TDriverConfig() + .SetEndpoint(kikimr.GetEndpoint()) + .SetAuthToken("user0@builtin"); + auto driver = TDriver(driverConfig); auto db = NYdb::NTable::TTableClient(driver); - - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; + + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; SELECT * FROM `/Root/TwoShard`; - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } - { - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; + } + { + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES - (10u, "One", -10); - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + (10u, "One", -10); + )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } + } driver.Stop(true); - } - - Y_UNIT_TEST(ReadSuccess) { + } + + Y_UNIT_TEST(ReadSuccess) { TKikimrRunner kikimr; - { - NYdb::NScheme::TPermissions permissions("user0@builtin", - {"ydb.deprecated.describe_schema", "ydb.deprecated.select_row"} - ); - auto schemeClient = kikimr.GetSchemeClient(); - auto result = schemeClient.ModifyPermissions("/Root/TwoShard", - NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) - ).ExtractValueSync(); - AssertSuccessResult(result); - } - - auto driverConfig = TDriverConfig() - .SetEndpoint(kikimr.GetEndpoint()) - .SetAuthToken("user0@builtin"); - auto driver = TDriver(driverConfig); + { + NYdb::NScheme::TPermissions permissions("user0@builtin", + {"ydb.deprecated.describe_schema", "ydb.deprecated.select_row"} + ); + auto schemeClient = kikimr.GetSchemeClient(); + auto result = schemeClient.ModifyPermissions("/Root/TwoShard", + NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) + ).ExtractValueSync(); + AssertSuccessResult(result); + } + + auto driverConfig = TDriverConfig() + .SetEndpoint(kikimr.GetEndpoint()) + .SetAuthToken("user0@builtin"); + auto driver = TDriver(driverConfig); auto db = NYdb::NTable::TTableClient(driver); - - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; + + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; SELECT * FROM `/Root/TwoShard`; - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - AssertSuccessResult(result); + )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + AssertSuccessResult(result); driver.Stop(true); - } - - Y_UNIT_TEST(WriteSuccess) { + } + + Y_UNIT_TEST(WriteSuccess) { TKikimrRunner kikimr; - { - NYdb::NScheme::TPermissions permissions("user0@builtin", - {"ydb.deprecated.describe_schema", "ydb.deprecated.update_row"} - ); - auto schemeClient = kikimr.GetSchemeClient(); - auto result = schemeClient.ModifyPermissions("/Root/TwoShard", - NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) - ).ExtractValueSync(); - AssertSuccessResult(result); - } - - auto driverConfig = TDriverConfig() - .SetEndpoint(kikimr.GetEndpoint()) - .SetAuthToken("user0@builtin"); - auto driver = TDriver(driverConfig); + { + NYdb::NScheme::TPermissions permissions("user0@builtin", + {"ydb.deprecated.describe_schema", "ydb.deprecated.update_row"} + ); + auto schemeClient = kikimr.GetSchemeClient(); + auto result = schemeClient.ModifyPermissions("/Root/TwoShard", + NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) + ).ExtractValueSync(); + AssertSuccessResult(result); + } + + auto driverConfig = TDriverConfig() + .SetEndpoint(kikimr.GetEndpoint()) + .SetAuthToken("user0@builtin"); + auto driver = TDriver(driverConfig); auto db = NYdb::NTable::TTableClient(driver); - - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; + + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES - (10u, "One", -10); - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - AssertSuccessResult(result); + (10u, "One", -10); + )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + AssertSuccessResult(result); driver.Stop(true); - } + } Y_UNIT_TEST(RecursiveCreateTableShouldSuccess) { TKikimrRunner kikimr; @@ -158,7 +158,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngineAcl) { driver.Stop(true); } -} - +} + } // namespace NKqp -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/kqp/ut/kqp_scan_ut.cpp b/ydb/core/kqp/ut/kqp_scan_ut.cpp index f5dcb82500..347ca35441 100644 --- a/ydb/core/kqp/ut/kqp_scan_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scan_ut.cpp @@ -1724,7 +1724,7 @@ Y_UNIT_TEST_SUITE(KqpScan) { UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", StreamResultToYson(it)); TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters); - UNIT_ASSERT_VALUES_EQUAL(1, counters.GetQueryTypeCounter(NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN)->Val()); + UNIT_ASSERT_VALUES_EQUAL(1, counters.GetQueryTypeCounter(NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN)->Val()); } Y_UNIT_TEST(DropRedundantSortByPk) { diff --git a/ydb/core/kqp/ut/kqp_sys_view_ut.cpp b/ydb/core/kqp/ut/kqp_sys_view_ut.cpp index 10437bf79e..1f8988517c 100644 --- a/ydb/core/kqp/ut/kqp_sys_view_ut.cpp +++ b/ydb/core/kqp/ut/kqp_sys_view_ut.cpp @@ -1,24 +1,24 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> - + #include <ydb/core/sys_view/service/query_history.h> - -namespace NKikimr { -namespace NKqp { - -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NYdb::NScheme; - -Y_UNIT_TEST_SUITE(KqpSystemView) { - + +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NYdb::NScheme; + +Y_UNIT_TEST_SUITE(KqpSystemView) { + Y_UNIT_TEST(Join) { return; // nodes table is currently switched off - + TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); + auto client = kikimr.GetTableClient(); while (true) { - auto it = client.StreamExecuteScanQuery( + auto it = client.StreamExecuteScanQuery( "select NodeId from `/Root/.sys/partition_stats` where Path = '/Root/KeyValue' limit 1" ).GetValueSync(); UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); @@ -31,7 +31,7 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { } } - auto it = client.StreamExecuteScanQuery(R"( + auto it = client.StreamExecuteScanQuery(R"( --!syntax_v1 select n.Host, ps.Path, ps.RowCount from `/Root/.sys/partition_stats` as ps @@ -44,42 +44,42 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { CompareYson(R"([[["::1"];["/Root/KeyValue"];[2u]]])", StreamResultToYson(it)); } - Y_UNIT_TEST(PartitionStatsSimple) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - - auto it = client.StreamExecuteScanQuery(R"( + Y_UNIT_TEST(PartitionStatsSimple) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + + auto it = client.StreamExecuteScanQuery(R"( SELECT OwnerId, PartIdx, Path, PathId FROM `/Root/.sys/partition_stats` - ORDER BY PathId, PartIdx; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - CompareYson(R"([ - [[72057594046644480u];[0u];["/Root/TwoShard"];[2u]]; - [[72057594046644480u];[1u];["/Root/TwoShard"];[2u]]; - [[72057594046644480u];[0u];["/Root/EightShard"];[3u]]; - [[72057594046644480u];[1u];["/Root/EightShard"];[3u]]; - [[72057594046644480u];[2u];["/Root/EightShard"];[3u]]; - [[72057594046644480u];[3u];["/Root/EightShard"];[3u]]; - [[72057594046644480u];[4u];["/Root/EightShard"];[3u]]; - [[72057594046644480u];[5u];["/Root/EightShard"];[3u]]; - [[72057594046644480u];[6u];["/Root/EightShard"];[3u]]; - [[72057594046644480u];[7u];["/Root/EightShard"];[3u]]; - [[72057594046644480u];[0u];["/Root/Logs"];[4u]]; - [[72057594046644480u];[1u];["/Root/Logs"];[4u]]; - [[72057594046644480u];[2u];["/Root/Logs"];[4u]]; - [[72057594046644480u];[0u];["/Root/BatchUpload"];[5u]]; - [[72057594046644480u];[1u];["/Root/BatchUpload"];[5u]]; - [[72057594046644480u];[2u];["/Root/BatchUpload"];[5u]]; - [[72057594046644480u];[3u];["/Root/BatchUpload"];[5u]]; - [[72057594046644480u];[4u];["/Root/BatchUpload"];[5u]]; - [[72057594046644480u];[5u];["/Root/BatchUpload"];[5u]]; - [[72057594046644480u];[6u];["/Root/BatchUpload"];[5u]]; - [[72057594046644480u];[7u];["/Root/BatchUpload"];[5u]]; - [[72057594046644480u];[8u];["/Root/BatchUpload"];[5u]]; - [[72057594046644480u];[9u];["/Root/BatchUpload"];[5u]]; + ORDER BY PathId, PartIdx; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + CompareYson(R"([ + [[72057594046644480u];[0u];["/Root/TwoShard"];[2u]]; + [[72057594046644480u];[1u];["/Root/TwoShard"];[2u]]; + [[72057594046644480u];[0u];["/Root/EightShard"];[3u]]; + [[72057594046644480u];[1u];["/Root/EightShard"];[3u]]; + [[72057594046644480u];[2u];["/Root/EightShard"];[3u]]; + [[72057594046644480u];[3u];["/Root/EightShard"];[3u]]; + [[72057594046644480u];[4u];["/Root/EightShard"];[3u]]; + [[72057594046644480u];[5u];["/Root/EightShard"];[3u]]; + [[72057594046644480u];[6u];["/Root/EightShard"];[3u]]; + [[72057594046644480u];[7u];["/Root/EightShard"];[3u]]; + [[72057594046644480u];[0u];["/Root/Logs"];[4u]]; + [[72057594046644480u];[1u];["/Root/Logs"];[4u]]; + [[72057594046644480u];[2u];["/Root/Logs"];[4u]]; + [[72057594046644480u];[0u];["/Root/BatchUpload"];[5u]]; + [[72057594046644480u];[1u];["/Root/BatchUpload"];[5u]]; + [[72057594046644480u];[2u];["/Root/BatchUpload"];[5u]]; + [[72057594046644480u];[3u];["/Root/BatchUpload"];[5u]]; + [[72057594046644480u];[4u];["/Root/BatchUpload"];[5u]]; + [[72057594046644480u];[5u];["/Root/BatchUpload"];[5u]]; + [[72057594046644480u];[6u];["/Root/BatchUpload"];[5u]]; + [[72057594046644480u];[7u];["/Root/BatchUpload"];[5u]]; + [[72057594046644480u];[8u];["/Root/BatchUpload"];[5u]]; + [[72057594046644480u];[9u];["/Root/BatchUpload"];[5u]]; [[72057594046644480u];[0u];["/Root/KeyValue"];[6u]]; [[72057594046644480u];[0u];["/Root/KeyValue2"];[7u]]; [[72057594046644480u];[0u];["/Root/Test"];[8u]]; @@ -87,24 +87,24 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { [[72057594046644480u];[1u];["/Root/Join1"];[9u]]; [[72057594046644480u];[0u];["/Root/Join2"];[10u]]; [[72057594046644480u];[1u];["/Root/Join2"];[10u]] - ])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(PartitionStatsRange1) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - + ])", StreamResultToYson(it)); + } + + Y_UNIT_TEST(PartitionStatsRange1) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + TString enablePredicateExtractor = R"( PRAGMA Kikimr.OptEnablePredicateExtract = "true"; )"; TString query = R"( - SELECT OwnerId, PathId, PartIdx, Path + SELECT OwnerId, PathId, PartIdx, Path FROM `/Root/.sys/partition_stats` WHERE OwnerId = 72057594046644480ul AND PathId > 5u AND PathId <= 9u - ORDER BY PathId, PartIdx; + ORDER BY PathId, PartIdx; )"; - + TString expectedYson = R"([ [[72057594046644480u];[6u];[0u];["/Root/KeyValue"]]; [[72057594046644480u];[7u];[0u];["/Root/KeyValue2"]]; @@ -120,22 +120,22 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { it = client.StreamExecuteScanQuery(enablePredicateExtractor + query).GetValueSync(); UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); CompareYson(expectedYson, StreamResultToYson(it)); - } - - Y_UNIT_TEST(PartitionStatsRange2) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); + } + + Y_UNIT_TEST(PartitionStatsRange2) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); TString enablePredicateExtractor = R"( PRAGMA Kikimr.OptEnablePredicateExtract = "true"; )"; - + TString query = R"( - SELECT OwnerId, PathId, PartIdx, Path + SELECT OwnerId, PathId, PartIdx, Path FROM `/Root/.sys/partition_stats` WHERE OwnerId = 72057594046644480ul AND PathId >= 6u AND PathId < 9u - ORDER BY PathId, PartIdx; + ORDER BY PathId, PartIdx; )"; - + TString expectedYson = R"([ [[72057594046644480u];[6u];[0u];["/Root/KeyValue"]]; [[72057594046644480u];[7u];[0u];["/Root/KeyValue2"]]; @@ -149,111 +149,111 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { it = client.StreamExecuteScanQuery(enablePredicateExtractor + query).GetValueSync(); UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); CompareYson(expectedYson, StreamResultToYson(it)); - } - - Y_UNIT_TEST(PartitionStatsRange3) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - - auto it = client.StreamExecuteScanQuery(R"( - SELECT OwnerId, PathId, PartIdx, Path + } + + Y_UNIT_TEST(PartitionStatsRange3) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + + auto it = client.StreamExecuteScanQuery(R"( + SELECT OwnerId, PathId, PartIdx, Path FROM `/Root/.sys/partition_stats` - WHERE OwnerId = 72057594046644480ul AND PathId = 5u AND PartIdx > 1u AND PartIdx < 7u - ORDER BY PathId, PartIdx; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - CompareYson(R"([ + WHERE OwnerId = 72057594046644480ul AND PathId = 5u AND PartIdx > 1u AND PartIdx < 7u + ORDER BY PathId, PartIdx; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + CompareYson(R"([ [[72057594046644480u];[5u];[2u];["/Root/BatchUpload"]]; [[72057594046644480u];[5u];[3u];["/Root/BatchUpload"]]; [[72057594046644480u];[5u];[4u];["/Root/BatchUpload"]]; [[72057594046644480u];[5u];[5u];["/Root/BatchUpload"]]; [[72057594046644480u];[5u];[6u];["/Root/BatchUpload"]]; - ])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(NodesSimple) { - return; // nodes table is currenty switched off - - TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 3); - auto client = kikimr.GetTableClient(); - - ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0); - - auto it = client.StreamExecuteScanQuery(R"( - SELECT NodeId, Host + ])", StreamResultToYson(it)); + } + + Y_UNIT_TEST(NodesSimple) { + return; // nodes table is currenty switched off + + TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 3); + auto client = kikimr.GetTableClient(); + + ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0); + + auto it = client.StreamExecuteScanQuery(R"( + SELECT NodeId, Host FROM `/Root/.sys/nodes`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - auto expected = Sprintf(R"([ + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + auto expected = Sprintf(R"([ [[%du];["::1"]]; [[%du];["::1"]]; [[%du];["::1"]] - ])", offset, offset + 1, offset + 2); - - CompareYson(expected, StreamResultToYson(it)); - } - - Y_UNIT_TEST(NodesRange1) { - return; // nodes table is currenty switched off - - TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 5); - auto client = kikimr.GetTableClient(); - - ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0); - - auto query = Sprintf(R"( - SELECT NodeId, Host + ])", offset, offset + 1, offset + 2); + + CompareYson(expected, StreamResultToYson(it)); + } + + Y_UNIT_TEST(NodesRange1) { + return; // nodes table is currenty switched off + + TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 5); + auto client = kikimr.GetTableClient(); + + ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0); + + auto query = Sprintf(R"( + SELECT NodeId, Host FROM `/Root/.sys/nodes` - WHERE NodeId >= %du AND NodeId <= %du - )", offset + 1, offset + 3); - - auto it = client.StreamExecuteScanQuery(query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - auto expected = Sprintf(R"([ + WHERE NodeId >= %du AND NodeId <= %du + )", offset + 1, offset + 3); + + auto it = client.StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + auto expected = Sprintf(R"([ [[%du];["::1"]]; [[%du];["::1"]]; [[%du];["::1"]] - ])", offset + 1, offset + 2, offset + 3); - - CompareYson(expected, StreamResultToYson(it)); - } - - Y_UNIT_TEST(NodesRange2) { - return; // nodes table is currenty switched off - - TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 5); - auto client = kikimr.GetTableClient(); - - ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0); - - auto query = Sprintf(R"( - SELECT NodeId, Host + ])", offset + 1, offset + 2, offset + 3); + + CompareYson(expected, StreamResultToYson(it)); + } + + Y_UNIT_TEST(NodesRange2) { + return; // nodes table is currenty switched off + + TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 5); + auto client = kikimr.GetTableClient(); + + ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0); + + auto query = Sprintf(R"( + SELECT NodeId, Host FROM `/Root/.sys/nodes` - WHERE NodeId > %du AND NodeId < %du - )", offset + 1, offset + 3); - - auto it = client.StreamExecuteScanQuery(query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - auto expected = Sprintf(R"([ + WHERE NodeId > %du AND NodeId < %du + )", offset + 1, offset + 3); + + auto it = client.StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + auto expected = Sprintf(R"([ [[%du];["::1"]]; - ])", offset + 2); - - CompareYson(expected, StreamResultToYson(it)); - } - + ])", offset + 2); + + CompareYson(expected, StreamResultToYson(it)); + } + Y_UNIT_TEST_NEW_ENGINE(QueryStatsSimple) { - auto checkTable = [&] (const TStringBuf tableName) { - TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 3); - - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - { + auto checkTable = [&] (const TStringBuf tableName) { + TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 3); + + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + { auto result = session.ExecuteDataQuery("SELECT 1;", TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -262,21 +262,21 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { auto result = session.ExecuteDataQuery(Q_(R"( SELECT * FROM `/Root/TwoShard` )"), TTxControl::BeginTx().CommitTx()).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - { + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + { auto result = session.ExecuteDataQuery(Q_(R"( SELECT * FROM `/Root/EightShard` )"), TTxControl::BeginTx().CommitTx()).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - TStringStream request; - request << "SELECT ReadBytes FROM " << tableName << " ORDER BY ReadBytes"; - - auto it = client.StreamExecuteScanQuery(request.Str()).GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + TStringStream request; + request << "SELECT ReadBytes FROM " << tableName << " ORDER BY ReadBytes"; + + auto it = client.StreamExecuteScanQuery(request.Str()).GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); TSet<ui64> readBytesSet; for (;;) { @@ -301,87 +301,87 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { UNIT_ASSERT(readBytesSet.contains(0)); // Pure UNIT_ASSERT(readBytesSet.contains(79)); // TwoShard UNIT_ASSERT(readBytesSet.contains(432)); // EightShard - }; - + }; + checkTable("`/Root/.sys/top_queries_by_read_bytes_one_minute`"); checkTable("`/Root/.sys/top_queries_by_read_bytes_one_hour`"); checkTable("`/Root/.sys/top_queries_by_duration_one_minute`"); checkTable("`/Root/.sys/top_queries_by_duration_one_hour`"); checkTable("`/Root/.sys/top_queries_by_cpu_time_one_minute`"); checkTable("`/Root/.sys/top_queries_by_cpu_time_one_hour`"); - } - - Y_UNIT_TEST(FailNavigate) { - TKikimrRunner kikimr("user0@builtin"); - auto client = kikimr.GetTableClient(); - - auto it = client.StreamExecuteScanQuery(R"( + } + + Y_UNIT_TEST(FailNavigate) { + TKikimrRunner kikimr("user0@builtin"); + auto client = kikimr.GetTableClient(); + + auto it = client.StreamExecuteScanQuery(R"( SELECT PathId FROM `/Root/.sys/partition_stats`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto streamPart = it.ReadNext().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); - } - - Y_UNIT_TEST(FailResolve) { - TKikimrRunner kikimr; - { - TPermissions permissions("user0@builtin", - {"ydb.deprecated.describe_schema"} - ); - auto schemeClient = kikimr.GetSchemeClient(); - auto result = schemeClient.ModifyPermissions("/Root", - TModifyPermissionsSettings().AddGrantPermissions(permissions) - ).ExtractValueSync(); - AssertSuccessResult(result); - } - - auto driverConfig = TDriverConfig() - .SetEndpoint(kikimr.GetEndpoint()) - .SetAuthToken("user0@builtin"); - auto driver = TDriver(driverConfig); - - TTableClient client(driver); - auto it = client.StreamExecuteScanQuery(R"( + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto streamPart = it.ReadNext().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); + } + + Y_UNIT_TEST(FailResolve) { + TKikimrRunner kikimr; + { + TPermissions permissions("user0@builtin", + {"ydb.deprecated.describe_schema"} + ); + auto schemeClient = kikimr.GetSchemeClient(); + auto result = schemeClient.ModifyPermissions("/Root", + TModifyPermissionsSettings().AddGrantPermissions(permissions) + ).ExtractValueSync(); + AssertSuccessResult(result); + } + + auto driverConfig = TDriverConfig() + .SetEndpoint(kikimr.GetEndpoint()) + .SetAuthToken("user0@builtin"); + auto driver = TDriver(driverConfig); + + TTableClient client(driver); + auto it = client.StreamExecuteScanQuery(R"( SELECT PathId FROM `/Root/.sys/partition_stats`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto streamPart = it.ReadNext().GetValueSync(); + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto streamPart = it.ReadNext().GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); driver.Stop(true); - } - - Y_UNIT_TEST(ReadSuccess) { - TKikimrRunner kikimr; - { - TPermissions permissions("user0@builtin", - {"ydb.deprecated.describe_schema", "ydb.deprecated.select_row"} - ); - auto schemeClient = kikimr.GetSchemeClient(); - auto result = schemeClient.ModifyPermissions("/Root", - TModifyPermissionsSettings().AddGrantPermissions(permissions) - ).ExtractValueSync(); - AssertSuccessResult(result); - } - - auto driverConfig = TDriverConfig() - .SetEndpoint(kikimr.GetEndpoint()) - .SetAuthToken("user0@builtin"); - auto driver = TDriver(driverConfig); - - TTableClient client(driver); - auto it = client.StreamExecuteScanQuery(R"( + } + + Y_UNIT_TEST(ReadSuccess) { + TKikimrRunner kikimr; + { + TPermissions permissions("user0@builtin", + {"ydb.deprecated.describe_schema", "ydb.deprecated.select_row"} + ); + auto schemeClient = kikimr.GetSchemeClient(); + auto result = schemeClient.ModifyPermissions("/Root", + TModifyPermissionsSettings().AddGrantPermissions(permissions) + ).ExtractValueSync(); + AssertSuccessResult(result); + } + + auto driverConfig = TDriverConfig() + .SetEndpoint(kikimr.GetEndpoint()) + .SetAuthToken("user0@builtin"); + auto driver = TDriver(driverConfig); + + TTableClient client(driver); + auto it = client.StreamExecuteScanQuery(R"( SELECT PathId FROM `/Root/.sys/partition_stats`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto streamPart = it.ReadNext().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SUCCESS); + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto streamPart = it.ReadNext().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SUCCESS); driver.Stop(true); - } -} - -} // namspace NKqp -} // namespace NKikimr + } +} + +} // namspace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/ut/ya.make b/ydb/core/kqp/ut/ya.make index 97135d645e..609ec3237f 100644 --- a/ydb/core/kqp/ut/ya.make +++ b/ydb/core/kqp/ut/ya.make @@ -19,7 +19,7 @@ ELSE() ENDIF() SRCS( - kqp_acl_ut.cpp + kqp_acl_ut.cpp kqp_arrow_in_channels_ut.cpp kqp_document_api_ut.cpp kqp_effects_perf_ut.cpp @@ -49,7 +49,7 @@ SRCS( kqp_sort_ut.cpp kqp_stats_ut.cpp kqp_sqlin_ut.cpp - kqp_sys_view_ut.cpp + kqp_sys_view_ut.cpp kqp_sys_col_ut.cpp kqp_table_predicate_ut.cpp kqp_tx_ut.cpp diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp index 6be79630f3..779381fd51 100644 --- a/ydb/core/mind/bscontroller/bsc.cpp +++ b/ydb/core/mind/bscontroller/bsc.cpp @@ -1,7 +1,7 @@ #include "impl.h" #include "config.h" #include "self_heal.h" -#include "sys_view.h" +#include "sys_view.h" namespace NKikimr { @@ -121,9 +121,9 @@ void TBlobStorageController::OnActivateExecutor(const TActorContext&) { // create stat processor StatProcessorActorId = Register(CreateStatProcessorActor()); - // create system views collector + // create system views collector SystemViewsCollectorId = Register(CreateSystemViewsCollector()); - + Executor()->RegisterExternalTabletCounters(TabletCountersPtr); if (!ResponsivenessPinger) { ResponsivenessPinger = new TTabletResponsivenessPinger(TabletCounters->Simple()[NBlobStorageController::COUNTER_RESPONSE_TIME_USEC], TDuration::Seconds(1)); diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h index 3ceeac6960..ba311a254c 100644 --- a/ydb/core/mind/bscontroller/impl.h +++ b/ydb/core/mind/bscontroller/impl.h @@ -1335,10 +1335,10 @@ private: friend class TConfigState; class TNodeWardenUpdateNotifier; - - struct TEvPrivate { - enum EEv { - EvUpdateSystemViews = EventSpaceBegin(TEvents::ES_PRIVATE), + + struct TEvPrivate { + enum EEv { + EvUpdateSystemViews = EventSpaceBegin(TEvents::ES_PRIVATE), EvUpdateSelfHealCounters, EvHostRecordsTimeToLiveExceeded, EvDropDonor, @@ -1346,9 +1346,9 @@ private: EvVSlotReadyUpdate, EvVSlotNotReadyHistogramUpdate, EvProcessIncomingEvent, - }; - - struct TEvUpdateSystemViews : public TEventLocal<TEvUpdateSystemViews, EvUpdateSystemViews> {}; + }; + + struct TEvUpdateSystemViews : public TEventLocal<TEvUpdateSystemViews, EvUpdateSystemViews> {}; struct TEvUpdateSelfHealCounters : TEventLocal<TEvUpdateSelfHealCounters, EvUpdateSelfHealCounters> {}; struct TEvHostRecordsTimeToLiveExceeded : TEventLocal<TEvHostRecordsTimeToLiveExceeded, EvHostRecordsTimeToLiveExceeded> {}; @@ -1359,10 +1359,10 @@ private: struct TEvScrub : TEventLocal<TEvScrub, EvScrub> {}; struct TEvVSlotReadyUpdate : TEventLocal<TEvVSlotReadyUpdate, EvVSlotReadyUpdate> {}; struct TEvVSlotNotReadyHistogramUpdate : TEventLocal<TEvVSlotNotReadyHistogramUpdate, EvVSlotNotReadyHistogramUpdate> {}; - }; - - static constexpr TDuration UpdateSystemViewsPeriod = TDuration::Seconds(5); - + }; + + static constexpr TDuration UpdateSystemViewsPeriod = TDuration::Seconds(5); + std::unordered_set<TPDiskId, THash<TPDiskId>> SysViewChangedPDisks; std::unordered_set<TVSlotId, THash<TVSlotId>> SysViewChangedVSlots; std::unordered_set<TGroupId, THash<TGroupId>> SysViewChangedGroups; @@ -1371,7 +1371,7 @@ private: IActor* CreateSystemViewsCollector(); void UpdateSystemViews(); - + bool CommitConfigUpdates(TConfigState& state, bool suppressFailModelChecking, bool suppressDegradedGroupsChecking, TTransactionContext& txc, TString *errorDescription); diff --git a/ydb/core/mind/bscontroller/sys_view.cpp b/ydb/core/mind/bscontroller/sys_view.cpp index c1153e42d9..ba9b50cde0 100644 --- a/ydb/core/mind/bscontroller/sys_view.cpp +++ b/ydb/core/mind/bscontroller/sys_view.cpp @@ -1,52 +1,52 @@ -#include "sys_view.h" +#include "sys_view.h" #include "group_geometry_info.h" - -namespace NKikimr::NBsController { - -using namespace NSysView; - -TPDiskId TransformKey(const NKikimrSysView::TPDiskKey& key) { - return TPDiskId(key.GetNodeId(), key.GetPDiskId()); -} - -void FillKey(NKikimrSysView::TPDiskKey* key, const TPDiskId& id) { - key->SetNodeId(id.NodeId); - key->SetPDiskId(id.PDiskId); -} - -TVSlotId TransformKey(const NKikimrSysView::TVSlotKey& key) { - return TVSlotId(key.GetNodeId(), key.GetPDiskId(), key.GetVSlotId()); -} - -void FillKey(NKikimrSysView::TVSlotKey* key, const TVSlotId& id) { - key->SetNodeId(id.NodeId); - key->SetPDiskId(id.PDiskId); - key->SetVSlotId(id.VSlotId); -} - -TGroupId TransformKey(const NKikimrSysView::TGroupKey& key) { - return key.GetGroupId(); -} - -void FillKey(NKikimrSysView::TGroupKey* key, const TGroupId& id) { - key->SetGroupId(id); -} - -TBlobStorageController::TBoxStoragePoolId TransformKey(const NKikimrSysView::TStoragePoolKey& key) { - return std::make_tuple(key.GetBoxId(), key.GetStoragePoolId()); -} - -void FillKey(NKikimrSysView::TStoragePoolKey* key, const TBlobStorageController::TBoxStoragePoolId& id) { - key->SetBoxId(std::get<0>(id)); - key->SetStoragePoolId(std::get<1>(id)); -} - + +namespace NKikimr::NBsController { + +using namespace NSysView; + +TPDiskId TransformKey(const NKikimrSysView::TPDiskKey& key) { + return TPDiskId(key.GetNodeId(), key.GetPDiskId()); +} + +void FillKey(NKikimrSysView::TPDiskKey* key, const TPDiskId& id) { + key->SetNodeId(id.NodeId); + key->SetPDiskId(id.PDiskId); +} + +TVSlotId TransformKey(const NKikimrSysView::TVSlotKey& key) { + return TVSlotId(key.GetNodeId(), key.GetPDiskId(), key.GetVSlotId()); +} + +void FillKey(NKikimrSysView::TVSlotKey* key, const TVSlotId& id) { + key->SetNodeId(id.NodeId); + key->SetPDiskId(id.PDiskId); + key->SetVSlotId(id.VSlotId); +} + +TGroupId TransformKey(const NKikimrSysView::TGroupKey& key) { + return key.GetGroupId(); +} + +void FillKey(NKikimrSysView::TGroupKey* key, const TGroupId& id) { + key->SetGroupId(id); +} + +TBlobStorageController::TBoxStoragePoolId TransformKey(const NKikimrSysView::TStoragePoolKey& key) { + return std::make_tuple(key.GetBoxId(), key.GetStoragePoolId()); +} + +void FillKey(NKikimrSysView::TStoragePoolKey* key, const TBlobStorageController::TBoxStoragePoolId& id) { + key->SetBoxId(std::get<0>(id)); + key->SetStoragePoolId(std::get<1>(id)); +} + struct TGroupDiskInfo { const NKikimrBlobStorage::TPDiskMetrics *PDiskMetrics; const NKikimrBlobStorage::TVDiskMetrics *VDiskMetrics; ui32 ExpectedSlotCount; }; - + void CalculateGroupUsageStats(NKikimrSysView::TGroupInfo *info, const std::vector<TGroupDiskInfo>& disks, TBlobStorageGroupType type) { ui64 allocatedSize = 0; @@ -75,8 +75,8 @@ void CalculateGroupUsageStats(NKikimrSysView::TGroupInfo *info, const std::vecto info->SetAvailableSize(b < a ? a - b : 0); } -class TSystemViewsCollector : public TActor<TSystemViewsCollector> { - TControllerSystemViewsState State; +class TSystemViewsCollector : public TActor<TSystemViewsCollector> { + TControllerSystemViewsState State; std::optional<std::vector<NKikimrSysView::TStorageStatsEntry>> StorageStats; std::vector<std::pair<TPDiskId, const NKikimrSysView::TPDiskInfo*>> PDiskIndex; std::vector<std::pair<TVSlotId, const NKikimrSysView::TVSlotInfo*>> VSlotIndex; @@ -88,32 +88,32 @@ class TSystemViewsCollector : public TActor<TSystemViewsCollector> { NMonitoring::TDynamicCounterPtr Counters; std::unordered_set<std::tuple<TString>> PDiskFilterCounters; std::unordered_set<std::tuple<TString, TString>> ErasureCounters; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::BSC_SYSTEM_VIEWS_COLLECTOR; - } - + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::BSC_SYSTEM_VIEWS_COLLECTOR; + } + TSystemViewsCollector(NMonitoring::TDynamicCounterPtr counters) - : TActor(&TSystemViewsCollector::StateWork) + : TActor(&TSystemViewsCollector::StateWork) , Counters(std::move(counters)) - {} - + {} + ~TSystemViewsCollector() { Counters->RemoveSubgroup("subsystem", "storage_stats"); } - STRICT_STFUNC(StateWork, - hFunc(TEvControllerUpdateSystemViews, Handle); - hFunc(TEvSysView::TEvGetPDisksRequest, Handle); - hFunc(TEvSysView::TEvGetVSlotsRequest, Handle); - hFunc(TEvSysView::TEvGetGroupsRequest, Handle); - hFunc(TEvSysView::TEvGetStoragePoolsRequest, Handle); + STRICT_STFUNC(StateWork, + hFunc(TEvControllerUpdateSystemViews, Handle); + hFunc(TEvSysView::TEvGetPDisksRequest, Handle); + hFunc(TEvSysView::TEvGetVSlotsRequest, Handle); + hFunc(TEvSysView::TEvGetGroupsRequest, Handle); + hFunc(TEvSysView::TEvGetStoragePoolsRequest, Handle); hFunc(TEvSysView::TEvGetStorageStatsRequest, Handle); - cFunc(TEvents::TSystem::Poison, PassAway); - ) - - void Handle(TEvControllerUpdateSystemViews::TPtr& ev) { + cFunc(TEvents::TSystem::Poison, PassAway); + ) + + void Handle(TEvControllerUpdateSystemViews::TPtr& ev) { auto *msg = ev->Get(); auto& newState = msg->State; Merge(State.PDisks, newState.PDisks, msg->DeletedPDisks, PDiskIndex); @@ -124,8 +124,8 @@ public: GroupReserveMin = msg->GroupReserveMin; GroupReservePart = msg->GroupReservePart; GenerateStorageStats(); - } - + } + template<typename TDest, typename TSrc, typename TDeleted, typename TIndex> void Merge(TDest& dest, TSrc& src, const TDeleted& deleted, TIndex& index) { if (!src.empty() || !deleted.empty()) { @@ -142,9 +142,9 @@ public: template <typename TResponse, typename TRequest, typename TMap, typename TIndex> void Reply(TRequest& request, const TMap& entries, TIndex& index) { - const auto& record = request->Get()->Record; + const auto& record = request->Get()->Record; auto response = MakeHolder<TResponse>(); - + if (index.empty() && !entries.empty()) { index.reserve(entries.size()); for (const auto& [key, value] : entries) { @@ -152,51 +152,51 @@ public: } std::sort(index.begin(), index.end()); } - + auto begin = index.begin(); auto end = index.end(); auto comp = [](const auto& kv, const auto& key) { return kv.first < key; }; - if (record.HasFrom()) { + if (record.HasFrom()) { auto from = TransformKey(record.GetFrom()); begin = std::lower_bound(index.begin(), index.end(), from, comp); if (begin != index.end() && begin->first == from && record.HasInclusiveFrom() && !record.GetInclusiveFrom()) { ++begin; - } - } - - if (record.HasTo()) { + } + } + + if (record.HasTo()) { auto to = TransformKey(record.GetTo()); end = std::lower_bound(index.begin(), index.end(), to, comp); if (end != index.end() && end->first == to && record.GetInclusiveTo()) { ++end; - } - } - + } + } + for (; begin < end; ++begin) { - auto* entry = response->Record.AddEntries(); + auto* entry = response->Record.AddEntries(); FillKey(entry->MutableKey(), begin->first); entry->MutableInfo()->CopyFrom(*begin->second); - } - - Send(request->Sender, response.Release()); - } - - void Handle(TEvSysView::TEvGetPDisksRequest::TPtr& ev) { + } + + Send(request->Sender, response.Release()); + } + + void Handle(TEvSysView::TEvGetPDisksRequest::TPtr& ev) { Reply<TEvSysView::TEvGetPDisksResponse>(ev, State.PDisks, PDiskIndex); - } - - void Handle(TEvSysView::TEvGetVSlotsRequest::TPtr& ev) { + } + + void Handle(TEvSysView::TEvGetVSlotsRequest::TPtr& ev) { Reply<TEvSysView::TEvGetVSlotsResponse>(ev, State.VSlots, VSlotIndex); - } - - void Handle(TEvSysView::TEvGetGroupsRequest::TPtr& ev) { + } + + void Handle(TEvSysView::TEvGetGroupsRequest::TPtr& ev) { Reply<TEvSysView::TEvGetGroupsResponse>(ev, State.Groups, GroupIndex); - } - - void Handle(TEvSysView::TEvGetStoragePoolsRequest::TPtr& ev) { + } + + void Handle(TEvSysView::TEvGetStoragePoolsRequest::TPtr& ev) { Reply<TEvSysView::TEvGetStoragePoolsResponse>(ev, State.StoragePools, StoragePoolIndex); - } + } void Handle(TEvSysView::TEvGetStorageStatsRequest::TPtr& ev) { auto response = std::make_unique<TEvSysView::TEvGetStorageStatsResponse>(); @@ -386,37 +386,37 @@ public: ->RemoveSubgroup("pdiskFilter", std::get<0>(item)); } } -}; - +}; + IActor* TBlobStorageController::CreateSystemViewsCollector() { return new TSystemViewsCollector(GetServiceCounters(AppData()->Counters, "storage_pool_stat")); -} - +} + void TBlobStorageController::ForwardToSystemViewsCollector(STATEFN_SIG) { TActivationContext::Send(ev->Forward(SystemViewsCollectorId)); -} - +} + void TBlobStorageController::Handle(TEvPrivate::TEvUpdateSystemViews::TPtr&) { UpdateSystemViews(); -} - -void CopyInfo(NKikimrSysView::TPDiskInfo* info, const THolder<TBlobStorageController::TPDiskInfo>& pDiskInfo) { - TPDiskCategory category(pDiskInfo->Kind); - info->SetType(category.TypeStrShort()); - info->SetKind(category.Kind()); +} + +void CopyInfo(NKikimrSysView::TPDiskInfo* info, const THolder<TBlobStorageController::TPDiskInfo>& pDiskInfo) { + TPDiskCategory category(pDiskInfo->Kind); + info->SetType(category.TypeStrShort()); + info->SetKind(category.Kind()); info->SetCategory(category); - info->SetPath(pDiskInfo->Path); - info->SetGuid(pDiskInfo->Guid); + info->SetPath(pDiskInfo->Path); + info->SetGuid(pDiskInfo->Guid); info->SetBoxId(pDiskInfo->BoxId); - if (pDiskInfo->SharedWithOs) { - info->SetSharedWithOs(*pDiskInfo->SharedWithOs); - } - if (pDiskInfo->ReadCentric) { - info->SetReadCentric(*pDiskInfo->ReadCentric); - } - info->SetAvailableSize(pDiskInfo->Metrics.GetAvailableSize()); - info->SetTotalSize(pDiskInfo->Metrics.GetTotalSize()); - info->SetStatusV2(NKikimrBlobStorage::EDriveStatus_Name(pDiskInfo->Status)); + if (pDiskInfo->SharedWithOs) { + info->SetSharedWithOs(*pDiskInfo->SharedWithOs); + } + if (pDiskInfo->ReadCentric) { + info->SetReadCentric(*pDiskInfo->ReadCentric); + } + info->SetAvailableSize(pDiskInfo->Metrics.GetAvailableSize()); + info->SetTotalSize(pDiskInfo->Metrics.GetTotalSize()); + info->SetStatusV2(NKikimrBlobStorage::EDriveStatus_Name(pDiskInfo->Status)); if (pDiskInfo->StatusTimestamp != TInstant::Zero()) { info->SetStatusChangeTimestamp(pDiskInfo->StatusTimestamp.GetValue()); } @@ -425,8 +425,8 @@ void CopyInfo(NKikimrSysView::TPDiskInfo* info, const THolder<TBlobStorageContro } info->SetExpectedSlotCount(pDiskInfo->ExpectedSlotCount); info->SetNumActiveSlots(pDiskInfo->NumActiveSlots + pDiskInfo->StaticSlotUsage); -} - +} + void SerializeVSlotInfo(NKikimrSysView::TVSlotInfo *pb, const TVDiskID& vdiskId, const NKikimrBlobStorage::TVDiskMetrics& m, NKikimrBlobStorage::EVDiskStatus status, NKikimrBlobStorage::TVDiskKind::EVDiskKind kind, bool isBeingDeleted) { pb->SetGroupId(vdiskId.GroupID); @@ -447,68 +447,68 @@ void SerializeVSlotInfo(NKikimrSysView::TVSlotInfo *pb, const TVDiskID& vdiskId, } } -void CopyInfo(NKikimrSysView::TVSlotInfo* info, const THolder<TBlobStorageController::TVSlotInfo>& vSlotInfo) { +void CopyInfo(NKikimrSysView::TVSlotInfo* info, const THolder<TBlobStorageController::TVSlotInfo>& vSlotInfo) { SerializeVSlotInfo(info, vSlotInfo->GetVDiskId(), vSlotInfo->Metrics, vSlotInfo->GetStatus(), vSlotInfo->Kind, vSlotInfo->IsBeingDeleted()); -} - -void CopyInfo(NKikimrSysView::TGroupInfo* info, const THolder<TBlobStorageController::TGroupInfo>& groupInfo) { - info->SetGeneration(groupInfo->Generation); - info->SetErasureSpeciesV2(TErasureType::ErasureSpeciesName(groupInfo->ErasureSpecies)); +} + +void CopyInfo(NKikimrSysView::TGroupInfo* info, const THolder<TBlobStorageController::TGroupInfo>& groupInfo) { + info->SetGeneration(groupInfo->Generation); + info->SetErasureSpeciesV2(TErasureType::ErasureSpeciesName(groupInfo->ErasureSpecies)); info->SetBoxId(std::get<0>(groupInfo->StoragePoolId)); info->SetStoragePoolId(std::get<1>(groupInfo->StoragePoolId)); - if (groupInfo->EncryptionMode) { - info->SetEncryptionMode(*groupInfo->EncryptionMode); - } - if (groupInfo->LifeCyclePhase) { - info->SetLifeCyclePhase(*groupInfo->LifeCyclePhase); - } + if (groupInfo->EncryptionMode) { + info->SetEncryptionMode(*groupInfo->EncryptionMode); + } + if (groupInfo->LifeCyclePhase) { + info->SetLifeCyclePhase(*groupInfo->LifeCyclePhase); + } std::vector<TGroupDiskInfo> disks; for (const auto& vslot : groupInfo->VDisksInGroup) { disks.push_back({&vslot->PDisk->Metrics, &vslot->Metrics, vslot->PDisk->ExpectedSlotCount}); - } + } CalculateGroupUsageStats(info, disks, TBlobStorageGroupType(groupInfo->ErasureSpecies)); - info->SetSeenOperational(groupInfo->SeenOperational); - const auto& latencyStats = groupInfo->LatencyStats; - if (latencyStats.PutTabletLog) { - info->SetPutTabletLogLatency(latencyStats.PutTabletLog->MicroSeconds()); - } - if (latencyStats.PutUserData) { - info->SetPutUserDataLatency(latencyStats.PutUserData->MicroSeconds()); - } - if (latencyStats.GetFast) { - info->SetGetFastLatency(latencyStats.GetFast->MicroSeconds()); - } -} - -void CopyInfo(NKikimrSysView::TStoragePoolInfo* info, const TBlobStorageController::TStoragePoolInfo& poolInfo) { - info->SetName(poolInfo.Name); - if (poolInfo.Generation) { - info->SetGeneration(*poolInfo.Generation); - } - info->SetErasureSpeciesV2(TErasureType::ErasureSpeciesName(poolInfo.ErasureSpecies)); - info->SetVDiskKindV2(NKikimrBlobStorage::TVDiskKind::EVDiskKind_Name(poolInfo.VDiskKind)); - info->SetKind(poolInfo.Kind); - info->SetNumGroups(poolInfo.NumGroups); - if (poolInfo.EncryptionMode) { - info->SetEncryptionMode(*poolInfo.EncryptionMode); - } - if (poolInfo.SchemeshardId) { - info->SetSchemeshardId(*poolInfo.SchemeshardId); - } - if (poolInfo.PathItemId) { - info->SetPathId(*poolInfo.PathItemId); - } + info->SetSeenOperational(groupInfo->SeenOperational); + const auto& latencyStats = groupInfo->LatencyStats; + if (latencyStats.PutTabletLog) { + info->SetPutTabletLogLatency(latencyStats.PutTabletLog->MicroSeconds()); + } + if (latencyStats.PutUserData) { + info->SetPutUserDataLatency(latencyStats.PutUserData->MicroSeconds()); + } + if (latencyStats.GetFast) { + info->SetGetFastLatency(latencyStats.GetFast->MicroSeconds()); + } +} + +void CopyInfo(NKikimrSysView::TStoragePoolInfo* info, const TBlobStorageController::TStoragePoolInfo& poolInfo) { + info->SetName(poolInfo.Name); + if (poolInfo.Generation) { + info->SetGeneration(*poolInfo.Generation); + } + info->SetErasureSpeciesV2(TErasureType::ErasureSpeciesName(poolInfo.ErasureSpecies)); + info->SetVDiskKindV2(NKikimrBlobStorage::TVDiskKind::EVDiskKind_Name(poolInfo.VDiskKind)); + info->SetKind(poolInfo.Kind); + info->SetNumGroups(poolInfo.NumGroups); + if (poolInfo.EncryptionMode) { + info->SetEncryptionMode(*poolInfo.EncryptionMode); + } + if (poolInfo.SchemeshardId) { + info->SetSchemeshardId(*poolInfo.SchemeshardId); + } + if (poolInfo.PathItemId) { + info->SetPathId(*poolInfo.PathItemId); + } info->SetPDiskFilter(TBlobStorageController::TStoragePoolInfo::TPDiskFilter::ToString(poolInfo.PDiskFilters)); TStringStream pdiskFilterData; Save(&pdiskFilterData, poolInfo.PDiskFilters); info->SetPDiskFilterData(pdiskFilterData.Str()); -} - +} + template<typename TDstMap, typename TDeletedSet, typename TSrcMap, typename TChangedSet> void CopyInfo(TDstMap& dst, TDeletedSet& deleted, const TSrcMap& src, TChangedSet& changed) { for (const auto& key : changed) { @@ -517,29 +517,29 @@ void CopyInfo(TDstMap& dst, TDeletedSet& deleted, const TSrcMap& src, TChangedSe } else { deleted.insert(key); } - } + } changed.clear(); -} - +} + void TBlobStorageController::UpdateSystemViews() { if (!AppData()->FeatureFlags.GetEnableSystemViews()) { - return; - } - + return; + } + if (!SysViewChangedPDisks.empty() || !SysViewChangedVSlots.empty() || !SysViewChangedGroups.empty() || !SysViewChangedStoragePools.empty()) { auto update = MakeHolder<TEvControllerUpdateSystemViews>(); update->HostRecords = HostRecords; update->GroupReserveMin = GroupReserveMin; update->GroupReservePart = GroupReservePart; - + auto& state = update->State; CopyInfo(state.PDisks, update->DeletedPDisks, PDisks, SysViewChangedPDisks); CopyInfo(state.VSlots, update->DeletedVSlots, VSlots, SysViewChangedVSlots); CopyInfo(state.Groups, update->DeletedGroups, GroupMap, SysViewChangedGroups); CopyInfo(state.StoragePools, update->DeletedStoragePools, StoragePools, SysViewChangedStoragePools); SysViewChangedSettings = false; - + // process static slots and static groups for (const auto& [pdiskId, pdisk] : StaticPDisks) { if (SysViewChangedPDisks.count(pdiskId) && !FindPDisk(pdiskId)) { @@ -579,7 +579,7 @@ void TBlobStorageController::UpdateSystemViews() { pb->SetLifeCyclePhase(group.GetLifeCyclePhase()); pb->SetSeenOperational(true); pb->SetErasureSpeciesV2(TBlobStorageGroupType::ErasureSpeciesName(group.GetErasureSpecies())); - + const NKikimrBlobStorage::TVDiskMetrics zero; std::vector<TGroupDiskInfo> disks; for (const auto& realm : group.GetRings()) { @@ -607,6 +607,6 @@ void TBlobStorageController::UpdateSystemViews() { } Schedule(UpdateSystemViewsPeriod, new TEvPrivate::TEvUpdateSystemViews); -} - -} // NKikimr::NBsController +} + +} // NKikimr::NBsController diff --git a/ydb/core/mind/bscontroller/sys_view.h b/ydb/core/mind/bscontroller/sys_view.h index a354aef9f4..9effe8c5bb 100644 --- a/ydb/core/mind/bscontroller/sys_view.h +++ b/ydb/core/mind/bscontroller/sys_view.h @@ -1,21 +1,21 @@ -#pragma once - -#include "impl.h" - -namespace NKikimr::NBsController { - -struct TControllerSystemViewsState { +#pragma once + +#include "impl.h" + +namespace NKikimr::NBsController { + +struct TControllerSystemViewsState { std::unordered_map<TPDiskId, NKikimrSysView::TPDiskInfo, THash<TPDiskId>> PDisks; std::unordered_map<TVSlotId, NKikimrSysView::TVSlotInfo, THash<TVSlotId>> VSlots; std::unordered_map<TGroupId, NKikimrSysView::TGroupInfo, THash<TGroupId>> Groups; std::unordered_map<TBlobStorageController::TBoxStoragePoolId, NKikimrSysView::TStoragePoolInfo, THash<TBlobStorageController::TBoxStoragePoolId>> StoragePools; -}; - -struct TEvControllerUpdateSystemViews : - TEventLocal<TEvControllerUpdateSystemViews, TEvBlobStorage::EvControllerUpdateSystemViews> -{ - TControllerSystemViewsState State; +}; + +struct TEvControllerUpdateSystemViews : + TEventLocal<TEvControllerUpdateSystemViews, TEvBlobStorage::EvControllerUpdateSystemViews> +{ + TControllerSystemViewsState State; std::unordered_set<TPDiskId, THash<TPDiskId>> DeletedPDisks; std::unordered_set<TVSlotId, THash<TVSlotId>> DeletedVSlots; std::unordered_set<TGroupId, THash<TGroupId>> DeletedGroups; @@ -23,7 +23,7 @@ struct TEvControllerUpdateSystemViews : TBlobStorageController::THostRecordMap HostRecords; ui32 GroupReserveMin; ui32 GroupReservePart; -}; - -} // NKikimr::NBsController - +}; + +} // NKikimr::NBsController + diff --git a/ydb/core/mind/bscontroller/ya.make b/ydb/core/mind/bscontroller/ya.make index f11b7ff686..275a4dc238 100644 --- a/ydb/core/mind/bscontroller/ya.make +++ b/ydb/core/mind/bscontroller/ya.make @@ -50,8 +50,8 @@ SRCS( stat_processor.cpp stat_processor.h storage_pool_stat.h - sys_view.cpp - sys_view.h + sys_view.cpp + sys_view.h table_merger.h types.h update_group_latencies.cpp diff --git a/ydb/core/mind/configured_tablet_bootstrapper.cpp b/ydb/core/mind/configured_tablet_bootstrapper.cpp index e72a6d5759..008acd2b40 100644 --- a/ydb/core/mind/configured_tablet_bootstrapper.cpp +++ b/ydb/core/mind/configured_tablet_bootstrapper.cpp @@ -211,9 +211,9 @@ TIntrusivePtr<TTabletSetupInfo> MakeTabletSetupInfo( case TTabletTypes::Kesus: createFunc = &NKesus::CreateKesusTablet; break; - case TTabletTypes::SysViewProcessor: - createFunc = &NSysView::CreateSysViewProcessor; - break; + case TTabletTypes::SysViewProcessor: + createFunc = &NSysView::CreateSysViewProcessor; + break; case TTabletTypes::TestShard: createFunc = &NTestShard::CreateTestShard; break; diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp index f895b84d01..58ca2ac3e4 100644 --- a/ydb/core/mind/hive/hive_impl.cpp +++ b/ydb/core/mind/hive/hive_impl.cpp @@ -2322,8 +2322,8 @@ STFUNC(THive::StateWork) { hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle); hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, Handle); - hFunc(NSysView::TEvSysView::TEvGetTabletIdsRequest, Handle); - hFunc(NSysView::TEvSysView::TEvGetTabletsRequest, Handle); + hFunc(NSysView::TEvSysView::TEvGetTabletIdsRequest, Handle); + hFunc(NSysView::TEvSysView::TEvGetTabletsRequest, Handle); default: if (!HandleDefaultEvents(ev, ctx)) { BLOG_W("THive::StateWork unhandled event type: " << ev->GetTypeRewrite() @@ -2387,89 +2387,89 @@ void THive::Handle(TEvHive::TEvConfigureHive::TPtr& ev) { } -void THive::Handle(NSysView::TEvSysView::TEvGetTabletIdsRequest::TPtr& ev) { - const auto& request = ev->Get()->Record; - auto fromId = request.GetFrom(); - auto toId = request.GetTo(); - - auto response = MakeHolder<NSysView::TEvSysView::TEvGetTabletIdsResponse>(); - auto& record = response->Record; - - for (const auto& [tabletId, _] : Tablets) { - if (tabletId >= fromId && tabletId <= toId) { - record.AddTabletIds(tabletId); - } - } - - Send(ev->Sender, response.Release()); -} - -void THive::Handle(NSysView::TEvSysView::TEvGetTabletsRequest::TPtr& ev) { - const auto& request = ev->Get()->Record; - - auto response = MakeHolder<NSysView::TEvSysView::TEvGetTabletsResponse>(); - auto& record = response->Record; - - auto limit = request.GetBatchSizeLimit(); - size_t count = 0; - - for (size_t i = 0; i < request.TabletIdsSize(); ++i) { - auto tabletId = request.GetTabletIds(i); - auto lookup = Tablets.find(tabletId); - if (lookup == Tablets.end()) { - continue; - } - - auto* entry = record.AddEntries(); - ++count; - - const auto& tablet = lookup->second; - auto tabletTypeName = TTabletTypes::TypeToStr(tablet.Type); - - entry->SetTabletId(tabletId); +void THive::Handle(NSysView::TEvSysView::TEvGetTabletIdsRequest::TPtr& ev) { + const auto& request = ev->Get()->Record; + auto fromId = request.GetFrom(); + auto toId = request.GetTo(); + + auto response = MakeHolder<NSysView::TEvSysView::TEvGetTabletIdsResponse>(); + auto& record = response->Record; + + for (const auto& [tabletId, _] : Tablets) { + if (tabletId >= fromId && tabletId <= toId) { + record.AddTabletIds(tabletId); + } + } + + Send(ev->Sender, response.Release()); +} + +void THive::Handle(NSysView::TEvSysView::TEvGetTabletsRequest::TPtr& ev) { + const auto& request = ev->Get()->Record; + + auto response = MakeHolder<NSysView::TEvSysView::TEvGetTabletsResponse>(); + auto& record = response->Record; + + auto limit = request.GetBatchSizeLimit(); + size_t count = 0; + + for (size_t i = 0; i < request.TabletIdsSize(); ++i) { + auto tabletId = request.GetTabletIds(i); + auto lookup = Tablets.find(tabletId); + if (lookup == Tablets.end()) { + continue; + } + + auto* entry = record.AddEntries(); + ++count; + + const auto& tablet = lookup->second; + auto tabletTypeName = TTabletTypes::TypeToStr(tablet.Type); + + entry->SetTabletId(tabletId); entry->SetFollowerId(0); - - entry->SetType(tabletTypeName); - entry->SetState(ETabletStateName(tablet.State)); - entry->SetVolatileState(TTabletInfo::EVolatileStateName(tablet.GetVolatileState())); - entry->SetBootState(tablet.BootState); - entry->SetGeneration(tablet.KnownGeneration); - entry->SetNodeId(tablet.NodeId); - - const auto& resourceValues = tablet.GetResourceValues(); - entry->SetCPU(resourceValues.GetCPU()); - entry->SetMemory(resourceValues.GetMemory()); - entry->SetNetwork(resourceValues.GetNetwork()); - + + entry->SetType(tabletTypeName); + entry->SetState(ETabletStateName(tablet.State)); + entry->SetVolatileState(TTabletInfo::EVolatileStateName(tablet.GetVolatileState())); + entry->SetBootState(tablet.BootState); + entry->SetGeneration(tablet.KnownGeneration); + entry->SetNodeId(tablet.NodeId); + + const auto& resourceValues = tablet.GetResourceValues(); + entry->SetCPU(resourceValues.GetCPU()); + entry->SetMemory(resourceValues.GetMemory()); + entry->SetNetwork(resourceValues.GetNetwork()); + for (const auto& follower : tablet.Followers) { - auto* entry = record.AddEntries(); - ++count; - - entry->SetTabletId(tabletId); + auto* entry = record.AddEntries(); + ++count; + + entry->SetTabletId(tabletId); entry->SetFollowerId(follower.Id); - - entry->SetType(tabletTypeName); - // state is null + + entry->SetType(tabletTypeName); + // state is null entry->SetVolatileState(TTabletInfo::EVolatileStateName(follower.GetVolatileState())); entry->SetBootState(follower.BootState); - // generation is null + // generation is null entry->SetNodeId(follower.NodeId); - + const auto& resourceValues = follower.GetResourceValues(); - entry->SetCPU(resourceValues.GetCPU()); - entry->SetMemory(resourceValues.GetMemory()); - entry->SetNetwork(resourceValues.GetNetwork()); - } - - if (count >= limit && i < request.TabletIdsSize() - 1) { - record.SetNextTabletId(request.GetTabletIds(i + 1)); - break; - } - } - - Send(ev->Sender, response.Release()); -} - + entry->SetCPU(resourceValues.GetCPU()); + entry->SetMemory(resourceValues.GetMemory()); + entry->SetNetwork(resourceValues.GetNetwork()); + } + + if (count >= limit && i < request.TabletIdsSize() - 1) { + record.SetNextTabletId(request.GetTabletIds(i + 1)); + break; + } + } + + Send(ev->Sender, response.Release()); +} + const TTabletMetricsAggregates& THive::GetDefaultResourceMetricsAggregates() const { return DefaultResourceMetricsAggregates; } diff --git a/ydb/core/mind/hive/hive_impl.h b/ydb/core/mind/hive/hive_impl.h index e2bf6cbad0..70c4530fee 100644 --- a/ydb/core/mind/hive/hive_impl.h +++ b/ydb/core/mind/hive/hive_impl.h @@ -496,8 +496,8 @@ protected: void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev); void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev); void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr& ev); - void Handle(NSysView::TEvSysView::TEvGetTabletIdsRequest::TPtr& ev); - void Handle(NSysView::TEvSysView::TEvGetTabletsRequest::TPtr& ev); + void Handle(NSysView::TEvSysView::TEvGetTabletIdsRequest::TPtr& ev); + void Handle(NSysView::TEvSysView::TEvGetTabletsRequest::TPtr& ev); protected: void RestartPipeTx(ui64 tabletId); diff --git a/ydb/core/mind/hive/hive_schema.h b/ydb/core/mind/hive/hive_schema.h index 364b6bfb2a..6289499cc3 100644 --- a/ydb/core/mind/hive/hive_schema.h +++ b/ydb/core/mind/hive/hive_schema.h @@ -214,19 +214,19 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns<TabletID, CPU, Memory, Network, Metrics>; }; - struct Metrics : Table<16> { - struct TabletID : Column<1, Tablet::ID::ColumnType> {}; + struct Metrics : Table<16> { + struct TabletID : Column<1, Tablet::ID::ColumnType> {}; struct FollowerID : Column<2, TabletFollowerTablet::FollowerID::ColumnType> {}; - struct ProtoMetrics : Column<3, NScheme::NTypeIds::String> { using Type = NKikimrTabletBase::TMetrics; }; - + struct ProtoMetrics : Column<3, NScheme::NTypeIds::String> { using Type = NKikimrTabletBase::TMetrics; }; + struct MaximumCPU : Column<100 + (int)NMetrics::EResource::CPU, NScheme::NTypeIds::String> { using Type = NKikimrMetricsProto::TMaximumValueUI64; }; struct MaximumMemory : Column<100 + (int)NMetrics::EResource::Memory, NScheme::NTypeIds::String> { using Type = NKikimrMetricsProto::TMaximumValueUI64; }; struct MaximumNetwork : Column<100 + (int)NMetrics::EResource::Network, NScheme::NTypeIds::String> { using Type = NKikimrMetricsProto::TMaximumValueUI64; }; using TKey = TableKey<TabletID, FollowerID>; using TColumns = TableColumns<TabletID, FollowerID, ProtoMetrics, MaximumCPU, MaximumMemory, MaximumNetwork>; - }; - + }; + struct TabletTypeMetrics : Table<13> { struct TabletType : Column<1, Tablet::TabletType::ColumnType> { using Type = TTabletTypes::EType; }; struct AllowedMetricIDs : Column<2, NScheme::NTypeIds::String> { using Type = TVector<i64>; }; @@ -294,7 +294,7 @@ struct Schema : NIceDb::Schema { TabletFollowerGroup, TabletFollowerTablet, TabletTypeMetrics, - Sequences, + Sequences, Metrics, SubDomain, BlockedOwner, diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp index 5f2fc40f66..7e9a83b354 100644 --- a/ydb/core/mind/hive/monitoring.cpp +++ b/ydb/core/mind/hive/monitoring.cpp @@ -1084,8 +1084,8 @@ public: return "BP"; case TTabletTypes::Kesus: return "K"; - case TTabletTypes::SysViewProcessor: - return "SV"; + case TTabletTypes::SysViewProcessor: + return "SV"; case TTabletTypes::FileStore: return "FS"; case TTabletTypes::TestShard: diff --git a/ydb/core/mind/hive/tx__delete_tablet_result.cpp b/ydb/core/mind/hive/tx__delete_tablet_result.cpp index 5543d1a3d2..bbe89acae4 100644 --- a/ydb/core/mind/hive/tx__delete_tablet_result.cpp +++ b/ydb/core/mind/hive/tx__delete_tablet_result.cpp @@ -27,7 +27,7 @@ public: if (Tablet != nullptr) { if (msg->Status == NKikimrProto::OK) { NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::Metrics>().Key(Tablet->Id, 0).Delete(); + db.Table<Schema::Metrics>().Key(Tablet->Id, 0).Delete(); for (const TTabletChannelInfo& channelInfo : Tablet->TabletStorageInfo->Channels) { for (const TTabletChannelInfo::THistoryEntry& historyInfo : channelInfo.History) { db.Table<Schema::TabletChannelGen>().Key(Tablet->Id, channelInfo.Channel, historyInfo.FromGeneration).Delete(); @@ -37,7 +37,7 @@ public: for (TFollowerTabletInfo& follower : Tablet->Followers) { auto fullTabletId = follower.GetFullTabletId(); db.Table<Schema::TabletFollowerTablet>().Key(fullTabletId).Delete(); - db.Table<Schema::Metrics>().Key(fullTabletId).Delete(); + db.Table<Schema::Metrics>().Key(fullTabletId).Delete(); } for (TFollowerGroup& group : Tablet->FollowerGroups) { db.Table<Schema::TabletFollowerGroup>().Key(Tablet->Id, group.Id).Delete(); diff --git a/ydb/core/mind/hive/tx__load_everything.cpp b/ydb/core/mind/hive/tx__load_everything.cpp index 3f98a0b354..e4c1cf1f51 100644 --- a/ydb/core/mind/hive/tx__load_everything.cpp +++ b/ydb/core/mind/hive/tx__load_everything.cpp @@ -45,7 +45,7 @@ public: auto tabletRowset = db.Table<Schema::Tablet>().Range().Select(); auto tabletChannelRowset = db.Table<Schema::TabletChannel>().Range().Select(); auto tabletChannelGenRowset = db.Table<Schema::TabletChannelGen>().Range().Select(); - auto metrics = db.Table<Schema::Metrics>().Range().Select(); + auto metrics = db.Table<Schema::Metrics>().Range().Select(); auto tabletFollowerGroupRowset = db.Table<Schema::TabletFollowerGroup>().Range().Select(); auto tabletFollowerRowset = db.Table<Schema::TabletFollowerTablet>().Range().Select(); auto tabletTypeAllowedMetrics = db.Table<Schema::TabletTypeMetrics>().Range().Select(); @@ -57,7 +57,7 @@ public: if (!tabletRowset.IsReady() || !tabletChannelRowset.IsReady() || !tabletChannelGenRowset.IsReady() - || !metrics.IsReady() + || !metrics.IsReady() || !tabletFollowerGroupRowset.IsReady() || !tabletFollowerRowset.IsReady() || !tabletTypeAllowedMetrics.IsReady() @@ -470,10 +470,10 @@ public: return false; } - auto metricsRowset = db.Table<Schema::Metrics>().Range(tabletId).Select(); - if (!metricsRowset.IsReady()) - return false; - while (!metricsRowset.EndOfSet()) { + auto metricsRowset = db.Table<Schema::Metrics>().Range(tabletId).Select(); + if (!metricsRowset.IsReady()) + return false; + while (!metricsRowset.EndOfSet()) { TFollowerId followerId = metricsRowset.GetValue<Schema::Metrics::FollowerID>(); auto* leaderOrFollower = tablet.FindTablet(followerId); if (leaderOrFollower) { @@ -482,11 +482,11 @@ public: leaderOrFollower->MutableResourceMetricsAggregates().MaximumNetwork.InitiaizeFrom(metricsRowset.GetValueOrDefault<Schema::Metrics::MaximumNetwork>()); // do not reorder leaderOrFollower->UpdateResourceUsage(metricsRowset.GetValueOrDefault<Schema::Metrics::ProtoMetrics>()); - } - if (!metricsRowset.Next()) - return false; - } - + } + if (!metricsRowset.Next()) + return false; + } + for (auto& pr : followersPerGroup) { TFollowerGroup& followerGroup(*pr.first); while (followerGroup.GetComputedFollowerCount(Self->GetDataCenters()) > pr.second) { diff --git a/ydb/core/mind/labels_maintainer.cpp b/ydb/core/mind/labels_maintainer.cpp index f3e1d04dbb..19138c7d23 100644 --- a/ydb/core/mind/labels_maintainer.cpp +++ b/ydb/core/mind/labels_maintainer.cpp @@ -246,26 +246,26 @@ private: oldGroup = oldGroup->GetSubgroup("subsystem", subSvc); TIntrusivePtr<NMonitoring::TDynamicCounters> serviceGroup = new NMonitoring::TDynamicCounters; TIntrusivePtr<NMonitoring::TDynamicCounters> curGroup = serviceGroup; - - const auto* actualLabels = &labels; - - TSmallVec<std::pair<TString, TString>> ydbLabels; - if (DatabaseAttributeSensorServices.contains(service)) { - // explicitly remove "slot" label for external services ("ydb") - ydbLabels = labels; - if (auto it = std::find_if(ydbLabels.begin(), ydbLabels.end(), [](auto& el){ return el.first == SLOT_LABEL; }); - it != ydbLabels.end()) - { - ydbLabels.erase(it); - actualLabels = &ydbLabels; - } - } - - for (size_t i = 0; i < actualLabels->size() - 1; ++i) { - curGroup = curGroup->GetSubgroup((*actualLabels)[i].first, (*actualLabels)[i].second); - } - curGroup->RegisterSubgroup(actualLabels->back().first, actualLabels->back().second, oldGroup); - + + const auto* actualLabels = &labels; + + TSmallVec<std::pair<TString, TString>> ydbLabels; + if (DatabaseAttributeSensorServices.contains(service)) { + // explicitly remove "slot" label for external services ("ydb") + ydbLabels = labels; + if (auto it = std::find_if(ydbLabels.begin(), ydbLabels.end(), [](auto& el){ return el.first == SLOT_LABEL; }); + it != ydbLabels.end()) + { + ydbLabels.erase(it); + actualLabels = &ydbLabels; + } + } + + for (size_t i = 0; i < actualLabels->size() - 1; ++i) { + curGroup = curGroup->GetSubgroup((*actualLabels)[i].first, (*actualLabels)[i].second); + } + curGroup->RegisterSubgroup(actualLabels->back().first, actualLabels->back().second, oldGroup); + auto rt = GetServiceCountersRoot(root, service); rt->ReplaceSubgroup(subSvc.empty() ? "counters" : "subsystem", subSvc.empty() ? svc : subSvc, serviceGroup); } diff --git a/ydb/core/mind/local.cpp b/ydb/core/mind/local.cpp index 34a697ff7c..09fc15f76d 100644 --- a/ydb/core/mind/local.cpp +++ b/ydb/core/mind/local.cpp @@ -631,15 +631,15 @@ class TLocalNodeRegistrar : public TActorBootstrapped<TLocalNodeRegistrar> { const auto& poolStats(info.GetPoolStats(AppData()->UserPoolId)); UserPoolUsage = poolStats.usage() * poolStats.threads() * 1000000; // uS } - if (info.HasMemoryUsedInAlloc()) { - MemUsage = info.GetMemoryUsedInAlloc(); + if (info.HasMemoryUsedInAlloc()) { + MemUsage = info.GetMemoryUsedInAlloc(); } /*if (info.HasMemoryUsed()) { MemUsage = info.GetMemoryUsed(); }*/ - + double usage = 0; - + if (info.HasMemoryLimit()) { if (MemLimit != info.GetMemoryLimit()) { MemLimit = info.GetMemoryLimit(); diff --git a/ydb/core/mind/tenant_ut_pool.cpp b/ydb/core/mind/tenant_ut_pool.cpp index d11bcfcf82..9ac6b6843c 100644 --- a/ydb/core/mind/tenant_ut_pool.cpp +++ b/ydb/core/mind/tenant_ut_pool.cpp @@ -186,18 +186,18 @@ void CheckLabels(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, allServices.insert(attrServices.begin(), attrServices.end()); for (auto &service : allServices) { - THashSet<TString> allLabels = GetDatabaseAttributeLabels(); - allLabels.insert(DATABASE_LABEL); - allLabels.insert(SLOT_LABEL); - + THashSet<TString> allLabels = GetDatabaseAttributeLabels(); + allLabels.insert(DATABASE_LABEL); + allLabels.insert(SLOT_LABEL); + THashMap<TString, TString> labels; - if (dbServices.contains(service)) { + if (dbServices.contains(service)) { labels.insert(dbLabels.begin(), dbLabels.end()); - if (attrServices.contains(service)) { - labels.erase(SLOT_LABEL); // no slot now - allLabels.erase(SLOT_LABEL); - } - } + if (attrServices.contains(service)) { + labels.erase(SLOT_LABEL); // no slot now + allLabels.erase(SLOT_LABEL); + } + } if (attrServices.contains(service)) labels.insert(attrLabels.begin(), attrLabels.end()); auto serviceGroup = GetServiceCounters(counters, service, false); @@ -496,17 +496,17 @@ Y_UNIT_TEST_SUITE(TTenantPoolTests) { runtime.WaitForHiveState({{{TENANT1_1_NAME, 1, 1, 1}}}); - auto &attrServices = GetDatabaseAttributeSensorServices(); + auto &attrServices = GetDatabaseAttributeSensorServices(); for (auto &service : services) { auto serviceGroup = GetServiceCounters(counters, service, false); auto tenantGroup = serviceGroup->FindSubgroup(DATABASE_LABEL, TENANT1_1_NAME); UNIT_ASSERT(tenantGroup); - TIntrusivePtr<NMonitoring::TDynamicCounters> slotGroup; - if (attrServices.contains(service)) { - slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "slot-1"); - } else { - slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "slot-1"); - } + TIntrusivePtr<NMonitoring::TDynamicCounters> slotGroup; + if (attrServices.contains(service)) { + slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "slot-1"); + } else { + slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "slot-1"); + } UNIT_ASSERT(slotGroup); auto counter = slotGroup->GetCounter("counter", true); UNIT_ASSERT(*counter == 0); @@ -585,18 +585,18 @@ Y_UNIT_TEST_SUITE(TTenantPoolTests) { runtime.WaitForHiveState({{{TENANT1_1_NAME, 1, 1, 1}}}); - auto &attrServices = GetDatabaseAttributeSensorServices(); - + auto &attrServices = GetDatabaseAttributeSensorServices(); + for (auto &service : services) { auto serviceGroup = GetServiceCounters(counters, service, false); auto tenantGroup = serviceGroup->FindSubgroup(DATABASE_LABEL, TENANT1_1_NAME); UNIT_ASSERT(tenantGroup); - NMonitoring::TDynamicCounterPtr slotGroup; - if (attrServices.contains(service)) { - slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "slot-1"); - } else { - slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "slot-1"); - } + NMonitoring::TDynamicCounterPtr slotGroup; + if (attrServices.contains(service)) { + slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "slot-1"); + } else { + slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "slot-1"); + } UNIT_ASSERT(slotGroup); auto counter = slotGroup->GetCounter("counter", true); UNIT_ASSERT(*counter == 0); @@ -612,12 +612,12 @@ Y_UNIT_TEST_SUITE(TTenantPoolTests) { auto serviceGroup = GetServiceCounters(counters, service, false); auto tenantGroup = serviceGroup->FindSubgroup(DATABASE_LABEL, CanonizePath(DOMAIN1_NAME)); UNIT_ASSERT(tenantGroup); - NMonitoring::TDynamicCounterPtr slotGroup; - if (attrServices.contains(service)) { - slotGroup = tenantGroup; - } else { - slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static"); - } + NMonitoring::TDynamicCounterPtr slotGroup; + if (attrServices.contains(service)) { + slotGroup = tenantGroup; + } else { + slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static"); + } UNIT_ASSERT(slotGroup); auto counter = slotGroup->GetCounter("counter", true); UNIT_ASSERT(*counter == 0); @@ -633,12 +633,12 @@ Y_UNIT_TEST_SUITE(TTenantPoolTests) { auto serviceGroup = GetServiceCounters(counters, service, false); auto tenantGroup = serviceGroup->FindSubgroup(DATABASE_LABEL, "<none>"); UNIT_ASSERT(tenantGroup); - NMonitoring::TDynamicCounterPtr slotGroup; - if (attrServices.contains(service)) { - slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "unassigned"); - } else { - slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "unassigned"); - } + NMonitoring::TDynamicCounterPtr slotGroup; + if (attrServices.contains(service)) { + slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "unassigned"); + } else { + slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "unassigned"); + } UNIT_ASSERT(slotGroup); auto counter = slotGroup->GetCounter("counter", true); UNIT_ASSERT(*counter == 0); diff --git a/ydb/core/mon_alloc/monitor.cpp b/ydb/core/mon_alloc/monitor.cpp index 30bd2218d9..8afc7852c9 100644 --- a/ydb/core/mon_alloc/monitor.cpp +++ b/ydb/core/mon_alloc/monitor.cpp @@ -1,5 +1,5 @@ #include "monitor.h" -#include "tcmalloc.h" +#include "tcmalloc.h" #include <ydb/core/base/counters.h> #include <ydb/core/base/appdata.h> @@ -8,7 +8,7 @@ #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/prof/tag.h> -#include <library/cpp/html/pcdata/pcdata.h> +#include <library/cpp/html/pcdata/pcdata.h> #include <library/cpp/lfalloc/dbg_info/dbg_info.h> #include <library/cpp/malloc/api/malloc.h> #include <library/cpp/monlib/dynamic_counters/counters.h> @@ -72,7 +72,7 @@ namespace NKikimr { } void Update() override { -#ifdef PROFILE_MEMORY_ALLOCATIONS +#ifdef PROFILE_MEMORY_ALLOCATIONS int maxTag = 0; int numSizes = 0; auto info = NAllocDbg::GetPerTagAllocInfo(true, maxTag, numSizes); @@ -142,21 +142,21 @@ namespace NKikimr { bySize.Set(bucket.Count, bucket.Size); } } -#endif +#endif } - void Dump(IOutputStream& out, const TString& relPath) override { - Y_UNUSED(relPath); -#ifdef PROFILE_MEMORY_ALLOCATIONS + void Dump(IOutputStream& out, const TString& relPath) override { + Y_UNUSED(relPath); +#ifdef PROFILE_MEMORY_ALLOCATIONS int maxTag = 0; int numSizes = 0; auto info = NAllocDbg::GetPerTagAllocInfo(true, maxTag, numSizes); HTML(out) { - H3() { - out << "LFAlloc" << Endl; - } - out << "<hr>" << Endl; + H3() { + out << "LFAlloc" << Endl; + } + out << "<hr>" << Endl; TABLE_SORTABLE_CLASS("table") { TABLEHEAD() { TABLER() { @@ -244,15 +244,15 @@ namespace NKikimr { "{$('[data-toggle=\"tooltip\"]').tooltip();}</script>"; } } -#else - HTML(out) { - H3() { - out << "LFAlloc" << Endl; - } - out << "<hr>" << Endl; - out << "PROFILE_MEMORY_ALLOCATIONS is off" << Endl; - } -#endif +#else + HTML(out) { + H3() { + out << "LFAlloc" << Endl; + } + out << "<hr>" << Endl; + out << "PROFILE_MEMORY_ALLOCATIONS is off" << Endl; + } +#endif } }; @@ -267,7 +267,7 @@ namespace NKikimr { } void Update() override { -#ifdef PROFILE_MEMORY_ALLOCATIONS +#ifdef PROFILE_MEMORY_ALLOCATIONS using namespace NYT::NYTAlloc; size_t maxTag = NProfiling::GetTagsCount(); @@ -295,12 +295,12 @@ namespace NKikimr { *perTag = usages[tag]; } -#endif +#endif } - void Dump(IOutputStream& out, const TString& relPath) override { - Y_UNUSED(relPath); -#ifdef PROFILE_MEMORY_ALLOCATIONS + void Dump(IOutputStream& out, const TString& relPath) override { + Y_UNUSED(relPath); +#ifdef PROFILE_MEMORY_ALLOCATIONS using namespace NYT::NYTAlloc; size_t maxTag = NProfiling::GetTagsCount(); @@ -312,10 +312,10 @@ namespace NKikimr { GetMemoryUsageForTags(tags.data(), tags.size(), usages.data()); HTML(out) { - H3() { - out << "YTAlloc" << Endl; - } - out << "<hr>" << Endl; + H3() { + out << "YTAlloc" << Endl; + } + out << "<hr>" << Endl; TABLE_SORTABLE_CLASS("table") { TABLEHEAD() { TABLER() { @@ -354,15 +354,15 @@ namespace NKikimr { } } } -#else - HTML(out) { - H3() { - out << "YTAlloc" << Endl; - } - out << "<hr>" << Endl; - out << "PROFILE_MEMORY_ALLOCATIONS is off" << Endl; - } -#endif +#else + HTML(out) { + H3() { + out << "YTAlloc" << Endl; + } + out << "<hr>" << Endl; + out << "PROFILE_MEMORY_ALLOCATIONS is off" << Endl; + } +#endif } }; @@ -370,50 +370,50 @@ namespace NKikimr { void Update() override { } - void Dump(IOutputStream& out, const TString& relPath) override { + void Dump(IOutputStream& out, const TString& relPath) override { Y_UNUSED(out); - Y_UNUSED(relPath); + Y_UNUSED(relPath); } }; std::unique_ptr<IAllocMonitor> CreateAllocMonitor(TDynamicCountersPtr group) { const auto& info = NMalloc::MallocInfo(); - TStringBuf name(info.Name); + TStringBuf name(info.Name); - std::unique_ptr<IAllocMonitor> monitor; + std::unique_ptr<IAllocMonitor> monitor; if (name.StartsWith("lf")) { - monitor = std::make_unique<TLfAllocMonitor>(std::move(group)); + monitor = std::make_unique<TLfAllocMonitor>(std::move(group)); } else if (name.StartsWith("yt")) { - monitor = std::make_unique<TYtAllocMonitor>(std::move(group)); - } else if (name.StartsWith("tc")) { - monitor = std::move(CreateTcMallocMonitor(std::move(group))); + monitor = std::make_unique<TYtAllocMonitor>(std::move(group)); + } else if (name.StartsWith("tc")) { + monitor = std::move(CreateTcMallocMonitor(std::move(group))); } - - return monitor ? std::move(monitor) : std::make_unique<TFakeAllocMonitor>(); + + return monitor ? std::move(monitor) : std::make_unique<TFakeAllocMonitor>(); } class TMemProfMonitor: public TActorBootstrapped<TMemProfMonitor> { - struct TDumpLogConfig { - static constexpr double RssUsageHard = 0.9; - static constexpr double RssUsageSoft = 0.85; - static constexpr TDuration RepeatInterval = TDuration::Seconds(10); - static constexpr TDuration DumpInterval = TDuration::Minutes(10); - }; - - enum { - EvDumpLogStats = EventSpaceBegin(TEvents::ES_PRIVATE), - EvEnd - }; - - struct TEvDumpLogStats : public TEventLocal<TEvDumpLogStats, EvDumpLogStats> {}; - + struct TDumpLogConfig { + static constexpr double RssUsageHard = 0.9; + static constexpr double RssUsageSoft = 0.85; + static constexpr TDuration RepeatInterval = TDuration::Seconds(10); + static constexpr TDuration DumpInterval = TDuration::Minutes(10); + }; + + enum { + EvDumpLogStats = EventSpaceBegin(TEvents::ES_PRIVATE), + EvEnd + }; + + struct TEvDumpLogStats : public TEventLocal<TEvDumpLogStats, EvDumpLogStats> {}; + const TDuration Interval; const std::unique_ptr<IAllocMonitor> AllocMonitor; - TInstant LogMemoryStatsTime = TInstant::Now() - TDumpLogConfig::DumpInterval; - - bool IsDangerous = false; - + TInstant LogMemoryStatsTime = TInstant::Now() - TDumpLogConfig::DumpInterval; + + bool IsDangerous = false; + public: static constexpr EActivityType ActorActivityType() { return ACTORLIB_STATS; @@ -422,7 +422,7 @@ namespace NKikimr { TMemProfMonitor(TDuration interval, std::unique_ptr<IAllocMonitor> allocMonitor) : Interval(interval) , AllocMonitor(std::move(allocMonitor)) - {} + {} void Bootstrap(const TActorContext& ctx) { NActors::TMon* mon = AppData(ctx)->Mon; @@ -433,14 +433,14 @@ namespace NKikimr { return; } - auto* indexPage = mon->RegisterIndexPage("memory", "Memory"); + auto* indexPage = mon->RegisterIndexPage("memory", "Memory"); mon->RegisterActorPage( - indexPage, "statistics", "Statistics", + indexPage, "statistics", "Statistics", false, ctx.ExecutorThread.ActorSystem, ctx.SelfID); - AllocMonitor->RegisterPages(mon, ctx.ExecutorThread.ActorSystem, ctx.SelfID); - AllocMonitor->RegisterControls(AppData(ctx)->Icb); - + AllocMonitor->RegisterPages(mon, ctx.ExecutorThread.ActorSystem, ctx.SelfID); + AllocMonitor->RegisterControls(AppData(ctx)->Icb); + Become(&TThis::StateWork); ctx.Schedule(Interval, new TEvents::TEvWakeup()); } @@ -449,65 +449,65 @@ namespace NKikimr { STFUNC(StateWork) { switch (ev->GetTypeRewrite()) { CFunc(TEvents::TEvWakeup::EventType, HandleWakeup); - HFunc(TEvDumpLogStats, HandleDump); + HFunc(TEvDumpLogStats, HandleDump); HFunc(NMon::TEvHttpInfo, HandleHttpInfo); } } - void LogMemoryStats(const TActorContext& ctx, size_t limit) noexcept { - LogMemoryStatsTime = TInstant::Now(); - TStringStream out; - AllocMonitor->DumpForLog(out, limit); - - TVector<TString> split; - Split(out.Str(), "\n", split); - for (const auto& line : split) { - LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, line); - } - } - - void LogMemoryStatsIfNeeded(const TActorContext& ctx) noexcept { - auto memoryUsage = TAllocState::GetMemoryUsage(); - - if (IsDangerous && memoryUsage < TDumpLogConfig::RssUsageSoft) { - IsDangerous = false; - } else if (!IsDangerous && memoryUsage > TDumpLogConfig::RssUsageHard) { - if (TInstant::Now() - LogMemoryStatsTime > TDumpLogConfig::DumpInterval) { - IsDangerous = true; - LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, - "RSS usage " << memoryUsage * 100. << "%"); - LogMemoryStats(ctx, 256); - ctx.Schedule(TDumpLogConfig::RepeatInterval, new TEvDumpLogStats); - } - } - } - + void LogMemoryStats(const TActorContext& ctx, size_t limit) noexcept { + LogMemoryStatsTime = TInstant::Now(); + TStringStream out; + AllocMonitor->DumpForLog(out, limit); + + TVector<TString> split; + Split(out.Str(), "\n", split); + for (const auto& line : split) { + LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, line); + } + } + + void LogMemoryStatsIfNeeded(const TActorContext& ctx) noexcept { + auto memoryUsage = TAllocState::GetMemoryUsage(); + + if (IsDangerous && memoryUsage < TDumpLogConfig::RssUsageSoft) { + IsDangerous = false; + } else if (!IsDangerous && memoryUsage > TDumpLogConfig::RssUsageHard) { + if (TInstant::Now() - LogMemoryStatsTime > TDumpLogConfig::DumpInterval) { + IsDangerous = true; + LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, + "RSS usage " << memoryUsage * 100. << "%"); + LogMemoryStats(ctx, 256); + ctx.Schedule(TDumpLogConfig::RepeatInterval, new TEvDumpLogStats); + } + } + } + void HandleWakeup(const TActorContext& ctx) noexcept { AllocMonitor->Update(); - LogMemoryStatsIfNeeded(ctx); + LogMemoryStatsIfNeeded(ctx); ctx.Schedule(Interval, new TEvents::TEvWakeup()); } - void HandleDump(TEvDumpLogStats::TPtr&, const TActorContext& ctx) noexcept { - if (IsDangerous) { - auto memoryUsage = TAllocState::GetMemoryUsage(); - LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, - "RSS usage " << memoryUsage * 100. << "%"); - LogMemoryStats(ctx, 256); - } - } - + void HandleDump(TEvDumpLogStats::TPtr&, const TActorContext& ctx) noexcept { + if (IsDangerous) { + auto memoryUsage = TAllocState::GetMemoryUsage(); + LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, + "RSS usage " << memoryUsage * 100. << "%"); + LogMemoryStats(ctx, 256); + } + } + void HandleHttpInfo(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) noexcept { - TString action = ev->Get()->Request.GetParams().Get("action"); - TString relPath(ev->Get()->Request.GetPath()); + TString action = ev->Get()->Request.GetParams().Get("action"); + TString relPath(ev->Get()->Request.GetPath()); TStringStream out; - if (action == "log" && relPath == "/memory/heap") { - LogMemoryStats(ctx, 2048); - out << "<p>Output dumped to log</p>" - << "<p><a href=\"/memory/heap\">Return</a></p>\n"; - } else { - AllocMonitor->Dump(out, relPath); - } + if (action == "log" && relPath == "/memory/heap") { + LogMemoryStats(ctx, 2048); + out << "<p>Output dumped to log</p>" + << "<p><a href=\"/memory/heap\">Return</a></p>\n"; + } else { + AllocMonitor->Dump(out, relPath); + } ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(out.Str())); } }; diff --git a/ydb/core/mon_alloc/monitor.h b/ydb/core/mon_alloc/monitor.h index c1e44be27a..f6a8404b65 100644 --- a/ydb/core/mon_alloc/monitor.h +++ b/ydb/core/mon_alloc/monitor.h @@ -1,49 +1,49 @@ #pragma once #include <ydb/core/control/immediate_control_board_impl.h> - + #include <library/cpp/actors/core/defs.h> #include <library/cpp/actors/core/actor.h> #include <library/cpp/monlib/dynamic_counters/counters.h> -namespace NActors { - class TMon; -} - +namespace NActors { + class TMon; +} + namespace NKikimr { inline NActors::TActorId MakeMemProfMonitorID(ui32 node = 0) { char x[12] = {'m', 'e', 'm', 'p', 'r', 'o', 'f', 'm', 'o', 'n', 'i', 't'}; return NActors::TActorId(node, TStringBuf(x, 12)); } - struct IAllocMonitor { - virtual ~IAllocMonitor() = default; - - virtual void RegisterPages( - NActors::TMon* mon, - NActors::TActorSystem* actorSystem, - NActors::TActorId actorId) - { - Y_UNUSED(mon); - Y_UNUSED(actorSystem); - Y_UNUSED(actorId); - } - - virtual void RegisterControls(TIntrusivePtr<TControlBoard> icb) { - Y_UNUSED(icb); - } - - virtual void Update() = 0; - - virtual void Dump(IOutputStream& out, const TString& relPath) = 0; - - virtual void DumpForLog(IOutputStream& out, size_t limit) { - Y_UNUSED(out); - Y_UNUSED(limit); - } - }; - + struct IAllocMonitor { + virtual ~IAllocMonitor() = default; + + virtual void RegisterPages( + NActors::TMon* mon, + NActors::TActorSystem* actorSystem, + NActors::TActorId actorId) + { + Y_UNUSED(mon); + Y_UNUSED(actorSystem); + Y_UNUSED(actorId); + } + + virtual void RegisterControls(TIntrusivePtr<TControlBoard> icb) { + Y_UNUSED(icb); + } + + virtual void Update() = 0; + + virtual void Dump(IOutputStream& out, const TString& relPath) = 0; + + virtual void DumpForLog(IOutputStream& out, size_t limit) { + Y_UNUSED(out); + Y_UNUSED(limit); + } + }; + NActors::IActor* CreateMemProfMonitor( ui32 intervalSec, TIntrusivePtr<NMonitoring::TDynamicCounters> counters); diff --git a/ydb/core/mon_alloc/profiler.cpp b/ydb/core/mon_alloc/profiler.cpp index b4464e5129..bfff45c180 100644 --- a/ydb/core/mon_alloc/profiler.cpp +++ b/ydb/core/mon_alloc/profiler.cpp @@ -1,13 +1,13 @@ #include "profiler.h" -#include "tcmalloc.h" +#include "tcmalloc.h" #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/mon.h> -#include <library/cpp/actors/core/log.h> +#include <library/cpp/actors/core/log.h> #include <library/cpp/actors/prof/tag.h> #include <library/cpp/html/pcdata/pcdata.h> -#include <library/cpp/malloc/api/malloc.h> +#include <library/cpp/malloc/api/malloc.h> #include <library/cpp/monlib/service/pages/templates.h> #if defined(PROFILE_MEMORY_ALLOCATIONS) @@ -49,8 +49,8 @@ namespace NActors { NAllocProfiler::StartAllocationSampling(true); } - void Stop(IOutputStream& out, size_t limit, bool forLog) override { - Y_UNUSED(forLog); + void Stop(IOutputStream& out, size_t limit, bool forLog) override { + Y_UNUSED(forLog); TAllocDumper dumper(out); NAllocProfiler::StopAllocationSampling(dumper, limit); } @@ -66,8 +66,8 @@ namespace NActors { SetEnabled(true); } - void Stop(IOutputStream& out, size_t limit, bool forLog) override { - Y_UNUSED(forLog); + void Stop(IOutputStream& out, size_t limit, bool forLog) override { + Y_UNUSED(forLog); DumpStats(out, limit); SetEnabled(false); } @@ -165,9 +165,9 @@ namespace NActors { BeginProfiling(); } - void Stop(IOutputStream& out, size_t limit, bool forLog) override { + void Stop(IOutputStream& out, size_t limit, bool forLog) override { Y_UNUSED(limit); - Y_UNUSED(forLog); + Y_UNUSED(forLog); char* buf = nullptr; size_t len = 0; @@ -188,35 +188,35 @@ namespace NActors { void Start() override { } - void Stop(IOutputStream& out, size_t limit, bool forLog) override { + void Stop(IOutputStream& out, size_t limit, bool forLog) override { Y_UNUSED(out); Y_UNUSED(limit); - Y_UNUSED(forLog); + Y_UNUSED(forLog); } }; std::unique_ptr<IProfilerLogic> CreateProfiler() { const auto& info = NMalloc::MallocInfo(); - TStringBuf name(info.Name); + TStringBuf name(info.Name); - std::unique_ptr<IProfilerLogic> profiler; - -#if defined(PROFILE_MEMORY_ALLOCATIONS) + std::unique_ptr<IProfilerLogic> profiler; + +#if defined(PROFILE_MEMORY_ALLOCATIONS) if (name.StartsWith("lf")) { - profiler = std::make_unique<TLfAllocProfiler>(); + profiler = std::make_unique<TLfAllocProfiler>(); } else if (name.StartsWith("yt")) { - profiler = std::make_unique<TYtAllocProfiler>(); + profiler = std::make_unique<TYtAllocProfiler>(); } #endif // PROFILE_MEMORY_ALLOCATIONS - if (name.StartsWith("tc")) { - profiler = std::move(NKikimr::CreateTcMallocProfiler()); - } - - if (profiler) { - return std::move(profiler); - } - + if (name.StartsWith("tc")) { + profiler = std::move(NKikimr::CreateTcMallocProfiler()); + } + + if (profiler) { + return std::move(profiler); + } + #if defined(EXEC_PROFILER_ENABLED) return std::make_unique<TExecProfiler>(); #endif // EXEC_PROFILER_ENABLED @@ -265,8 +265,8 @@ namespace NActors { } } - void DumpProfilingTimes(IOutputStream& out); - + void DumpProfilingTimes(IOutputStream& out); + void HandleStart(TEvProfiler::TEvStart::TPtr& ev, const TActorContext& ctx); void HandleStop(TEvProfiler::TEvStop::TPtr& ev, const TActorContext& ctx); void HandleMonInfo(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx); @@ -274,18 +274,18 @@ namespace NActors { bool StartProfiler(); bool StopProfiler(TString& profile); bool StopProfilerFile(TString& outFileName, TString& err); - bool StopProfilerDumpToLog(const TActorContext& ctx); + bool StopProfilerDumpToLog(const TActorContext& ctx); void OutputControlHtml(IOutputStream& os, const TString& action, bool isOk); void OutputResultHtml(IOutputStream& os, const TString& action, const TString& profile, const TString& fileName, bool isOk, const TString& err); }; - void TProfilerActor::DumpProfilingTimes(IOutputStream& out) { - out << "Profiling started: " << StartTime << Endl - << "Profiling stopped: " << StopTime << Endl - << Endl; - } - + void TProfilerActor::DumpProfilingTimes(IOutputStream& out) { + out << "Profiling started: " << StartTime << Endl + << "Profiling stopped: " << StopTime << Endl + << Endl; + } + void TProfilerActor::HandleStart(TEvProfiler::TEvStart::TPtr& ev, const NActors::TActorContext& ctx) { bool isOk = StartProfiler(); ctx.Send(ev->Sender, new TEvProfiler::TEvStartResult(ev->Get()->Cookie(), isOk)); @@ -310,8 +310,8 @@ namespace NActors { isOk = StopProfiler(profile); } else if (action == "stop-save") { isOk = StopProfilerFile(fileName, err); - } else if (action == "stop-log") { - isOk = StopProfilerDumpToLog(ctx); + } else if (action == "stop-log") { + isOk = StopProfilerDumpToLog(ctx); } TStringStream out; @@ -359,34 +359,34 @@ namespace NActors { } TStringOutput out(profile); - DumpProfilingTimes(out); - Profiler->Stop(out, 256, false); + DumpProfilingTimes(out); + Profiler->Stop(out, 256, false); + return true; + } + + bool TProfilerActor::StopProfilerDumpToLog(const TActorContext& ctx) { + if (!IsProfiling) { + return false; + } + + IsProfiling = false; + StopTime = TInstant::Now(); + if (DynamicCounters) { + *Counters.IsProfiling = 0; + } + + TStringStream out; + DumpProfilingTimes(out); + Profiler->Stop(out, 2048, true); + + TVector<TString> split; + Split(out.Str(), "\n", split); + for (const auto& line : split) { + LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, line); + } return true; } - bool TProfilerActor::StopProfilerDumpToLog(const TActorContext& ctx) { - if (!IsProfiling) { - return false; - } - - IsProfiling = false; - StopTime = TInstant::Now(); - if (DynamicCounters) { - *Counters.IsProfiling = 0; - } - - TStringStream out; - DumpProfilingTimes(out); - Profiler->Stop(out, 2048, true); - - TVector<TString> split; - Split(out.Str(), "\n", split); - for (const auto& line : split) { - LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, line); - } - return true; - } - bool TProfilerActor::StopProfilerFile(TString& outFileName, TString& err) { if (!IsProfiling) { outFileName.clear(); @@ -404,8 +404,8 @@ namespace NActors { try { TFileOutput out(outFileName); - DumpProfilingTimes(out); - Profiler->Stop(out, 4096, false); + DumpProfilingTimes(out); + Profiler->Stop(out, 4096, false); } catch (const yexception& e) { err = "Failed to dump profile: "; err += e.what(); @@ -420,7 +420,7 @@ namespace NActors { os << "<p>"; if (IsProfiling) { os << "<a class=\"btn btn-primary\" href=\"?action=stop-display\">Stop & display</a> \n"; - os << "<a class=\"btn btn-primary\" href=\"?action=stop-log\">Stop & dump to log</a>\n"; + os << "<a class=\"btn btn-primary\" href=\"?action=stop-log\">Stop & dump to log</a>\n"; os << "<a class=\"btn btn-primary\" href=\"?action=stop-save\">Stop & save to file</a>\n"; } else { os << "<a class=\"btn btn-primary\" href=\"?action=start\">Start</a>\n"; @@ -450,12 +450,12 @@ namespace NActors { os << "filename: " << fileName; os << "</p><p>Error: " << err << "</p>"; } - } else if (action == "stop-log") { - if (isOk) { - os << "<p>Output dumped to log</p>"; - } else { - os << "<p>Error stopping profiling.</p>"; - } + } else if (action == "stop-log") { + if (isOk) { + os << "<p>Output dumped to log</p>"; + } else { + os << "<p>Error stopping profiling.</p>"; + } } else if (action == "start") { if (isOk) { os << "<p>Profiling started OK.</p>"; diff --git a/ydb/core/mon_alloc/profiler.h b/ydb/core/mon_alloc/profiler.h index 6769418a66..4fba8941da 100644 --- a/ydb/core/mon_alloc/profiler.h +++ b/ydb/core/mon_alloc/profiler.h @@ -107,7 +107,7 @@ namespace NActors { struct IProfilerLogic { virtual ~IProfilerLogic() = default; virtual void Start() = 0; - virtual void Stop(IOutputStream& out, size_t limit, bool forLog) = 0; + virtual void Stop(IOutputStream& out, size_t limit, bool forLog) = 0; }; inline TActorId MakeProfilerID(ui32 nodeId) { diff --git a/ydb/core/mon_alloc/stats.cpp b/ydb/core/mon_alloc/stats.cpp index 6052ed8f6b..050822241c 100644 --- a/ydb/core/mon_alloc/stats.cpp +++ b/ydb/core/mon_alloc/stats.cpp @@ -1,11 +1,11 @@ #include "stats.h" -#include "tcmalloc.h" +#include "tcmalloc.h" #include <ydb/core/base/counters.h> #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/process_stats.h> +#include <library/cpp/actors/core/process_stats.h> #include <library/cpp/lfalloc/dbg_info/dbg_info.h> #include <library/cpp/malloc/api/malloc.h> #include <library/cpp/monlib/dynamic_counters/counters.h> @@ -203,18 +203,18 @@ namespace NKikimr { std::unique_ptr<IAllocStats> CreateAllocStats(TDynamicCountersPtr group) { const auto& info = NMalloc::MallocInfo(); - TStringBuf name(info.Name); + TStringBuf name(info.Name); - std::unique_ptr<IAllocStats> stats; + std::unique_ptr<IAllocStats> stats; if (name.StartsWith("lf")) { - stats = std::make_unique<TLfAllocStats>(std::move(group)); + stats = std::make_unique<TLfAllocStats>(std::move(group)); } else if (name.StartsWith("yt")) { - stats = std::make_unique<TYtAllocStats>(std::move(group)); - } else if (name.StartsWith("tc")) { - stats = std::move(CreateTcMallocStats(std::move(group))); + stats = std::make_unique<TYtAllocStats>(std::move(group)); + } else if (name.StartsWith("tc")) { + stats = std::move(CreateTcMallocStats(std::move(group))); } - - return stats ? std::move(stats) : std::make_unique<TFakeAllocStats>(); + + return stats ? std::move(stats) : std::make_unique<TFakeAllocStats>(); } class TMemStatsCollector: public TActorBootstrapped<TMemStatsCollector> { @@ -250,39 +250,39 @@ namespace NKikimr { ctx.Schedule(Interval, new TEvents::TEvWakeup()); } }; - - struct TLfAllocState: public IAllocState { - ui64 GetAllocatedMemoryEstimate() const override { - ui64 result = 0; - result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SYSTEM_ALLOC); - result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SYSTEM_FREE); - result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SMALL_ALLOC); - result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SMALL_FREE); - result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_LARGE_ALLOC); - result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_LARGE_FREE); - return result; - } - }; - - struct TFakeAllocState: public IAllocState { - ui64 GetAllocatedMemoryEstimate() const override { - return 0; - } - }; - - std::unique_ptr<IAllocState> CreateAllocState() { - const auto& info = NMalloc::MallocInfo(); - TStringBuf name(info.Name); - - std::unique_ptr<IAllocState> state; - if (name.StartsWith("lf")) { - state = std::make_unique<TLfAllocState>(); - } else if (name.StartsWith("tc")) { - state = std::move(CreateTcMallocState()); - } - - return state ? std::move(state) : std::make_unique<TFakeAllocState>(); - } + + struct TLfAllocState: public IAllocState { + ui64 GetAllocatedMemoryEstimate() const override { + ui64 result = 0; + result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SYSTEM_ALLOC); + result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SYSTEM_FREE); + result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SMALL_ALLOC); + result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SMALL_FREE); + result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_LARGE_ALLOC); + result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_LARGE_FREE); + return result; + } + }; + + struct TFakeAllocState: public IAllocState { + ui64 GetAllocatedMemoryEstimate() const override { + return 0; + } + }; + + std::unique_ptr<IAllocState> CreateAllocState() { + const auto& info = NMalloc::MallocInfo(); + TStringBuf name(info.Name); + + std::unique_ptr<IAllocState> state; + if (name.StartsWith("lf")) { + state = std::make_unique<TLfAllocState>(); + } else if (name.StartsWith("tc")) { + state = std::move(CreateTcMallocState()); + } + + return state ? std::move(state) : std::make_unique<TFakeAllocState>(); + } } IActor* CreateMemStatsCollector(ui32 intervalSec, TDynamicCountersPtr counters) { @@ -290,19 +290,19 @@ namespace NKikimr { TDuration::Seconds(intervalSec), CreateAllocStats(GetServiceCounters(counters, "utils"))); } - - std::unique_ptr<IAllocState> TAllocState::AllocState = CreateAllocState(); - - ui64 TAllocState::GetAllocatedMemoryEstimate() { - return AllocState->GetAllocatedMemoryEstimate(); - } - - double TAllocState::GetMemoryUsage() { - NActors::TProcStat procStat; - procStat.Fill(getpid()); - if (!procStat.CGroupMemLim) { - return 0; - } - return (double)procStat.AnonRss / procStat.CGroupMemLim; - } + + std::unique_ptr<IAllocState> TAllocState::AllocState = CreateAllocState(); + + ui64 TAllocState::GetAllocatedMemoryEstimate() { + return AllocState->GetAllocatedMemoryEstimate(); + } + + double TAllocState::GetMemoryUsage() { + NActors::TProcStat procStat; + procStat.Fill(getpid()); + if (!procStat.CGroupMemLim) { + return 0; + } + return (double)procStat.AnonRss / procStat.CGroupMemLim; + } } diff --git a/ydb/core/mon_alloc/stats.h b/ydb/core/mon_alloc/stats.h index a992541245..a5528e5afd 100644 --- a/ydb/core/mon_alloc/stats.h +++ b/ydb/core/mon_alloc/stats.h @@ -6,25 +6,25 @@ #include <library/cpp/monlib/dynamic_counters/counters.h> namespace NKikimr { - struct IAllocStats { - virtual ~IAllocStats() = default; - virtual void Update() = 0; - }; - + struct IAllocStats { + virtual ~IAllocStats() = default; + virtual void Update() = 0; + }; + NActors::IActor* CreateMemStatsCollector( ui32 intervalSec, TIntrusivePtr<NMonitoring::TDynamicCounters> counters); - - - struct IAllocState { - virtual ~IAllocState() = default; - virtual ui64 GetAllocatedMemoryEstimate() const = 0; - }; - - struct TAllocState { - static std::unique_ptr<IAllocState> AllocState; - - static ui64 GetAllocatedMemoryEstimate(); - static double GetMemoryUsage(); - }; + + + struct IAllocState { + virtual ~IAllocState() = default; + virtual ui64 GetAllocatedMemoryEstimate() const = 0; + }; + + struct TAllocState { + static std::unique_ptr<IAllocState> AllocState; + + static ui64 GetAllocatedMemoryEstimate(); + static double GetMemoryUsage(); + }; } diff --git a/ydb/core/mon_alloc/tcmalloc.cpp b/ydb/core/mon_alloc/tcmalloc.cpp index 7c7890d25f..6c8689205f 100644 --- a/ydb/core/mon_alloc/tcmalloc.cpp +++ b/ydb/core/mon_alloc/tcmalloc.cpp @@ -1,791 +1,791 @@ -#include "tcmalloc.h" - -#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h> - -#include <library/cpp/actors/prof/tag.h> -#include <library/cpp/cache/cache.h> -#include <library/cpp/html/pcdata/pcdata.h> -#include <library/cpp/monlib/service/pages/templates.h> - +#include "tcmalloc.h" + +#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h> + +#include <library/cpp/actors/prof/tag.h> +#include <library/cpp/cache/cache.h> +#include <library/cpp/html/pcdata/pcdata.h> +#include <library/cpp/monlib/service/pages/templates.h> + #include <ydb/core/mon/mon.h> - -#include <util/stream/format.h> - -using namespace NActors; - -namespace NKikimr { - -using TDynamicCountersPtr = TIntrusivePtr<NMonitoring::TDynamicCounters>; -using TDynamicCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; -using THistogramPtr = NMonitoring::THistogramPtr; - -static void FormatPrettyNumber(IOutputStream& out, ssize_t val) { - if (val == 0) { - out << "0"; - return; - } - if (val < 0) { - out << "- "; - } - ui64 value = val < 0 ? -val : val; - TSmallVec<ui64> parts; - while (value > 0) { - auto head = value / 1000; - parts.push_back(value - head * 1000); - value = head; - } - out << parts.back(); - parts.pop_back(); - while (!parts.empty()) { - out << Sprintf(" %03" PRIu64, parts.back()); - parts.pop_back(); - } -} - -static void FormatSize(IOutputStream& out, size_t size) { - if (size < (1 << 10)) { - out << size; - } else if (size < (1 << 20)) { - out << size << " (" - << Sprintf("%.1f", size / float(1 << 10)) << " KiB)"; - } else { - out << size << " (" - << Sprintf("%.1f", size / float(1 << 20)) << " MiB)"; - } -} - -static ui64 GetProperty( - const std::map<std::string, tcmalloc::MallocExtension::Property>& properties, - const char* name) -{ - auto found = properties.find(name); - return found != properties.end() ? found->second.value : 0; -} - -static ui64 GetProperty(const char* name) { - const auto properties = tcmalloc::MallocExtension::GetProperties(); - return GetProperty(properties, name); -} - -static ui64 GetCachesSize( - const std::map<std::string, tcmalloc::MallocExtension::Property>& properties) -{ - return - GetProperty(properties, "tcmalloc.page_heap_free") + - GetProperty(properties, "tcmalloc.central_cache_free") + - GetProperty(properties, "tcmalloc.transfer_cache_free") + - GetProperty(properties, "tcmalloc.sharded_transfer_cache_free") + - GetProperty(properties, "tcmalloc.cpu_free") + - GetProperty(properties, "tcmalloc.thread_cache_free"); -} - -struct TAllocationStats { - size_t Size; - size_t Count; - size_t RequestedSize; - - static constexpr size_t MaxSizeIndex = 32; - size_t Sizes[MaxSizeIndex]; - size_t Counts[MaxSizeIndex]; - - static constexpr ui32 MaxTag = 1024; - size_t TagSizes[MaxTag]; - size_t TagCounts[MaxTag]; - - TAllocationStats() { - Zero(*this); - } - - static size_t SizeUpperBound(size_t index) { - return 1 << std::min(MaxSizeIndex - 1, index); - } - - static size_t GetIndex(size_t size) { - size_t index = 0; - if (size > 0) { - index = std::min(MaxSizeIndex - 1, (size_t)GetValueBitCount(size - 1)); - } - return index; - } - - static const char* GetTagName(ui32 tag) { - const char* name = NProfiling::GetTag(tag); - return name ? name : "_DEFAULT"; - } - - void Add(const tcmalloc::Profile::Sample& sample) { - Size += sample.sum; - Count += sample.count; - RequestedSize += sample.count * sample.requested_size; - - auto index = GetIndex(sample.allocated_size); - Sizes[index] += sample.sum; - Counts[index] += sample.count; - - ui32 tag = reinterpret_cast<uintptr_t>(sample.user_data); - tag = (tag < MaxTag) ? tag : 0; - TagSizes[tag] += sample.sum; - TagCounts[tag] += sample.count; - } - - void DumpSizeStats(IOutputStream& out, const char* marker, const char* sep) const { - out << marker << sep << sep; - for (size_t b = 0; b < MaxSizeIndex; ++b) { - if (b == MaxSizeIndex - 1) { - out << "inf : "; - } else { - out << "<= " << Sprintf("%-10" PRIu64, SizeUpperBound(b)) << ": "; - } - if (!Sizes[b]) { - out << "0" << sep; - continue; - } - out << "size "; - FormatSize(out, Sizes[b]); - out << ", count " << Counts[b] << sep; - } - out << Endl; - } - - void DumpTagStats(IOutputStream& out, const char* marker, const char* sep) const { - out << marker << sep << sep; - for (ui32 t = 0; t < MaxTag; ++t) { - if (!TagSizes[t]) { - continue; - } - out << Sprintf("%-32s", TAllocationStats::GetTagName(t)) << ": size "; - FormatSize(out, TagSizes[t]); - out << ", count " << TagCounts[t] << sep; - } - out << Endl; - } -}; - - -class TAllocationAnalyzer { - const tcmalloc::Profile Profile; - - struct TSymbol { - const void* Address; - TString Name; - }; - TLFUCache<void*, TSymbol> SymbolCache; - - struct TStackKey { - int Depth; - void* Stack[tcmalloc::Profile::Sample::kMaxStackDepth]; - - explicit TStackKey(const tcmalloc::Profile::Sample& sample) { - Depth = std::min(sample.depth, tcmalloc::Profile::Sample::kMaxStackDepth); - std::memcpy(Stack, sample.stack, Depth * sizeof(void*)); - } - - bool operator==(const TStackKey& key) const { - if (Depth != key.Depth) { - return false; - } - return std::memcmp(Stack, key.Stack, Depth * sizeof(void*)) == 0; - } - - struct THash { - size_t operator()(const TStackKey& key) const { - std::string_view view((char*)key.Stack, key.Depth * sizeof(void*)); - return std::hash<std::string_view>()(view); - } - }; - }; - - struct TStackStats { - std::vector<tcmalloc::Profile::Sample> Samples; - size_t Size = 0; - size_t Count = 0; - ui32 Tag = 0; - }; - - std::unordered_map<TStackKey, TStackStats, TStackKey::THash> Stacks; - size_t SampleCount = 0; - size_t Size = 0; - size_t Count = 0; - size_t RequestedSize = 0; - - bool Prepared = false; - -private: - void PrintBackTrace(IOutputStream& out, void* const* stack, size_t sz, - const char* sep) - { - char name[1024]; - for (size_t i = 0; i < sz; ++i) { - TSymbol symbol; - auto it = SymbolCache.Find(stack[i]); - if (it != SymbolCache.End()) { - symbol = it.Value(); - } else { - TResolvedSymbol rs = ResolveSymbol(stack[i], name, sizeof(name)); - symbol = {rs.NearestSymbol, rs.Name}; - SymbolCache.Insert(stack[i], symbol); - } - - out << Hex((intptr_t)stack[i], HF_FULL) << " " << symbol.Name; - intptr_t offset = (intptr_t)stack[i] - (intptr_t)symbol.Address; - if (offset) - out << " +" << offset; - out << sep; - } - } - - void PrintSample(IOutputStream& out, const tcmalloc::Profile::Sample* sample, - const char* sep) const - { - out << sample->sum << " = " - << sample->count << " * " - << sample->allocated_size << " (" - << sample->requested_size << ")" - << sep; - } - - void PrintStack(IOutputStream& out, const TStackStats* stats, size_t sampleCountLimit, - const char* marker, const char* sep) - { - std::vector<const tcmalloc::Profile::Sample*> samples; - samples.reserve(stats->Samples.size()); - size_t size = 0; - size_t count = 0; - - for (const auto& sample : stats->Samples) { - samples.push_back(&sample); - size += sample.sum; - count += sample.count; - } - - std::sort(samples.begin(), samples.end(), [] (const auto* l, const auto* r) { - return l->sum > r->sum; - }); - - out << marker << sep << sep << "size "; - FormatSize(out, size); - out << ", count " << count - << ", " << TAllocationStats::GetTagName(stats->Tag) << sep; - - size_t i = 0; - for (const auto* sample : samples) { - PrintSample(out, sample, sep); - if (++i >= sampleCountLimit) { - break; - } - } - - if (samples.size() > sampleCountLimit) { - out << "....TRUNCATED, first " << sampleCountLimit << " samples are shown" << sep; - } - out << sep; - - if (samples.size()) { - const auto& sample = samples.front(); - PrintBackTrace(out, sample->stack, sample->depth, sep); - } - out << Endl; - } - - void PrintAllocatorDetails(IOutputStream& out, const char* sep) const { - const auto properties = tcmalloc::MallocExtension::GetProperties(); - out << sep - << "Physical memory used: "; - FormatSize(out, GetProperty(properties, "generic.physical_memory_used")); - out << sep - << " Application: "; - FormatSize(out, GetProperty(properties, "generic.bytes_in_use_by_app")); - out << sep - << " Allocator caches: "; - FormatSize(out, GetCachesSize(properties)); - out << sep - << " Allocator metadata: "; - FormatSize(out, GetProperty(properties, "tcmalloc.metadata_bytes")); - out << sep; - } - -public: - explicit TAllocationAnalyzer(tcmalloc::Profile&& profile) - : Profile(std::move(profile)) - , SymbolCache(2048) - {} - - void Prepare(TAllocationStats* allocationStats) { - Y_VERIFY(!Prepared); - - TAllocationStats allocations; - - Profile.Iterate([&] (const tcmalloc::Profile::Sample& sample) { - if (!sample.sum) { - return; - } - - TStackKey key(sample); - auto& stats = Stacks[key]; - stats.Samples.push_back(sample); - stats.Size += sample.sum; - stats.Count += sample.count; - stats.Tag = reinterpret_cast<uintptr_t>(sample.user_data); - - allocations.Add(sample); - - ++SampleCount; - }); - - Size = allocations.Size; - Count = allocations.Count; - RequestedSize = allocations.RequestedSize; - - if (allocationStats) { - *allocationStats = allocations; - } - - Prepared = true; - } - - void Dump(IOutputStream& out, size_t stackCountLimit, size_t sampleCountLimit, - bool isHeap, bool forLog) - { - Y_VERIFY(Prepared); - - const char* marker = forLog ? - (isHeap ? "HEAP" : "PROFILE") : "--------------------------------"; - const char* sep = forLog ? " | " : "\n"; - - out << marker << sep << sep; - out << "Stack count: " << Stacks.size() << sep - << "Sample count: " << SampleCount << sep - << "Allocated bytes: "; - FormatSize(out, Size); - out << sep - << "Allocation count: " << Count << sep; - - if (isHeap) { - out << sep - << "Requested bytes: "; - FormatSize(out, RequestedSize); - out << sep - << "Wasted bytes: "; - FormatSize(out, Size - RequestedSize); - out << sep; - - PrintAllocatorDetails(out, sep); - } - - if (!forLog) { - out << sep - << "Details: size = count * allocated_size (requested_size)" << sep; - } - - out << Endl; - - std::vector<const TStackStats*> stats; - stats.reserve(Stacks.size()); - for (auto& [key, stackStats] : Stacks) { - stats.push_back(&stackStats); - } - - std::sort(stats.begin(), stats.end(), [] (const auto* l, const auto* r) { - return l->Size > r->Size; - }); - - size_t i = 0; - for (const auto* stackStats : stats) { - PrintStack(out, stackStats, sampleCountLimit, marker, sep); - if (++i >= stackCountLimit) { - break; - } - } - - if (stats.size() > stackCountLimit) { - out << "....TRUNCATED, first " << stackCountLimit << " stacks are shown" << sep; - } - out << Endl; - } -}; - - -class TTcMallocStats : public IAllocStats { - TDynamicCountersPtr CounterGroup; - - static constexpr const char* Names[] = { - "generic.virtual_memory_used", - "generic.physical_memory_used", - "generic.bytes_in_use_by_app", - "tcmalloc.page_heap_free", - "tcmalloc.metadata_bytes", - "tcmalloc.thread_cache_count", - "tcmalloc.central_cache_free", - "tcmalloc.transfer_cache_free", - "tcmalloc.sharded_transfer_cache_free", - "tcmalloc.cpu_free", - "tcmalloc.per_cpu_caches_active", - "tcmalloc.thread_cache_free", - "tcmalloc.page_heap_unmapped", - }; - - std::unordered_map<TString, TDynamicCounterPtr> Counters; - - TDynamicCounterPtr CachesSize; - -public: - explicit TTcMallocStats(TDynamicCountersPtr group) { - CounterGroup = group->GetSubgroup("component", "tcmalloc"); - for (auto name : Names) { - Counters[name] = CounterGroup->GetCounter(name, false); - } - CachesSize = CounterGroup->GetCounter("tcmalloc.caches_bytes", false); - } - - void Update() override { - auto properties = tcmalloc::MallocExtension::GetProperties(); - for (auto name : Names) { - if (auto it = properties.find(name); it != properties.end()) { - Counters[name]->Set(it->second.value); - } - } - CachesSize->Set(GetCachesSize(properties)); - } -}; - - -class TTcMallocState : public IAllocState { -public: - ui64 GetAllocatedMemoryEstimate() const override { - return GetProperty("generic.physical_memory_used"); - } -}; - - -class TTcMallocMonitor : public IAllocMonitor { - TDynamicCountersPtr CounterGroup; - - THistogramPtr SizeHistogram; - THistogramPtr CountHistogram; - std::unordered_map<ui32, std::pair<TDynamicCounterPtr, TDynamicCounterPtr>> TaggedCounters; - - struct TControls { - static constexpr size_t MaxSamplingRate = 4ll << 30; - - TControlWrapper ProfileSamplingRate; - TControlWrapper GuardedSamplingRate; - TControlWrapper BackgroundReleaseRate; - TControlWrapper MemoryLimit; - - TControls() - : ProfileSamplingRate(tcmalloc::MallocExtension::GetProfileSamplingRate(), - 64 << 10, MaxSamplingRate) - , GuardedSamplingRate(MaxSamplingRate, - 64 << 10, MaxSamplingRate) - , BackgroundReleaseRate(0, - 0, 64 << 20) - , MemoryLimit(0, - 0, std::numeric_limits<i64>::max()) - {} - - void Register(TIntrusivePtr<TControlBoard> icb) { - icb->RegisterSharedControl(ProfileSamplingRate, "TCMallocControls.ProfileSamplingRate"); - icb->RegisterSharedControl(GuardedSamplingRate, "TCMallocControls.GuardedSamplingRate"); - icb->RegisterSharedControl(BackgroundReleaseRate, "TCMallocControls.BackgroundReleaseRate"); - icb->RegisterSharedControl(MemoryLimit, "TCMallocControls.MemoryLimit"); - } - }; - TControls Controls; - -private: - void UpdateCounters() { - auto profile = tcmalloc::MallocExtension::SnapshotCurrent(tcmalloc::ProfileType::kHeap); - - TAllocationAnalyzer analyzer(std::move(profile)); - TAllocationStats allocationStats; - analyzer.Prepare(&allocationStats); - - auto updateHistogram = [] (THistogramPtr histogram, size_t* numbers) { - histogram->Reset(); - auto snapshot = histogram->Snapshot(); - auto count = std::min(snapshot->Count(), (ui32)TAllocationStats::MaxSizeIndex); - for (ui32 b = 0; b < count; ++b) { - histogram->Collect(snapshot->UpperBound(b), numbers[b]); - } - }; - - updateHistogram(SizeHistogram, allocationStats.Sizes); - updateHistogram(CountHistogram, allocationStats.Counts); - - for (ui32 t = 0; t < TAllocationStats::MaxTag; ++t) { - auto found = TaggedCounters.find(t); - if (!allocationStats.TagSizes[t] && found == TaggedCounters.end()) { - continue; - } - - TDynamicCounterPtr size, count; - - if (found == TaggedCounters.end()) { - auto name = TAllocationStats::GetTagName(t); - auto group = CounterGroup->GetSubgroup("activity", name); - size = group->GetCounter("tcmalloc.sampled_size_by_activity", false); - count = group->GetCounter("tcmalloc.sampled_count_by_activity", false); - TaggedCounters.emplace(t, std::make_pair(size, count)); - } else { - size = found->second.first; - count = found->second.second; - } - - size->Set(allocationStats.TagSizes[t]); - count->Set(allocationStats.TagCounts[t]); - } - } - - void UpdateControls() { - tcmalloc::MallocExtension::SetProfileSamplingRate(Controls.ProfileSamplingRate); - - if (Controls.GuardedSamplingRate != TControls::MaxSamplingRate) { - tcmalloc::MallocExtension::ActivateGuardedSampling(); - } - tcmalloc::MallocExtension::SetGuardedSamplingRate(Controls.GuardedSamplingRate); - - tcmalloc::MallocExtension::BytesPerSecond rate{(size_t)Controls.BackgroundReleaseRate}; - tcmalloc::MallocExtension::SetBackgroundReleaseRate(rate); - - tcmalloc::MallocExtension::MemoryLimit limit; - limit.hard = false; - limit.limit = Controls.MemoryLimit ? - (size_t)Controls.MemoryLimit : std::numeric_limits<size_t>::max(); - tcmalloc::MallocExtension::SetMemoryLimit(limit); - } - - void DumpCurrent(IOutputStream& out, tcmalloc::ProfileType type, - size_t stackCountLimit = 256, size_t sampleCountLimit = 1024, bool forLog = false) - { - auto start = TInstant::Now(); - auto profile = tcmalloc::MallocExtension::SnapshotCurrent(type); - auto end = TInstant::Now(); - - TAllocationAnalyzer analyzer(std::move(profile)); - TAllocationStats allocationStats; - analyzer.Prepare(&allocationStats); - - bool isHeap = (type == tcmalloc::ProfileType::kHeap); - - if (forLog) { - const char* marker = "HEAP"; - const char* sep = " | "; - out << marker << sep - << "Snapshot calculation time: " << (end - start).MicroSeconds() << " us" << sep - << Endl; - allocationStats.DumpSizeStats(out, marker, sep); - allocationStats.DumpTagStats(out, marker, sep); - analyzer.Dump(out, stackCountLimit, sampleCountLimit, isHeap, true); - return; - } - - HTML(out) { - if (isHeap) { - out << "<p><a class=\"btn btn-primary\" href=\"?action=log\">Dump to log</a></p>\n"; - } - TABLE_SORTABLE_CLASS("table") { - TABLEHEAD() { - TABLER() { - TABLEH() { out << "upper bound"; } - TABLEH() { out << "size"; } - TABLEH() { out << "count"; } - } - out << Endl; - } - TABLEBODY() { - for (size_t b = 0; b < TAllocationStats::MaxSizeIndex; ++b) { - if (!allocationStats.Sizes[b]) { - continue; - } - TABLER() { - if (b == TAllocationStats::MaxSizeIndex - 1) { - TABLED() { out << "inf"; } - } else { - TABLED() { out << TAllocationStats::SizeUpperBound(b); } - } - TABLED() { out << allocationStats.Sizes[b]; } - TABLED() { out << allocationStats.Counts[b]; } - } - out << Endl; - } - TABLER() { - TABLED() { out << "<b>total</b>"; } - TABLED() { out << "<b>" << allocationStats.Size << "</b>"; } - TABLED() { out << "<b>" << allocationStats.Count << "</b>"; } - } - out << Endl; - } - } - TABLE_SORTABLE_CLASS("table") { - TABLEHEAD() { - TABLER() { - TABLEH() { out << "activity"; } - TABLEH() { out << "size"; } - TABLEH() { out << "count"; } - } - out << Endl; - } - TABLEBODY() { - for (ui32 t = 0; t < TAllocationStats::MaxTag; ++t) { - if (!allocationStats.TagSizes[t]) { - continue; - } - TABLER() { - TABLED() { out << TAllocationStats::GetTagName(t); } - TABLED() { out << allocationStats.TagSizes[t]; } - TABLED() { out << allocationStats.TagCounts[t]; } - } - out << Endl; - } - } - } - PRE() { - out << "Snapshot calculation time: " << (end - start).MicroSeconds() << " us" << Endl - << Endl; - analyzer.Dump(out, stackCountLimit, sampleCountLimit, isHeap, false); - } - } - } - -public: - explicit TTcMallocMonitor(TDynamicCountersPtr group) { - CounterGroup = group->GetSubgroup("component", "tcmalloc"); - - SizeHistogram = CounterGroup->GetHistogram("tcmalloc.sampled_size", - NMonitoring::ExponentialHistogram(TAllocationStats::MaxSizeIndex, 2, 1), false); - - CountHistogram = CounterGroup->GetHistogram("tcmalloc.sampled_count", - NMonitoring::ExponentialHistogram(TAllocationStats::MaxSizeIndex, 2, 1), false); - } - - void RegisterPages(TMon* mon, TActorSystem* actorSystem, TActorId actorId) override { - auto* indexPage = mon->FindIndexPage("memory"); - if (!indexPage) { - return; - } - mon->RegisterActorPage(indexPage, "heap", "Heap", false, actorSystem, actorId); - mon->RegisterActorPage(indexPage, "peakheap", "Peak heap", false, actorSystem, actorId); - mon->RegisterActorPage(indexPage, "fragmentation", "Fragmentation", false, actorSystem, actorId); - } - - void RegisterControls(TIntrusivePtr<TControlBoard> icb) override { - Controls.Register(icb); - } - - void Update() override { - UpdateCounters(); - UpdateControls(); - } - - void Dump(IOutputStream& out, const TString& relPath) override { - if (relPath == "/memory/heap") { - DumpCurrent(out, tcmalloc::ProfileType::kHeap); - return; - } else if (relPath == "/memory/peakheap") { - DumpCurrent(out, tcmalloc::ProfileType::kPeakHeap); - return; - } else if (relPath == "/memory/fragmentation") { - DumpCurrent(out, tcmalloc::ProfileType::kFragmentation); - return; - } - - auto properties = tcmalloc::MallocExtension::GetProperties(); - auto stats = tcmalloc::MallocExtension::GetStats(); - - HTML(out) { - H3() { - out << "TCMalloc" << Endl; - } - out << "<hr>" << Endl; - H4() { - out << "Allocator properties" << Endl; - } - TABLE_CLASS("table") { - TABLEHEAD() { - TABLER() { - TABLEH() { out << "Property"; } - TABLEH() { out << "Value"; } - } - out << Endl; - } - TABLEBODY() { - for (const auto& [name, value] : properties) { - TABLER() { - TABLED() { out << name; } - TABLED() { FormatPrettyNumber(out, value.value); } - } - out << Endl; - } - } - } - out << "<hr>" << Endl; - H4() { - out << "<a href=\"https://github.com/google/tcmalloc/blob/master/docs/stats.md\">Internal stats</a>" << Endl; - } - PRE() { - out << EncodeHtmlPcdata(stats) << Endl; - } - } - } - - void DumpForLog(IOutputStream& out, size_t limit) override { - auto properties = tcmalloc::MallocExtension::GetProperties(); - auto stats = tcmalloc::MallocExtension::GetStats(); - - const char* sep = " | "; - out << "======== TCMALLOC PROPERTIES" << sep; - for (const auto& [name, value] : properties) { - out << name << ": " << value.value << sep; - } - out << Endl; - - out << "======== TCMALLOC STATISTICS" << Endl; - out << stats << Endl; - - out << "======== TCMALLOC HEAP" << Endl; - DumpCurrent(out, tcmalloc::ProfileType::kHeap, limit, 128, true); - } -}; - - -class TTcMallocProfiler : public IProfilerLogic { - tcmalloc::MallocExtension::AllocationProfilingToken Token; - -public: - void Start() override { - Token = tcmalloc::MallocExtension::StartAllocationProfiling(); - } - - void Stop(IOutputStream& out, size_t countLimit, bool forLog) override { - auto profile = std::move(Token).Stop(); - - TAllocationAnalyzer analyzer(std::move(profile)); - TAllocationStats allocationStats; - analyzer.Prepare(&allocationStats); - - const char* marker = forLog ? "PROFILE" : "--------------------------------"; - const char* sep = forLog ? " | " : "\n"; - allocationStats.DumpSizeStats(out, marker, sep); - allocationStats.DumpTagStats(out, marker, sep); - analyzer.Dump(out, countLimit, 1024, false, forLog); - } -}; - - -std::unique_ptr<IAllocStats> CreateTcMallocStats(TDynamicCountersPtr group) { - return std::make_unique<TTcMallocStats>(std::move(group)); -} - -std::unique_ptr<IAllocState> CreateTcMallocState() { - return std::make_unique<TTcMallocState>(); -} - -std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor(TDynamicCountersPtr group) { - return std::make_unique<TTcMallocMonitor>(std::move(group)); -} - -std::unique_ptr<IProfilerLogic> CreateTcMallocProfiler() { - return std::make_unique<TTcMallocProfiler>(); -} - -} + +#include <util/stream/format.h> + +using namespace NActors; + +namespace NKikimr { + +using TDynamicCountersPtr = TIntrusivePtr<NMonitoring::TDynamicCounters>; +using TDynamicCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; +using THistogramPtr = NMonitoring::THistogramPtr; + +static void FormatPrettyNumber(IOutputStream& out, ssize_t val) { + if (val == 0) { + out << "0"; + return; + } + if (val < 0) { + out << "- "; + } + ui64 value = val < 0 ? -val : val; + TSmallVec<ui64> parts; + while (value > 0) { + auto head = value / 1000; + parts.push_back(value - head * 1000); + value = head; + } + out << parts.back(); + parts.pop_back(); + while (!parts.empty()) { + out << Sprintf(" %03" PRIu64, parts.back()); + parts.pop_back(); + } +} + +static void FormatSize(IOutputStream& out, size_t size) { + if (size < (1 << 10)) { + out << size; + } else if (size < (1 << 20)) { + out << size << " (" + << Sprintf("%.1f", size / float(1 << 10)) << " KiB)"; + } else { + out << size << " (" + << Sprintf("%.1f", size / float(1 << 20)) << " MiB)"; + } +} + +static ui64 GetProperty( + const std::map<std::string, tcmalloc::MallocExtension::Property>& properties, + const char* name) +{ + auto found = properties.find(name); + return found != properties.end() ? found->second.value : 0; +} + +static ui64 GetProperty(const char* name) { + const auto properties = tcmalloc::MallocExtension::GetProperties(); + return GetProperty(properties, name); +} + +static ui64 GetCachesSize( + const std::map<std::string, tcmalloc::MallocExtension::Property>& properties) +{ + return + GetProperty(properties, "tcmalloc.page_heap_free") + + GetProperty(properties, "tcmalloc.central_cache_free") + + GetProperty(properties, "tcmalloc.transfer_cache_free") + + GetProperty(properties, "tcmalloc.sharded_transfer_cache_free") + + GetProperty(properties, "tcmalloc.cpu_free") + + GetProperty(properties, "tcmalloc.thread_cache_free"); +} + +struct TAllocationStats { + size_t Size; + size_t Count; + size_t RequestedSize; + + static constexpr size_t MaxSizeIndex = 32; + size_t Sizes[MaxSizeIndex]; + size_t Counts[MaxSizeIndex]; + + static constexpr ui32 MaxTag = 1024; + size_t TagSizes[MaxTag]; + size_t TagCounts[MaxTag]; + + TAllocationStats() { + Zero(*this); + } + + static size_t SizeUpperBound(size_t index) { + return 1 << std::min(MaxSizeIndex - 1, index); + } + + static size_t GetIndex(size_t size) { + size_t index = 0; + if (size > 0) { + index = std::min(MaxSizeIndex - 1, (size_t)GetValueBitCount(size - 1)); + } + return index; + } + + static const char* GetTagName(ui32 tag) { + const char* name = NProfiling::GetTag(tag); + return name ? name : "_DEFAULT"; + } + + void Add(const tcmalloc::Profile::Sample& sample) { + Size += sample.sum; + Count += sample.count; + RequestedSize += sample.count * sample.requested_size; + + auto index = GetIndex(sample.allocated_size); + Sizes[index] += sample.sum; + Counts[index] += sample.count; + + ui32 tag = reinterpret_cast<uintptr_t>(sample.user_data); + tag = (tag < MaxTag) ? tag : 0; + TagSizes[tag] += sample.sum; + TagCounts[tag] += sample.count; + } + + void DumpSizeStats(IOutputStream& out, const char* marker, const char* sep) const { + out << marker << sep << sep; + for (size_t b = 0; b < MaxSizeIndex; ++b) { + if (b == MaxSizeIndex - 1) { + out << "inf : "; + } else { + out << "<= " << Sprintf("%-10" PRIu64, SizeUpperBound(b)) << ": "; + } + if (!Sizes[b]) { + out << "0" << sep; + continue; + } + out << "size "; + FormatSize(out, Sizes[b]); + out << ", count " << Counts[b] << sep; + } + out << Endl; + } + + void DumpTagStats(IOutputStream& out, const char* marker, const char* sep) const { + out << marker << sep << sep; + for (ui32 t = 0; t < MaxTag; ++t) { + if (!TagSizes[t]) { + continue; + } + out << Sprintf("%-32s", TAllocationStats::GetTagName(t)) << ": size "; + FormatSize(out, TagSizes[t]); + out << ", count " << TagCounts[t] << sep; + } + out << Endl; + } +}; + + +class TAllocationAnalyzer { + const tcmalloc::Profile Profile; + + struct TSymbol { + const void* Address; + TString Name; + }; + TLFUCache<void*, TSymbol> SymbolCache; + + struct TStackKey { + int Depth; + void* Stack[tcmalloc::Profile::Sample::kMaxStackDepth]; + + explicit TStackKey(const tcmalloc::Profile::Sample& sample) { + Depth = std::min(sample.depth, tcmalloc::Profile::Sample::kMaxStackDepth); + std::memcpy(Stack, sample.stack, Depth * sizeof(void*)); + } + + bool operator==(const TStackKey& key) const { + if (Depth != key.Depth) { + return false; + } + return std::memcmp(Stack, key.Stack, Depth * sizeof(void*)) == 0; + } + + struct THash { + size_t operator()(const TStackKey& key) const { + std::string_view view((char*)key.Stack, key.Depth * sizeof(void*)); + return std::hash<std::string_view>()(view); + } + }; + }; + + struct TStackStats { + std::vector<tcmalloc::Profile::Sample> Samples; + size_t Size = 0; + size_t Count = 0; + ui32 Tag = 0; + }; + + std::unordered_map<TStackKey, TStackStats, TStackKey::THash> Stacks; + size_t SampleCount = 0; + size_t Size = 0; + size_t Count = 0; + size_t RequestedSize = 0; + + bool Prepared = false; + +private: + void PrintBackTrace(IOutputStream& out, void* const* stack, size_t sz, + const char* sep) + { + char name[1024]; + for (size_t i = 0; i < sz; ++i) { + TSymbol symbol; + auto it = SymbolCache.Find(stack[i]); + if (it != SymbolCache.End()) { + symbol = it.Value(); + } else { + TResolvedSymbol rs = ResolveSymbol(stack[i], name, sizeof(name)); + symbol = {rs.NearestSymbol, rs.Name}; + SymbolCache.Insert(stack[i], symbol); + } + + out << Hex((intptr_t)stack[i], HF_FULL) << " " << symbol.Name; + intptr_t offset = (intptr_t)stack[i] - (intptr_t)symbol.Address; + if (offset) + out << " +" << offset; + out << sep; + } + } + + void PrintSample(IOutputStream& out, const tcmalloc::Profile::Sample* sample, + const char* sep) const + { + out << sample->sum << " = " + << sample->count << " * " + << sample->allocated_size << " (" + << sample->requested_size << ")" + << sep; + } + + void PrintStack(IOutputStream& out, const TStackStats* stats, size_t sampleCountLimit, + const char* marker, const char* sep) + { + std::vector<const tcmalloc::Profile::Sample*> samples; + samples.reserve(stats->Samples.size()); + size_t size = 0; + size_t count = 0; + + for (const auto& sample : stats->Samples) { + samples.push_back(&sample); + size += sample.sum; + count += sample.count; + } + + std::sort(samples.begin(), samples.end(), [] (const auto* l, const auto* r) { + return l->sum > r->sum; + }); + + out << marker << sep << sep << "size "; + FormatSize(out, size); + out << ", count " << count + << ", " << TAllocationStats::GetTagName(stats->Tag) << sep; + + size_t i = 0; + for (const auto* sample : samples) { + PrintSample(out, sample, sep); + if (++i >= sampleCountLimit) { + break; + } + } + + if (samples.size() > sampleCountLimit) { + out << "....TRUNCATED, first " << sampleCountLimit << " samples are shown" << sep; + } + out << sep; + + if (samples.size()) { + const auto& sample = samples.front(); + PrintBackTrace(out, sample->stack, sample->depth, sep); + } + out << Endl; + } + + void PrintAllocatorDetails(IOutputStream& out, const char* sep) const { + const auto properties = tcmalloc::MallocExtension::GetProperties(); + out << sep + << "Physical memory used: "; + FormatSize(out, GetProperty(properties, "generic.physical_memory_used")); + out << sep + << " Application: "; + FormatSize(out, GetProperty(properties, "generic.bytes_in_use_by_app")); + out << sep + << " Allocator caches: "; + FormatSize(out, GetCachesSize(properties)); + out << sep + << " Allocator metadata: "; + FormatSize(out, GetProperty(properties, "tcmalloc.metadata_bytes")); + out << sep; + } + +public: + explicit TAllocationAnalyzer(tcmalloc::Profile&& profile) + : Profile(std::move(profile)) + , SymbolCache(2048) + {} + + void Prepare(TAllocationStats* allocationStats) { + Y_VERIFY(!Prepared); + + TAllocationStats allocations; + + Profile.Iterate([&] (const tcmalloc::Profile::Sample& sample) { + if (!sample.sum) { + return; + } + + TStackKey key(sample); + auto& stats = Stacks[key]; + stats.Samples.push_back(sample); + stats.Size += sample.sum; + stats.Count += sample.count; + stats.Tag = reinterpret_cast<uintptr_t>(sample.user_data); + + allocations.Add(sample); + + ++SampleCount; + }); + + Size = allocations.Size; + Count = allocations.Count; + RequestedSize = allocations.RequestedSize; + + if (allocationStats) { + *allocationStats = allocations; + } + + Prepared = true; + } + + void Dump(IOutputStream& out, size_t stackCountLimit, size_t sampleCountLimit, + bool isHeap, bool forLog) + { + Y_VERIFY(Prepared); + + const char* marker = forLog ? + (isHeap ? "HEAP" : "PROFILE") : "--------------------------------"; + const char* sep = forLog ? " | " : "\n"; + + out << marker << sep << sep; + out << "Stack count: " << Stacks.size() << sep + << "Sample count: " << SampleCount << sep + << "Allocated bytes: "; + FormatSize(out, Size); + out << sep + << "Allocation count: " << Count << sep; + + if (isHeap) { + out << sep + << "Requested bytes: "; + FormatSize(out, RequestedSize); + out << sep + << "Wasted bytes: "; + FormatSize(out, Size - RequestedSize); + out << sep; + + PrintAllocatorDetails(out, sep); + } + + if (!forLog) { + out << sep + << "Details: size = count * allocated_size (requested_size)" << sep; + } + + out << Endl; + + std::vector<const TStackStats*> stats; + stats.reserve(Stacks.size()); + for (auto& [key, stackStats] : Stacks) { + stats.push_back(&stackStats); + } + + std::sort(stats.begin(), stats.end(), [] (const auto* l, const auto* r) { + return l->Size > r->Size; + }); + + size_t i = 0; + for (const auto* stackStats : stats) { + PrintStack(out, stackStats, sampleCountLimit, marker, sep); + if (++i >= stackCountLimit) { + break; + } + } + + if (stats.size() > stackCountLimit) { + out << "....TRUNCATED, first " << stackCountLimit << " stacks are shown" << sep; + } + out << Endl; + } +}; + + +class TTcMallocStats : public IAllocStats { + TDynamicCountersPtr CounterGroup; + + static constexpr const char* Names[] = { + "generic.virtual_memory_used", + "generic.physical_memory_used", + "generic.bytes_in_use_by_app", + "tcmalloc.page_heap_free", + "tcmalloc.metadata_bytes", + "tcmalloc.thread_cache_count", + "tcmalloc.central_cache_free", + "tcmalloc.transfer_cache_free", + "tcmalloc.sharded_transfer_cache_free", + "tcmalloc.cpu_free", + "tcmalloc.per_cpu_caches_active", + "tcmalloc.thread_cache_free", + "tcmalloc.page_heap_unmapped", + }; + + std::unordered_map<TString, TDynamicCounterPtr> Counters; + + TDynamicCounterPtr CachesSize; + +public: + explicit TTcMallocStats(TDynamicCountersPtr group) { + CounterGroup = group->GetSubgroup("component", "tcmalloc"); + for (auto name : Names) { + Counters[name] = CounterGroup->GetCounter(name, false); + } + CachesSize = CounterGroup->GetCounter("tcmalloc.caches_bytes", false); + } + + void Update() override { + auto properties = tcmalloc::MallocExtension::GetProperties(); + for (auto name : Names) { + if (auto it = properties.find(name); it != properties.end()) { + Counters[name]->Set(it->second.value); + } + } + CachesSize->Set(GetCachesSize(properties)); + } +}; + + +class TTcMallocState : public IAllocState { +public: + ui64 GetAllocatedMemoryEstimate() const override { + return GetProperty("generic.physical_memory_used"); + } +}; + + +class TTcMallocMonitor : public IAllocMonitor { + TDynamicCountersPtr CounterGroup; + + THistogramPtr SizeHistogram; + THistogramPtr CountHistogram; + std::unordered_map<ui32, std::pair<TDynamicCounterPtr, TDynamicCounterPtr>> TaggedCounters; + + struct TControls { + static constexpr size_t MaxSamplingRate = 4ll << 30; + + TControlWrapper ProfileSamplingRate; + TControlWrapper GuardedSamplingRate; + TControlWrapper BackgroundReleaseRate; + TControlWrapper MemoryLimit; + + TControls() + : ProfileSamplingRate(tcmalloc::MallocExtension::GetProfileSamplingRate(), + 64 << 10, MaxSamplingRate) + , GuardedSamplingRate(MaxSamplingRate, + 64 << 10, MaxSamplingRate) + , BackgroundReleaseRate(0, + 0, 64 << 20) + , MemoryLimit(0, + 0, std::numeric_limits<i64>::max()) + {} + + void Register(TIntrusivePtr<TControlBoard> icb) { + icb->RegisterSharedControl(ProfileSamplingRate, "TCMallocControls.ProfileSamplingRate"); + icb->RegisterSharedControl(GuardedSamplingRate, "TCMallocControls.GuardedSamplingRate"); + icb->RegisterSharedControl(BackgroundReleaseRate, "TCMallocControls.BackgroundReleaseRate"); + icb->RegisterSharedControl(MemoryLimit, "TCMallocControls.MemoryLimit"); + } + }; + TControls Controls; + +private: + void UpdateCounters() { + auto profile = tcmalloc::MallocExtension::SnapshotCurrent(tcmalloc::ProfileType::kHeap); + + TAllocationAnalyzer analyzer(std::move(profile)); + TAllocationStats allocationStats; + analyzer.Prepare(&allocationStats); + + auto updateHistogram = [] (THistogramPtr histogram, size_t* numbers) { + histogram->Reset(); + auto snapshot = histogram->Snapshot(); + auto count = std::min(snapshot->Count(), (ui32)TAllocationStats::MaxSizeIndex); + for (ui32 b = 0; b < count; ++b) { + histogram->Collect(snapshot->UpperBound(b), numbers[b]); + } + }; + + updateHistogram(SizeHistogram, allocationStats.Sizes); + updateHistogram(CountHistogram, allocationStats.Counts); + + for (ui32 t = 0; t < TAllocationStats::MaxTag; ++t) { + auto found = TaggedCounters.find(t); + if (!allocationStats.TagSizes[t] && found == TaggedCounters.end()) { + continue; + } + + TDynamicCounterPtr size, count; + + if (found == TaggedCounters.end()) { + auto name = TAllocationStats::GetTagName(t); + auto group = CounterGroup->GetSubgroup("activity", name); + size = group->GetCounter("tcmalloc.sampled_size_by_activity", false); + count = group->GetCounter("tcmalloc.sampled_count_by_activity", false); + TaggedCounters.emplace(t, std::make_pair(size, count)); + } else { + size = found->second.first; + count = found->second.second; + } + + size->Set(allocationStats.TagSizes[t]); + count->Set(allocationStats.TagCounts[t]); + } + } + + void UpdateControls() { + tcmalloc::MallocExtension::SetProfileSamplingRate(Controls.ProfileSamplingRate); + + if (Controls.GuardedSamplingRate != TControls::MaxSamplingRate) { + tcmalloc::MallocExtension::ActivateGuardedSampling(); + } + tcmalloc::MallocExtension::SetGuardedSamplingRate(Controls.GuardedSamplingRate); + + tcmalloc::MallocExtension::BytesPerSecond rate{(size_t)Controls.BackgroundReleaseRate}; + tcmalloc::MallocExtension::SetBackgroundReleaseRate(rate); + + tcmalloc::MallocExtension::MemoryLimit limit; + limit.hard = false; + limit.limit = Controls.MemoryLimit ? + (size_t)Controls.MemoryLimit : std::numeric_limits<size_t>::max(); + tcmalloc::MallocExtension::SetMemoryLimit(limit); + } + + void DumpCurrent(IOutputStream& out, tcmalloc::ProfileType type, + size_t stackCountLimit = 256, size_t sampleCountLimit = 1024, bool forLog = false) + { + auto start = TInstant::Now(); + auto profile = tcmalloc::MallocExtension::SnapshotCurrent(type); + auto end = TInstant::Now(); + + TAllocationAnalyzer analyzer(std::move(profile)); + TAllocationStats allocationStats; + analyzer.Prepare(&allocationStats); + + bool isHeap = (type == tcmalloc::ProfileType::kHeap); + + if (forLog) { + const char* marker = "HEAP"; + const char* sep = " | "; + out << marker << sep + << "Snapshot calculation time: " << (end - start).MicroSeconds() << " us" << sep + << Endl; + allocationStats.DumpSizeStats(out, marker, sep); + allocationStats.DumpTagStats(out, marker, sep); + analyzer.Dump(out, stackCountLimit, sampleCountLimit, isHeap, true); + return; + } + + HTML(out) { + if (isHeap) { + out << "<p><a class=\"btn btn-primary\" href=\"?action=log\">Dump to log</a></p>\n"; + } + TABLE_SORTABLE_CLASS("table") { + TABLEHEAD() { + TABLER() { + TABLEH() { out << "upper bound"; } + TABLEH() { out << "size"; } + TABLEH() { out << "count"; } + } + out << Endl; + } + TABLEBODY() { + for (size_t b = 0; b < TAllocationStats::MaxSizeIndex; ++b) { + if (!allocationStats.Sizes[b]) { + continue; + } + TABLER() { + if (b == TAllocationStats::MaxSizeIndex - 1) { + TABLED() { out << "inf"; } + } else { + TABLED() { out << TAllocationStats::SizeUpperBound(b); } + } + TABLED() { out << allocationStats.Sizes[b]; } + TABLED() { out << allocationStats.Counts[b]; } + } + out << Endl; + } + TABLER() { + TABLED() { out << "<b>total</b>"; } + TABLED() { out << "<b>" << allocationStats.Size << "</b>"; } + TABLED() { out << "<b>" << allocationStats.Count << "</b>"; } + } + out << Endl; + } + } + TABLE_SORTABLE_CLASS("table") { + TABLEHEAD() { + TABLER() { + TABLEH() { out << "activity"; } + TABLEH() { out << "size"; } + TABLEH() { out << "count"; } + } + out << Endl; + } + TABLEBODY() { + for (ui32 t = 0; t < TAllocationStats::MaxTag; ++t) { + if (!allocationStats.TagSizes[t]) { + continue; + } + TABLER() { + TABLED() { out << TAllocationStats::GetTagName(t); } + TABLED() { out << allocationStats.TagSizes[t]; } + TABLED() { out << allocationStats.TagCounts[t]; } + } + out << Endl; + } + } + } + PRE() { + out << "Snapshot calculation time: " << (end - start).MicroSeconds() << " us" << Endl + << Endl; + analyzer.Dump(out, stackCountLimit, sampleCountLimit, isHeap, false); + } + } + } + +public: + explicit TTcMallocMonitor(TDynamicCountersPtr group) { + CounterGroup = group->GetSubgroup("component", "tcmalloc"); + + SizeHistogram = CounterGroup->GetHistogram("tcmalloc.sampled_size", + NMonitoring::ExponentialHistogram(TAllocationStats::MaxSizeIndex, 2, 1), false); + + CountHistogram = CounterGroup->GetHistogram("tcmalloc.sampled_count", + NMonitoring::ExponentialHistogram(TAllocationStats::MaxSizeIndex, 2, 1), false); + } + + void RegisterPages(TMon* mon, TActorSystem* actorSystem, TActorId actorId) override { + auto* indexPage = mon->FindIndexPage("memory"); + if (!indexPage) { + return; + } + mon->RegisterActorPage(indexPage, "heap", "Heap", false, actorSystem, actorId); + mon->RegisterActorPage(indexPage, "peakheap", "Peak heap", false, actorSystem, actorId); + mon->RegisterActorPage(indexPage, "fragmentation", "Fragmentation", false, actorSystem, actorId); + } + + void RegisterControls(TIntrusivePtr<TControlBoard> icb) override { + Controls.Register(icb); + } + + void Update() override { + UpdateCounters(); + UpdateControls(); + } + + void Dump(IOutputStream& out, const TString& relPath) override { + if (relPath == "/memory/heap") { + DumpCurrent(out, tcmalloc::ProfileType::kHeap); + return; + } else if (relPath == "/memory/peakheap") { + DumpCurrent(out, tcmalloc::ProfileType::kPeakHeap); + return; + } else if (relPath == "/memory/fragmentation") { + DumpCurrent(out, tcmalloc::ProfileType::kFragmentation); + return; + } + + auto properties = tcmalloc::MallocExtension::GetProperties(); + auto stats = tcmalloc::MallocExtension::GetStats(); + + HTML(out) { + H3() { + out << "TCMalloc" << Endl; + } + out << "<hr>" << Endl; + H4() { + out << "Allocator properties" << Endl; + } + TABLE_CLASS("table") { + TABLEHEAD() { + TABLER() { + TABLEH() { out << "Property"; } + TABLEH() { out << "Value"; } + } + out << Endl; + } + TABLEBODY() { + for (const auto& [name, value] : properties) { + TABLER() { + TABLED() { out << name; } + TABLED() { FormatPrettyNumber(out, value.value); } + } + out << Endl; + } + } + } + out << "<hr>" << Endl; + H4() { + out << "<a href=\"https://github.com/google/tcmalloc/blob/master/docs/stats.md\">Internal stats</a>" << Endl; + } + PRE() { + out << EncodeHtmlPcdata(stats) << Endl; + } + } + } + + void DumpForLog(IOutputStream& out, size_t limit) override { + auto properties = tcmalloc::MallocExtension::GetProperties(); + auto stats = tcmalloc::MallocExtension::GetStats(); + + const char* sep = " | "; + out << "======== TCMALLOC PROPERTIES" << sep; + for (const auto& [name, value] : properties) { + out << name << ": " << value.value << sep; + } + out << Endl; + + out << "======== TCMALLOC STATISTICS" << Endl; + out << stats << Endl; + + out << "======== TCMALLOC HEAP" << Endl; + DumpCurrent(out, tcmalloc::ProfileType::kHeap, limit, 128, true); + } +}; + + +class TTcMallocProfiler : public IProfilerLogic { + tcmalloc::MallocExtension::AllocationProfilingToken Token; + +public: + void Start() override { + Token = tcmalloc::MallocExtension::StartAllocationProfiling(); + } + + void Stop(IOutputStream& out, size_t countLimit, bool forLog) override { + auto profile = std::move(Token).Stop(); + + TAllocationAnalyzer analyzer(std::move(profile)); + TAllocationStats allocationStats; + analyzer.Prepare(&allocationStats); + + const char* marker = forLog ? "PROFILE" : "--------------------------------"; + const char* sep = forLog ? " | " : "\n"; + allocationStats.DumpSizeStats(out, marker, sep); + allocationStats.DumpTagStats(out, marker, sep); + analyzer.Dump(out, countLimit, 1024, false, forLog); + } +}; + + +std::unique_ptr<IAllocStats> CreateTcMallocStats(TDynamicCountersPtr group) { + return std::make_unique<TTcMallocStats>(std::move(group)); +} + +std::unique_ptr<IAllocState> CreateTcMallocState() { + return std::make_unique<TTcMallocState>(); +} + +std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor(TDynamicCountersPtr group) { + return std::make_unique<TTcMallocMonitor>(std::move(group)); +} + +std::unique_ptr<IProfilerLogic> CreateTcMallocProfiler() { + return std::make_unique<TTcMallocProfiler>(); +} + +} diff --git a/ydb/core/mon_alloc/tcmalloc.h b/ydb/core/mon_alloc/tcmalloc.h index d394b32c9b..db69ea16a8 100644 --- a/ydb/core/mon_alloc/tcmalloc.h +++ b/ydb/core/mon_alloc/tcmalloc.h @@ -1,20 +1,20 @@ -#pragma once - -#include "monitor.h" -#include "profiler.h" -#include "stats.h" - -namespace NKikimr { - -std::unique_ptr<IAllocStats> CreateTcMallocStats( - TIntrusivePtr<NMonitoring::TDynamicCounters> group); - -std::unique_ptr<IAllocState> CreateTcMallocState(); - -std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor( - TIntrusivePtr<NMonitoring::TDynamicCounters> group); - -std::unique_ptr<NActors::IProfilerLogic> CreateTcMallocProfiler(); - - -} +#pragma once + +#include "monitor.h" +#include "profiler.h" +#include "stats.h" + +namespace NKikimr { + +std::unique_ptr<IAllocStats> CreateTcMallocStats( + TIntrusivePtr<NMonitoring::TDynamicCounters> group); + +std::unique_ptr<IAllocState> CreateTcMallocState(); + +std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor( + TIntrusivePtr<NMonitoring::TDynamicCounters> group); + +std::unique_ptr<NActors::IProfilerLogic> CreateTcMallocProfiler(); + + +} diff --git a/ydb/core/mon_alloc/tcmalloc_null.cpp b/ydb/core/mon_alloc/tcmalloc_null.cpp index a4de4d5dcc..e5139fb8b8 100644 --- a/ydb/core/mon_alloc/tcmalloc_null.cpp +++ b/ydb/core/mon_alloc/tcmalloc_null.cpp @@ -1,27 +1,27 @@ -#include "tcmalloc.h" - -namespace NKikimr { - -std::unique_ptr<IAllocStats> CreateTcMallocStats( - TIntrusivePtr<NMonitoring::TDynamicCounters> group) -{ - Y_UNUSED(group); - return {}; -} - -std::unique_ptr<IAllocState> CreateTcMallocState() { - return {}; -} - -std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor( - TIntrusivePtr<NMonitoring::TDynamicCounters> group) -{ - Y_UNUSED(group); - return {}; -} - -std::unique_ptr<NActors::IProfilerLogic> CreateTcMallocProfiler() { - return {}; -} - -} +#include "tcmalloc.h" + +namespace NKikimr { + +std::unique_ptr<IAllocStats> CreateTcMallocStats( + TIntrusivePtr<NMonitoring::TDynamicCounters> group) +{ + Y_UNUSED(group); + return {}; +} + +std::unique_ptr<IAllocState> CreateTcMallocState() { + return {}; +} + +std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor( + TIntrusivePtr<NMonitoring::TDynamicCounters> group) +{ + Y_UNUSED(group); + return {}; +} + +std::unique_ptr<NActors::IProfilerLogic> CreateTcMallocProfiler() { + return {}; +} + +} diff --git a/ydb/core/mon_alloc/ya.make b/ydb/core/mon_alloc/ya.make index 8b60e88262..972ea082f3 100644 --- a/ydb/core/mon_alloc/ya.make +++ b/ydb/core/mon_alloc/ya.make @@ -34,10 +34,10 @@ IF (ALLOCATOR == "TCMALLOC_256K") PEERDIR( contrib/libs/tcmalloc ) -ELSE() +ELSE() SRCS( tcmalloc_null.cpp ) -ENDIF() - +ENDIF() + END() diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index 2f997e69a3..a08ccbf865 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -1469,14 +1469,14 @@ message TEvTestLoadRequest { TStockWorkload Stock = 13; } } - - message TMemoryLoadStart { - optional uint64 Tag = 1; - optional uint32 DurationSeconds = 2; - optional uint64 BlockSize = 3; - optional uint64 IntervalUs = 4; - } - + + message TMemoryLoadStart { + optional uint64 Tag = 1; + optional uint32 DurationSeconds = 2; + optional uint64 BlockSize = 3; + optional uint64 IntervalUs = 4; + } + optional uint64 Cookie = 1; oneof Command { TLoadStart LoadStart = 2; @@ -1487,7 +1487,7 @@ message TEvTestLoadRequest { TPDiskLogLoadStart PDiskLogLoadStart = 7; TKeyValueLoadStart KeyValueLoadStart = 8; TKqpLoadStart KqpLoadStart = 9; - TMemoryLoadStart MemoryLoadStart = 10; + TMemoryLoadStart MemoryLoadStart = 10; } } diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index f3dc06a28c..d64169d4fc 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -628,7 +628,7 @@ message TFeatureFlags { optional bool AllowHugeKeyValueDeletes = 13 [default = true]; // delete when all clients limit deletes per request optional bool SendSchemaVersionToDatashard = 14 [default = true]; // deprecated: always true optional bool EnableSchemeBoardCache = 15 [default = true]; // deprecated: always true - optional bool EnableSystemViews = 16 [default = true]; + optional bool EnableSystemViews = 16 [default = true]; optional bool EnableExternalHive = 17 [default = true]; optional bool UseSchemeBoardCacheForSchemeRequests = 18 [default = true]; // deprecated: always true optional bool CompileMinikqlWithVersion = 19 [default = true]; // deprecated: always true @@ -637,7 +637,7 @@ message TFeatureFlags { optional bool EnableOfflineSlaves = 22 [default = true]; // deprecated: always true optional bool CheckDatabaseAccessPermission = 23 [default = false]; optional bool AllowOnlineIndexBuild = 24 [default = true]; // deprecated: always true - optional bool EnablePersistentQueryStats = 25 [default = false]; + optional bool EnablePersistentQueryStats = 25 [default = false]; optional bool DisableDataShardBarrier = 26 [default = false]; optional bool EnablePutBatchingForBlobStorage = 27 [default = true]; optional bool EnableKqpWideFlow = 28 [default = true]; // deprecated: always true @@ -651,7 +651,7 @@ message TFeatureFlags { optional bool EnableGracefulShutdown = 36 [default = true]; optional bool EnableDrainOnShutdown = 37 [default = true]; optional bool EnableConfigurationCache = 38 [default = false]; - optional bool EnableDbCounters = 39 [default = false]; + optional bool EnableDbCounters = 39 [default = false]; optional bool EnableClockGettimeForUserCpuAccounting = 40 [default = false]; optional bool EnableAsyncIndexes = 41 [default = true]; optional bool AllowStreamExecuteYqlScript = 42 [default = true]; diff --git a/ydb/core/protos/counters_schemeshard.proto b/ydb/core/protos/counters_schemeshard.proto index 4d6adcbbdf..a839915c98 100644 --- a/ydb/core/protos/counters_schemeshard.proto +++ b/ydb/core/protos/counters_schemeshard.proto @@ -85,7 +85,7 @@ enum ESimpleCounters { COUNTER_IN_FLIGHT_OPS_TxAlterSolomonVolume = 71 [(CounterOpts) = {Name: "InFlightOps/AlterSolomonVolume"}]; COUNTER_IN_FLIGHT_OPS_TxDropLock = 72 [(CounterOpts) = {Name: "InFlightOps/DropLock"}]; COUNTER_IN_FLIGHT_OPS_TxDropTableIndexAtMainTable = 73 [(CounterOpts) = {Name: "InFlightOps/DropTableIndexAtMainTable"}]; - COUNTER_SYS_VIEW_PROCESSOR_COUNT = 74 [(CounterOpts) = {Name: "SysViewProcessors"}]; + COUNTER_SYS_VIEW_PROCESSOR_COUNT = 74 [(CounterOpts) = {Name: "SysViewProcessors"}]; COUNTER_FILESTORE_COUNT = 75 [(CounterOpts) = {Name: "FileStores"}]; COUNTER_FILESTORE_SHARD_COUNT = 76 [(CounterOpts) = {Name: "FileStoreShards"}]; diff --git a/ydb/core/protos/counters_sysview_processor.proto b/ydb/core/protos/counters_sysview_processor.proto index e7df0b8334..63c03cb653 100644 --- a/ydb/core/protos/counters_sysview_processor.proto +++ b/ydb/core/protos/counters_sysview_processor.proto @@ -1,17 +1,17 @@ import "ydb/core/protos/counters.proto"; - -package NKikimr.NSysView; - -option java_package = "ru.yandex.kikimr.proto"; - -option (TabletTypeName) = "SysViewProcessor"; - -enum ETxTypes { - TXTYPE_INIT_SCHEMA = 0 [(TxTypeOpts) = {Name: "TxInitSchema"}]; - TXTYPE_INIT = 1 [(TxTypeOpts) = {Name: "TxInit"}]; - TXTYPE_CONFIGURE = 2 [(TxTypeOpts) = {Name: "TxConfigure"}]; - TXTYPE_COLLECT = 3 [(TxTypeOpts) = {Name: "TxCollect"}]; - TXTYPE_AGGREGATE = 4 [(TxTypeOpts) = {Name: "TxAggregate"}]; - TXTYPE_INTERVAL_SUMMARY = 5 [(TxTypeOpts) = {Name: "TxIntervalSummary"}]; - TXTYPE_INTERVAL_METRICS = 6 [(TxTypeOpts) = {Name: "TxIntervalMetrics"}]; -} + +package NKikimr.NSysView; + +option java_package = "ru.yandex.kikimr.proto"; + +option (TabletTypeName) = "SysViewProcessor"; + +enum ETxTypes { + TXTYPE_INIT_SCHEMA = 0 [(TxTypeOpts) = {Name: "TxInitSchema"}]; + TXTYPE_INIT = 1 [(TxTypeOpts) = {Name: "TxInit"}]; + TXTYPE_CONFIGURE = 2 [(TxTypeOpts) = {Name: "TxConfigure"}]; + TXTYPE_COLLECT = 3 [(TxTypeOpts) = {Name: "TxCollect"}]; + TXTYPE_AGGREGATE = 4 [(TxTypeOpts) = {Name: "TxAggregate"}]; + TXTYPE_INTERVAL_SUMMARY = 5 [(TxTypeOpts) = {Name: "TxIntervalSummary"}]; + TXTYPE_INTERVAL_METRICS = 6 [(TxTypeOpts) = {Name: "TxIntervalMetrics"}]; +} diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto index 141a0f4945..dc06523e5c 100644 --- a/ydb/core/protos/flat_tx_scheme.proto +++ b/ydb/core/protos/flat_tx_scheme.proto @@ -310,7 +310,7 @@ message TEvSyncTenantSchemeShard { optional uint64 UserAttributesVersion = 7; optional uint64 TenantHive = 8; - optional uint64 TenantSysViewProcessor = 9; + optional uint64 TenantSysViewProcessor = 9; optional string TenantRootACL = 10; } @@ -330,7 +330,7 @@ message TEvUpdateTenantSchemeShard { optional uint64 UserAttributesVersion = 9; optional uint64 TenantHive = 10; - optional uint64 TenantSysViewProcessor = 11; + optional uint64 TenantSysViewProcessor = 11; optional NKikimrSubDomains.TSchemeQuotas DeclaredSchemeQuotas = 12; optional Ydb.Cms.DatabaseQuotas DatabaseQuotas = 14; diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto index e5fa3054b5..b96713ca29 100644 --- a/ydb/core/protos/node_whiteboard.proto +++ b/ydb/core/protos/node_whiteboard.proto @@ -286,7 +286,7 @@ message TSystemStateInfo { optional uint64 MemoryUsed = 26; optional uint64 MemoryLimit = 27; optional EConfigState ConfigState = 28 [default = Consistent]; - optional uint64 MemoryUsedInAlloc = 29; + optional uint64 MemoryUsedInAlloc = 29; optional double MaxDiskUsage = 30; optional NActorsInterconnect.TNodeLocation Location = 31; } diff --git a/ydb/core/protos/query_stats.proto b/ydb/core/protos/query_stats.proto index d97cd3ae6b..453b303401 100644 --- a/ydb/core/protos/query_stats.proto +++ b/ydb/core/protos/query_stats.proto @@ -28,7 +28,7 @@ message TTableAccessStats { optional TReadOpStats SelectRange = 3; optional TWriteOpStats UpdateRow = 4; optional TWriteOpStats EraseRow = 5; - optional uint64 ShardCount = 6; + optional uint64 ShardCount = 6; } message TPerShardStats { diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index 7276c61c86..c17c8a7dc3 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -271,9 +271,9 @@ enum EServiceKikimr { DATASHARD_RESTORE = 801; IMPORT = 802; S3_WRAPPER = 803; - - // System views - SYSTEM_VIEWS = 900; + + // System views + SYSTEM_VIEWS = 900; BUILD_INDEX = 1000; @@ -693,7 +693,7 @@ message TActivity { INTERCONNECT_DESTRUCT_ACTOR = 378; BS_MIGRATION_ACTOR = 379; SQS_GARBAGE_COLLECTOR_ACTOR = 380; - SYSTEM_VIEW_SERVICE = 381; + SYSTEM_VIEW_SERVICE = 381; CMS_SENTINEL_ACTOR = 382; CMS_SENTINEL_CONFIG_UPDATER_ACTOR = 383; CMS_SENTINEL_STATE_UPDATER_ACTOR = 384; @@ -838,12 +838,12 @@ message TActivity { BS_RESTORE_CORRUPTED_BLOB_ACTOR = 533; YQ_CONTROL_PLANE_STORAGE_ACTOR = 534; YQ_CONTROL_PLANE_PROXY_ACTOR = 535; - DB_WATCHER_ACTOR = 495; + DB_WATCHER_ACTOR = 495; YQL_PENDING_FETCHER_ACTOR = 536; YQL_PINGER_ACTOR = 537; YQL_PRIVATE_PROXY_ACTOR = 538; YQL_PRIVATE_PING_TASK_ACTOR = 539; - EXT_COUNTERS_UPDATER_ACTOR = 540; + EXT_COUNTERS_UPDATER_ACTOR = 540; YQL_PRIVATE_GET_TASK_ACTOR = 541; YQL_PRIVATE_WRITE_TASK_ACTOR = 542; YQL_NODES_MANAGER_ACTOR = 543; diff --git a/ydb/core/protos/subdomains.proto b/ydb/core/protos/subdomains.proto index 22d7d53d85..5d665ff381 100644 --- a/ydb/core/protos/subdomains.proto +++ b/ydb/core/protos/subdomains.proto @@ -19,7 +19,7 @@ message TSubDomainSettings { optional bool ExternalSchemeShard = 7 [default = false]; optional bool ExternalHive = 8 [default = false]; optional TDomainKey ResourcesDomainKey = 9; - optional bool ExternalSysViewProcessor = 10 [default = false]; + optional bool ExternalSysViewProcessor = 10 [default = false]; optional TSchemeQuotas DeclaredSchemeQuotas = 11; optional Ydb.Cms.DatabaseQuotas DatabaseQuotas = 12; } @@ -33,7 +33,7 @@ message TProcessingParams { optional fixed64 SchemeShard = 6; optional fixed64 Hive = 7; - optional fixed64 SysViewProcessor = 8; + optional fixed64 SysViewProcessor = 8; //put there SubSchemeShard and SubHive at the future } diff --git a/ydb/core/protos/sys_view.proto b/ydb/core/protos/sys_view.proto index 94b3095e6d..cfc0366272 100644 --- a/ydb/core/protos/sys_view.proto +++ b/ydb/core/protos/sys_view.proto @@ -1,85 +1,85 @@ -package NKikimrSysView; - -option java_package = "ru.yandex.kikimr.proto"; - +package NKikimrSysView; + +option java_package = "ru.yandex.kikimr.proto"; + import "ydb/core/protos/tablet.proto"; - -message TPartitionStatsKey { - optional uint64 OwnerId = 1; - optional uint64 PathId = 2; - optional uint64 PartIdx = 3; -} - + +message TPartitionStatsKey { + optional uint64 OwnerId = 1; + optional uint64 PathId = 2; + optional uint64 PartIdx = 3; +} + message TTtlStats { optional uint64 LastRunTime = 1; // milliseconds since epoch optional uint64 LastRowsProcessed = 2; optional uint64 LastRowsErased = 3; } -message TPartitionStats { - optional uint64 DataSize = 1; - optional uint64 RowCount = 2; - optional uint64 IndexSize = 3; - optional double CPUCores = 4; - - optional fixed64 TabletId = 5; - optional uint32 NodeId = 6; - - optional uint64 StartTime = 7; // milliseconds since epoch - optional uint64 AccessTime = 8; - optional uint64 UpdateTime = 9; - - optional uint64 InFlightTxCount = 10; - - optional uint64 RowUpdates = 11; - optional uint64 RowDeletes = 12; - optional uint64 RowReads = 13; - optional uint64 RangeReads = 14; - optional uint64 RangeReadRows = 15; - - optional uint64 ImmediateTxCompleted = 16; - optional uint64 PlannedTxCompleted = 17; - optional uint64 TxRejectedByOverload = 18; - optional uint64 TxRejectedBySpace = 19; +message TPartitionStats { + optional uint64 DataSize = 1; + optional uint64 RowCount = 2; + optional uint64 IndexSize = 3; + optional double CPUCores = 4; + + optional fixed64 TabletId = 5; + optional uint32 NodeId = 6; + + optional uint64 StartTime = 7; // milliseconds since epoch + optional uint64 AccessTime = 8; + optional uint64 UpdateTime = 9; + + optional uint64 InFlightTxCount = 10; + + optional uint64 RowUpdates = 11; + optional uint64 RowDeletes = 12; + optional uint64 RowReads = 13; + optional uint64 RangeReads = 14; + optional uint64 RangeReadRows = 15; + + optional uint64 ImmediateTxCompleted = 16; + optional uint64 PlannedTxCompleted = 17; + optional uint64 TxRejectedByOverload = 18; + optional uint64 TxRejectedBySpace = 19; optional TTtlStats TtlStats = 20; -} - -message TPartitionStatsResult { - optional TPartitionStatsKey Key = 1; - optional TPartitionStats Stats = 2; - optional string Path = 3; -} - -message TEvGetPartitionStats { - optional uint64 DomainKeyOwnerId = 1; - optional uint64 DomainKeyPathId = 2; - - optional TPartitionStatsKey From = 3; - optional bool FromInclusive = 4; - - optional TPartitionStatsKey To = 5; - optional bool ToInclusive = 6; - - optional bool IncludePathColumn = 7; -} - -message TEvGetPartitionStatsResult { - repeated TPartitionStatsResult Stats = 1; - optional TPartitionStatsKey Next = 2; - optional bool LastBatch = 3; - optional bool Overloaded = 4; -} - +} + +message TPartitionStatsResult { + optional TPartitionStatsKey Key = 1; + optional TPartitionStats Stats = 2; + optional string Path = 3; +} + +message TEvGetPartitionStats { + optional uint64 DomainKeyOwnerId = 1; + optional uint64 DomainKeyPathId = 2; + + optional TPartitionStatsKey From = 3; + optional bool FromInclusive = 4; + + optional TPartitionStatsKey To = 5; + optional bool ToInclusive = 6; + + optional bool IncludePathColumn = 7; +} + +message TEvGetPartitionStatsResult { + repeated TPartitionStatsResult Stats = 1; + optional TPartitionStatsKey Next = 2; + optional bool LastBatch = 3; + optional bool Overloaded = 4; +} + message TQueryDataStats { optional uint64 ReadRows = 1; optional uint64 ReadBytes = 2; optional uint64 UpdateRows = 3; optional uint64 UpdateBytes = 4; optional uint64 DeleteRows = 5; - optional uint64 DeleteBytes = 6; // deprecated - optional uint64 PartitionCount = 7; -} + optional uint64 DeleteBytes = 6; // deprecated + optional uint64 PartitionCount = 7; +} message TStatsAggr { optional uint64 Min = 1; @@ -88,229 +88,229 @@ message TStatsAggr { optional uint64 Cnt = 4; } -message TQueryStats { - optional string QueryText = 1; - optional uint64 QueryTextHash = 2; - optional uint64 DurationMs = 3; - optional uint64 EndTimeMs = 4; +message TQueryStats { + optional string QueryText = 1; + optional uint64 QueryTextHash = 2; + optional uint64 DurationMs = 3; + optional uint64 EndTimeMs = 4; optional TQueryDataStats Stats = 5; - optional string UserSID = 6; - optional uint64 ParametersSize = 7; - optional uint64 CompileDurationMs = 8; - optional bool FromQueryCache = 9; - optional uint32 NodeId = 10; + optional string UserSID = 6; + optional uint64 ParametersSize = 7; + optional uint64 CompileDurationMs = 8; + optional bool FromQueryCache = 9; + optional uint32 NodeId = 10; optional TStatsAggr ShardsCpuTimeUs = 11; optional TStatsAggr ComputeCpuTimeUs = 12; - optional uint64 CompileCpuTimeUs = 13; - optional uint64 ProcessCpuTimeUs = 14; - optional uint64 TotalCpuTimeUs = 15; - optional string Type = 16; - optional uint64 RequestUnits = 17; -} - -message TTopQueryStats { - repeated TQueryStats Stats = 1; -} - -enum EStatsType { - TOP_DURATION_ONE_MINUTE = 1; - TOP_DURATION_ONE_HOUR = 2; - TOP_READ_BYTES_ONE_MINUTE = 3; - TOP_READ_BYTES_ONE_HOUR = 4; - TOP_CPU_TIME_ONE_MINUTE = 5; - TOP_CPU_TIME_ONE_HOUR = 6; - METRICS_ONE_MINUTE = 7; - METRICS_ONE_HOUR = 8; - TOP_REQUEST_UNITS_ONE_MINUTE = 9; - TOP_REQUEST_UNITS_ONE_HOUR = 10; -} - -message TEvGetQueryStats { - optional EStatsType StatsType = 1; - optional uint64 StartBucket = 2; - optional string TenantName = 3; -} - -message THashedQueryText { - optional uint64 Hash = 1; - optional string Text = 2; -} - -message TEvGetQueryStatsResult { - repeated TTopQueryStats Buckets = 1; - optional uint64 LastBucket = 2; - optional uint64 BucketSizeMs = 3; - repeated THashedQueryText QueryTexts = 4; -} - -message TPDiskKey { - optional uint32 NodeId = 1; - optional uint32 PDiskId = 2; -} - -message TPDiskInfo { - optional string Type = 1; - optional uint64 Kind = 2; - optional string Path = 3; - optional uint64 Guid = 4; - optional uint64 BoxId = 5; - optional bool SharedWithOs = 6; - optional bool ReadCentric = 7; - optional uint64 AvailableSize = 8; - optional uint64 TotalSize = 9; - reserved 10; + optional uint64 CompileCpuTimeUs = 13; + optional uint64 ProcessCpuTimeUs = 14; + optional uint64 TotalCpuTimeUs = 15; + optional string Type = 16; + optional uint64 RequestUnits = 17; +} + +message TTopQueryStats { + repeated TQueryStats Stats = 1; +} + +enum EStatsType { + TOP_DURATION_ONE_MINUTE = 1; + TOP_DURATION_ONE_HOUR = 2; + TOP_READ_BYTES_ONE_MINUTE = 3; + TOP_READ_BYTES_ONE_HOUR = 4; + TOP_CPU_TIME_ONE_MINUTE = 5; + TOP_CPU_TIME_ONE_HOUR = 6; + METRICS_ONE_MINUTE = 7; + METRICS_ONE_HOUR = 8; + TOP_REQUEST_UNITS_ONE_MINUTE = 9; + TOP_REQUEST_UNITS_ONE_HOUR = 10; +} + +message TEvGetQueryStats { + optional EStatsType StatsType = 1; + optional uint64 StartBucket = 2; + optional string TenantName = 3; +} + +message THashedQueryText { + optional uint64 Hash = 1; + optional string Text = 2; +} + +message TEvGetQueryStatsResult { + repeated TTopQueryStats Buckets = 1; + optional uint64 LastBucket = 2; + optional uint64 BucketSizeMs = 3; + repeated THashedQueryText QueryTexts = 4; +} + +message TPDiskKey { + optional uint32 NodeId = 1; + optional uint32 PDiskId = 2; +} + +message TPDiskInfo { + optional string Type = 1; + optional uint64 Kind = 2; + optional string Path = 3; + optional uint64 Guid = 4; + optional uint64 BoxId = 5; + optional bool SharedWithOs = 6; + optional bool ReadCentric = 7; + optional uint64 AvailableSize = 8; + optional uint64 TotalSize = 9; + reserved 10; reserved 11; - optional string StatusV2 = 12; + optional string StatusV2 = 12; optional uint64 StatusChangeTimestamp = 13; optional uint64 EnforcedDynamicSlotSize = 14; optional uint32 ExpectedSlotCount = 15; optional uint32 NumActiveSlots = 16; optional uint64 Category = 17; - // metrics ? - // physical location ? - // config ? -} - -message TPDiskEntry { - optional TPDiskKey Key = 1; - optional TPDiskInfo Info = 2; -} - -message TEvGetPDisksRequest { - optional TPDiskKey From = 1; - optional bool InclusiveFrom = 2; - - optional TPDiskKey To = 3; - optional bool InclusiveTo = 4; -} - -message TEvGetPDisksResponse { - repeated TPDiskEntry Entries = 1; -} - -message TVSlotKey { - optional uint32 NodeId = 1; - optional uint32 PDiskId = 2; - optional uint32 VSlotId = 3; -} - -message TVSlotInfo { - reserved 1; - optional uint32 GroupId = 2; - optional uint32 GroupGeneration = 3; + // metrics ? + // physical location ? + // config ? +} + +message TPDiskEntry { + optional TPDiskKey Key = 1; + optional TPDiskInfo Info = 2; +} + +message TEvGetPDisksRequest { + optional TPDiskKey From = 1; + optional bool InclusiveFrom = 2; + + optional TPDiskKey To = 3; + optional bool InclusiveTo = 4; +} + +message TEvGetPDisksResponse { + repeated TPDiskEntry Entries = 1; +} + +message TVSlotKey { + optional uint32 NodeId = 1; + optional uint32 PDiskId = 2; + optional uint32 VSlotId = 3; +} + +message TVSlotInfo { + reserved 1; + optional uint32 GroupId = 2; + optional uint32 GroupGeneration = 3; optional uint32 FailRealm = 4; - optional uint32 FailDomain = 5; - optional uint32 VDisk = 6; - optional uint64 AllocatedSize = 7; - optional uint64 AvailableSize = 8; - reserved 9; + optional uint32 FailDomain = 5; + optional uint32 VDisk = 6; + optional uint64 AllocatedSize = 7; + optional uint64 AvailableSize = 8; + reserved 9; reserved 10; - optional string StatusV2 = 12; - optional string Kind = 13; + optional string StatusV2 = 12; + optional string Kind = 13; optional bool IsBeingDeleted = 14; - // metrics ? -} - -message TVSlotEntry { - optional TVSlotKey Key = 1; - optional TVSlotInfo Info = 2; -} - -message TEvGetVSlotsRequest { - optional TVSlotKey From = 1; - optional bool InclusiveFrom = 2; - - optional TVSlotKey To = 3; - optional bool InclusiveTo = 4; -} - -message TEvGetVSlotsResponse { - repeated TVSlotEntry Entries = 1; -} - -message TGroupKey { - optional uint32 GroupId = 1; -} - -message TGroupInfo { - optional uint32 Generation = 1; - reserved 2; - optional uint64 BoxId = 3; - optional uint64 StoragePoolId = 4; - optional uint32 EncryptionMode = 5; - optional uint32 LifeCyclePhase = 6; - optional uint64 AllocatedSize = 7; - optional uint64 AvailableSize = 8; + // metrics ? +} + +message TVSlotEntry { + optional TVSlotKey Key = 1; + optional TVSlotInfo Info = 2; +} + +message TEvGetVSlotsRequest { + optional TVSlotKey From = 1; + optional bool InclusiveFrom = 2; + + optional TVSlotKey To = 3; + optional bool InclusiveTo = 4; +} + +message TEvGetVSlotsResponse { + repeated TVSlotEntry Entries = 1; +} + +message TGroupKey { + optional uint32 GroupId = 1; +} + +message TGroupInfo { + optional uint32 Generation = 1; + reserved 2; + optional uint64 BoxId = 3; + optional uint64 StoragePoolId = 4; + optional uint32 EncryptionMode = 5; + optional uint32 LifeCyclePhase = 6; + optional uint64 AllocatedSize = 7; + optional uint64 AvailableSize = 8; reserved 9; reserved 10; - optional bool SeenOperational = 11; - optional string ErasureSpeciesV2 = 12; - optional uint64 PutTabletLogLatency = 13; - optional uint64 PutUserDataLatency = 14; - optional uint64 GetFastLatency = 15; - // owner ? - // storage pool name ? - // desired disk categories ? - // down/persisted down ? - // metrics ? -} - -message TGroupEntry { - optional TGroupKey Key = 1; - optional TGroupInfo Info = 2; -} - -message TEvGetGroupsRequest { - optional TGroupKey From = 1; - optional bool InclusiveFrom = 2; - - optional TGroupKey To = 3; - optional bool InclusiveTo = 4; -} - -message TEvGetGroupsResponse { - repeated TGroupEntry Entries = 1; -} - -message TStoragePoolKey { - optional uint64 BoxId = 1; - optional uint64 StoragePoolId = 2; -} - -message TStoragePoolInfo { - optional string Name = 1; - optional uint64 Generation = 2; - reserved 3; - reserved 4; - optional string Kind = 5; - optional uint32 NumGroups = 6; - optional uint32 EncryptionMode = 7; - optional uint64 SchemeshardId = 8; - optional uint64 PathId = 9; - optional string ErasureSpeciesV2 = 10; - optional string VDiskKindV2 = 12; + optional bool SeenOperational = 11; + optional string ErasureSpeciesV2 = 12; + optional uint64 PutTabletLogLatency = 13; + optional uint64 PutUserDataLatency = 14; + optional uint64 GetFastLatency = 15; + // owner ? + // storage pool name ? + // desired disk categories ? + // down/persisted down ? + // metrics ? +} + +message TGroupEntry { + optional TGroupKey Key = 1; + optional TGroupInfo Info = 2; +} + +message TEvGetGroupsRequest { + optional TGroupKey From = 1; + optional bool InclusiveFrom = 2; + + optional TGroupKey To = 3; + optional bool InclusiveTo = 4; +} + +message TEvGetGroupsResponse { + repeated TGroupEntry Entries = 1; +} + +message TStoragePoolKey { + optional uint64 BoxId = 1; + optional uint64 StoragePoolId = 2; +} + +message TStoragePoolInfo { + optional string Name = 1; + optional uint64 Generation = 2; + reserved 3; + reserved 4; + optional string Kind = 5; + optional uint32 NumGroups = 6; + optional uint32 EncryptionMode = 7; + optional uint64 SchemeshardId = 8; + optional uint64 PathId = 9; + optional string ErasureSpeciesV2 = 10; + optional string VDiskKindV2 = 12; optional string PDiskFilter = 13; optional bytes PDiskFilterData = 14; - // metrics ? -} - -message TStoragePoolEntry { - optional TStoragePoolKey Key = 1; - optional TStoragePoolInfo Info = 2; -} - -message TEvGetStoragePoolsRequest { - optional TStoragePoolKey From = 1; - optional bool InclusiveFrom = 2; - - optional TStoragePoolKey To = 3; - optional bool InclusiveTo = 4; -} - -message TEvGetStoragePoolsResponse { - repeated TStoragePoolEntry Entries = 1; -} - + // metrics ? +} + +message TStoragePoolEntry { + optional TStoragePoolKey Key = 1; + optional TStoragePoolInfo Info = 2; +} + +message TEvGetStoragePoolsRequest { + optional TStoragePoolKey From = 1; + optional bool InclusiveFrom = 2; + + optional TStoragePoolKey To = 3; + optional bool InclusiveTo = 4; +} + +message TEvGetStoragePoolsResponse { + repeated TStoragePoolEntry Entries = 1; +} + message TStorageStatsEntry { reserved 1; optional string PDiskFilter = 2; @@ -330,216 +330,216 @@ message TEvGetStorageStatsResponse { repeated TStorageStatsEntry Entries = 1; } -message TTabletEntry { - optional fixed64 TabletId = 1; +message TTabletEntry { + optional fixed64 TabletId = 1; optional uint32 FollowerId = 2; - optional string Type = 3; - optional string State = 4; - optional string VolatileState = 5; - optional string BootState = 6; - optional uint32 Generation = 7; - optional uint32 NodeId = 8; - optional uint64 CPU = 9; - optional uint64 Memory = 10; - optional uint64 Network = 11; -} - -message TEvGetTabletIdsRequest { - optional fixed64 From = 1; - optional fixed64 To = 2; -} - -message TEvGetTabletIdsResponse { - repeated fixed64 TabletIds = 1; -} - -message TEvGetTabletsRequest { - repeated fixed64 TabletIds = 1; - optional uint32 BatchSizeLimit = 2; -} - -message TEvGetTabletsResponse { - repeated TTabletEntry Entries = 1; - optional fixed64 NextTabletId = 2; -} - -// ---- Query metrics - -message TQueryMetrics { - message TMetrics { - optional uint64 Sum = 1; - optional uint64 Min = 2; - optional uint64 Max = 3; - } - - optional fixed64 QueryTextHash = 1; - optional uint64 Count = 2; - - optional TMetrics CpuTimeUs = 3; - optional TMetrics DurationUs = 4; - optional TMetrics ReadRows = 5; - optional TMetrics ReadBytes = 6; - optional TMetrics UpdateRows = 7; - optional TMetrics UpdateBytes = 8; - optional TMetrics DeleteRows = 9; - optional TMetrics RequestUnits = 10; -} - -// node -> sysview processor tablet -// per node {hash -> cpu} mapping -message TEvIntervalQuerySummary { - optional string Database = 1; - optional uint64 IntervalEndUs = 2; - optional uint32 NodeId = 3; - - message TQuerySet { - repeated fixed64 Hashes = 1; // hashes - repeated uint64 Values = 2; // top by values - } - - optional TQuerySet Metrics = 4; - optional TQuerySet TopByDuration = 5; - optional TQuerySet TopByReadBytes = 6; - optional TQuerySet TopByCpuTime = 7; - optional TQuerySet TopByRequestUnits = 8; -} - -// sysview processor tablet -> node -// only selected queries with top cpu -message TEvGetIntervalMetricsRequest { - optional string Database = 1; - optional uint64 IntervalEndUs = 2; - repeated fixed64 Metrics = 3; // request full query metrics - repeated fixed64 QueryTextsToGet = 4; // request query texts - repeated fixed64 TopByDuration = 5; // request interval top - repeated fixed64 TopByReadBytes = 6; // request interval top - repeated fixed64 TopByCpuTime = 7; // request interval top - repeated fixed64 TopByRequestUnits = 8; // request interval top -} - -// node -> sysview processor tablet -// query metrics and texts -message TEvGetIntervalMetricsResponse { - message TQueryText { - optional fixed64 Hash = 1; - optional string Text = 2; - } - - optional uint64 IntervalEndUs = 1; - repeated TQueryMetrics Metrics = 2; - repeated TQueryText QueryTexts = 3; - repeated TQueryStats TopByDuration = 4; - repeated TQueryStats TopByReadBytes = 5; - repeated TQueryStats TopByCpuTime = 6; - repeated TQueryStats TopByRequestUnits = 7; -} - -message TEvConfigureProcessor { - optional string Database = 1; -} - -message TQueryMetricsKey { - optional uint64 IntervalEndUs = 1; - optional uint32 Rank = 2; -} - -message TQueryMetricsEntry { - optional TQueryMetricsKey Key = 1; - optional TQueryMetrics Metrics = 2; - optional string QueryText = 3; -} - -message TQueryStatsEntry { - optional TQueryMetricsKey Key = 1; - optional TQueryStats Stats = 2; -} - -message TEvGetQueryMetricsRequest { - optional TQueryMetricsKey From = 1; - optional bool InclusiveFrom = 2; - - optional TQueryMetricsKey To = 3; - optional bool InclusiveTo = 4; - - optional EStatsType Type = 5; -} - -// reply to TEvGetQueryMetricsRequest -message TEvGetQueryMetricsResponse { - repeated TQueryMetricsEntry Entries = 1; - optional TQueryMetricsKey Next = 2; - optional bool LastBatch = 3; - optional bool Overloaded = 4; -} - -// reply to TEvGetQueryMetricsRequest -message TEvGetQueryStatsResponse { - repeated TQueryStatsEntry Entries = 1; - optional TQueryMetricsKey Next = 2; - optional bool LastBatch = 3; - optional bool Overloaded = 4; -} - -// ---- Database counters - -// generic set of counters with stable enumeration order in each category -// simple values are absolute -// cumulative and histogram values may be absolute or diffs depending on the context -// only index-value pairs with non-zero values are stored in diff mode -message TDbCounters { - message THistogram { - repeated uint64 Buckets = 1; - optional uint64 BucketsCount = 2; // total number of buckets in diff mode - } - - repeated uint64 Simple = 1; - repeated uint64 Cumulative = 2; - optional uint64 CumulativeCount = 3; // total number of cumulative counters in diff mode - repeated THistogram Histogram = 4; -} - -message TDbTabletCounters { - optional NKikimrTabletBase.TTabletTypes.EType Type = 1; - optional TDbCounters ExecutorCounters = 2; - optional TDbCounters AppCounters = 3; - optional TDbCounters MaxExecutorCounters = 4; - optional TDbCounters MaxAppCounters = 5; -} - -message TDbGRpcCounters { - optional string GRpcService = 1; - optional string GRpcRequest = 2; - optional TDbCounters RequestCounters = 3; -} - -message TDbServiceCounters { - optional TDbCounters Main = 1; - repeated TDbTabletCounters TabletCounters = 2; - repeated TDbGRpcCounters GRpcCounters = 3; -} - -enum EDbCountersService { - KQP = 1; - TABLETS = 2; - GRPC = 3; - RESERVED_1 = 4; - RESERVED_2 = 5; -} - -// node -> sysview processor tablet -// cumulative values are diffs since last confirmed state -message TEvSendDbCountersRequest { - message TServiceCounters { - optional EDbCountersService Service = 1; - optional TDbServiceCounters Counters = 2; - } - repeated TServiceCounters ServiceCounters = 1; - optional uint64 NodeId = 2; - optional uint64 Generation = 3; -} - -message TEvSendDbCountersResponse { - optional string Database = 1; - optional uint64 Generation = 2; // confirmed generation -} - + optional string Type = 3; + optional string State = 4; + optional string VolatileState = 5; + optional string BootState = 6; + optional uint32 Generation = 7; + optional uint32 NodeId = 8; + optional uint64 CPU = 9; + optional uint64 Memory = 10; + optional uint64 Network = 11; +} + +message TEvGetTabletIdsRequest { + optional fixed64 From = 1; + optional fixed64 To = 2; +} + +message TEvGetTabletIdsResponse { + repeated fixed64 TabletIds = 1; +} + +message TEvGetTabletsRequest { + repeated fixed64 TabletIds = 1; + optional uint32 BatchSizeLimit = 2; +} + +message TEvGetTabletsResponse { + repeated TTabletEntry Entries = 1; + optional fixed64 NextTabletId = 2; +} + +// ---- Query metrics + +message TQueryMetrics { + message TMetrics { + optional uint64 Sum = 1; + optional uint64 Min = 2; + optional uint64 Max = 3; + } + + optional fixed64 QueryTextHash = 1; + optional uint64 Count = 2; + + optional TMetrics CpuTimeUs = 3; + optional TMetrics DurationUs = 4; + optional TMetrics ReadRows = 5; + optional TMetrics ReadBytes = 6; + optional TMetrics UpdateRows = 7; + optional TMetrics UpdateBytes = 8; + optional TMetrics DeleteRows = 9; + optional TMetrics RequestUnits = 10; +} + +// node -> sysview processor tablet +// per node {hash -> cpu} mapping +message TEvIntervalQuerySummary { + optional string Database = 1; + optional uint64 IntervalEndUs = 2; + optional uint32 NodeId = 3; + + message TQuerySet { + repeated fixed64 Hashes = 1; // hashes + repeated uint64 Values = 2; // top by values + } + + optional TQuerySet Metrics = 4; + optional TQuerySet TopByDuration = 5; + optional TQuerySet TopByReadBytes = 6; + optional TQuerySet TopByCpuTime = 7; + optional TQuerySet TopByRequestUnits = 8; +} + +// sysview processor tablet -> node +// only selected queries with top cpu +message TEvGetIntervalMetricsRequest { + optional string Database = 1; + optional uint64 IntervalEndUs = 2; + repeated fixed64 Metrics = 3; // request full query metrics + repeated fixed64 QueryTextsToGet = 4; // request query texts + repeated fixed64 TopByDuration = 5; // request interval top + repeated fixed64 TopByReadBytes = 6; // request interval top + repeated fixed64 TopByCpuTime = 7; // request interval top + repeated fixed64 TopByRequestUnits = 8; // request interval top +} + +// node -> sysview processor tablet +// query metrics and texts +message TEvGetIntervalMetricsResponse { + message TQueryText { + optional fixed64 Hash = 1; + optional string Text = 2; + } + + optional uint64 IntervalEndUs = 1; + repeated TQueryMetrics Metrics = 2; + repeated TQueryText QueryTexts = 3; + repeated TQueryStats TopByDuration = 4; + repeated TQueryStats TopByReadBytes = 5; + repeated TQueryStats TopByCpuTime = 6; + repeated TQueryStats TopByRequestUnits = 7; +} + +message TEvConfigureProcessor { + optional string Database = 1; +} + +message TQueryMetricsKey { + optional uint64 IntervalEndUs = 1; + optional uint32 Rank = 2; +} + +message TQueryMetricsEntry { + optional TQueryMetricsKey Key = 1; + optional TQueryMetrics Metrics = 2; + optional string QueryText = 3; +} + +message TQueryStatsEntry { + optional TQueryMetricsKey Key = 1; + optional TQueryStats Stats = 2; +} + +message TEvGetQueryMetricsRequest { + optional TQueryMetricsKey From = 1; + optional bool InclusiveFrom = 2; + + optional TQueryMetricsKey To = 3; + optional bool InclusiveTo = 4; + + optional EStatsType Type = 5; +} + +// reply to TEvGetQueryMetricsRequest +message TEvGetQueryMetricsResponse { + repeated TQueryMetricsEntry Entries = 1; + optional TQueryMetricsKey Next = 2; + optional bool LastBatch = 3; + optional bool Overloaded = 4; +} + +// reply to TEvGetQueryMetricsRequest +message TEvGetQueryStatsResponse { + repeated TQueryStatsEntry Entries = 1; + optional TQueryMetricsKey Next = 2; + optional bool LastBatch = 3; + optional bool Overloaded = 4; +} + +// ---- Database counters + +// generic set of counters with stable enumeration order in each category +// simple values are absolute +// cumulative and histogram values may be absolute or diffs depending on the context +// only index-value pairs with non-zero values are stored in diff mode +message TDbCounters { + message THistogram { + repeated uint64 Buckets = 1; + optional uint64 BucketsCount = 2; // total number of buckets in diff mode + } + + repeated uint64 Simple = 1; + repeated uint64 Cumulative = 2; + optional uint64 CumulativeCount = 3; // total number of cumulative counters in diff mode + repeated THistogram Histogram = 4; +} + +message TDbTabletCounters { + optional NKikimrTabletBase.TTabletTypes.EType Type = 1; + optional TDbCounters ExecutorCounters = 2; + optional TDbCounters AppCounters = 3; + optional TDbCounters MaxExecutorCounters = 4; + optional TDbCounters MaxAppCounters = 5; +} + +message TDbGRpcCounters { + optional string GRpcService = 1; + optional string GRpcRequest = 2; + optional TDbCounters RequestCounters = 3; +} + +message TDbServiceCounters { + optional TDbCounters Main = 1; + repeated TDbTabletCounters TabletCounters = 2; + repeated TDbGRpcCounters GRpcCounters = 3; +} + +enum EDbCountersService { + KQP = 1; + TABLETS = 2; + GRPC = 3; + RESERVED_1 = 4; + RESERVED_2 = 5; +} + +// node -> sysview processor tablet +// cumulative values are diffs since last confirmed state +message TEvSendDbCountersRequest { + message TServiceCounters { + optional EDbCountersService Service = 1; + optional TDbServiceCounters Counters = 2; + } + repeated TServiceCounters ServiceCounters = 1; + optional uint64 NodeId = 2; + optional uint64 Generation = 3; +} + +message TEvSendDbCountersResponse { + optional string Database = 1; + optional uint64 Generation = 2; // confirmed generation +} + diff --git a/ydb/core/protos/tablet.proto b/ydb/core/protos/tablet.proto index cc849726f1..f0bec238e7 100644 --- a/ydb/core/protos/tablet.proto +++ b/ydb/core/protos/tablet.proto @@ -39,7 +39,7 @@ message TTabletTypes { Kesus = 29; BlockStorePartition2 = 30; BlockStoreDiskRegistry = 31; - SysViewProcessor = 32; + SysViewProcessor = 32; FileStore = 33; OlapShard = 34; ColumnShard = 35; diff --git a/ydb/core/protos/tablet_database.proto b/ydb/core/protos/tablet_database.proto index afa5d6899b..a6658688a6 100644 --- a/ydb/core/protos/tablet_database.proto +++ b/ydb/core/protos/tablet_database.proto @@ -43,7 +43,7 @@ message TExecutorSettings { optional TCompactionPolicy CompactionPolicy = 10; repeated TLocalityGroupPolicy LocalityGroupPolicy = 11; - reserved 100; //optional bool ExposeCounters = 100; + reserved 100; //optional bool ExposeCounters = 100; } message TRuntimeSettings { diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto index 11379e3fce..89863b2540 100644 --- a/ydb/core/protos/tx_datashard.proto +++ b/ydb/core/protos/tx_datashard.proto @@ -749,9 +749,9 @@ message TEvPeriodicTableStats { optional uint32 ShardState = 7; repeated uint64 UserTablePartOwners = 8; repeated uint64 SysTablesPartOwners = 9; - - optional uint32 NodeId = 10; - optional uint64 StartTime = 11; // milliseconds since epoch + + optional uint32 NodeId = 10; + optional uint64 StartTime = 11; // milliseconds since epoch optional uint64 TableOwnerId = 12; } diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 09c5c38df3..70bb65514c 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -49,7 +49,7 @@ SRCS( counters_replication.proto counters_schemeshard.proto counters_sequenceshard.proto - counters_sysview_processor.proto + counters_sysview_processor.proto counters_testshard.proto counters_tx_proxy.proto counters_mediator.proto @@ -128,7 +128,7 @@ SRCS( ydb_table_impl.proto scheme_board.proto scheme_board_mon.proto - sys_view.proto + sys_view.proto ) GENERATE_ENUM_SERIALIZATION(blobstorage_pdisk_config.pb.h) diff --git a/ydb/core/scheme/scheme_tabledefs.h b/ydb/core/scheme/scheme_tabledefs.h index 5ce1f7a1cf..190233fdee 100644 --- a/ydb/core/scheme/scheme_tabledefs.h +++ b/ydb/core/scheme/scheme_tabledefs.h @@ -19,18 +19,18 @@ using TSchemaVersion = ui64; // ident for table, must be unique in selected scope // for global transactions ownerid is tabletid of owning schemeshard and tableid is counter designated by schemeshard -// SysViewInfo is not empty for system views attached to corresponding table - +// SysViewInfo is not empty for system views attached to corresponding table + struct TTableId { - TPathId PathId; - TString SysViewInfo; + TPathId PathId; + TString SysViewInfo; TSchemaVersion SchemaVersion = 0; - TTableId() = default; - + TTableId() = default; + // raw ctors TTableId(ui64 ownerId, ui64 tableId, const TString& sysViewInfo, ui64 schemaVersion) - : PathId(ownerId, tableId) + : PathId(ownerId, tableId) , SysViewInfo(sysViewInfo) , SchemaVersion(schemaVersion) {} @@ -65,17 +65,17 @@ struct TTableId { {} explicit operator bool() const noexcept { - return bool(PathId); + return bool(PathId); } bool HasSamePath(const TTableId &x) const noexcept { - return PathId == x.PathId && SysViewInfo == x.SysViewInfo; + return PathId == x.PathId && SysViewInfo == x.SysViewInfo; + } + + bool IsSystemView() const noexcept { + return !SysViewInfo.empty(); } - bool IsSystemView() const noexcept { - return !SysViewInfo.empty(); - } - bool operator==(const TTableId& x) const { return PathId == x.PathId && SysViewInfo == x.SysViewInfo && SchemaVersion == x.SchemaVersion; } @@ -97,12 +97,12 @@ struct TTableId { } ui64 PathHash() const noexcept { - auto hash = PathId.Hash(); + auto hash = PathId.Hash(); if (SysViewInfo) { - hash = CombineHashes(hash, THash<TString>()(SysViewInfo)); + hash = CombineHashes(hash, THash<TString>()(SysViewInfo)); } - return hash; + return hash; } }; @@ -580,7 +580,7 @@ struct TSecurityObject : TAtomicRefCount<TSecurityObject>, NACLib::TSecurityObje }; // key description of one minikql operation -class TKeyDesc : TNonCopyable { +class TKeyDesc : TNonCopyable { public: // what we do on row? diff --git a/ydb/core/sys_view/common/common.h b/ydb/core/sys_view/common/common.h index 2785e6b446..c88107063a 100644 --- a/ydb/core/sys_view/common/common.h +++ b/ydb/core/sys_view/common/common.h @@ -1,32 +1,32 @@ -#pragma once - -#include <library/cpp/actors/core/log.h> - -#if defined SVLOG_T || \ - defined SVLOG_D || \ - defined SVLOG_I || \ - defined SVLOG_N || \ - defined SVLOG_W || \ - defined SVLOG_E || \ - defined SVLOG_CRIT -#error log macro redefinition -#endif - -#define SVLOG_T(stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) -#define SVLOG_D(stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) -#define SVLOG_I(stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) -#define SVLOG_N(stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) -#define SVLOG_W(stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) -#define SVLOG_E(stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) -#define SVLOG_CRIT(stream) LOG_CRIT_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) - -namespace NKikimr { -namespace NSysView { - -enum class EProcessorMode { - MINUTE, // aggregate stats every minute - FAST // fast mode for tests -}; - -} // NSysView -} // NKikimr +#pragma once + +#include <library/cpp/actors/core/log.h> + +#if defined SVLOG_T || \ + defined SVLOG_D || \ + defined SVLOG_I || \ + defined SVLOG_N || \ + defined SVLOG_W || \ + defined SVLOG_E || \ + defined SVLOG_CRIT +#error log macro redefinition +#endif + +#define SVLOG_T(stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) +#define SVLOG_D(stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) +#define SVLOG_I(stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) +#define SVLOG_N(stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) +#define SVLOG_W(stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) +#define SVLOG_E(stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) +#define SVLOG_CRIT(stream) LOG_CRIT_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream) + +namespace NKikimr { +namespace NSysView { + +enum class EProcessorMode { + MINUTE, // aggregate stats every minute + FAST // fast mode for tests +}; + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/db_counters.h b/ydb/core/sys_view/common/db_counters.h index 3e00068a04..0a1850e78a 100644 --- a/ydb/core/sys_view/common/db_counters.h +++ b/ydb/core/sys_view/common/db_counters.h @@ -1,79 +1,79 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/sys_view.pb.h> - -#include <util/generic/hash.h> - -namespace NKikimr { -namespace NSysView { - -class TDbServiceCounters { - NKikimrSysView::TDbServiceCounters ProtoCounters; - - THashMap<NKikimrTabletBase::TTabletTypes::EType, NKikimrSysView::TDbTabletCounters*> ByTabletType; - - using TGRpcRequestDesc = std::pair<TString, TString>; - THashMap<TGRpcRequestDesc, NKikimrSysView::TDbGRpcCounters*> ByGRpcRequest; - -public: - void Swap(TDbServiceCounters& other) { - ProtoCounters.Swap(&other.ProtoCounters); - ByTabletType.swap(other.ByTabletType); - ByGRpcRequest.swap(other.ByGRpcRequest); - } - - NKikimrSysView::TDbServiceCounters& Proto() { return ProtoCounters; } - const NKikimrSysView::TDbServiceCounters& Proto() const { return ProtoCounters; } - - NKikimrSysView::TDbTabletCounters* FindTabletCounters( - NKikimrTabletBase::TTabletTypes::EType tabletType) const - { - if (auto it = ByTabletType.find(tabletType); it != ByTabletType.end()) { - return it->second; - } - return {}; - } - - NKikimrSysView::TDbTabletCounters* FindOrAddTabletCounters( - NKikimrTabletBase::TTabletTypes::EType tabletType) - { - if (auto it = ByTabletType.find(tabletType); it != ByTabletType.end()) { - return it->second; - } - - auto* counters = ProtoCounters.AddTabletCounters(); - counters->SetType(tabletType); - ByTabletType[tabletType] = counters; - - return counters; - } - - NKikimrSysView::TDbGRpcCounters* FindGRpcCounters( - const TString& grpcService, const TString& grpcRequest) const - { - auto key = std::make_pair(grpcService, grpcRequest); - if (auto it = ByGRpcRequest.find(key); it != ByGRpcRequest.end()) { - return it->second; - } - return {}; - } - - NKikimrSysView::TDbGRpcCounters* FindOrAddGRpcCounters( - const TString& grpcService, const TString& grpcRequest) - { - auto key = std::make_pair(grpcService, grpcRequest); - if (auto it = ByGRpcRequest.find(key); it != ByGRpcRequest.end()) { - return it->second; - } - - auto* counters = ProtoCounters.AddGRpcCounters(); - counters->SetGRpcService(grpcService); - counters->SetGRpcRequest(grpcRequest); - ByGRpcRequest[key] = counters; - - return counters; - } -}; - -} // NSysView -} // NKikimr + +#include <util/generic/hash.h> + +namespace NKikimr { +namespace NSysView { + +class TDbServiceCounters { + NKikimrSysView::TDbServiceCounters ProtoCounters; + + THashMap<NKikimrTabletBase::TTabletTypes::EType, NKikimrSysView::TDbTabletCounters*> ByTabletType; + + using TGRpcRequestDesc = std::pair<TString, TString>; + THashMap<TGRpcRequestDesc, NKikimrSysView::TDbGRpcCounters*> ByGRpcRequest; + +public: + void Swap(TDbServiceCounters& other) { + ProtoCounters.Swap(&other.ProtoCounters); + ByTabletType.swap(other.ByTabletType); + ByGRpcRequest.swap(other.ByGRpcRequest); + } + + NKikimrSysView::TDbServiceCounters& Proto() { return ProtoCounters; } + const NKikimrSysView::TDbServiceCounters& Proto() const { return ProtoCounters; } + + NKikimrSysView::TDbTabletCounters* FindTabletCounters( + NKikimrTabletBase::TTabletTypes::EType tabletType) const + { + if (auto it = ByTabletType.find(tabletType); it != ByTabletType.end()) { + return it->second; + } + return {}; + } + + NKikimrSysView::TDbTabletCounters* FindOrAddTabletCounters( + NKikimrTabletBase::TTabletTypes::EType tabletType) + { + if (auto it = ByTabletType.find(tabletType); it != ByTabletType.end()) { + return it->second; + } + + auto* counters = ProtoCounters.AddTabletCounters(); + counters->SetType(tabletType); + ByTabletType[tabletType] = counters; + + return counters; + } + + NKikimrSysView::TDbGRpcCounters* FindGRpcCounters( + const TString& grpcService, const TString& grpcRequest) const + { + auto key = std::make_pair(grpcService, grpcRequest); + if (auto it = ByGRpcRequest.find(key); it != ByGRpcRequest.end()) { + return it->second; + } + return {}; + } + + NKikimrSysView::TDbGRpcCounters* FindOrAddGRpcCounters( + const TString& grpcService, const TString& grpcRequest) + { + auto key = std::make_pair(grpcService, grpcRequest); + if (auto it = ByGRpcRequest.find(key); it != ByGRpcRequest.end()) { + return it->second; + } + + auto* counters = ProtoCounters.AddGRpcCounters(); + counters->SetGRpcService(grpcService); + counters->SetGRpcRequest(grpcRequest); + ByGRpcRequest[key] = counters; + + return counters; + } +}; + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/events.h b/ydb/core/sys_view/common/events.h index b1b481451d..6e19aadc2b 100644 --- a/ydb/core/sys_view/common/events.h +++ b/ydb/core/sys_view/common/events.h @@ -1,341 +1,341 @@ -#pragma once - -#include "utils.h" -#include "db_counters.h" - +#pragma once + +#include "utils.h" +#include "db_counters.h" + #include <ydb/core/base/events.h> #include <ydb/core/base/pathid.h> #include <library/cpp/actors/core/events.h> #include <ydb/core/protos/sys_view.pb.h> - -namespace NKikimr { -namespace NSysView { - -using TShardIdx = std::pair<ui64, ui64>; - -class IDbCounters : public TThrRefBase { -public: - virtual void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) = 0; - virtual void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) = 0; -}; - -struct TEvSysView { - enum EEv { - EvSendPartitionStats = EventSpaceBegin(TKikimrEvents::ES_SYSTEM_VIEW), - EvSetPartitioning, - EvRemoveTable, - EvGetPartitionStats, - EvGetPartitionStatsResult, - - EvCollectQueryStats, - EvGetQueryStats, - EvGetQueryStatsResult, - - EvGetScanLimiter, - EvGetScanLimiterResult, - - EvGetPDisksRequest, - EvGetPDisksResponse, - EvGetVSlotsRequest, - EvGetVSlotsResponse, - EvGetGroupsRequest, - EvGetGroupsResponse, - EvGetStoragePoolsRequest, - EvGetStoragePoolsResponse, - - EvGetTabletIdsRequest, - EvGetTabletIdsResponse, - EvGetTabletsRequest, - EvGetTabletsResponse, - - EvConfigureProcessor, - EvIntervalQuerySummary, - EvGetIntervalMetricsRequest, - EvGetIntervalMetricsResponse, - EvGetQueryMetricsRequest, - EvGetQueryMetricsResponse, - EvGetQueryStatsResponse, - - EvRegisterDbCounters, - EvSendDbCountersRequest, - EvSendDbCountersResponse, - EvWatchDatabase, - + +namespace NKikimr { +namespace NSysView { + +using TShardIdx = std::pair<ui64, ui64>; + +class IDbCounters : public TThrRefBase { +public: + virtual void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) = 0; + virtual void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) = 0; +}; + +struct TEvSysView { + enum EEv { + EvSendPartitionStats = EventSpaceBegin(TKikimrEvents::ES_SYSTEM_VIEW), + EvSetPartitioning, + EvRemoveTable, + EvGetPartitionStats, + EvGetPartitionStatsResult, + + EvCollectQueryStats, + EvGetQueryStats, + EvGetQueryStatsResult, + + EvGetScanLimiter, + EvGetScanLimiterResult, + + EvGetPDisksRequest, + EvGetPDisksResponse, + EvGetVSlotsRequest, + EvGetVSlotsResponse, + EvGetGroupsRequest, + EvGetGroupsResponse, + EvGetStoragePoolsRequest, + EvGetStoragePoolsResponse, + + EvGetTabletIdsRequest, + EvGetTabletIdsResponse, + EvGetTabletsRequest, + EvGetTabletsResponse, + + EvConfigureProcessor, + EvIntervalQuerySummary, + EvGetIntervalMetricsRequest, + EvGetIntervalMetricsResponse, + EvGetQueryMetricsRequest, + EvGetQueryMetricsResponse, + EvGetQueryStatsResponse, + + EvRegisterDbCounters, + EvSendDbCountersRequest, + EvSendDbCountersResponse, + EvWatchDatabase, + EvUpdateTtlStats, EvGetStorageStatsRequest, EvGetStorageStatsResponse, - EvEnd, - }; - - struct TEvSendPartitionStats : public TEventLocal< - TEvSendPartitionStats, - EvSendPartitionStats> - { - TPathId DomainKey; - TPathId PathId; - TShardIdx ShardIdx; - - NKikimrSysView::TPartitionStats Stats; - - TEvSendPartitionStats(TPathId domainKey, TPathId pathId, TShardIdx shardIdx) - : DomainKey(domainKey) - , PathId(pathId) - , ShardIdx(shardIdx) - {} - }; - - struct TEvSetPartitioning : public TEventLocal< - TEvSetPartitioning, - EvSetPartitioning> - { - TPathId DomainKey; - TPathId PathId; - TString Path; - TVector<TShardIdx> ShardIndices; - - TEvSetPartitioning(TPathId domainKey, TPathId pathId, const TStringBuf path) - : DomainKey(domainKey) - , PathId(pathId) - , Path(path) - {} - }; - - struct TEvRemoveTable : public TEventLocal< - TEvRemoveTable, - EvRemoveTable> - { - TPathId DomainKey; - TPathId PathId; - - TEvRemoveTable(TPathId domainKey, TPathId pathId) - : DomainKey(domainKey) - , PathId(pathId) - {} - }; - - struct TEvGetPartitionStats : public TEventPB< - TEvGetPartitionStats, - NKikimrSysView::TEvGetPartitionStats, - EvGetPartitionStats> - {}; - - struct TEvGetPartitionStatsResult : public TEventPB< - TEvGetPartitionStatsResult, - NKikimrSysView::TEvGetPartitionStatsResult, - EvGetPartitionStatsResult> - {}; - - // KQP -> SysViewService - struct TEvCollectQueryStats : public TEventLocal< - TEvCollectQueryStats, - EvCollectQueryStats> - { - NKikimrSysView::TQueryStats QueryStats; - TString Database; - }; - - // top queries scan -> SysViewService - struct TEvGetQueryStats : public TEventPB< - TEvGetQueryStats, - NKikimrSysView::TEvGetQueryStats, - EvGetQueryStats> - {}; - - // SysViewService -> top queries scan - struct TEvGetQueryStatsResult : public TEventPB< - TEvGetQueryStatsResult, - NKikimrSysView::TEvGetQueryStatsResult, - EvGetQueryStatsResult> - {}; - - struct TEvGetScanLimiter : public TEventLocal< - TEvGetScanLimiter, - EvGetScanLimiter> - {}; - - struct TEvGetScanLimiterResult : public TEventLocal< - TEvGetScanLimiterResult, - EvGetScanLimiterResult> - { - TIntrusivePtr<TScanLimiter> ScanLimiter; - }; - - struct TEvGetPDisksRequest : public TEventPB< - TEvGetPDisksRequest, - NKikimrSysView::TEvGetPDisksRequest, - EvGetPDisksRequest> - {}; - - struct TEvGetPDisksResponse : public TEventPB< - TEvGetPDisksResponse, - NKikimrSysView::TEvGetPDisksResponse, - EvGetPDisksResponse> - {}; - - struct TEvGetVSlotsRequest : public TEventPB< - TEvGetVSlotsRequest, - NKikimrSysView::TEvGetVSlotsRequest, - EvGetVSlotsRequest> - {}; - - struct TEvGetVSlotsResponse : public TEventPB< - TEvGetVSlotsResponse, - NKikimrSysView::TEvGetVSlotsResponse, - EvGetVSlotsResponse> - {}; - - struct TEvGetGroupsRequest : public TEventPB< - TEvGetGroupsRequest, - NKikimrSysView::TEvGetGroupsRequest, - EvGetGroupsRequest> - {}; - - struct TEvGetGroupsResponse : public TEventPB< - TEvGetGroupsResponse, - NKikimrSysView::TEvGetGroupsResponse, - EvGetGroupsResponse> - {}; - - struct TEvGetStoragePoolsRequest : public TEventPB< - TEvGetStoragePoolsRequest, - NKikimrSysView::TEvGetStoragePoolsRequest, - EvGetStoragePoolsRequest> - {}; - - struct TEvGetStoragePoolsResponse : public TEventPB< - TEvGetStoragePoolsResponse, - NKikimrSysView::TEvGetStoragePoolsResponse, - EvGetStoragePoolsResponse> - {}; - - struct TEvGetTabletIdsRequest : public TEventPB< - TEvGetTabletIdsRequest, - NKikimrSysView::TEvGetTabletIdsRequest, - EvGetTabletIdsRequest> - {}; - - struct TEvGetTabletIdsResponse : public TEventPB< - TEvGetTabletIdsResponse, - NKikimrSysView::TEvGetTabletIdsResponse, - EvGetTabletIdsResponse> - {}; - - struct TEvGetTabletsRequest : public TEventPB< - TEvGetTabletsRequest, - NKikimrSysView::TEvGetTabletsRequest, - EvGetTabletsRequest> - {}; - - struct TEvGetTabletsResponse : public TEventPB< - TEvGetTabletsResponse, - NKikimrSysView::TEvGetTabletsResponse, - EvGetTabletsResponse> - {}; - - struct TEvConfigureProcessor : public TEventPB< - TEvConfigureProcessor, - NKikimrSysView::TEvConfigureProcessor, - EvConfigureProcessor> - { - TEvConfigureProcessor() = default; - explicit TEvConfigureProcessor(const TString& database) { - Record.SetDatabase(database); - } - }; - - struct TEvIntervalQuerySummary : public TEventPB< - TEvIntervalQuerySummary, - NKikimrSysView::TEvIntervalQuerySummary, - EvIntervalQuerySummary> - {}; - - struct TEvGetIntervalMetricsRequest : public TEventPB< - TEvGetIntervalMetricsRequest, - NKikimrSysView::TEvGetIntervalMetricsRequest, - EvGetIntervalMetricsRequest> - {}; - - struct TEvGetIntervalMetricsResponse : public TEventPB< - TEvGetIntervalMetricsResponse, - NKikimrSysView::TEvGetIntervalMetricsResponse, - EvGetIntervalMetricsResponse> - {}; - - struct TEvGetQueryMetricsRequest : public TEventPB< - TEvGetQueryMetricsRequest, - NKikimrSysView::TEvGetQueryMetricsRequest, - EvGetQueryMetricsRequest> - {}; - - struct TEvGetQueryMetricsResponse : public TEventPB< - TEvGetQueryMetricsResponse, - NKikimrSysView::TEvGetQueryMetricsResponse, - EvGetQueryMetricsResponse> - {}; - - struct TEvGetQueryStatsResponse : public TEventPB< - TEvGetQueryStatsResponse, - NKikimrSysView::TEvGetQueryStatsResponse, - EvGetQueryStatsResponse> - {}; - - struct TEvRegisterDbCounters : public TEventLocal< - TEvRegisterDbCounters, - EvRegisterDbCounters> - { - NKikimrSysView::EDbCountersService Service; - TString Database; - TPathId PathId; // database path id, for tablet counters - TIntrusivePtr<IDbCounters> Counters; - - TEvRegisterDbCounters( - NKikimrSysView::EDbCountersService service, - const TString& database, - TIntrusivePtr<IDbCounters> counters) - : Service(service) - , Database(database) - , Counters(counters) - {} - - TEvRegisterDbCounters( - NKikimrSysView::EDbCountersService service, - TPathId pathId, - TIntrusivePtr<IDbCounters> counters) - : Service(service) - , PathId(pathId) - , Counters(counters) - {} - }; - - struct TEvSendDbCountersRequest : public TEventPB< - TEvSendDbCountersRequest, - NKikimrSysView::TEvSendDbCountersRequest, - EvSendDbCountersRequest> - {}; - - struct TEvSendDbCountersResponse : public TEventPB< - TEvSendDbCountersResponse, - NKikimrSysView::TEvSendDbCountersResponse, - EvSendDbCountersResponse> - {}; - - struct TEvWatchDatabase : public TEventLocal< - TEvWatchDatabase, - EvWatchDatabase> - { - TString Database; - TPathId PathId; - - explicit TEvWatchDatabase(const TString& database) - : Database(database) - {} - - explicit TEvWatchDatabase(TPathId pathId) - : PathId(pathId) - {} - }; + EvEnd, + }; + + struct TEvSendPartitionStats : public TEventLocal< + TEvSendPartitionStats, + EvSendPartitionStats> + { + TPathId DomainKey; + TPathId PathId; + TShardIdx ShardIdx; + + NKikimrSysView::TPartitionStats Stats; + + TEvSendPartitionStats(TPathId domainKey, TPathId pathId, TShardIdx shardIdx) + : DomainKey(domainKey) + , PathId(pathId) + , ShardIdx(shardIdx) + {} + }; + + struct TEvSetPartitioning : public TEventLocal< + TEvSetPartitioning, + EvSetPartitioning> + { + TPathId DomainKey; + TPathId PathId; + TString Path; + TVector<TShardIdx> ShardIndices; + + TEvSetPartitioning(TPathId domainKey, TPathId pathId, const TStringBuf path) + : DomainKey(domainKey) + , PathId(pathId) + , Path(path) + {} + }; + + struct TEvRemoveTable : public TEventLocal< + TEvRemoveTable, + EvRemoveTable> + { + TPathId DomainKey; + TPathId PathId; + + TEvRemoveTable(TPathId domainKey, TPathId pathId) + : DomainKey(domainKey) + , PathId(pathId) + {} + }; + + struct TEvGetPartitionStats : public TEventPB< + TEvGetPartitionStats, + NKikimrSysView::TEvGetPartitionStats, + EvGetPartitionStats> + {}; + + struct TEvGetPartitionStatsResult : public TEventPB< + TEvGetPartitionStatsResult, + NKikimrSysView::TEvGetPartitionStatsResult, + EvGetPartitionStatsResult> + {}; + + // KQP -> SysViewService + struct TEvCollectQueryStats : public TEventLocal< + TEvCollectQueryStats, + EvCollectQueryStats> + { + NKikimrSysView::TQueryStats QueryStats; + TString Database; + }; + + // top queries scan -> SysViewService + struct TEvGetQueryStats : public TEventPB< + TEvGetQueryStats, + NKikimrSysView::TEvGetQueryStats, + EvGetQueryStats> + {}; + + // SysViewService -> top queries scan + struct TEvGetQueryStatsResult : public TEventPB< + TEvGetQueryStatsResult, + NKikimrSysView::TEvGetQueryStatsResult, + EvGetQueryStatsResult> + {}; + + struct TEvGetScanLimiter : public TEventLocal< + TEvGetScanLimiter, + EvGetScanLimiter> + {}; + + struct TEvGetScanLimiterResult : public TEventLocal< + TEvGetScanLimiterResult, + EvGetScanLimiterResult> + { + TIntrusivePtr<TScanLimiter> ScanLimiter; + }; + + struct TEvGetPDisksRequest : public TEventPB< + TEvGetPDisksRequest, + NKikimrSysView::TEvGetPDisksRequest, + EvGetPDisksRequest> + {}; + + struct TEvGetPDisksResponse : public TEventPB< + TEvGetPDisksResponse, + NKikimrSysView::TEvGetPDisksResponse, + EvGetPDisksResponse> + {}; + + struct TEvGetVSlotsRequest : public TEventPB< + TEvGetVSlotsRequest, + NKikimrSysView::TEvGetVSlotsRequest, + EvGetVSlotsRequest> + {}; + + struct TEvGetVSlotsResponse : public TEventPB< + TEvGetVSlotsResponse, + NKikimrSysView::TEvGetVSlotsResponse, + EvGetVSlotsResponse> + {}; + + struct TEvGetGroupsRequest : public TEventPB< + TEvGetGroupsRequest, + NKikimrSysView::TEvGetGroupsRequest, + EvGetGroupsRequest> + {}; + + struct TEvGetGroupsResponse : public TEventPB< + TEvGetGroupsResponse, + NKikimrSysView::TEvGetGroupsResponse, + EvGetGroupsResponse> + {}; + + struct TEvGetStoragePoolsRequest : public TEventPB< + TEvGetStoragePoolsRequest, + NKikimrSysView::TEvGetStoragePoolsRequest, + EvGetStoragePoolsRequest> + {}; + + struct TEvGetStoragePoolsResponse : public TEventPB< + TEvGetStoragePoolsResponse, + NKikimrSysView::TEvGetStoragePoolsResponse, + EvGetStoragePoolsResponse> + {}; + + struct TEvGetTabletIdsRequest : public TEventPB< + TEvGetTabletIdsRequest, + NKikimrSysView::TEvGetTabletIdsRequest, + EvGetTabletIdsRequest> + {}; + + struct TEvGetTabletIdsResponse : public TEventPB< + TEvGetTabletIdsResponse, + NKikimrSysView::TEvGetTabletIdsResponse, + EvGetTabletIdsResponse> + {}; + + struct TEvGetTabletsRequest : public TEventPB< + TEvGetTabletsRequest, + NKikimrSysView::TEvGetTabletsRequest, + EvGetTabletsRequest> + {}; + + struct TEvGetTabletsResponse : public TEventPB< + TEvGetTabletsResponse, + NKikimrSysView::TEvGetTabletsResponse, + EvGetTabletsResponse> + {}; + + struct TEvConfigureProcessor : public TEventPB< + TEvConfigureProcessor, + NKikimrSysView::TEvConfigureProcessor, + EvConfigureProcessor> + { + TEvConfigureProcessor() = default; + explicit TEvConfigureProcessor(const TString& database) { + Record.SetDatabase(database); + } + }; + + struct TEvIntervalQuerySummary : public TEventPB< + TEvIntervalQuerySummary, + NKikimrSysView::TEvIntervalQuerySummary, + EvIntervalQuerySummary> + {}; + + struct TEvGetIntervalMetricsRequest : public TEventPB< + TEvGetIntervalMetricsRequest, + NKikimrSysView::TEvGetIntervalMetricsRequest, + EvGetIntervalMetricsRequest> + {}; + + struct TEvGetIntervalMetricsResponse : public TEventPB< + TEvGetIntervalMetricsResponse, + NKikimrSysView::TEvGetIntervalMetricsResponse, + EvGetIntervalMetricsResponse> + {}; + + struct TEvGetQueryMetricsRequest : public TEventPB< + TEvGetQueryMetricsRequest, + NKikimrSysView::TEvGetQueryMetricsRequest, + EvGetQueryMetricsRequest> + {}; + + struct TEvGetQueryMetricsResponse : public TEventPB< + TEvGetQueryMetricsResponse, + NKikimrSysView::TEvGetQueryMetricsResponse, + EvGetQueryMetricsResponse> + {}; + + struct TEvGetQueryStatsResponse : public TEventPB< + TEvGetQueryStatsResponse, + NKikimrSysView::TEvGetQueryStatsResponse, + EvGetQueryStatsResponse> + {}; + + struct TEvRegisterDbCounters : public TEventLocal< + TEvRegisterDbCounters, + EvRegisterDbCounters> + { + NKikimrSysView::EDbCountersService Service; + TString Database; + TPathId PathId; // database path id, for tablet counters + TIntrusivePtr<IDbCounters> Counters; + + TEvRegisterDbCounters( + NKikimrSysView::EDbCountersService service, + const TString& database, + TIntrusivePtr<IDbCounters> counters) + : Service(service) + , Database(database) + , Counters(counters) + {} + + TEvRegisterDbCounters( + NKikimrSysView::EDbCountersService service, + TPathId pathId, + TIntrusivePtr<IDbCounters> counters) + : Service(service) + , PathId(pathId) + , Counters(counters) + {} + }; + + struct TEvSendDbCountersRequest : public TEventPB< + TEvSendDbCountersRequest, + NKikimrSysView::TEvSendDbCountersRequest, + EvSendDbCountersRequest> + {}; + + struct TEvSendDbCountersResponse : public TEventPB< + TEvSendDbCountersResponse, + NKikimrSysView::TEvSendDbCountersResponse, + EvSendDbCountersResponse> + {}; + + struct TEvWatchDatabase : public TEventLocal< + TEvWatchDatabase, + EvWatchDatabase> + { + TString Database; + TPathId PathId; + + explicit TEvWatchDatabase(const TString& database) + : Database(database) + {} + + explicit TEvWatchDatabase(TPathId pathId) + : PathId(pathId) + {} + }; struct TEvUpdateTtlStats : public TEventLocal< TEvUpdateTtlStats, @@ -361,7 +361,7 @@ struct TEvSysView { struct TEvGetStorageStatsResponse : TEventPB<TEvGetStorageStatsResponse, NKikimrSysView::TEvGetStorageStatsResponse, EvGetStorageStatsResponse> {}; -}; - -} // NSysView -} // NKikimr +}; + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/keys.h b/ydb/core/sys_view/common/keys.h index 96b420f70d..b200e7abb0 100644 --- a/ydb/core/sys_view/common/keys.h +++ b/ydb/core/sys_view/common/keys.h @@ -1,48 +1,48 @@ -#pragma once - +#pragma once + #include <ydb/core/scheme/scheme_tabledefs.h> - -namespace NKikimr { -namespace NSysView { - -template <size_t N, typename TKey, typename TField> -void SetField(TKey&, TField); - -template <size_t N, typename TKey, typename TField> -void ConvertField(TKey& key, const TConstArrayRef<TCell>& cells) { - if (cells.size() > N) { - if (!cells[N].IsNull()) { - SetField<N>(key, cells[N].AsValue<TField>()); - } else { - SetField<N>(key, Min<TField>()); - } - } else { - SetField<N>(key, Max<TField>()); - } -} - -template <size_t N, typename TKey> -void ConvertKey(TKey&, const TConstArrayRef<TCell>&) { -} - -template <size_t N, typename TKey, typename TField, typename... TFields> -void ConvertKey(TKey& key, const TConstArrayRef<TCell>& cells) { - ConvertField<N, TKey, TField>(key, cells); - ConvertKey<N + 1, TKey, TFields...>(key, cells); -} - -template <typename TRange, typename... TFields> -void ConvertKeyRange(TRange& protoRange, const TSerializedTableRange& tableRange) { - ConvertKey<0, decltype(*protoRange.MutableFrom()), TFields...>( - *protoRange.MutableFrom(), tableRange.From.GetCells()); - - protoRange.SetInclusiveFrom(tableRange.FromInclusive); - - ConvertKey<0, decltype(*protoRange.MutableTo()), TFields...>( - *protoRange.MutableTo(), tableRange.To.GetCells()); - - protoRange.SetInclusiveTo(tableRange.To.GetCells().empty() ? true : tableRange.ToInclusive); -} - -} // NSysView -} // NKikimr + +namespace NKikimr { +namespace NSysView { + +template <size_t N, typename TKey, typename TField> +void SetField(TKey&, TField); + +template <size_t N, typename TKey, typename TField> +void ConvertField(TKey& key, const TConstArrayRef<TCell>& cells) { + if (cells.size() > N) { + if (!cells[N].IsNull()) { + SetField<N>(key, cells[N].AsValue<TField>()); + } else { + SetField<N>(key, Min<TField>()); + } + } else { + SetField<N>(key, Max<TField>()); + } +} + +template <size_t N, typename TKey> +void ConvertKey(TKey&, const TConstArrayRef<TCell>&) { +} + +template <size_t N, typename TKey, typename TField, typename... TFields> +void ConvertKey(TKey& key, const TConstArrayRef<TCell>& cells) { + ConvertField<N, TKey, TField>(key, cells); + ConvertKey<N + 1, TKey, TFields...>(key, cells); +} + +template <typename TRange, typename... TFields> +void ConvertKeyRange(TRange& protoRange, const TSerializedTableRange& tableRange) { + ConvertKey<0, decltype(*protoRange.MutableFrom()), TFields...>( + *protoRange.MutableFrom(), tableRange.From.GetCells()); + + protoRange.SetInclusiveFrom(tableRange.FromInclusive); + + ConvertKey<0, decltype(*protoRange.MutableTo()), TFields...>( + *protoRange.MutableTo(), tableRange.To.GetCells()); + + protoRange.SetInclusiveTo(tableRange.To.GetCells().empty() ? true : tableRange.ToInclusive); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/path.h b/ydb/core/sys_view/common/path.h index d917ca9038..5f17df91a4 100644 --- a/ydb/core/sys_view/common/path.h +++ b/ydb/core/sys_view/common/path.h @@ -1,11 +1,11 @@ -#pragma once - -#include <util/generic/strbuf.h> - -namespace NKikimr { -namespace NSysView { - +#pragma once + +#include <util/generic/strbuf.h> + +namespace NKikimr { +namespace NSysView { + constexpr TStringBuf SysPathName = ".sys"; - -} // NSysView -} // NKikimr + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/scan_actor_base_impl.h b/ydb/core/sys_view/common/scan_actor_base_impl.h index ed0b3f4c12..298a8efbc0 100644 --- a/ydb/core/sys_view/common/scan_actor_base_impl.h +++ b/ydb/core/sys_view/common/scan_actor_base_impl.h @@ -1,72 +1,72 @@ -#pragma once - -#include "utils.h" - +#pragma once + +#include "utils.h" + #include <ydb/core/kqp/kqp_compute.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/mind/tenant_node_enumeration.h> #include <ydb/core/sys_view/service/sysview_service.h> #include <ydb/core/base/appdata.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> - + #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/tx/schemeshard/schemeshard.h> - -namespace NKikimr { -namespace NSysView { - -template <typename TDerived> -class TScanActorBase : public TActorBootstrapped<TDerived> { -public: - using TBase = TActorBootstrapped<TDerived>; - + +namespace NKikimr { +namespace NSysView { + +template <typename TDerived> +class TScanActorBase : public TActorBootstrapped<TDerived> { +public: + using TBase = TActorBootstrapped<TDerived>; + TScanActorBase(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) - : OwnerActorId(ownerId) - , ScanId(scanId) - , TableId(tableId) - , TableRange(tableRange) - , Columns(columns.begin(), columns.end()) - {} - - void Bootstrap(const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::SYSTEM_VIEWS, - "Scan started, actor: " << TBase::SelfId() - << ", owner: " << OwnerActorId - << ", scan id: " << ScanId - << ", table id: " << TableId); - - auto sysViewServiceId = MakeSysViewServiceID(TBase::SelfId().NodeId()); - TBase::Send(sysViewServiceId, new TEvSysView::TEvGetScanLimiter()); - - TBase::Schedule(Timeout, new TEvents::TEvWakeup()); - TBase::Become(&TDerived::StateLimiter); - } - -protected: - void SendBatch(THolder<NKqp::TEvKqpCompute::TEvScanData> batch) { - LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS, - "Sending scan batch, actor: " << TBase::SelfId() - << ", row count: " << batch->Rows.size() - << ", finished: " << batch->Finished); - - bool finished = batch->Finished; - TBase::Send(OwnerActorId, batch.Release()); - if (finished) { - this->PassAway(); - } - } - - void HandleTimeout() { - ReplyErrorAndDie(Ydb::StatusIds::TIMEOUT, "System view: timeout"); - } - + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) + : OwnerActorId(ownerId) + , ScanId(scanId) + , TableId(tableId) + , TableRange(tableRange) + , Columns(columns.begin(), columns.end()) + {} + + void Bootstrap(const TActorContext& ctx) { + LOG_INFO_S(ctx, NKikimrServices::SYSTEM_VIEWS, + "Scan started, actor: " << TBase::SelfId() + << ", owner: " << OwnerActorId + << ", scan id: " << ScanId + << ", table id: " << TableId); + + auto sysViewServiceId = MakeSysViewServiceID(TBase::SelfId().NodeId()); + TBase::Send(sysViewServiceId, new TEvSysView::TEvGetScanLimiter()); + + TBase::Schedule(Timeout, new TEvents::TEvWakeup()); + TBase::Become(&TDerived::StateLimiter); + } + +protected: + void SendBatch(THolder<NKqp::TEvKqpCompute::TEvScanData> batch) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS, + "Sending scan batch, actor: " << TBase::SelfId() + << ", row count: " << batch->Rows.size() + << ", finished: " << batch->Finished); + + bool finished = batch->Finished; + TBase::Send(OwnerActorId, batch.Release()); + if (finished) { + this->PassAway(); + } + } + + void HandleTimeout() { + ReplyErrorAndDie(Ydb::StatusIds::TIMEOUT, "System view: timeout"); + } + void HandleAbortExecution(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev) { LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS, "Got abort execution event, actor: " << TBase::SelfId() @@ -79,273 +79,273 @@ protected: this->PassAway(); } - void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const TString& message) { - LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS, - "Scan error, actor: " << TBase::SelfId() - << ", owner: " << OwnerActorId - << ", scan id: " << ScanId - << ", table id: " << TableId - << ", error: " << message); - - auto error = MakeHolder<NKqp::TEvKqpCompute::TEvScanError>(); + void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const TString& message) { + LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS, + "Scan error, actor: " << TBase::SelfId() + << ", owner: " << OwnerActorId + << ", scan id: " << ScanId + << ", table id: " << TableId + << ", error: " << message); + + auto error = MakeHolder<NKqp::TEvKqpCompute::TEvScanError>(); error->Record.SetStatus(status); IssueToMessage(NYql::TIssue(message), error->Record.MutableIssues()->Add()); - - TBase::Send(OwnerActorId, error.Release()); - - this->PassAway(); - } - - void ReplyEmptyAndDie() { - auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); - batch->Finished = true; - TBase::Send(OwnerActorId, batch.Release()); - - this->PassAway(); - } - - void PassAway() override { - LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS, - "Scan finished, actor: " << TBase::SelfId() - << ", owner: " << OwnerActorId - << ", scan id: " << ScanId - << ", table id: " << TableId); - - if (AllowedByLimiter) { - ScanLimiter->Dec(); - } - - TBase::PassAway(); - } - - ui64 GetBSControllerId() { - auto domainInfo = AppData()->DomainsInfo; - if (domainInfo->Domains.empty()) { - ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Invalid domain info"); - return 0; - } - - auto domain = domainInfo->Domains.begin()->second; - auto defaultSSGroup = domainInfo->GetDefaultStateStorageGroup(domain->DomainUid); - return MakeBSControllerID(defaultSSGroup); - } - - template <typename TResponse, typename TEntry, typename TExtractorsMap, bool BatchSupport = false> - void ReplyBatch(typename TResponse::TPtr& ev) { - static TExtractorsMap extractors; - - const auto& record = ev->Get()->Record; - auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); - - TVector<TCell> cells; - for (const auto& entry : record.GetEntries()) { - for (auto column : Columns) { - auto extractor = extractors.find(column.Tag); - if (extractor == extractors.end()) { - cells.push_back(TCell()); - } else { - cells.push_back(extractor->second(entry)); - } - } - - TArrayRef<const TCell> ref(cells); + + TBase::Send(OwnerActorId, error.Release()); + + this->PassAway(); + } + + void ReplyEmptyAndDie() { + auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); + batch->Finished = true; + TBase::Send(OwnerActorId, batch.Release()); + + this->PassAway(); + } + + void PassAway() override { + LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS, + "Scan finished, actor: " << TBase::SelfId() + << ", owner: " << OwnerActorId + << ", scan id: " << ScanId + << ", table id: " << TableId); + + if (AllowedByLimiter) { + ScanLimiter->Dec(); + } + + TBase::PassAway(); + } + + ui64 GetBSControllerId() { + auto domainInfo = AppData()->DomainsInfo; + if (domainInfo->Domains.empty()) { + ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Invalid domain info"); + return 0; + } + + auto domain = domainInfo->Domains.begin()->second; + auto defaultSSGroup = domainInfo->GetDefaultStateStorageGroup(domain->DomainUid); + return MakeBSControllerID(defaultSSGroup); + } + + template <typename TResponse, typename TEntry, typename TExtractorsMap, bool BatchSupport = false> + void ReplyBatch(typename TResponse::TPtr& ev) { + static TExtractorsMap extractors; + + const auto& record = ev->Get()->Record; + auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); + + TVector<TCell> cells; + for (const auto& entry : record.GetEntries()) { + for (auto column : Columns) { + auto extractor = extractors.find(column.Tag); + if (extractor == extractors.end()) { + cells.push_back(TCell()); + } else { + cells.push_back(extractor->second(entry)); + } + } + + TArrayRef<const TCell> ref(cells); batch->Rows.emplace_back(TOwnedCellVec::Make(ref)); - cells.clear(); - } - - if constexpr (BatchSupport) { - batch->Finished = record.GetLastBatch(); - } else { - batch->Finished = true; - } - - SendBatch(std::move(batch)); - } - -private: - virtual void ProceedToScan() = 0; - - void ReplyLimiterFailedAndDie() { - ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED, "System view: concurrent scans limit exceeded"); - } - - void ReplyNavigateFailedAndDie() { - ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "System view: navigate failed"); - } - - void ReplyLookupFailedAndDie() { - ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "System view: tenant nodes lookup failed"); - } - - void HandleScanAck(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { - switch (FailState) { - case LIMITER_FAILED: - ReplyLimiterFailedAndDie(); - break; - case NAVIGATE_FAILED: - ReplyNavigateFailedAndDie(); - break; - default: - AckReceived = true; - break; - } - } - - void HandleLimiter(TEvSysView::TEvGetScanLimiterResult::TPtr& ev) { - ScanLimiter = ev->Get()->ScanLimiter; - - if (!ScanLimiter->Inc()) { - FailState = LIMITER_FAILED; - if (AckReceived) { - ReplyLimiterFailedAndDie(); - } - return; - } - - AllowedByLimiter = true; - - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - - auto request = MakeHolder<TNavigate>(); - request->ResultSet.push_back({}); - auto& entry = request->ResultSet.back(); - - entry.TableId = TTableId(TableId.PathId.OwnerId, TableId.PathId.LocalPathId); // domain or subdomain - entry.Operation = TNavigate::EOp::OpPath; - entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; - - TBase::Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); - TBase::Become(&TDerived::StateNavigate); - } - - void HandleNavigate(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - + cells.clear(); + } + + if constexpr (BatchSupport) { + batch->Finished = record.GetLastBatch(); + } else { + batch->Finished = true; + } + + SendBatch(std::move(batch)); + } + +private: + virtual void ProceedToScan() = 0; + + void ReplyLimiterFailedAndDie() { + ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED, "System view: concurrent scans limit exceeded"); + } + + void ReplyNavigateFailedAndDie() { + ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "System view: navigate failed"); + } + + void ReplyLookupFailedAndDie() { + ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "System view: tenant nodes lookup failed"); + } + + void HandleScanAck(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { + switch (FailState) { + case LIMITER_FAILED: + ReplyLimiterFailedAndDie(); + break; + case NAVIGATE_FAILED: + ReplyNavigateFailedAndDie(); + break; + default: + AckReceived = true; + break; + } + } + + void HandleLimiter(TEvSysView::TEvGetScanLimiterResult::TPtr& ev) { + ScanLimiter = ev->Get()->ScanLimiter; + + if (!ScanLimiter->Inc()) { + FailState = LIMITER_FAILED; + if (AckReceived) { + ReplyLimiterFailedAndDie(); + } + return; + } + + AllowedByLimiter = true; + + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + + auto request = MakeHolder<TNavigate>(); + request->ResultSet.push_back({}); + auto& entry = request->ResultSet.back(); + + entry.TableId = TTableId(TableId.PathId.OwnerId, TableId.PathId.LocalPathId); // domain or subdomain + entry.Operation = TNavigate::EOp::OpPath; + entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; + + TBase::Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); + TBase::Become(&TDerived::StateNavigate); + } + + void HandleNavigate(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + THolder<NSchemeCache::TSchemeCacheNavigate> request(ev->Get()->Request.Release()); - Y_VERIFY(request->ResultSet.size() == 1); - - auto& entry = request->ResultSet.back(); - if (entry.Status != TNavigate::EStatus::Ok) { - FailState = NAVIGATE_FAILED; - if (AckReceived) { - ReplyNavigateFailedAndDie(); - } - return; - } - + Y_VERIFY(request->ResultSet.size() == 1); + + auto& entry = request->ResultSet.back(); + if (entry.Status != TNavigate::EStatus::Ok) { + FailState = NAVIGATE_FAILED; + if (AckReceived) { + ReplyNavigateFailedAndDie(); + } + return; + } + SchemeShardId = entry.DomainInfo->ExtractSchemeShard(); - - if (entry.DomainInfo->Params.HasSysViewProcessor()) { - SysViewProcessorId = entry.DomainInfo->Params.GetSysViewProcessor(); - } - - if (entry.DomainInfo->Params.HasHive()) { - HiveId = entry.DomainInfo->Params.GetHive(); - } else { - auto domainInfo = AppData()->DomainsInfo; - if (domainInfo->Domains.empty()) { - ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Invalid domain info"); - return; - } - auto domain = domainInfo->Domains.begin()->second; - HiveId = domainInfo->GetHive(domain->DefaultHiveUid); - } - - DomainKey = entry.DomainInfo->DomainKey; - - TenantName = CanonizePath(entry.Path); - - TBase::Register(CreateTenantNodeEnumerationLookup(TBase::SelfId(), TenantName)); - TBase::Become(&TDerived::StateLookup); - } - - void HandleLookup(TEvTenantNodeEnumerator::TEvLookupResult::TPtr& ev) { - if (ev->Get()->Success) { - for (auto& node : ev->Get()->AssignedNodes) { - TenantNodes.insert(node); - } - } - - LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS, - "Scan prepared, actor: " << TBase::SelfId() - << ", schemeshard id: " << SchemeShardId - << ", hive id: " << HiveId - << ", tenant name: " << TenantName - << ", domain key: " << DomainKey - << ", tenant node count: " << TenantNodes.size()); - - ProceedToScan(); - } - - STFUNC(StateLimiter) { - switch (ev->GetTypeRewrite()) { - hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck); - hFunc(TEvSysView::TEvGetScanLimiterResult, HandleLimiter); - cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); - cFunc(TEvents::TEvPoison::EventType, this->PassAway); - default: - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - - STFUNC(StateNavigate) { - switch (ev->GetTypeRewrite()) { - hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck); - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNavigate); + + if (entry.DomainInfo->Params.HasSysViewProcessor()) { + SysViewProcessorId = entry.DomainInfo->Params.GetSysViewProcessor(); + } + + if (entry.DomainInfo->Params.HasHive()) { + HiveId = entry.DomainInfo->Params.GetHive(); + } else { + auto domainInfo = AppData()->DomainsInfo; + if (domainInfo->Domains.empty()) { + ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Invalid domain info"); + return; + } + auto domain = domainInfo->Domains.begin()->second; + HiveId = domainInfo->GetHive(domain->DefaultHiveUid); + } + + DomainKey = entry.DomainInfo->DomainKey; + + TenantName = CanonizePath(entry.Path); + + TBase::Register(CreateTenantNodeEnumerationLookup(TBase::SelfId(), TenantName)); + TBase::Become(&TDerived::StateLookup); + } + + void HandleLookup(TEvTenantNodeEnumerator::TEvLookupResult::TPtr& ev) { + if (ev->Get()->Success) { + for (auto& node : ev->Get()->AssignedNodes) { + TenantNodes.insert(node); + } + } + + LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS, + "Scan prepared, actor: " << TBase::SelfId() + << ", schemeshard id: " << SchemeShardId + << ", hive id: " << HiveId + << ", tenant name: " << TenantName + << ", domain key: " << DomainKey + << ", tenant node count: " << TenantNodes.size()); + + ProceedToScan(); + } + + STFUNC(StateLimiter) { + switch (ev->GetTypeRewrite()) { + hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck); + hFunc(TEvSysView::TEvGetScanLimiterResult, HandleLimiter); + cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); + cFunc(TEvents::TEvPoison::EventType, this->PassAway); + default: + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + + STFUNC(StateNavigate) { + switch (ev->GetTypeRewrite()) { + hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck); + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNavigate); hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution); - cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); - cFunc(TEvents::TEvPoison::EventType, this->PassAway); - default: - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - - STFUNC(StateLookup) { - switch (ev->GetTypeRewrite()) { - hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck); - hFunc(TEvTenantNodeEnumerator::TEvLookupResult, HandleLookup); + cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); + cFunc(TEvents::TEvPoison::EventType, this->PassAway); + default: + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + + STFUNC(StateLookup) { + switch (ev->GetTypeRewrite()) { + hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck); + hFunc(TEvTenantNodeEnumerator::TEvLookupResult, HandleLookup); hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution); - cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); - cFunc(TEvents::TEvPoison::EventType, this->PassAway); - default: - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - -protected: - static constexpr TDuration Timeout = TDuration::Seconds(60); - + cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); + cFunc(TEvents::TEvPoison::EventType, this->PassAway); + default: + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + +protected: + static constexpr TDuration Timeout = TDuration::Seconds(60); + const TActorId OwnerActorId; - const ui32 ScanId; - const TTableId TableId; - TSerializedTableRange TableRange; - TSmallVec<NMiniKQL::TKqpComputeContextBase::TColumn> Columns; - - ui64 SchemeShardId = 0; - TPathId DomainKey; - TString TenantName; - THashSet<ui32> TenantNodes; - ui64 HiveId = 0; - ui64 SysViewProcessorId = 0; - - bool AckReceived = false; - - bool BatchRequestInFlight = false; - -private: - enum EFailState { - OK, - LIMITER_FAILED, - NAVIGATE_FAILED - } FailState = OK; - - TIntrusivePtr<TScanLimiter> ScanLimiter; - bool AllowedByLimiter = false; -}; - - -} // NSysView -} // NKikimr + const ui32 ScanId; + const TTableId TableId; + TSerializedTableRange TableRange; + TSmallVec<NMiniKQL::TKqpComputeContextBase::TColumn> Columns; + + ui64 SchemeShardId = 0; + TPathId DomainKey; + TString TenantName; + THashSet<ui32> TenantNodes; + ui64 HiveId = 0; + ui64 SysViewProcessorId = 0; + + bool AckReceived = false; + + bool BatchRequestInFlight = false; + +private: + enum EFailState { + OK, + LIMITER_FAILED, + NAVIGATE_FAILED + } FailState = OK; + + TIntrusivePtr<TScanLimiter> ScanLimiter; + bool AllowedByLimiter = false; +}; + + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/schema.cpp b/ydb/core/sys_view/common/schema.cpp index a83113078f..1917ea35a4 100644 --- a/ydb/core/sys_view/common/schema.cpp +++ b/ydb/core/sys_view/common/schema.cpp @@ -1,91 +1,91 @@ -#include "schema.h" - +#include "schema.h" + #include <ydb/core/base/appdata.h> - -namespace NKikimr { -namespace NSysView { - -bool MaybeSystemViewPath(const TVector<TString>& path) { - auto length = path.size(); - // minimal system view path should be /Root/.sys/view - // only one level after ".sys" is allowed at the moment - if (length < 3 || path[length - 2] != SysPathName) { - return false; - } - return true; -} - -bool MaybeSystemViewFolderPath(const TVector<TString>& path) { - if (path.size() < 2 || path.back() != SysPathName) { - return false; - } - return true; -} - -class TSystemViewResolver : public ISystemViewResolver { -public: - TSystemViewResolver() { - RegisterSystemViews(); - } - - bool IsSystemViewPath(const TVector<TString>& path, TSystemViewPath& sysViewPath) const override final { - if (MaybeSystemViewFolderPath(path)) { - TVector<TString> realPath(path.begin(), path.end() - 1); - sysViewPath.Parent = std::move(realPath); - sysViewPath.ViewName = SysPathName; - return true; - - } else if (MaybeSystemViewPath(path)) { - auto maybeSystemViewName = path.back(); - if (!DomainSystemViews.contains(maybeSystemViewName) && + +namespace NKikimr { +namespace NSysView { + +bool MaybeSystemViewPath(const TVector<TString>& path) { + auto length = path.size(); + // minimal system view path should be /Root/.sys/view + // only one level after ".sys" is allowed at the moment + if (length < 3 || path[length - 2] != SysPathName) { + return false; + } + return true; +} + +bool MaybeSystemViewFolderPath(const TVector<TString>& path) { + if (path.size() < 2 || path.back() != SysPathName) { + return false; + } + return true; +} + +class TSystemViewResolver : public ISystemViewResolver { +public: + TSystemViewResolver() { + RegisterSystemViews(); + } + + bool IsSystemViewPath(const TVector<TString>& path, TSystemViewPath& sysViewPath) const override final { + if (MaybeSystemViewFolderPath(path)) { + TVector<TString> realPath(path.begin(), path.end() - 1); + sysViewPath.Parent = std::move(realPath); + sysViewPath.ViewName = SysPathName; + return true; + + } else if (MaybeSystemViewPath(path)) { + auto maybeSystemViewName = path.back(); + if (!DomainSystemViews.contains(maybeSystemViewName) && !SubDomainSystemViews.contains(maybeSystemViewName) && !OlapStoreSystemViews.contains(maybeSystemViewName) && !OlapTableSystemViews.contains(maybeSystemViewName)) - { - return false; - } - TVector<TString> realPath(path.begin(), path.end() - 2); - sysViewPath.Parent = std::move(realPath); - sysViewPath.ViewName = path.back(); - return true; - } - return false; - } - - TMaybe<TSchema> GetSystemViewSchema(const TStringBuf viewName, ETarget target) const override final { - const TSchema* view = nullptr; - switch (target) { - case ETarget::Domain: - view = DomainSystemViews.FindPtr(viewName); - break; - case ETarget::SubDomain: - view = SubDomainSystemViews.FindPtr(viewName); - break; + { + return false; + } + TVector<TString> realPath(path.begin(), path.end() - 2); + sysViewPath.Parent = std::move(realPath); + sysViewPath.ViewName = path.back(); + return true; + } + return false; + } + + TMaybe<TSchema> GetSystemViewSchema(const TStringBuf viewName, ETarget target) const override final { + const TSchema* view = nullptr; + switch (target) { + case ETarget::Domain: + view = DomainSystemViews.FindPtr(viewName); + break; + case ETarget::SubDomain: + view = SubDomainSystemViews.FindPtr(viewName); + break; case ETarget::OlapStore: view = OlapStoreSystemViews.FindPtr(viewName); break; case ETarget::OlapTable: view = OlapTableSystemViews.FindPtr(viewName); break; - } - return view ? TMaybe<TSchema>(*view) : Nothing(); - } - - TVector<TString> GetSystemViewNames(ETarget target) const override { - TVector<TString> result; - switch (target) { - case ETarget::Domain: - result.reserve(DomainSystemViews.size()); - for (const auto& [name, _] : DomainSystemViews) { - result.push_back(name); - } - break; - case ETarget::SubDomain: - result.reserve(SubDomainSystemViews.size()); - for (const auto& [name, _] : SubDomainSystemViews) { - result.push_back(name); - } - break; + } + return view ? TMaybe<TSchema>(*view) : Nothing(); + } + + TVector<TString> GetSystemViewNames(ETarget target) const override { + TVector<TString> result; + switch (target) { + case ETarget::Domain: + result.reserve(DomainSystemViews.size()); + for (const auto& [name, _] : DomainSystemViews) { + result.push_back(name); + } + break; + case ETarget::SubDomain: + result.reserve(SubDomainSystemViews.size()); + for (const auto& [name, _] : SubDomainSystemViews) { + result.push_back(name); + } + break; case ETarget::OlapStore: result.reserve(OlapStoreSystemViews.size()); for (const auto& [name, _] : OlapStoreSystemViews) { @@ -98,87 +98,87 @@ public: result.push_back(name); } break; - } - return result; - } - -private: - template <typename Table> - struct TSchemaFiller { - - template <typename...> - struct TFiller; - - template <typename Column> - struct TFiller<Column> { - static void Fill(TSchema& schema) { - schema.Columns[Column::ColumnId] = TSysTables::TTableColumnInfo( - Table::template TableColumns<Column>::GetColumnName(), - Column::ColumnId, Column::ColumnType, -1); - } - }; - - template <typename Column, typename... Columns> - struct TFiller<Column, Columns...> { - static void Fill(TSchema& schema) { - TFiller<Column>::Fill(schema); - TFiller<Columns...>::Fill(schema); - } - }; - - template <typename... Columns> - using TColumnsType = typename Table::template TableColumns<Columns...>; - - template <typename... Columns> - static void FillColumns(TSchema& schema, TColumnsType<Columns...>) { - TFiller<Columns...>::Fill(schema); - } - - template <typename...> - struct TKeyFiller; - - template <typename Key> - struct TKeyFiller<Key> { - static void Fill(TSchema& schema, i32 index) { - auto& column = schema.Columns[Key::ColumnId]; - column.KeyOrder = index; - schema.KeyColumnTypes.push_back(column.PType); - } - }; - - template <typename Key, typename... Keys> - struct TKeyFiller<Key, Keys...> { - static void Fill(TSchema& schema, i32 index) { - TKeyFiller<Key>::Fill(schema, index); - TKeyFiller<Keys...>::Fill(schema, index + 1); - } - }; - - template <typename... Keys> - using TKeysType = typename Table::template TableKey<Keys...>; - - template <typename... Keys> - static void FillKeys(TSchema& schema, TKeysType<Keys...>) { - TKeyFiller<Keys...>::Fill(schema, 0); - } - - static void Fill(TSchema& schema) { - FillColumns(schema, typename Table::TColumns()); - FillKeys(schema, typename Table::TKey()); - } - }; - - template <typename Table> - void RegisterSystemView(const TStringBuf& name) { - TSchemaFiller<Table>::Fill(DomainSystemViews[name]); - TSchemaFiller<Table>::Fill(SubDomainSystemViews[name]); - } - - template <typename Table> - void RegisterDomainSystemView(const TStringBuf& name) { - TSchemaFiller<Table>::Fill(DomainSystemViews[name]); - } - + } + return result; + } + +private: + template <typename Table> + struct TSchemaFiller { + + template <typename...> + struct TFiller; + + template <typename Column> + struct TFiller<Column> { + static void Fill(TSchema& schema) { + schema.Columns[Column::ColumnId] = TSysTables::TTableColumnInfo( + Table::template TableColumns<Column>::GetColumnName(), + Column::ColumnId, Column::ColumnType, -1); + } + }; + + template <typename Column, typename... Columns> + struct TFiller<Column, Columns...> { + static void Fill(TSchema& schema) { + TFiller<Column>::Fill(schema); + TFiller<Columns...>::Fill(schema); + } + }; + + template <typename... Columns> + using TColumnsType = typename Table::template TableColumns<Columns...>; + + template <typename... Columns> + static void FillColumns(TSchema& schema, TColumnsType<Columns...>) { + TFiller<Columns...>::Fill(schema); + } + + template <typename...> + struct TKeyFiller; + + template <typename Key> + struct TKeyFiller<Key> { + static void Fill(TSchema& schema, i32 index) { + auto& column = schema.Columns[Key::ColumnId]; + column.KeyOrder = index; + schema.KeyColumnTypes.push_back(column.PType); + } + }; + + template <typename Key, typename... Keys> + struct TKeyFiller<Key, Keys...> { + static void Fill(TSchema& schema, i32 index) { + TKeyFiller<Key>::Fill(schema, index); + TKeyFiller<Keys...>::Fill(schema, index + 1); + } + }; + + template <typename... Keys> + using TKeysType = typename Table::template TableKey<Keys...>; + + template <typename... Keys> + static void FillKeys(TSchema& schema, TKeysType<Keys...>) { + TKeyFiller<Keys...>::Fill(schema, 0); + } + + static void Fill(TSchema& schema) { + FillColumns(schema, typename Table::TColumns()); + FillKeys(schema, typename Table::TKey()); + } + }; + + template <typename Table> + void RegisterSystemView(const TStringBuf& name) { + TSchemaFiller<Table>::Fill(DomainSystemViews[name]); + TSchemaFiller<Table>::Fill(SubDomainSystemViews[name]); + } + + template <typename Table> + void RegisterDomainSystemView(const TStringBuf& name) { + TSchemaFiller<Table>::Fill(DomainSystemViews[name]); + } + template <typename Table> void RegisterOlapStoreSystemView(const TStringBuf& name) { TSchemaFiller<Table>::Fill(OlapStoreSystemViews[name]); @@ -189,45 +189,45 @@ private: TSchemaFiller<Table>::Fill(OlapTableSystemViews[name]); } - void RegisterSystemViews() { - RegisterSystemView<Schema::PartitionStats>(PartitionStatsName); - - // 'nodes' table is currently switched off - // RegisterSystemView<Schema::Nodes>(NodesName); - - RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1MinuteName); - RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1HourName); - RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1MinuteName); - RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1HourName); - RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1MinuteName); - RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1HourName); - RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1MinuteName); - RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1HourName); - - RegisterDomainSystemView<Schema::PDisks>(PDisksName); - RegisterDomainSystemView<Schema::VSlots>(VSlotsName); - RegisterDomainSystemView<Schema::Groups>(GroupsName); - RegisterDomainSystemView<Schema::StoragePools>(StoragePoolsName); + void RegisterSystemViews() { + RegisterSystemView<Schema::PartitionStats>(PartitionStatsName); + + // 'nodes' table is currently switched off + // RegisterSystemView<Schema::Nodes>(NodesName); + + RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1MinuteName); + RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1HourName); + RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1MinuteName); + RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1HourName); + RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1MinuteName); + RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1HourName); + RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1MinuteName); + RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1HourName); + + RegisterDomainSystemView<Schema::PDisks>(PDisksName); + RegisterDomainSystemView<Schema::VSlots>(VSlotsName); + RegisterDomainSystemView<Schema::Groups>(GroupsName); + RegisterDomainSystemView<Schema::StoragePools>(StoragePoolsName); RegisterDomainSystemView<Schema::StorageStats>(StorageStatsName); - - RegisterDomainSystemView<Schema::Tablets>(TabletsName); - - RegisterSystemView<Schema::QueryMetrics>(QueryMetricsName); + + RegisterDomainSystemView<Schema::Tablets>(TabletsName); + + RegisterSystemView<Schema::QueryMetrics>(QueryMetricsName); RegisterOlapStoreSystemView<Schema::PrimaryIndexStats>(StorePrimaryIndexStatsName); RegisterOlapTableSystemView<Schema::PrimaryIndexStats>(TablePrimaryIndexStatsName); - } - -private: - THashMap<TString, TSchema> DomainSystemViews; - THashMap<TString, TSchema> SubDomainSystemViews; + } + +private: + THashMap<TString, TSchema> DomainSystemViews; + THashMap<TString, TSchema> SubDomainSystemViews; THashMap<TString, TSchema> OlapStoreSystemViews; THashMap<TString, TSchema> OlapTableSystemViews; -}; - -ISystemViewResolver* CreateSystemViewResolver() { - return new TSystemViewResolver(); -} - -} // NSysView -} // NKikimr +}; + +ISystemViewResolver* CreateSystemViewResolver() { + return new TSystemViewResolver(); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h index 96f15695a0..f7a4f191e1 100644 --- a/ydb/core/sys_view/common/schema.h +++ b/ydb/core/sys_view/common/schema.h @@ -1,384 +1,384 @@ -#pragma once - -#include "path.h" - +#pragma once + +#include "path.h" + #include <ydb/core/tablet_flat/flat_cxx_database.h> #include <ydb/core/tx/datashard/sys_tables.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + constexpr TStringBuf PartitionStatsName = "partition_stats"; constexpr TStringBuf NodesName = "nodes"; - + constexpr TStringBuf TopQueriesByDuration1MinuteName = "top_queries_by_duration_one_minute"; constexpr TStringBuf TopQueriesByDuration1HourName = "top_queries_by_duration_one_hour"; constexpr TStringBuf TopQueriesByReadBytes1MinuteName = "top_queries_by_read_bytes_one_minute"; constexpr TStringBuf TopQueriesByReadBytes1HourName = "top_queries_by_read_bytes_one_hour"; constexpr TStringBuf TopQueriesByCpuTime1MinuteName = "top_queries_by_cpu_time_one_minute"; constexpr TStringBuf TopQueriesByCpuTime1HourName = "top_queries_by_cpu_time_one_hour"; -constexpr TStringBuf TopQueriesByRequestUnits1MinuteName = "top_queries_by_request_units_one_minute"; -constexpr TStringBuf TopQueriesByRequestUnits1HourName = "top_queries_by_request_units_one_hour"; - +constexpr TStringBuf TopQueriesByRequestUnits1MinuteName = "top_queries_by_request_units_one_minute"; +constexpr TStringBuf TopQueriesByRequestUnits1HourName = "top_queries_by_request_units_one_hour"; + constexpr TStringBuf PDisksName = "ds_pdisks"; constexpr TStringBuf VSlotsName = "ds_vslots"; constexpr TStringBuf GroupsName = "ds_groups"; constexpr TStringBuf StoragePoolsName = "ds_storage_pools"; constexpr TStringBuf StorageStatsName = "ds_storage_stats"; - + constexpr TStringBuf TabletsName = "hive_tablets"; - + constexpr TStringBuf QueryMetricsName = "query_metrics_one_minute"; - + constexpr TStringBuf StorePrimaryIndexStatsName = "store_primary_index_stats"; constexpr TStringBuf TablePrimaryIndexStatsName = "primary_index_stats"; -struct Schema : NIceDb::Schema { - struct PartitionStats : Table<1> { - struct OwnerId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct PathId : Column<2, NScheme::NTypeIds::Uint64> {}; - struct PartIdx : Column<3, NScheme::NTypeIds::Uint64> {}; - struct DataSize : Column<4, NScheme::NTypeIds::Uint64> {}; - struct RowCount : Column<5, NScheme::NTypeIds::Uint64> {}; - struct IndexSize : Column<6, NScheme::NTypeIds::Uint64> {}; - struct CPUCores : Column<7, NScheme::NTypeIds::Double> {}; - struct TabletId : Column<8, NScheme::NTypeIds::Uint64> {}; - struct Path : Column<9, NScheme::NTypeIds::Utf8> {}; - struct NodeId : Column<10, NScheme::NTypeIds::Uint32> {}; - struct StartTime : Column<11, NScheme::NTypeIds::Timestamp> {}; - struct AccessTime : Column<12, NScheme::NTypeIds::Timestamp> {}; - struct UpdateTime : Column<13, NScheme::NTypeIds::Timestamp> {}; - struct InFlightTxCount : Column<14, NScheme::NTypeIds::Uint32> {}; - struct RowUpdates : Column<15, NScheme::NTypeIds::Uint64> {}; - struct RowDeletes : Column<16, NScheme::NTypeIds::Uint64> {}; - struct RowReads : Column<17, NScheme::NTypeIds::Uint64> {}; - struct RangeReads : Column<18, NScheme::NTypeIds::Uint64> {}; - struct RangeReadRows : Column<19, NScheme::NTypeIds::Uint64> {}; - struct ImmediateTxCompleted : Column<20, NScheme::NTypeIds::Uint64> {}; - struct CoordinatedTxCompleted : Column<21, NScheme::NTypeIds::Uint64> {}; - struct TxRejectedByOverload : Column<22, NScheme::NTypeIds::Uint64> {}; - struct TxRejectedByOutOfStorage : Column<23, NScheme::NTypeIds::Uint64> {}; +struct Schema : NIceDb::Schema { + struct PartitionStats : Table<1> { + struct OwnerId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct PathId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct PartIdx : Column<3, NScheme::NTypeIds::Uint64> {}; + struct DataSize : Column<4, NScheme::NTypeIds::Uint64> {}; + struct RowCount : Column<5, NScheme::NTypeIds::Uint64> {}; + struct IndexSize : Column<6, NScheme::NTypeIds::Uint64> {}; + struct CPUCores : Column<7, NScheme::NTypeIds::Double> {}; + struct TabletId : Column<8, NScheme::NTypeIds::Uint64> {}; + struct Path : Column<9, NScheme::NTypeIds::Utf8> {}; + struct NodeId : Column<10, NScheme::NTypeIds::Uint32> {}; + struct StartTime : Column<11, NScheme::NTypeIds::Timestamp> {}; + struct AccessTime : Column<12, NScheme::NTypeIds::Timestamp> {}; + struct UpdateTime : Column<13, NScheme::NTypeIds::Timestamp> {}; + struct InFlightTxCount : Column<14, NScheme::NTypeIds::Uint32> {}; + struct RowUpdates : Column<15, NScheme::NTypeIds::Uint64> {}; + struct RowDeletes : Column<16, NScheme::NTypeIds::Uint64> {}; + struct RowReads : Column<17, NScheme::NTypeIds::Uint64> {}; + struct RangeReads : Column<18, NScheme::NTypeIds::Uint64> {}; + struct RangeReadRows : Column<19, NScheme::NTypeIds::Uint64> {}; + struct ImmediateTxCompleted : Column<20, NScheme::NTypeIds::Uint64> {}; + struct CoordinatedTxCompleted : Column<21, NScheme::NTypeIds::Uint64> {}; + struct TxRejectedByOverload : Column<22, NScheme::NTypeIds::Uint64> {}; + struct TxRejectedByOutOfStorage : Column<23, NScheme::NTypeIds::Uint64> {}; struct LastTtlRunTime : Column<24, NScheme::NTypeIds::Timestamp> {}; struct LastTtlRowsProcessed : Column<25, NScheme::NTypeIds::Uint64> {}; struct LastTtlRowsErased : Column<26, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<OwnerId, PathId, PartIdx>; - using TColumns = TableColumns< - OwnerId, - PathId, - PartIdx, - DataSize, - RowCount, - IndexSize, - CPUCores, - TabletId, - Path, - NodeId, - AccessTime, - UpdateTime, - StartTime, - InFlightTxCount, - RowUpdates, - RowDeletes, - RowReads, - RangeReads, - RangeReadRows, - ImmediateTxCompleted, - CoordinatedTxCompleted, - TxRejectedByOverload, + + using TKey = TableKey<OwnerId, PathId, PartIdx>; + using TColumns = TableColumns< + OwnerId, + PathId, + PartIdx, + DataSize, + RowCount, + IndexSize, + CPUCores, + TabletId, + Path, + NodeId, + AccessTime, + UpdateTime, + StartTime, + InFlightTxCount, + RowUpdates, + RowDeletes, + RowReads, + RangeReads, + RangeReadRows, + ImmediateTxCompleted, + CoordinatedTxCompleted, + TxRejectedByOverload, TxRejectedByOutOfStorage, LastTtlRunTime, LastTtlRowsProcessed, LastTtlRowsErased>; - }; - - struct Nodes : Table<2> { - struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; - struct Address : Column<2, NScheme::NTypeIds::String> {}; - struct Host : Column<3, NScheme::NTypeIds::String> {}; - struct Port : Column<4, NScheme::NTypeIds::Uint32> {}; - struct StartTime : Column<5, NScheme::NTypeIds::Timestamp> {}; - struct UpTime : Column<6, NScheme::NTypeIds::Interval> {}; - - using TKey = TableKey<NodeId>; - using TColumns = TableColumns< - NodeId, - Host, - Address, - Port, - StartTime, - UpTime>; - }; - - struct QueryStats : Table<3> { - struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; - struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; - struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {}; - struct Duration : Column<4, NScheme::NTypeIds::Interval> {}; - struct EndTime : Column<5, NScheme::NTypeIds::Timestamp> {}; - struct ReadRows : Column<6, NScheme::NTypeIds::Uint64> {}; - struct ReadBytes : Column<7, NScheme::NTypeIds::Uint64> {}; - struct UpdateRows : Column<8, NScheme::NTypeIds::Uint64> {}; - struct UpdateBytes : Column<9, NScheme::NTypeIds::Uint64> {}; - struct DeleteRows : Column<10, NScheme::NTypeIds::Uint64> {}; - struct DeleteBytes : Column<11, NScheme::NTypeIds::Uint64> {}; // deprecated, always 0 - struct Partitions : Column<12, NScheme::NTypeIds::Uint64> {}; - struct UserSID : Column<13, NScheme::NTypeIds::String> {}; - struct ParametersSize : Column<14, NScheme::NTypeIds::Uint64> {}; - struct CompileDuration : Column<15, NScheme::NTypeIds::Interval> {}; - struct FromQueryCache : Column<16, NScheme::NTypeIds::Bool> {}; - struct CPUTime : Column<17, NScheme::NTypeIds::Uint64> {}; - struct ShardCount : Column<18, NScheme::NTypeIds::Uint64> {}; - struct SumShardCPUTime : Column<19, NScheme::NTypeIds::Uint64> {}; - struct MinShardCPUTime : Column<20, NScheme::NTypeIds::Uint64> {}; - struct MaxShardCPUTime : Column<21, NScheme::NTypeIds::Uint64> {}; - struct ComputeNodesCount : Column<22, NScheme::NTypeIds::Uint64> {}; - struct SumComputeCPUTime : Column<23, NScheme::NTypeIds::Uint64> {}; - struct MinComputeCPUTime : Column<24, NScheme::NTypeIds::Uint64> {}; - struct MaxComputeCPUTime : Column<25, NScheme::NTypeIds::Uint64> {}; - struct CompileCPUTime : Column<26, NScheme::NTypeIds::Uint64> {}; - struct ProcessCPUTime : Column<27, NScheme::NTypeIds::Uint64> {}; - struct TypeCol : Column<28, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } }; - struct RequestUnits : Column<29, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<IntervalEnd, Rank>; - using TColumns = TableColumns< - IntervalEnd, - Rank, - QueryText, - Duration, - EndTime, - ReadRows, - ReadBytes, - UpdateRows, - UpdateBytes, - DeleteRows, - DeleteBytes, - Partitions, - UserSID, - ParametersSize, - CompileDuration, - FromQueryCache, - CPUTime, - ShardCount, - SumShardCPUTime, - MinShardCPUTime, - MaxShardCPUTime, - ComputeNodesCount, - SumComputeCPUTime, - MinComputeCPUTime, - MaxComputeCPUTime, - CompileCPUTime, - ProcessCPUTime, - TypeCol, - RequestUnits>; - }; - - struct PDisks : Table<4> { - struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; - struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; - struct TypeCol : Column<3, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } }; - struct Kind : Column<4, NScheme::NTypeIds::Uint64> {}; - struct Path : Column<5, NScheme::NTypeIds::String> {}; - struct Guid : Column<6, NScheme::NTypeIds::Uint64> {}; - struct BoxId : Column<7, NScheme::NTypeIds::Uint32> {}; - struct SharedWithOS : Column<8, NScheme::NTypeIds::Bool> {}; - struct ReadCentric : Column<9, NScheme::NTypeIds::Bool> {}; - struct AvailableSize : Column<10, NScheme::NTypeIds::Uint64> {}; - struct TotalSize : Column<11, NScheme::NTypeIds::Uint64> {}; - struct Status : Column<12, NScheme::NTypeIds::String> {}; + }; + + struct Nodes : Table<2> { + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct Address : Column<2, NScheme::NTypeIds::String> {}; + struct Host : Column<3, NScheme::NTypeIds::String> {}; + struct Port : Column<4, NScheme::NTypeIds::Uint32> {}; + struct StartTime : Column<5, NScheme::NTypeIds::Timestamp> {}; + struct UpTime : Column<6, NScheme::NTypeIds::Interval> {}; + + using TKey = TableKey<NodeId>; + using TColumns = TableColumns< + NodeId, + Host, + Address, + Port, + StartTime, + UpTime>; + }; + + struct QueryStats : Table<3> { + struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; + struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; + struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Duration : Column<4, NScheme::NTypeIds::Interval> {}; + struct EndTime : Column<5, NScheme::NTypeIds::Timestamp> {}; + struct ReadRows : Column<6, NScheme::NTypeIds::Uint64> {}; + struct ReadBytes : Column<7, NScheme::NTypeIds::Uint64> {}; + struct UpdateRows : Column<8, NScheme::NTypeIds::Uint64> {}; + struct UpdateBytes : Column<9, NScheme::NTypeIds::Uint64> {}; + struct DeleteRows : Column<10, NScheme::NTypeIds::Uint64> {}; + struct DeleteBytes : Column<11, NScheme::NTypeIds::Uint64> {}; // deprecated, always 0 + struct Partitions : Column<12, NScheme::NTypeIds::Uint64> {}; + struct UserSID : Column<13, NScheme::NTypeIds::String> {}; + struct ParametersSize : Column<14, NScheme::NTypeIds::Uint64> {}; + struct CompileDuration : Column<15, NScheme::NTypeIds::Interval> {}; + struct FromQueryCache : Column<16, NScheme::NTypeIds::Bool> {}; + struct CPUTime : Column<17, NScheme::NTypeIds::Uint64> {}; + struct ShardCount : Column<18, NScheme::NTypeIds::Uint64> {}; + struct SumShardCPUTime : Column<19, NScheme::NTypeIds::Uint64> {}; + struct MinShardCPUTime : Column<20, NScheme::NTypeIds::Uint64> {}; + struct MaxShardCPUTime : Column<21, NScheme::NTypeIds::Uint64> {}; + struct ComputeNodesCount : Column<22, NScheme::NTypeIds::Uint64> {}; + struct SumComputeCPUTime : Column<23, NScheme::NTypeIds::Uint64> {}; + struct MinComputeCPUTime : Column<24, NScheme::NTypeIds::Uint64> {}; + struct MaxComputeCPUTime : Column<25, NScheme::NTypeIds::Uint64> {}; + struct CompileCPUTime : Column<26, NScheme::NTypeIds::Uint64> {}; + struct ProcessCPUTime : Column<27, NScheme::NTypeIds::Uint64> {}; + struct TypeCol : Column<28, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } }; + struct RequestUnits : Column<29, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<IntervalEnd, Rank>; + using TColumns = TableColumns< + IntervalEnd, + Rank, + QueryText, + Duration, + EndTime, + ReadRows, + ReadBytes, + UpdateRows, + UpdateBytes, + DeleteRows, + DeleteBytes, + Partitions, + UserSID, + ParametersSize, + CompileDuration, + FromQueryCache, + CPUTime, + ShardCount, + SumShardCPUTime, + MinShardCPUTime, + MaxShardCPUTime, + ComputeNodesCount, + SumComputeCPUTime, + MinComputeCPUTime, + MaxComputeCPUTime, + CompileCPUTime, + ProcessCPUTime, + TypeCol, + RequestUnits>; + }; + + struct PDisks : Table<4> { + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct TypeCol : Column<3, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } }; + struct Kind : Column<4, NScheme::NTypeIds::Uint64> {}; + struct Path : Column<5, NScheme::NTypeIds::String> {}; + struct Guid : Column<6, NScheme::NTypeIds::Uint64> {}; + struct BoxId : Column<7, NScheme::NTypeIds::Uint32> {}; + struct SharedWithOS : Column<8, NScheme::NTypeIds::Bool> {}; + struct ReadCentric : Column<9, NScheme::NTypeIds::Bool> {}; + struct AvailableSize : Column<10, NScheme::NTypeIds::Uint64> {}; + struct TotalSize : Column<11, NScheme::NTypeIds::Uint64> {}; + struct Status : Column<12, NScheme::NTypeIds::String> {}; //struct StopFactor : Column<13, NScheme::NTypeIds::Double> {}; struct StatusChangeTimestamp : Column<14, NScheme::NTypeIds::Timestamp> {}; struct ExpectedSlotCount : Column<15, NScheme::NTypeIds::Uint32> {}; struct NumActiveSlots : Column<16, NScheme::NTypeIds::Uint32> {}; - - using TKey = TableKey<NodeId, PDiskId>; - using TColumns = TableColumns< - NodeId, - PDiskId, - TypeCol, - Kind, - Path, - Guid, - BoxId, - SharedWithOS, - ReadCentric, - AvailableSize, - TotalSize, + + using TKey = TableKey<NodeId, PDiskId>; + using TColumns = TableColumns< + NodeId, + PDiskId, + TypeCol, + Kind, + Path, + Guid, + BoxId, + SharedWithOS, + ReadCentric, + AvailableSize, + TotalSize, Status, StatusChangeTimestamp, ExpectedSlotCount, NumActiveSlots>; - }; - - struct VSlots : Table<5> { - struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; - struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; - struct VSlotId : Column<3, NScheme::NTypeIds::Uint32> {}; - //struct Category : Column<4, NScheme::NTypeIds::Uint64> {}; - struct GroupId : Column<5, NScheme::NTypeIds::Uint32> {}; - struct GroupGeneration : Column<6, NScheme::NTypeIds::Uint32> {}; - //struct Ring : Column<7, NScheme::NTypeIds::Uint32> {}; - struct FailDomain : Column<8, NScheme::NTypeIds::Uint32> {}; - struct VDisk : Column<9, NScheme::NTypeIds::Uint32> {}; - struct AllocatedSize : Column<10, NScheme::NTypeIds::Uint64> {}; - struct AvailableSize : Column<11, NScheme::NTypeIds::Uint64> {}; - struct Status : Column<12, NScheme::NTypeIds::String> {}; + }; + + struct VSlots : Table<5> { + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct VSlotId : Column<3, NScheme::NTypeIds::Uint32> {}; + //struct Category : Column<4, NScheme::NTypeIds::Uint64> {}; + struct GroupId : Column<5, NScheme::NTypeIds::Uint32> {}; + struct GroupGeneration : Column<6, NScheme::NTypeIds::Uint32> {}; + //struct Ring : Column<7, NScheme::NTypeIds::Uint32> {}; + struct FailDomain : Column<8, NScheme::NTypeIds::Uint32> {}; + struct VDisk : Column<9, NScheme::NTypeIds::Uint32> {}; + struct AllocatedSize : Column<10, NScheme::NTypeIds::Uint64> {}; + struct AvailableSize : Column<11, NScheme::NTypeIds::Uint64> {}; + struct Status : Column<12, NScheme::NTypeIds::String> {}; //struct StopFactor : Column<13, NScheme::NTypeIds::Double> {}; - struct Kind : Column<14, NScheme::NTypeIds::String> {}; - struct FailRealm : Column<15, NScheme::NTypeIds::Uint32> {}; - - using TKey = TableKey<NodeId, PDiskId, VSlotId>; - using TColumns = TableColumns< - NodeId, - PDiskId, - VSlotId, - GroupId, - GroupGeneration, - FailDomain, - VDisk, - AllocatedSize, - AvailableSize, - Status, - Kind, - FailRealm>; - }; - - struct Groups : Table<6> { - struct GroupId : Column<1, NScheme::NTypeIds::Uint32> {}; - struct Generation : Column<2, NScheme::NTypeIds::Uint32> {}; - struct ErasureSpecies : Column<3, NScheme::NTypeIds::String> {}; - struct BoxId : Column<4, NScheme::NTypeIds::Uint64> {}; - struct StoragePoolId : Column<5, NScheme::NTypeIds::Uint64> {}; - struct EncryptionMode : Column<6, NScheme::NTypeIds::Uint32> {}; - struct LifeCyclePhase : Column<7, NScheme::NTypeIds::Uint32> {}; - struct AllocatedSize : Column<8, NScheme::NTypeIds::Uint64> {}; - struct AvailableSize : Column<9, NScheme::NTypeIds::Uint64> {}; + struct Kind : Column<14, NScheme::NTypeIds::String> {}; + struct FailRealm : Column<15, NScheme::NTypeIds::Uint32> {}; + + using TKey = TableKey<NodeId, PDiskId, VSlotId>; + using TColumns = TableColumns< + NodeId, + PDiskId, + VSlotId, + GroupId, + GroupGeneration, + FailDomain, + VDisk, + AllocatedSize, + AvailableSize, + Status, + Kind, + FailRealm>; + }; + + struct Groups : Table<6> { + struct GroupId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct Generation : Column<2, NScheme::NTypeIds::Uint32> {}; + struct ErasureSpecies : Column<3, NScheme::NTypeIds::String> {}; + struct BoxId : Column<4, NScheme::NTypeIds::Uint64> {}; + struct StoragePoolId : Column<5, NScheme::NTypeIds::Uint64> {}; + struct EncryptionMode : Column<6, NScheme::NTypeIds::Uint32> {}; + struct LifeCyclePhase : Column<7, NScheme::NTypeIds::Uint32> {}; + struct AllocatedSize : Column<8, NScheme::NTypeIds::Uint64> {}; + struct AvailableSize : Column<9, NScheme::NTypeIds::Uint64> {}; //struct Usage : Column<10, NScheme::NTypeIds::Double> {}; //struct StopFactor : Column<11, NScheme::NTypeIds::Double> {}; - struct SeenOperational : Column<12, NScheme::NTypeIds::Bool> {}; - struct PutTabletLogLatency : Column<13, NScheme::NTypeIds::Interval> {}; - struct PutUserDataLatency : Column<14, NScheme::NTypeIds::Interval> {}; - struct GetFastLatency : Column<15, NScheme::NTypeIds::Interval> {}; - - using TKey = TableKey<GroupId>; - using TColumns = TableColumns< - GroupId, - Generation, - ErasureSpecies, - BoxId, - StoragePoolId, - EncryptionMode, - LifeCyclePhase, - AllocatedSize, - AvailableSize, - SeenOperational, - PutTabletLogLatency, - PutUserDataLatency, - GetFastLatency>; - }; - - struct StoragePools : Table<7> { - struct BoxId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct StoragePoolId : Column<2, NScheme::NTypeIds::Uint64> {}; - struct Name : Column<3, NScheme::NTypeIds::String> {}; - struct Generation : Column<4, NScheme::NTypeIds::Uint64> {}; - struct ErasureSpecies : Column<5, NScheme::NTypeIds::String> {}; - struct VDiskKind : Column<6, NScheme::NTypeIds::String> {}; - struct Kind : Column<7, NScheme::NTypeIds::String> {}; - struct NumGroups : Column<8, NScheme::NTypeIds::Uint32> {}; - struct EncryptionMode : Column<9, NScheme::NTypeIds::Uint32> {}; - struct SchemeshardId : Column<10, NScheme::NTypeIds::Uint64> {}; - struct PathId : Column<11, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<BoxId, StoragePoolId>; - using TColumns = TableColumns< - BoxId, - StoragePoolId, - Name, - Generation, - ErasureSpecies, - VDiskKind, - Kind, - NumGroups, - EncryptionMode, - SchemeshardId, - PathId>; - }; - - struct Tablets : Table<8> { - struct TabletId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct SeenOperational : Column<12, NScheme::NTypeIds::Bool> {}; + struct PutTabletLogLatency : Column<13, NScheme::NTypeIds::Interval> {}; + struct PutUserDataLatency : Column<14, NScheme::NTypeIds::Interval> {}; + struct GetFastLatency : Column<15, NScheme::NTypeIds::Interval> {}; + + using TKey = TableKey<GroupId>; + using TColumns = TableColumns< + GroupId, + Generation, + ErasureSpecies, + BoxId, + StoragePoolId, + EncryptionMode, + LifeCyclePhase, + AllocatedSize, + AvailableSize, + SeenOperational, + PutTabletLogLatency, + PutUserDataLatency, + GetFastLatency>; + }; + + struct StoragePools : Table<7> { + struct BoxId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct StoragePoolId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Name : Column<3, NScheme::NTypeIds::String> {}; + struct Generation : Column<4, NScheme::NTypeIds::Uint64> {}; + struct ErasureSpecies : Column<5, NScheme::NTypeIds::String> {}; + struct VDiskKind : Column<6, NScheme::NTypeIds::String> {}; + struct Kind : Column<7, NScheme::NTypeIds::String> {}; + struct NumGroups : Column<8, NScheme::NTypeIds::Uint32> {}; + struct EncryptionMode : Column<9, NScheme::NTypeIds::Uint32> {}; + struct SchemeshardId : Column<10, NScheme::NTypeIds::Uint64> {}; + struct PathId : Column<11, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<BoxId, StoragePoolId>; + using TColumns = TableColumns< + BoxId, + StoragePoolId, + Name, + Generation, + ErasureSpecies, + VDiskKind, + Kind, + NumGroups, + EncryptionMode, + SchemeshardId, + PathId>; + }; + + struct Tablets : Table<8> { + struct TabletId : Column<1, NScheme::NTypeIds::Uint64> {}; struct FollowerId : Column<2, NScheme::NTypeIds::Uint32> {}; - struct TypeCol : Column<3, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } }; - struct State : Column<4, NScheme::NTypeIds::String> {}; - struct VolatileState : Column<5, NScheme::NTypeIds::String> {}; - struct BootState : Column<6, NScheme::NTypeIds::String> {}; - struct Generation : Column<7, NScheme::NTypeIds::Uint32> {}; - struct NodeId : Column<8, NScheme::NTypeIds::Uint32> {}; - struct CPU : Column<9, NScheme::NTypeIds::Double> {}; - struct Memory : Column<10, NScheme::NTypeIds::Uint64> {}; - struct Network : Column<11, NScheme::NTypeIds::Uint64> {}; - + struct TypeCol : Column<3, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } }; + struct State : Column<4, NScheme::NTypeIds::String> {}; + struct VolatileState : Column<5, NScheme::NTypeIds::String> {}; + struct BootState : Column<6, NScheme::NTypeIds::String> {}; + struct Generation : Column<7, NScheme::NTypeIds::Uint32> {}; + struct NodeId : Column<8, NScheme::NTypeIds::Uint32> {}; + struct CPU : Column<9, NScheme::NTypeIds::Double> {}; + struct Memory : Column<10, NScheme::NTypeIds::Uint64> {}; + struct Network : Column<11, NScheme::NTypeIds::Uint64> {}; + using TKey = TableKey<TabletId, FollowerId>; - using TColumns = TableColumns< - TabletId, + using TColumns = TableColumns< + TabletId, FollowerId, - TypeCol, - State, - VolatileState, - BootState, - Generation, - NodeId, - CPU, - Memory, - Network>; - }; - - struct QueryMetrics : Table<9> { - struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; - struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; - struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {}; - struct Count : Column<4, NScheme::NTypeIds::Uint64> {}; - struct SumCPUTime : Column<5, NScheme::NTypeIds::Uint64> {}; - struct MinCPUTime : Column<6, NScheme::NTypeIds::Uint64> {}; - struct MaxCPUTime : Column<7, NScheme::NTypeIds::Uint64> {}; - struct SumDuration : Column<8, NScheme::NTypeIds::Interval> {}; - struct MinDuration : Column<9, NScheme::NTypeIds::Interval> {}; - struct MaxDuration : Column<10, NScheme::NTypeIds::Interval> {}; - struct MinReadRows : Column<11, NScheme::NTypeIds::Uint64> {}; - struct MaxReadRows : Column<12, NScheme::NTypeIds::Uint64> {}; - struct SumReadRows : Column<13, NScheme::NTypeIds::Uint64> {}; - struct MinReadBytes : Column<14, NScheme::NTypeIds::Uint64> {}; - struct MaxReadBytes : Column<15, NScheme::NTypeIds::Uint64> {}; - struct SumReadBytes : Column<16, NScheme::NTypeIds::Uint64> {}; - struct MinUpdateRows : Column<17, NScheme::NTypeIds::Uint64> {}; - struct MaxUpdateRows : Column<18, NScheme::NTypeIds::Uint64> {}; - struct SumUpdateRows : Column<19, NScheme::NTypeIds::Uint64> {}; - struct MinUpdateBytes : Column<20, NScheme::NTypeIds::Uint64> {}; - struct MaxUpdateBytes : Column<21, NScheme::NTypeIds::Uint64> {}; - struct SumUpdateBytes : Column<22, NScheme::NTypeIds::Uint64> {}; - struct MinDeleteRows : Column<23, NScheme::NTypeIds::Uint64> {}; - struct MaxDeleteRows : Column<24, NScheme::NTypeIds::Uint64> {}; - struct SumDeleteRows : Column<25, NScheme::NTypeIds::Uint64> {}; - struct MinRequestUnits: Column<26, NScheme::NTypeIds::Uint64> {}; - struct MaxRequestUnits: Column<27, NScheme::NTypeIds::Uint64> {}; - struct SumRequestUnits: Column<28, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<IntervalEnd, Rank>; - using TColumns = TableColumns< - IntervalEnd, - Rank, - QueryText, - Count, - SumCPUTime, MinCPUTime, MaxCPUTime, - SumDuration, MinDuration, MaxDuration, - SumReadRows, MinReadRows, MaxReadRows, - SumReadBytes, MinReadBytes, MaxReadBytes, - SumUpdateRows, MinUpdateRows, MaxUpdateRows, - SumUpdateBytes, MinUpdateBytes, MaxUpdateBytes, - SumDeleteRows, MinDeleteRows, MaxDeleteRows, - SumRequestUnits, MinRequestUnits, MaxRequestUnits>; - }; + TypeCol, + State, + VolatileState, + BootState, + Generation, + NodeId, + CPU, + Memory, + Network>; + }; + + struct QueryMetrics : Table<9> { + struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; + struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; + struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Count : Column<4, NScheme::NTypeIds::Uint64> {}; + struct SumCPUTime : Column<5, NScheme::NTypeIds::Uint64> {}; + struct MinCPUTime : Column<6, NScheme::NTypeIds::Uint64> {}; + struct MaxCPUTime : Column<7, NScheme::NTypeIds::Uint64> {}; + struct SumDuration : Column<8, NScheme::NTypeIds::Interval> {}; + struct MinDuration : Column<9, NScheme::NTypeIds::Interval> {}; + struct MaxDuration : Column<10, NScheme::NTypeIds::Interval> {}; + struct MinReadRows : Column<11, NScheme::NTypeIds::Uint64> {}; + struct MaxReadRows : Column<12, NScheme::NTypeIds::Uint64> {}; + struct SumReadRows : Column<13, NScheme::NTypeIds::Uint64> {}; + struct MinReadBytes : Column<14, NScheme::NTypeIds::Uint64> {}; + struct MaxReadBytes : Column<15, NScheme::NTypeIds::Uint64> {}; + struct SumReadBytes : Column<16, NScheme::NTypeIds::Uint64> {}; + struct MinUpdateRows : Column<17, NScheme::NTypeIds::Uint64> {}; + struct MaxUpdateRows : Column<18, NScheme::NTypeIds::Uint64> {}; + struct SumUpdateRows : Column<19, NScheme::NTypeIds::Uint64> {}; + struct MinUpdateBytes : Column<20, NScheme::NTypeIds::Uint64> {}; + struct MaxUpdateBytes : Column<21, NScheme::NTypeIds::Uint64> {}; + struct SumUpdateBytes : Column<22, NScheme::NTypeIds::Uint64> {}; + struct MinDeleteRows : Column<23, NScheme::NTypeIds::Uint64> {}; + struct MaxDeleteRows : Column<24, NScheme::NTypeIds::Uint64> {}; + struct SumDeleteRows : Column<25, NScheme::NTypeIds::Uint64> {}; + struct MinRequestUnits: Column<26, NScheme::NTypeIds::Uint64> {}; + struct MaxRequestUnits: Column<27, NScheme::NTypeIds::Uint64> {}; + struct SumRequestUnits: Column<28, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<IntervalEnd, Rank>; + using TColumns = TableColumns< + IntervalEnd, + Rank, + QueryText, + Count, + SumCPUTime, MinCPUTime, MaxCPUTime, + SumDuration, MinDuration, MaxDuration, + SumReadRows, MinReadRows, MaxReadRows, + SumReadBytes, MinReadBytes, MaxReadBytes, + SumUpdateRows, MinUpdateRows, MaxUpdateRows, + SumUpdateBytes, MinUpdateBytes, MaxUpdateBytes, + SumDeleteRows, MinDeleteRows, MaxDeleteRows, + SumRequestUnits, MinRequestUnits, MaxRequestUnits>; + }; struct PrimaryIndexStats : Table<10> { struct PathId : Column<1, NScheme::NTypeIds::Uint64> {}; @@ -418,40 +418,40 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns<PDiskFilter, ErasureSpecies, CurrentGroupsCreated, CurrentAllocatedSize, CurrentAvailableSize, AvailableGroupsToCreate, AvailableSizeToCreate>; }; -}; - -bool MaybeSystemViewPath(const TVector<TString>& path); -bool MaybeSystemViewFolderPath(const TVector<TString>& path); - -class ISystemViewResolver { -public: - virtual ~ISystemViewResolver() = default; - - enum class ETarget : ui8 { - Domain, +}; + +bool MaybeSystemViewPath(const TVector<TString>& path); +bool MaybeSystemViewFolderPath(const TVector<TString>& path); + +class ISystemViewResolver { +public: + virtual ~ISystemViewResolver() = default; + + enum class ETarget : ui8 { + Domain, SubDomain, OlapStore, OlapTable - }; - - struct TSystemViewPath { - TVector<TString> Parent; - TString ViewName; - }; - - struct TSchema { - THashMap<NTable::TTag, TSysTables::TTableColumnInfo> Columns; - TVector<NScheme::TTypeId> KeyColumnTypes; - }; - - virtual bool IsSystemViewPath(const TVector<TString>& path, TSystemViewPath& sysViewPath) const = 0; - - virtual TMaybe<TSchema> GetSystemViewSchema(const TStringBuf viewName, ETarget target) const = 0; - - virtual TVector<TString> GetSystemViewNames(ETarget target) const = 0; -}; - -ISystemViewResolver* CreateSystemViewResolver(); - -} // NSysView -} // NKikimr + }; + + struct TSystemViewPath { + TVector<TString> Parent; + TString ViewName; + }; + + struct TSchema { + THashMap<NTable::TTag, TSysTables::TTableColumnInfo> Columns; + TVector<NScheme::TTypeId> KeyColumnTypes; + }; + + virtual bool IsSystemViewPath(const TVector<TString>& path, TSystemViewPath& sysViewPath) const = 0; + + virtual TMaybe<TSchema> GetSystemViewSchema(const TStringBuf viewName, ETarget target) const = 0; + + virtual TVector<TString> GetSystemViewNames(ETarget target) const = 0; +}; + +ISystemViewResolver* CreateSystemViewResolver(); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/utils.h b/ydb/core/sys_view/common/utils.h index 799aa16075..47a367f348 100644 --- a/ydb/core/sys_view/common/utils.h +++ b/ydb/core/sys_view/common/utils.h @@ -1,40 +1,40 @@ -#pragma once - -#include <util/generic/ptr.h> -#include <util/stream/output.h> - -namespace NKikimr { -namespace NSysView { - -class TScanLimiter : public TThrRefBase { -public: - explicit TScanLimiter(i64 limit) - : Limit(limit) - {} - - bool Inc() { - i64 newValue; - i64 prevValue; - do { - prevValue = AtomicGet(Count); - Y_VERIFY(prevValue >= 0); - if (Limit && prevValue >= Limit) { - return false; - } - newValue = prevValue + 1; - } while (!AtomicCas(&Count, newValue, prevValue)); - return true; - } - - void Dec() { - i64 newValue = AtomicDecrement(Count); - Y_VERIFY(newValue >= 0); - } - -private: - const i64 Limit; - TAtomic Count = 0; -}; - -} // NSysView -} // NKikimr +#pragma once + +#include <util/generic/ptr.h> +#include <util/stream/output.h> + +namespace NKikimr { +namespace NSysView { + +class TScanLimiter : public TThrRefBase { +public: + explicit TScanLimiter(i64 limit) + : Limit(limit) + {} + + bool Inc() { + i64 newValue; + i64 prevValue; + do { + prevValue = AtomicGet(Count); + Y_VERIFY(prevValue >= 0); + if (Limit && prevValue >= Limit) { + return false; + } + newValue = prevValue + 1; + } while (!AtomicCas(&Count, newValue, prevValue)); + return true; + } + + void Dec() { + i64 newValue = AtomicDecrement(Count); + Y_VERIFY(newValue >= 0); + } + +private: + const i64 Limit; + TAtomic Count = 0; +}; + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/ya.make b/ydb/core/sys_view/common/ya.make index b26b22c278..b4ec91b3d8 100644 --- a/ydb/core/sys_view/common/ya.make +++ b/ydb/core/sys_view/common/ya.make @@ -1,26 +1,26 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - common.h - events.h - keys.h - path.h - scan_actor_base_impl.h - schema.h - schema.cpp - utils.h -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + common.h + events.h + keys.h + path.h + scan_actor_base_impl.h + schema.h + schema.cpp + utils.h +) + +PEERDIR( library/cpp/actors/core ydb/core/base ydb/core/protos ydb/core/tablet_flat -) - -END() +) + +END() diff --git a/ydb/core/sys_view/nodes/nodes.cpp b/ydb/core/sys_view/nodes/nodes.cpp index d06091779b..a2322de9f9 100644 --- a/ydb/core/sys_view/nodes/nodes.cpp +++ b/ydb/core/sys_view/nodes/nodes.cpp @@ -1,243 +1,243 @@ -#include "nodes.h" - +#include "nodes.h" + #include <ydb/core/sys_view/common/events.h> #include <ydb/core/sys_view/common/schema.h> #include <ydb/core/sys_view/common/scan_actor_base_impl.h> #include <ydb/core/node_whiteboard/node_whiteboard.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/core/hfunc.h> - -namespace NKikimr { -namespace NSysView { - -using namespace NActors; -using namespace NNodeWhiteboard; - -class TNodesScan : public TScanActorBase<TNodesScan> { -public: - using TBase = TScanActorBase<TNodesScan>; - + +namespace NKikimr { +namespace NSysView { + +using namespace NActors; +using namespace NNodeWhiteboard; + +class TNodesScan : public TScanActorBase<TNodesScan> { +public: + using TBase = TScanActorBase<TNodesScan>; + static constexpr auto ActorActivityType() { - return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; - } - + return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; + } + TNodesScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) - : TBase(ownerId, scanId, tableId, tableRange, columns) - { - const auto& cellsFrom = TableRange.From.GetCells(); - if (cellsFrom.size() == 1 && !cellsFrom[0].IsNull()) { - NodeIdFrom = cellsFrom[0].AsValue<ui32>(); - if (!TableRange.FromInclusive) { - if (NodeIdFrom < std::numeric_limits<ui32>::max()) { - ++NodeIdFrom; - } else { - IsEmptyRange = true; - } - } - } - - const auto& cellsTo = TableRange.To.GetCells(); - if (cellsTo.size() == 1 && !cellsTo[0].IsNull()) { - NodeIdTo = cellsTo[0].AsValue<ui32>(); - if (!TableRange.ToInclusive) { - if (NodeIdTo > 0) { - --NodeIdTo; - } else { - IsEmptyRange = true; - } - } - } - } - - STFUNC(StateScan) { - switch (ev->GetTypeRewrite()) { - hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); - hFunc(TEvInterconnect::TEvNodesInfo, Handle); - hFunc(TEvWhiteboard::TEvSystemStateResponse, Handle); - hFunc(TEvents::TEvUndelivered, Undelivered); - hFunc(TEvInterconnect::TEvNodeConnected, Connected); - hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected); + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) + : TBase(ownerId, scanId, tableId, tableRange, columns) + { + const auto& cellsFrom = TableRange.From.GetCells(); + if (cellsFrom.size() == 1 && !cellsFrom[0].IsNull()) { + NodeIdFrom = cellsFrom[0].AsValue<ui32>(); + if (!TableRange.FromInclusive) { + if (NodeIdFrom < std::numeric_limits<ui32>::max()) { + ++NodeIdFrom; + } else { + IsEmptyRange = true; + } + } + } + + const auto& cellsTo = TableRange.To.GetCells(); + if (cellsTo.size() == 1 && !cellsTo[0].IsNull()) { + NodeIdTo = cellsTo[0].AsValue<ui32>(); + if (!TableRange.ToInclusive) { + if (NodeIdTo > 0) { + --NodeIdTo; + } else { + IsEmptyRange = true; + } + } + } + } + + STFUNC(StateScan) { + switch (ev->GetTypeRewrite()) { + hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); + hFunc(TEvInterconnect::TEvNodesInfo, Handle); + hFunc(TEvWhiteboard::TEvSystemStateResponse, Handle); + hFunc(TEvents::TEvUndelivered, Undelivered); + hFunc(TEvInterconnect::TEvNodeConnected, Connected); + hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected); hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution); - cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); - cFunc(TEvents::TEvPoison::EventType, PassAway); - default: - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "NSysView::TNodesScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - -private: - void ProceedToScan() override { - Become(&TNodesScan::StateScan); - if (AckReceived) { - StartScan(); - } - } - - void StartScan() { - if (IsEmptyRange) { - ReplyEmptyAndDie(); - return; - } - + cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); + cFunc(TEvents::TEvPoison::EventType, PassAway); + default: + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "NSysView::TNodesScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + +private: + void ProceedToScan() override { + Become(&TNodesScan::StateScan); + if (AckReceived) { + StartScan(); + } + } + + void StartScan() { + if (IsEmptyRange) { + ReplyEmptyAndDie(); + return; + } + const TActorId nameserviceId = GetNameserviceActorId(); - Send(nameserviceId, new TEvInterconnect::TEvListNodes()); - } - - void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { - StartScan(); - } - - void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) { - THolder<TEvInterconnect::TEvNodesInfo> nodesInfo = ev->Release(); - - for (const auto& info : nodesInfo->Nodes) { - auto nodeId = info.NodeId; - - if (TenantNodes.find(nodeId) != TenantNodes.end() && - nodeId >= NodeIdFrom && - nodeId <= NodeIdTo) - { + Send(nameserviceId, new TEvInterconnect::TEvListNodes()); + } + + void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { + StartScan(); + } + + void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) { + THolder<TEvInterconnect::TEvNodesInfo> nodesInfo = ev->Release(); + + for (const auto& info : nodesInfo->Nodes) { + auto nodeId = info.NodeId; + + if (TenantNodes.find(nodeId) != TenantNodes.end() && + nodeId >= NodeIdFrom && + nodeId <= NodeIdTo) + { TActorId whiteboardId = MakeNodeWhiteboardServiceId(nodeId); - auto request = MakeHolder<TEvWhiteboard::TEvSystemStateRequest>(); - - Send(whiteboardId, request.Release(), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); - - NodesInfo.emplace(nodeId, info); - } - } - - if (NodesInfo.empty()) { - ReplyEmptyAndDie(); - } - } - - void Handle(TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) { - ui32 nodeId = ev.Get()->Cookie; - WBSystemInfo[nodeId] = ev->Release(); - RequestDone(); - } - - void Undelivered(TEvents::TEvUndelivered::TPtr& ev) { - if (ev->Get()->SourceType == TEvWhiteboard::EvSystemStateRequest) { - ui32 nodeId = ev.Get()->Cookie; - WBSystemInfo[nodeId] = nullptr; - RequestDone(); - } - } - - void Connected(TEvInterconnect::TEvNodeConnected::TPtr&) { - } - - void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { - ui32 nodeId = ev->Get()->NodeId; - WBSystemInfo[nodeId] = nullptr; - RequestDone(); - } - - void RequestDone() { - if (NodesInfo.size() != WBSystemInfo.size()) { - return; - } - - using TNodeInfo = TEvInterconnect::TNodeInfo; - using TWBInfo = TEvWhiteboard::TEvSystemStateResponse; - using TExtractor = std::function<TCell(const TNodeInfo&, const TWBInfo*)>; - using TSchema = Schema::Nodes; - - struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { - static TMaybe<ui64> GetStartTime(const TWBInfo* wbInfo) { - if (!wbInfo || wbInfo->Record.SystemStateInfoSize() != 1) { - return {}; - } - const auto& systemState = wbInfo->Record.GetSystemStateInfo(0); - if (!systemState.HasStartTime()) { - return {}; - } - return systemState.GetStartTime(); - }; - - TExtractorsMap() { - insert({TSchema::NodeId::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) { - return TCell::Make<ui32>(info.NodeId); - }}); - insert({TSchema::Address::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) { - return TCell(info.Address.data(), info.Address.size()); - }}); - insert({TSchema::Host::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) { - return TCell(info.Host.data(), info.Host.size()); - }}); - insert({TSchema::Port::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) { - return TCell::Make<ui32>(info.Port); - }}); - insert({TSchema::StartTime::ColumnId, [] (const TNodeInfo&, const TWBInfo* wbInfo) { - auto time = TExtractorsMap::GetStartTime(wbInfo); - return time ? TCell::Make<ui64>(*time * 1000u) : TCell(); - }}); - insert({TSchema::UpTime::ColumnId, [] (const TNodeInfo&, const TWBInfo* wbInfo) { - auto time = TExtractorsMap::GetStartTime(wbInfo); - if (!time) { - return TCell(); - } - auto interval = TInstant::Now().MicroSeconds() - *time * 1000; - return TCell::Make<i64>(interval > 0 ? interval : 0); - }}); - } - }; - static TExtractorsMap extractors; - - auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); - batch->Finished = true; - - TVector<TCell> cells; - for (const auto& info : NodesInfo) { - auto nodeId = info.first; - auto* wbInfo = WBSystemInfo[nodeId].Get(); - - for (auto column : Columns) { - auto extractor = extractors.find(column.Tag); - if (extractor == extractors.end()) { - cells.push_back(TCell()); - } else { - cells.push_back(extractor->second(info.second, wbInfo)); - } - } - - TArrayRef<const TCell> ref(cells); + auto request = MakeHolder<TEvWhiteboard::TEvSystemStateRequest>(); + + Send(whiteboardId, request.Release(), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); + + NodesInfo.emplace(nodeId, info); + } + } + + if (NodesInfo.empty()) { + ReplyEmptyAndDie(); + } + } + + void Handle(TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) { + ui32 nodeId = ev.Get()->Cookie; + WBSystemInfo[nodeId] = ev->Release(); + RequestDone(); + } + + void Undelivered(TEvents::TEvUndelivered::TPtr& ev) { + if (ev->Get()->SourceType == TEvWhiteboard::EvSystemStateRequest) { + ui32 nodeId = ev.Get()->Cookie; + WBSystemInfo[nodeId] = nullptr; + RequestDone(); + } + } + + void Connected(TEvInterconnect::TEvNodeConnected::TPtr&) { + } + + void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { + ui32 nodeId = ev->Get()->NodeId; + WBSystemInfo[nodeId] = nullptr; + RequestDone(); + } + + void RequestDone() { + if (NodesInfo.size() != WBSystemInfo.size()) { + return; + } + + using TNodeInfo = TEvInterconnect::TNodeInfo; + using TWBInfo = TEvWhiteboard::TEvSystemStateResponse; + using TExtractor = std::function<TCell(const TNodeInfo&, const TWBInfo*)>; + using TSchema = Schema::Nodes; + + struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { + static TMaybe<ui64> GetStartTime(const TWBInfo* wbInfo) { + if (!wbInfo || wbInfo->Record.SystemStateInfoSize() != 1) { + return {}; + } + const auto& systemState = wbInfo->Record.GetSystemStateInfo(0); + if (!systemState.HasStartTime()) { + return {}; + } + return systemState.GetStartTime(); + }; + + TExtractorsMap() { + insert({TSchema::NodeId::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) { + return TCell::Make<ui32>(info.NodeId); + }}); + insert({TSchema::Address::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) { + return TCell(info.Address.data(), info.Address.size()); + }}); + insert({TSchema::Host::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) { + return TCell(info.Host.data(), info.Host.size()); + }}); + insert({TSchema::Port::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) { + return TCell::Make<ui32>(info.Port); + }}); + insert({TSchema::StartTime::ColumnId, [] (const TNodeInfo&, const TWBInfo* wbInfo) { + auto time = TExtractorsMap::GetStartTime(wbInfo); + return time ? TCell::Make<ui64>(*time * 1000u) : TCell(); + }}); + insert({TSchema::UpTime::ColumnId, [] (const TNodeInfo&, const TWBInfo* wbInfo) { + auto time = TExtractorsMap::GetStartTime(wbInfo); + if (!time) { + return TCell(); + } + auto interval = TInstant::Now().MicroSeconds() - *time * 1000; + return TCell::Make<i64>(interval > 0 ? interval : 0); + }}); + } + }; + static TExtractorsMap extractors; + + auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); + batch->Finished = true; + + TVector<TCell> cells; + for (const auto& info : NodesInfo) { + auto nodeId = info.first; + auto* wbInfo = WBSystemInfo[nodeId].Get(); + + for (auto column : Columns) { + auto extractor = extractors.find(column.Tag); + if (extractor == extractors.end()) { + cells.push_back(TCell()); + } else { + cells.push_back(extractor->second(info.second, wbInfo)); + } + } + + TArrayRef<const TCell> ref(cells); batch->Rows.emplace_back(TOwnedCellVec::Make(ref)); - cells.clear(); - } - - SendBatch(std::move(batch)); - } - - void PassAway() override { - for (const auto& info : NodesInfo) { - Send(TActivationContext::InterconnectProxy(info.first), new TEvents::TEvUnsubscribe()); - } - TBase::PassAway(); - } - -private: - ui32 NodeIdFrom = 0; - ui32 NodeIdTo = std::numeric_limits<ui32>::max(); - bool IsEmptyRange = false; - - TMap<ui32, TEvInterconnect::TNodeInfo> NodesInfo; - THashMap<TNodeId, THolder<TEvWhiteboard::TEvSystemStateResponse>> WBSystemInfo; -}; - + cells.clear(); + } + + SendBatch(std::move(batch)); + } + + void PassAway() override { + for (const auto& info : NodesInfo) { + Send(TActivationContext::InterconnectProxy(info.first), new TEvents::TEvUnsubscribe()); + } + TBase::PassAway(); + } + +private: + ui32 NodeIdFrom = 0; + ui32 NodeIdTo = std::numeric_limits<ui32>::max(); + bool IsEmptyRange = false; + + TMap<ui32, TEvInterconnect::TNodeInfo> NodesInfo; + THashMap<TNodeId, THolder<TEvWhiteboard::TEvSystemStateResponse>> WBSystemInfo; +}; + THolder<IActor> CreateNodesScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ return MakeHolder<TNodesScan>(ownerId, scanId, tableId, tableRange, columns); -} - -} // NSysView -} // NKikimr +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/nodes/nodes.h b/ydb/core/sys_view/nodes/nodes.h index e118555f28..8f1192250b 100644 --- a/ydb/core/sys_view/nodes/nodes.h +++ b/ydb/core/sys_view/nodes/nodes.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + THolder<IActor> CreateNodesScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/nodes/ya.make b/ydb/core/sys_view/nodes/ya.make index 02276dd4f6..3dc17dd412 100644 --- a/ydb/core/sys_view/nodes/ya.make +++ b/ydb/core/sys_view/nodes/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - nodes.h - nodes.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + nodes.h + nodes.cpp +) + +PEERDIR( library/cpp/actors/core ydb/core/base ydb/core/kqp/runtime ydb/core/sys_view/common -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/sys_view/partition_stats/partition_stats.cpp b/ydb/core/sys_view/partition_stats/partition_stats.cpp index 74e562f185..d6ea9ff73b 100644 --- a/ydb/core/sys_view/partition_stats/partition_stats.cpp +++ b/ydb/core/sys_view/partition_stats/partition_stats.cpp @@ -1,104 +1,104 @@ -#include "partition_stats.h" - +#include "partition_stats.h" + #include <ydb/core/sys_view/common/events.h> #include <ydb/core/sys_view/common/schema.h> #include <ydb/core/sys_view/common/scan_actor_base_impl.h> #include <ydb/core/base/tablet_pipecache.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> #include <library/cpp/actors/core/hfunc.h> - -namespace NKikimr { -namespace NSysView { - -using namespace NActors; - -class TPartitionStatsCollector : public TActor<TPartitionStatsCollector> { -public: + +namespace NKikimr { +namespace NSysView { + +using namespace NActors; + +class TPartitionStatsCollector : public TActor<TPartitionStatsCollector> { +public: static constexpr auto ActorActivityType() { - return NKikimrServices::TActivity::SYSTEM_VIEW_PART_STATS_COLLECTOR; - } - - explicit TPartitionStatsCollector(size_t batchSize, size_t pendingRequestsLimit) - : TActor(&TPartitionStatsCollector::StateWork) - , BatchSize(batchSize) - , PendingRequestsLimit(pendingRequestsLimit) - {} - - STFUNC(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvSysView::TEvSetPartitioning, Handle); - hFunc(TEvSysView::TEvRemoveTable, Handle); - hFunc(TEvSysView::TEvSendPartitionStats, Handle); + return NKikimrServices::TActivity::SYSTEM_VIEW_PART_STATS_COLLECTOR; + } + + explicit TPartitionStatsCollector(size_t batchSize, size_t pendingRequestsLimit) + : TActor(&TPartitionStatsCollector::StateWork) + , BatchSize(batchSize) + , PendingRequestsLimit(pendingRequestsLimit) + {} + + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvSysView::TEvSetPartitioning, Handle); + hFunc(TEvSysView::TEvRemoveTable, Handle); + hFunc(TEvSysView::TEvSendPartitionStats, Handle); hFunc(TEvSysView::TEvUpdateTtlStats, Handle); - hFunc(TEvSysView::TEvGetPartitionStats, Handle); - hFunc(TEvPrivate::TEvProcess, Handle); - cFunc(TEvents::TEvPoison::EventType, PassAway); - default: - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "NSysView::TPartitionStatsCollector: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - -private: - struct TEvPrivate { - enum EEv { - EvProcess = EventSpaceBegin(TEvents::ES_PRIVATE), - - EvEnd - }; - - struct TEvProcess : public TEventLocal<TEvProcess, EvProcess> {}; - }; - - void Handle(TEvSysView::TEvSetPartitioning::TPtr& ev) { - const auto& domainKey = ev->Get()->DomainKey; - const auto& pathId = ev->Get()->PathId; - - auto& tables = DomainTables[domainKey]; - auto it = tables.find(pathId); - if (it != tables.end()) { - auto& oldPartitions = it->second.Partitions; - THashMap<TShardIdx, NKikimrSysView::TPartitionStats> newPartitions; - - for (auto shardIdx : ev->Get()->ShardIndices) { - auto old = oldPartitions.find(shardIdx); - if (old != oldPartitions.end()) { - newPartitions[shardIdx] = old->second; - } - } - - oldPartitions.swap(newPartitions); - it->second.ShardIndices.swap(ev->Get()->ShardIndices); - it->second.Path = ev->Get()->Path; - - } else { - auto& table = tables[pathId]; - table.ShardIndices.swap(ev->Get()->ShardIndices); - table.Path = ev->Get()->Path; - } - } - - void Handle(TEvSysView::TEvRemoveTable::TPtr& ev) { - const auto& domainKey = ev->Get()->DomainKey; - const auto& pathId = ev->Get()->PathId; - - auto& tables = DomainTables[domainKey]; - tables.erase(pathId); - } - - void Handle(TEvSysView::TEvSendPartitionStats::TPtr& ev) { - const auto& domainKey = ev->Get()->DomainKey; - const auto& pathId = ev->Get()->PathId; - const auto& shardIdx = ev->Get()->ShardIdx; - - auto& tables = DomainTables[domainKey]; - auto it = tables.find(pathId); - if (it == tables.end()) { - return; - } - + hFunc(TEvSysView::TEvGetPartitionStats, Handle); + hFunc(TEvPrivate::TEvProcess, Handle); + cFunc(TEvents::TEvPoison::EventType, PassAway); + default: + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "NSysView::TPartitionStatsCollector: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + +private: + struct TEvPrivate { + enum EEv { + EvProcess = EventSpaceBegin(TEvents::ES_PRIVATE), + + EvEnd + }; + + struct TEvProcess : public TEventLocal<TEvProcess, EvProcess> {}; + }; + + void Handle(TEvSysView::TEvSetPartitioning::TPtr& ev) { + const auto& domainKey = ev->Get()->DomainKey; + const auto& pathId = ev->Get()->PathId; + + auto& tables = DomainTables[domainKey]; + auto it = tables.find(pathId); + if (it != tables.end()) { + auto& oldPartitions = it->second.Partitions; + THashMap<TShardIdx, NKikimrSysView::TPartitionStats> newPartitions; + + for (auto shardIdx : ev->Get()->ShardIndices) { + auto old = oldPartitions.find(shardIdx); + if (old != oldPartitions.end()) { + newPartitions[shardIdx] = old->second; + } + } + + oldPartitions.swap(newPartitions); + it->second.ShardIndices.swap(ev->Get()->ShardIndices); + it->second.Path = ev->Get()->Path; + + } else { + auto& table = tables[pathId]; + table.ShardIndices.swap(ev->Get()->ShardIndices); + table.Path = ev->Get()->Path; + } + } + + void Handle(TEvSysView::TEvRemoveTable::TPtr& ev) { + const auto& domainKey = ev->Get()->DomainKey; + const auto& pathId = ev->Get()->PathId; + + auto& tables = DomainTables[domainKey]; + tables.erase(pathId); + } + + void Handle(TEvSysView::TEvSendPartitionStats::TPtr& ev) { + const auto& domainKey = ev->Get()->DomainKey; + const auto& pathId = ev->Get()->PathId; + const auto& shardIdx = ev->Get()->ShardIdx; + + auto& tables = DomainTables[domainKey]; + auto it = tables.find(pathId); + if (it == tables.end()) { + return; + } + auto& oldStats = it->second.Partitions[shardIdx]; auto& newStats = ev->Get()->Stats; @@ -107,8 +107,8 @@ private: } oldStats.Swap(&newStats); - } - + } + void Handle(TEvSysView::TEvUpdateTtlStats::TPtr& ev) { const auto& domainKey = ev->Get()->DomainKey; const auto& pathId = ev->Get()->PathId; @@ -123,373 +123,373 @@ private: it->second.Partitions[shardIdx].MutableTtlStats()->Swap(&ev->Get()->Stats); } - void Handle(TEvSysView::TEvGetPartitionStats::TPtr& ev) { - if (PendingRequests.size() >= PendingRequestsLimit) { - auto result = MakeHolder<TEvSysView::TEvGetPartitionStatsResult>(); - result->Record.SetOverloaded(true); - Send(ev->Sender, std::move(result)); - return; - } - - PendingRequests.push(std::move(ev)); - - if (!ProcessInFly) { - Send(SelfId(), new TEvPrivate::TEvProcess()); - ProcessInFly = true; - } - } - - void Handle(TEvPrivate::TEvProcess::TPtr&) { - ProcessInFly = false; - - if (PendingRequests.empty()) { - return; - } - - TEvSysView::TEvGetPartitionStats::TPtr request = std::move(PendingRequests.front()); - PendingRequests.pop(); - - if (!PendingRequests.empty()) { - Send(SelfId(), new TEvPrivate::TEvProcess()); - ProcessInFly = true; - } - - auto& record = request->Get()->Record; - - auto result = MakeHolder<TEvSysView::TEvGetPartitionStatsResult>(); - result->Record.SetLastBatch(true); - - if (!record.HasDomainKeyOwnerId() || !record.HasDomainKeyPathId()) { - Send(request->Sender, std::move(result)); - return; - } - - auto domainKey = TPathId(record.GetDomainKeyOwnerId(), record.GetDomainKeyPathId()); - auto itTables = DomainTables.find(domainKey); - if (itTables == DomainTables.end()) { - Send(request->Sender, std::move(result)); - return; - } - auto& tables = itTables->second; - - auto it = tables.begin(); - auto itEnd = tables.end(); - - bool fromInclusive = record.HasFromInclusive() && record.GetFromInclusive(); - bool toInclusive = record.HasToInclusive() && record.GetToInclusive(); - - TPathId fromPathId; - TPathId toPathId; - - ui64 startPartIdx = 0; - ui64 endPartIdx = 0; - - auto& from = record.GetFrom(); - - if (from.HasOwnerId()) { - if (from.HasPathId()) { - fromPathId = TPathId(from.GetOwnerId(), from.GetPathId()); - if (fromInclusive || from.HasPartIdx()) { - it = tables.lower_bound(fromPathId); - if (it != tables.end() && it->first == fromPathId && from.GetPartIdx()) { - startPartIdx = from.GetPartIdx(); - if (!fromInclusive) { - ++startPartIdx; - } - } - } else { - it = tables.upper_bound(fromPathId); - } - } else { - if (fromInclusive) { - fromPathId = TPathId(from.GetOwnerId(), 0); - it = tables.lower_bound(fromPathId); - } else { - fromPathId = TPathId(from.GetOwnerId(), std::numeric_limits<ui64>::max()); - it = tables.upper_bound(fromPathId); - } - } - } - - auto& to = record.GetTo(); - - if (to.HasOwnerId()) { - if (to.HasPathId()) { - toPathId = TPathId(to.GetOwnerId(), to.GetPathId()); - if (toInclusive || to.HasPartIdx()) { - itEnd = tables.upper_bound(toPathId); - if (to.HasPartIdx()) { - endPartIdx = to.GetPartIdx(); - if (toInclusive) { - ++endPartIdx; - } - } - } else { - itEnd = tables.lower_bound(toPathId); - } - } else { - if (toInclusive) { - toPathId = TPathId(to.GetOwnerId(), std::numeric_limits<ui64>::max()); - itEnd = tables.upper_bound(toPathId); - } else { - toPathId = TPathId(to.GetOwnerId(), 0); - itEnd = tables.lower_bound(toPathId); - } - } - } - - bool includePathColumn = !record.HasIncludePathColumn() || record.GetIncludePathColumn(); - - for (size_t count = 0; count < BatchSize && it != itEnd; ++it) { - auto& pathId = it->first; - const auto& tableStats = it->second; - - ui64 end = tableStats.ShardIndices.size(); - if (to.HasPartIdx() && pathId == toPathId) { - end = std::min(endPartIdx, end); - } - - bool batchFinished = false; - - for (ui64 partIdx = startPartIdx; partIdx < end; ++partIdx) { - auto* stats = result->Record.AddStats(); - auto* key = stats->MutableKey(); - - key->SetOwnerId(pathId.OwnerId); - key->SetPathId(pathId.LocalPathId); - key->SetPartIdx(partIdx); - - if (includePathColumn) { - stats->SetPath(tableStats.Path); - } - - auto shardIdx = tableStats.ShardIndices[partIdx]; - auto part = tableStats.Partitions.find(shardIdx); - if (part != tableStats.Partitions.end()) { - *stats->MutableStats() = part->second; - } - - if (++count == BatchSize) { - auto* next = result->Record.MutableNext(); - next->SetOwnerId(pathId.OwnerId); - next->SetPathId(pathId.LocalPathId); - next->SetPartIdx(partIdx + 1); - result->Record.SetLastBatch(false); - batchFinished = true; - break; - } - } - - if (batchFinished) { - break; - } - - startPartIdx = 0; - } - - Send(request->Sender, std::move(result)); - } - -private: - const size_t BatchSize; - const size_t PendingRequestsLimit; - - struct TTableStats { - THashMap<TShardIdx, NKikimrSysView::TPartitionStats> Partitions; // shardIdx -> stats - TVector<TShardIdx> ShardIndices; - TString Path; - }; - - using TDomainTables = TMap<TPathId, TTableStats>; - THashMap<TPathId, TDomainTables> DomainTables; - - TQueue<TEvSysView::TEvGetPartitionStats::TPtr> PendingRequests; - bool ProcessInFly = false; -}; - -THolder<IActor> CreatePartitionStatsCollector(size_t batchSize, size_t pendingRequestsLimit) { + void Handle(TEvSysView::TEvGetPartitionStats::TPtr& ev) { + if (PendingRequests.size() >= PendingRequestsLimit) { + auto result = MakeHolder<TEvSysView::TEvGetPartitionStatsResult>(); + result->Record.SetOverloaded(true); + Send(ev->Sender, std::move(result)); + return; + } + + PendingRequests.push(std::move(ev)); + + if (!ProcessInFly) { + Send(SelfId(), new TEvPrivate::TEvProcess()); + ProcessInFly = true; + } + } + + void Handle(TEvPrivate::TEvProcess::TPtr&) { + ProcessInFly = false; + + if (PendingRequests.empty()) { + return; + } + + TEvSysView::TEvGetPartitionStats::TPtr request = std::move(PendingRequests.front()); + PendingRequests.pop(); + + if (!PendingRequests.empty()) { + Send(SelfId(), new TEvPrivate::TEvProcess()); + ProcessInFly = true; + } + + auto& record = request->Get()->Record; + + auto result = MakeHolder<TEvSysView::TEvGetPartitionStatsResult>(); + result->Record.SetLastBatch(true); + + if (!record.HasDomainKeyOwnerId() || !record.HasDomainKeyPathId()) { + Send(request->Sender, std::move(result)); + return; + } + + auto domainKey = TPathId(record.GetDomainKeyOwnerId(), record.GetDomainKeyPathId()); + auto itTables = DomainTables.find(domainKey); + if (itTables == DomainTables.end()) { + Send(request->Sender, std::move(result)); + return; + } + auto& tables = itTables->second; + + auto it = tables.begin(); + auto itEnd = tables.end(); + + bool fromInclusive = record.HasFromInclusive() && record.GetFromInclusive(); + bool toInclusive = record.HasToInclusive() && record.GetToInclusive(); + + TPathId fromPathId; + TPathId toPathId; + + ui64 startPartIdx = 0; + ui64 endPartIdx = 0; + + auto& from = record.GetFrom(); + + if (from.HasOwnerId()) { + if (from.HasPathId()) { + fromPathId = TPathId(from.GetOwnerId(), from.GetPathId()); + if (fromInclusive || from.HasPartIdx()) { + it = tables.lower_bound(fromPathId); + if (it != tables.end() && it->first == fromPathId && from.GetPartIdx()) { + startPartIdx = from.GetPartIdx(); + if (!fromInclusive) { + ++startPartIdx; + } + } + } else { + it = tables.upper_bound(fromPathId); + } + } else { + if (fromInclusive) { + fromPathId = TPathId(from.GetOwnerId(), 0); + it = tables.lower_bound(fromPathId); + } else { + fromPathId = TPathId(from.GetOwnerId(), std::numeric_limits<ui64>::max()); + it = tables.upper_bound(fromPathId); + } + } + } + + auto& to = record.GetTo(); + + if (to.HasOwnerId()) { + if (to.HasPathId()) { + toPathId = TPathId(to.GetOwnerId(), to.GetPathId()); + if (toInclusive || to.HasPartIdx()) { + itEnd = tables.upper_bound(toPathId); + if (to.HasPartIdx()) { + endPartIdx = to.GetPartIdx(); + if (toInclusive) { + ++endPartIdx; + } + } + } else { + itEnd = tables.lower_bound(toPathId); + } + } else { + if (toInclusive) { + toPathId = TPathId(to.GetOwnerId(), std::numeric_limits<ui64>::max()); + itEnd = tables.upper_bound(toPathId); + } else { + toPathId = TPathId(to.GetOwnerId(), 0); + itEnd = tables.lower_bound(toPathId); + } + } + } + + bool includePathColumn = !record.HasIncludePathColumn() || record.GetIncludePathColumn(); + + for (size_t count = 0; count < BatchSize && it != itEnd; ++it) { + auto& pathId = it->first; + const auto& tableStats = it->second; + + ui64 end = tableStats.ShardIndices.size(); + if (to.HasPartIdx() && pathId == toPathId) { + end = std::min(endPartIdx, end); + } + + bool batchFinished = false; + + for (ui64 partIdx = startPartIdx; partIdx < end; ++partIdx) { + auto* stats = result->Record.AddStats(); + auto* key = stats->MutableKey(); + + key->SetOwnerId(pathId.OwnerId); + key->SetPathId(pathId.LocalPathId); + key->SetPartIdx(partIdx); + + if (includePathColumn) { + stats->SetPath(tableStats.Path); + } + + auto shardIdx = tableStats.ShardIndices[partIdx]; + auto part = tableStats.Partitions.find(shardIdx); + if (part != tableStats.Partitions.end()) { + *stats->MutableStats() = part->second; + } + + if (++count == BatchSize) { + auto* next = result->Record.MutableNext(); + next->SetOwnerId(pathId.OwnerId); + next->SetPathId(pathId.LocalPathId); + next->SetPartIdx(partIdx + 1); + result->Record.SetLastBatch(false); + batchFinished = true; + break; + } + } + + if (batchFinished) { + break; + } + + startPartIdx = 0; + } + + Send(request->Sender, std::move(result)); + } + +private: + const size_t BatchSize; + const size_t PendingRequestsLimit; + + struct TTableStats { + THashMap<TShardIdx, NKikimrSysView::TPartitionStats> Partitions; // shardIdx -> stats + TVector<TShardIdx> ShardIndices; + TString Path; + }; + + using TDomainTables = TMap<TPathId, TTableStats>; + THashMap<TPathId, TDomainTables> DomainTables; + + TQueue<TEvSysView::TEvGetPartitionStats::TPtr> PendingRequests; + bool ProcessInFly = false; +}; + +THolder<IActor> CreatePartitionStatsCollector(size_t batchSize, size_t pendingRequestsLimit) { return MakeHolder<TPartitionStatsCollector>(batchSize, pendingRequestsLimit); -} - -class TPartitionStatsScan : public TScanActorBase<TPartitionStatsScan> { -public: - using TBase = TScanActorBase<TPartitionStatsScan>; - +} + +class TPartitionStatsScan : public TScanActorBase<TPartitionStatsScan> { +public: + using TBase = TScanActorBase<TPartitionStatsScan>; + static constexpr auto ActorActivityType() { - return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; - } - + return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; + } + TPartitionStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) - : TBase(ownerId, scanId, tableId, tableRange, columns) - { - auto extractKey = [] (NKikimrSysView::TPartitionStatsKey& key, const TConstArrayRef<TCell>& cells) { - if (cells.size() > 0 && !cells[0].IsNull()) { - key.SetOwnerId(cells[0].AsValue<ui64>()); - } - if (cells.size() > 1 && !cells[1].IsNull()) { - key.SetPathId(cells[1].AsValue<ui64>()); - } - if (cells.size() > 2 && !cells[2].IsNull()) { - key.SetPartIdx(cells[2].AsValue<ui64>()); - } - }; - - extractKey(From, TableRange.From.GetCells()); - FromInclusive = TableRange.FromInclusive; - - extractKey(To, TableRange.To.GetCells()); - ToInclusive = TableRange.ToInclusive; - - for (auto& column : columns) { - if (column.Tag == Schema::PartitionStats::Path::ColumnId) { - IncludePathColumn = true; - break; - } - } - } - - STFUNC(StateScan) { - switch (ev->GetTypeRewrite()) { - hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); - hFunc(TEvSysView::TEvGetPartitionStatsResult, Handle); - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) + : TBase(ownerId, scanId, tableId, tableRange, columns) + { + auto extractKey = [] (NKikimrSysView::TPartitionStatsKey& key, const TConstArrayRef<TCell>& cells) { + if (cells.size() > 0 && !cells[0].IsNull()) { + key.SetOwnerId(cells[0].AsValue<ui64>()); + } + if (cells.size() > 1 && !cells[1].IsNull()) { + key.SetPathId(cells[1].AsValue<ui64>()); + } + if (cells.size() > 2 && !cells[2].IsNull()) { + key.SetPartIdx(cells[2].AsValue<ui64>()); + } + }; + + extractKey(From, TableRange.From.GetCells()); + FromInclusive = TableRange.FromInclusive; + + extractKey(To, TableRange.To.GetCells()); + ToInclusive = TableRange.ToInclusive; + + for (auto& column : columns) { + if (column.Tag == Schema::PartitionStats::Path::ColumnId) { + IncludePathColumn = true; + break; + } + } + } + + STFUNC(StateScan) { + switch (ev->GetTypeRewrite()) { + hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); + hFunc(TEvSysView::TEvGetPartitionStatsResult, Handle); + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution); - cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); - cFunc(TEvents::TEvPoison::EventType, PassAway); - default: - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "NSysView::TPartitionStatsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - -private: - void ProceedToScan() override { - Become(&TThis::StateScan); - if (AckReceived) { - RequestBatch(); - } - } - - void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { - RequestBatch(); - } - - void RequestBatch() { - if (BatchRequestInFlight) { - return; - } - - auto request = MakeHolder<TEvSysView::TEvGetPartitionStats>(); - - request->Record.SetDomainKeyOwnerId(DomainKey.OwnerId); - request->Record.SetDomainKeyPathId(DomainKey.LocalPathId); - - request->Record.MutableFrom()->CopyFrom(From); - request->Record.SetFromInclusive(FromInclusive); - request->Record.MutableTo()->CopyFrom(To); - request->Record.SetToInclusive(ToInclusive); - - request->Record.SetIncludePathColumn(IncludePathColumn); - - auto pipeCache = MakePipePeNodeCacheID(false); - Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), SchemeShardId, true), - IEventHandle::FlagTrackDelivery); - - BatchRequestInFlight = true; - } - - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { - ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in partition stats scan"); - } - - void Handle(TEvSysView::TEvGetPartitionStatsResult::TPtr& ev) { - auto& record = ev->Get()->Record; - - if (record.HasOverloaded() && record.GetOverloaded()) { - ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Partition stats collector is overloaded"); - return; - } - - using TPartitionStats = NKikimrSysView::TPartitionStatsResult; - using TExtractor = std::function<TCell(const TPartitionStats&)>; - using TSchema = Schema::PartitionStats; - - struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { - TExtractorsMap() { - insert({TSchema::OwnerId::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetKey().GetOwnerId()); - }}); - insert({TSchema::PathId::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetKey().GetPathId()); - }}); - insert({TSchema::PartIdx::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetKey().GetPartIdx()); - }}); - insert({TSchema::DataSize::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetDataSize()); - }}); - insert({TSchema::RowCount::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetRowCount()); - }}); - insert({TSchema::IndexSize::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetIndexSize()); - }}); - insert({TSchema::CPUCores::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<double>(s.GetStats().GetCPUCores()); - }}); - insert({TSchema::TabletId::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetTabletId()); - }}); - insert({TSchema::Path::ColumnId, [] (const TPartitionStats& s) { - if (!s.HasPath()) { - return TCell(); - } - auto& path = s.GetPath(); - return TCell(path.data(), path.size()); - }}); - insert({TSchema::NodeId::ColumnId, [] (const TPartitionStats& s) { - return s.GetStats().HasNodeId() ? TCell::Make<ui32>(s.GetStats().GetNodeId()) : TCell(); - }}); - insert({TSchema::StartTime::ColumnId, [] (const TPartitionStats& s) { - return s.GetStats().HasStartTime() ? TCell::Make<ui64>(s.GetStats().GetStartTime() * 1000) : TCell(); - }}); - insert({TSchema::AccessTime::ColumnId, [] (const TPartitionStats& s) { - return s.GetStats().HasAccessTime() ? TCell::Make<ui64>(s.GetStats().GetAccessTime() * 1000) : TCell(); - }}); - insert({TSchema::UpdateTime::ColumnId, [] (const TPartitionStats& s) { - return s.GetStats().HasUpdateTime() ? TCell::Make<ui64>(s.GetStats().GetUpdateTime() * 1000) : TCell(); - }}); - insert({TSchema::InFlightTxCount::ColumnId, [] (const TPartitionStats& s) { + cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); + cFunc(TEvents::TEvPoison::EventType, PassAway); + default: + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "NSysView::TPartitionStatsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + +private: + void ProceedToScan() override { + Become(&TThis::StateScan); + if (AckReceived) { + RequestBatch(); + } + } + + void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { + RequestBatch(); + } + + void RequestBatch() { + if (BatchRequestInFlight) { + return; + } + + auto request = MakeHolder<TEvSysView::TEvGetPartitionStats>(); + + request->Record.SetDomainKeyOwnerId(DomainKey.OwnerId); + request->Record.SetDomainKeyPathId(DomainKey.LocalPathId); + + request->Record.MutableFrom()->CopyFrom(From); + request->Record.SetFromInclusive(FromInclusive); + request->Record.MutableTo()->CopyFrom(To); + request->Record.SetToInclusive(ToInclusive); + + request->Record.SetIncludePathColumn(IncludePathColumn); + + auto pipeCache = MakePipePeNodeCacheID(false); + Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), SchemeShardId, true), + IEventHandle::FlagTrackDelivery); + + BatchRequestInFlight = true; + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { + ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in partition stats scan"); + } + + void Handle(TEvSysView::TEvGetPartitionStatsResult::TPtr& ev) { + auto& record = ev->Get()->Record; + + if (record.HasOverloaded() && record.GetOverloaded()) { + ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Partition stats collector is overloaded"); + return; + } + + using TPartitionStats = NKikimrSysView::TPartitionStatsResult; + using TExtractor = std::function<TCell(const TPartitionStats&)>; + using TSchema = Schema::PartitionStats; + + struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { + TExtractorsMap() { + insert({TSchema::OwnerId::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetKey().GetOwnerId()); + }}); + insert({TSchema::PathId::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetKey().GetPathId()); + }}); + insert({TSchema::PartIdx::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetKey().GetPartIdx()); + }}); + insert({TSchema::DataSize::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetDataSize()); + }}); + insert({TSchema::RowCount::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetRowCount()); + }}); + insert({TSchema::IndexSize::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetIndexSize()); + }}); + insert({TSchema::CPUCores::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<double>(s.GetStats().GetCPUCores()); + }}); + insert({TSchema::TabletId::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetTabletId()); + }}); + insert({TSchema::Path::ColumnId, [] (const TPartitionStats& s) { + if (!s.HasPath()) { + return TCell(); + } + auto& path = s.GetPath(); + return TCell(path.data(), path.size()); + }}); + insert({TSchema::NodeId::ColumnId, [] (const TPartitionStats& s) { + return s.GetStats().HasNodeId() ? TCell::Make<ui32>(s.GetStats().GetNodeId()) : TCell(); + }}); + insert({TSchema::StartTime::ColumnId, [] (const TPartitionStats& s) { + return s.GetStats().HasStartTime() ? TCell::Make<ui64>(s.GetStats().GetStartTime() * 1000) : TCell(); + }}); + insert({TSchema::AccessTime::ColumnId, [] (const TPartitionStats& s) { + return s.GetStats().HasAccessTime() ? TCell::Make<ui64>(s.GetStats().GetAccessTime() * 1000) : TCell(); + }}); + insert({TSchema::UpdateTime::ColumnId, [] (const TPartitionStats& s) { + return s.GetStats().HasUpdateTime() ? TCell::Make<ui64>(s.GetStats().GetUpdateTime() * 1000) : TCell(); + }}); + insert({TSchema::InFlightTxCount::ColumnId, [] (const TPartitionStats& s) { return TCell::Make<ui32>(s.GetStats().GetInFlightTxCount()); - }}); - insert({TSchema::RowUpdates::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetRowUpdates()); - }}); - insert({TSchema::RowDeletes::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetRowDeletes()); - }}); - insert({TSchema::RowReads::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetRowReads()); - }}); - insert({TSchema::RangeReads::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetRangeReads()); - }}); - insert({TSchema::RangeReadRows::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetRangeReadRows()); - }}); - insert({TSchema::ImmediateTxCompleted::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetImmediateTxCompleted()); - }}); - insert({TSchema::CoordinatedTxCompleted::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetPlannedTxCompleted()); - }}); - insert({TSchema::TxRejectedByOverload::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetTxRejectedByOverload()); - }}); - insert({TSchema::TxRejectedByOutOfStorage::ColumnId, [] (const TPartitionStats& s) { - return TCell::Make<ui64>(s.GetStats().GetTxRejectedBySpace()); - }}); + }}); + insert({TSchema::RowUpdates::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetRowUpdates()); + }}); + insert({TSchema::RowDeletes::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetRowDeletes()); + }}); + insert({TSchema::RowReads::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetRowReads()); + }}); + insert({TSchema::RangeReads::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetRangeReads()); + }}); + insert({TSchema::RangeReadRows::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetRangeReadRows()); + }}); + insert({TSchema::ImmediateTxCompleted::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetImmediateTxCompleted()); + }}); + insert({TSchema::CoordinatedTxCompleted::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetPlannedTxCompleted()); + }}); + insert({TSchema::TxRejectedByOverload::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetTxRejectedByOverload()); + }}); + insert({TSchema::TxRejectedByOutOfStorage::ColumnId, [] (const TPartitionStats& s) { + return TCell::Make<ui64>(s.GetStats().GetTxRejectedBySpace()); + }}); insert({TSchema::LastTtlRunTime::ColumnId, [] (const TPartitionStats& s) { return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRunTime() * 1000) : TCell(); }}); @@ -499,59 +499,59 @@ private: insert({TSchema::LastTtlRowsErased::ColumnId, [] (const TPartitionStats& s) { return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRowsErased()) : TCell(); }}); - } - }; - static TExtractorsMap extractors; - - auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); - - TVector<TCell> cells; - for (const auto& s : record.GetStats()) { - for (auto& column : Columns) { - auto extractor = extractors.find(column.Tag); - if (extractor == extractors.end()) { - cells.push_back(TCell()); - } else { - cells.push_back(extractor->second(s)); - } - } - TArrayRef<const TCell> ref(cells); + } + }; + static TExtractorsMap extractors; + + auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); + + TVector<TCell> cells; + for (const auto& s : record.GetStats()) { + for (auto& column : Columns) { + auto extractor = extractors.find(column.Tag); + if (extractor == extractors.end()) { + cells.push_back(TCell()); + } else { + cells.push_back(extractor->second(s)); + } + } + TArrayRef<const TCell> ref(cells); batch->Rows.emplace_back(TOwnedCellVec::Make(ref)); - cells.clear(); - } - - batch->Finished = record.GetLastBatch(); - if (!batch->Finished) { - Y_VERIFY(record.HasNext()); - From = record.GetNext(); - FromInclusive = true; - } - - SendBatch(std::move(batch)); - - BatchRequestInFlight = false; - } - - void PassAway() override { - Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); - TBase::PassAway(); - } - -private: - NKikimrSysView::TPartitionStatsKey From; - bool FromInclusive = false; - - NKikimrSysView::TPartitionStatsKey To; - bool ToInclusive = false; - - bool IncludePathColumn = false; -}; - + cells.clear(); + } + + batch->Finished = record.GetLastBatch(); + if (!batch->Finished) { + Y_VERIFY(record.HasNext()); + From = record.GetNext(); + FromInclusive = true; + } + + SendBatch(std::move(batch)); + + BatchRequestInFlight = false; + } + + void PassAway() override { + Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + TBase::PassAway(); + } + +private: + NKikimrSysView::TPartitionStatsKey From; + bool FromInclusive = false; + + NKikimrSysView::TPartitionStatsKey To; + bool ToInclusive = false; + + bool IncludePathColumn = false; +}; + THolder<IActor> CreatePartitionStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ return MakeHolder<TPartitionStatsScan>(ownerId, scanId, tableId, tableRange, columns); -} - -} // NSysView -} // NKikimr +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/partition_stats/partition_stats.h b/ydb/core/sys_view/partition_stats/partition_stats.h index 48348b061d..dedf62906f 100644 --- a/ydb/core/sys_view/partition_stats/partition_stats.h +++ b/ydb/core/sys_view/partition_stats/partition_stats.h @@ -1,19 +1,19 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - -constexpr size_t STATS_COLLECTOR_BATCH_SIZE = 5000; -constexpr size_t STATS_COLLECTOR_QUEUE_SIZE_LIMIT = 10; - -THolder<IActor> CreatePartitionStatsCollector( - size_t batchSize = STATS_COLLECTOR_BATCH_SIZE, - size_t pendingRequestsCount = STATS_COLLECTOR_QUEUE_SIZE_LIMIT); - + +namespace NKikimr { +namespace NSysView { + +constexpr size_t STATS_COLLECTOR_BATCH_SIZE = 5000; +constexpr size_t STATS_COLLECTOR_QUEUE_SIZE_LIMIT = 10; + +THolder<IActor> CreatePartitionStatsCollector( + size_t batchSize = STATS_COLLECTOR_BATCH_SIZE, + size_t pendingRequestsCount = STATS_COLLECTOR_QUEUE_SIZE_LIMIT); + THolder<IActor> CreatePartitionStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/partition_stats/partition_stats_ut.cpp b/ydb/core/sys_view/partition_stats/partition_stats_ut.cpp index 25baccca3f..c123012948 100644 --- a/ydb/core/sys_view/partition_stats/partition_stats_ut.cpp +++ b/ydb/core/sys_view/partition_stats/partition_stats_ut.cpp @@ -1,200 +1,200 @@ -#include "partition_stats.h" - +#include "partition_stats.h" + #include <ydb/core/testlib/actors/test_runtime.h> #include <ydb/core/base/appdata.h> #include <ydb/core/sys_view/common/events.h> - + #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { -namespace NSysView { - -Y_UNIT_TEST_SUITE(PartitionStats) { - - TTestActorRuntime::TEgg MakeEgg() - { - return { new TAppData(0, 0, 0, 0, { }, nullptr, nullptr, nullptr, nullptr), nullptr, nullptr }; - } - - void TestCollector(size_t batchSize) { - TTestActorRuntime runtime; - runtime.Initialize(MakeEgg()); - - auto collector = CreatePartitionStatsCollector(batchSize); - auto collectorId = runtime.Register(collector.Release()); - auto sender = runtime.AllocateEdgeActor(); - - auto domainKey = TPathId(1, 1); - - for (ui64 ownerId = 0; ownerId <= 1; ++ownerId) { - for (ui64 pathId = 0; pathId <= 1; ++pathId) { - auto id = TPathId(ownerId, pathId); - auto setPartitioning = MakeHolder<TEvSysView::TEvSetPartitioning>(domainKey, id, ""); - setPartitioning->ShardIndices.push_back(TShardIdx(ownerId * 3 + pathId, 0)); - setPartitioning->ShardIndices.push_back(TShardIdx(ownerId * 3 + pathId, 1)); + +namespace NKikimr { +namespace NSysView { + +Y_UNIT_TEST_SUITE(PartitionStats) { + + TTestActorRuntime::TEgg MakeEgg() + { + return { new TAppData(0, 0, 0, 0, { }, nullptr, nullptr, nullptr, nullptr), nullptr, nullptr }; + } + + void TestCollector(size_t batchSize) { + TTestActorRuntime runtime; + runtime.Initialize(MakeEgg()); + + auto collector = CreatePartitionStatsCollector(batchSize); + auto collectorId = runtime.Register(collector.Release()); + auto sender = runtime.AllocateEdgeActor(); + + auto domainKey = TPathId(1, 1); + + for (ui64 ownerId = 0; ownerId <= 1; ++ownerId) { + for (ui64 pathId = 0; pathId <= 1; ++pathId) { + auto id = TPathId(ownerId, pathId); + auto setPartitioning = MakeHolder<TEvSysView::TEvSetPartitioning>(domainKey, id, ""); + setPartitioning->ShardIndices.push_back(TShardIdx(ownerId * 3 + pathId, 0)); + setPartitioning->ShardIndices.push_back(TShardIdx(ownerId * 3 + pathId, 1)); runtime.Send(new IEventHandle(collectorId, TActorId(), setPartitioning.Release())); - } - } - - auto test = [&] ( - TMaybe<ui64> fromOwnerId, TMaybe<ui64> fromPathId, TMaybe<ui64> fromPartIdx, bool fromInc, - TMaybe<ui64> toOwnerId, TMaybe<ui64> toPathId, TMaybe<ui64> toPartIdx, bool toInc, - std::initializer_list<std::tuple<ui64, ui64, ui64>> check) - { - auto get = MakeHolder<TEvSysView::TEvGetPartitionStats>(); - auto& record = get->Record; - record.SetDomainKeyOwnerId(domainKey.OwnerId); - record.SetDomainKeyPathId(domainKey.LocalPathId); - - if (fromOwnerId) { - record.MutableFrom()->SetOwnerId(*fromOwnerId); - } - if (fromPathId) { - record.MutableFrom()->SetPathId(*fromPathId); - } - if (fromPartIdx) { - record.MutableFrom()->SetPartIdx(*fromPartIdx); - } - record.SetFromInclusive(fromInc); - - if (toOwnerId) { - record.MutableTo()->SetOwnerId(*toOwnerId); - } - if (toPathId) { - record.MutableTo()->SetPathId(*toPathId); - } - if (toPartIdx) { - record.MutableTo()->SetPartIdx(*toPartIdx); - } - record.SetToInclusive(toInc); - - auto checkIt = check.begin(); - - while (true) { - runtime.Send(new IEventHandle(collectorId, sender, get.Release())); - - TAutoPtr<IEventHandle> handle; - auto result = runtime.GrabEdgeEvent<TEvSysView::TEvGetPartitionStatsResult>(handle); - - for (size_t i = 0; i < result->Record.StatsSize(); ++i, ++checkIt) { - auto& stats = result->Record.GetStats(i); - UNIT_ASSERT(checkIt != check.end()); - UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetOwnerId(), std::get<0>(*checkIt)); - UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetPathId(), std::get<1>(*checkIt)); - UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetPartIdx(), std::get<2>(*checkIt)); - } - - if (result->Record.GetLastBatch()) { - break; - } - - get = MakeHolder<TEvSysView::TEvGetPartitionStats>(); - auto& record = get->Record; - record.SetDomainKeyOwnerId(domainKey.OwnerId); - record.SetDomainKeyPathId(domainKey.LocalPathId); - - record.MutableFrom()->CopyFrom(result->Record.GetNext()); - record.SetFromInclusive(true); - - if (toOwnerId) { - record.MutableTo()->SetOwnerId(*toOwnerId); - } - if (toPathId) { - record.MutableTo()->SetPathId(*toPathId); - } - if (toPartIdx) { - record.MutableTo()->SetPartIdx(*toPartIdx); - } - record.SetToInclusive(toInc); - } - - UNIT_ASSERT(checkIt == check.end()); - }; - - test({}, {}, {}, false, {}, {}, {}, false, { - {0, 0, 0}, - {0, 0, 1}, - {0, 1, 0}, - {0, 1, 1}, - {1, 0, 0}, - {1, 0, 1}, - {1, 1, 0}, - {1, 1, 1}, - }); - - test(0, {}, {}, true, 1, {}, {}, false, { - {0, 0, 0}, - {0, 0, 1}, - {0, 1, 0}, - {0, 1, 1}, - }); - - test(0, {}, {}, false, 1, {}, {}, true, { - {1, 0, 0}, - {1, 0, 1}, - {1, 1, 0}, - {1, 1, 1}, - }); - - test(0, 1, {}, true, 1, 1, {}, false, { - {0, 1, 0}, - {0, 1, 1}, - {1, 0, 0}, - {1, 0, 1}, - }); - - test(0, 0, {}, false, 1, 0, {}, true, { - {0, 1, 0}, - {0, 1, 1}, - {1, 0, 0}, - {1, 0, 1}, - }); - - test(0, 0, 1, true, 1, 0, 1, false, { - {0, 0, 1}, - {0, 1, 0}, - {0, 1, 1}, - {1, 0, 0}, - }); - - test(0, 0, 1, false, 1, 0, 1, true, { - {0, 1, 0}, - {0, 1, 1}, - {1, 0, 0}, - {1, 0, 1}, - }); - } - - Y_UNIT_TEST(Collector) { - for (size_t batchSize = 1; batchSize < 9; ++batchSize) { - TestCollector(batchSize); - } - } - - Y_UNIT_TEST(CollectorOverload) { - TTestActorRuntime runtime; - runtime.Initialize(MakeEgg()); - - auto collector = CreatePartitionStatsCollector(1, 0); - auto collectorId = runtime.Register(collector.Release()); - auto sender = runtime.AllocateEdgeActor(); - - auto domainKey = TPathId(1, 1); - - auto get = MakeHolder<TEvSysView::TEvGetPartitionStats>(); - auto& record = get->Record; - record.SetDomainKeyOwnerId(domainKey.OwnerId); - record.SetDomainKeyPathId(domainKey.LocalPathId); - - runtime.Send(new IEventHandle(collectorId, sender, get.Release())); - - TAutoPtr<IEventHandle> handle; - auto result = runtime.GrabEdgeEvent<TEvSysView::TEvGetPartitionStatsResult>(handle); - - UNIT_ASSERT_VALUES_EQUAL(result->Record.GetOverloaded(), true); - } - -} - -} // NSysView -} // NKikimr + } + } + + auto test = [&] ( + TMaybe<ui64> fromOwnerId, TMaybe<ui64> fromPathId, TMaybe<ui64> fromPartIdx, bool fromInc, + TMaybe<ui64> toOwnerId, TMaybe<ui64> toPathId, TMaybe<ui64> toPartIdx, bool toInc, + std::initializer_list<std::tuple<ui64, ui64, ui64>> check) + { + auto get = MakeHolder<TEvSysView::TEvGetPartitionStats>(); + auto& record = get->Record; + record.SetDomainKeyOwnerId(domainKey.OwnerId); + record.SetDomainKeyPathId(domainKey.LocalPathId); + + if (fromOwnerId) { + record.MutableFrom()->SetOwnerId(*fromOwnerId); + } + if (fromPathId) { + record.MutableFrom()->SetPathId(*fromPathId); + } + if (fromPartIdx) { + record.MutableFrom()->SetPartIdx(*fromPartIdx); + } + record.SetFromInclusive(fromInc); + + if (toOwnerId) { + record.MutableTo()->SetOwnerId(*toOwnerId); + } + if (toPathId) { + record.MutableTo()->SetPathId(*toPathId); + } + if (toPartIdx) { + record.MutableTo()->SetPartIdx(*toPartIdx); + } + record.SetToInclusive(toInc); + + auto checkIt = check.begin(); + + while (true) { + runtime.Send(new IEventHandle(collectorId, sender, get.Release())); + + TAutoPtr<IEventHandle> handle; + auto result = runtime.GrabEdgeEvent<TEvSysView::TEvGetPartitionStatsResult>(handle); + + for (size_t i = 0; i < result->Record.StatsSize(); ++i, ++checkIt) { + auto& stats = result->Record.GetStats(i); + UNIT_ASSERT(checkIt != check.end()); + UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetOwnerId(), std::get<0>(*checkIt)); + UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetPathId(), std::get<1>(*checkIt)); + UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetPartIdx(), std::get<2>(*checkIt)); + } + + if (result->Record.GetLastBatch()) { + break; + } + + get = MakeHolder<TEvSysView::TEvGetPartitionStats>(); + auto& record = get->Record; + record.SetDomainKeyOwnerId(domainKey.OwnerId); + record.SetDomainKeyPathId(domainKey.LocalPathId); + + record.MutableFrom()->CopyFrom(result->Record.GetNext()); + record.SetFromInclusive(true); + + if (toOwnerId) { + record.MutableTo()->SetOwnerId(*toOwnerId); + } + if (toPathId) { + record.MutableTo()->SetPathId(*toPathId); + } + if (toPartIdx) { + record.MutableTo()->SetPartIdx(*toPartIdx); + } + record.SetToInclusive(toInc); + } + + UNIT_ASSERT(checkIt == check.end()); + }; + + test({}, {}, {}, false, {}, {}, {}, false, { + {0, 0, 0}, + {0, 0, 1}, + {0, 1, 0}, + {0, 1, 1}, + {1, 0, 0}, + {1, 0, 1}, + {1, 1, 0}, + {1, 1, 1}, + }); + + test(0, {}, {}, true, 1, {}, {}, false, { + {0, 0, 0}, + {0, 0, 1}, + {0, 1, 0}, + {0, 1, 1}, + }); + + test(0, {}, {}, false, 1, {}, {}, true, { + {1, 0, 0}, + {1, 0, 1}, + {1, 1, 0}, + {1, 1, 1}, + }); + + test(0, 1, {}, true, 1, 1, {}, false, { + {0, 1, 0}, + {0, 1, 1}, + {1, 0, 0}, + {1, 0, 1}, + }); + + test(0, 0, {}, false, 1, 0, {}, true, { + {0, 1, 0}, + {0, 1, 1}, + {1, 0, 0}, + {1, 0, 1}, + }); + + test(0, 0, 1, true, 1, 0, 1, false, { + {0, 0, 1}, + {0, 1, 0}, + {0, 1, 1}, + {1, 0, 0}, + }); + + test(0, 0, 1, false, 1, 0, 1, true, { + {0, 1, 0}, + {0, 1, 1}, + {1, 0, 0}, + {1, 0, 1}, + }); + } + + Y_UNIT_TEST(Collector) { + for (size_t batchSize = 1; batchSize < 9; ++batchSize) { + TestCollector(batchSize); + } + } + + Y_UNIT_TEST(CollectorOverload) { + TTestActorRuntime runtime; + runtime.Initialize(MakeEgg()); + + auto collector = CreatePartitionStatsCollector(1, 0); + auto collectorId = runtime.Register(collector.Release()); + auto sender = runtime.AllocateEdgeActor(); + + auto domainKey = TPathId(1, 1); + + auto get = MakeHolder<TEvSysView::TEvGetPartitionStats>(); + auto& record = get->Record; + record.SetDomainKeyOwnerId(domainKey.OwnerId); + record.SetDomainKeyPathId(domainKey.LocalPathId); + + runtime.Send(new IEventHandle(collectorId, sender, get.Release())); + + TAutoPtr<IEventHandle> handle; + auto result = runtime.GrabEdgeEvent<TEvSysView::TEvGetPartitionStatsResult>(handle); + + UNIT_ASSERT_VALUES_EQUAL(result->Record.GetOverloaded(), true); + } + +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/partition_stats/ut/ya.make b/ydb/core/sys_view/partition_stats/ut/ya.make index 41ddcb3165..01f5bbd1c2 100644 --- a/ydb/core/sys_view/partition_stats/ut/ya.make +++ b/ydb/core/sys_view/partition_stats/ut/ya.make @@ -1,25 +1,25 @@ UNITTEST_FOR(ydb/core/sys_view/partition_stats) - -OWNER( - monster - g:kikimr -) - -FORK_SUBTESTS() - -SIZE(MEDIUM) - -TIMEOUT(600) - -PEERDIR( + +OWNER( + monster + g:kikimr +) + +FORK_SUBTESTS() + +SIZE(MEDIUM) + +TIMEOUT(600) + +PEERDIR( library/cpp/testing/unittest ydb/core/testlib -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - partition_stats_ut.cpp -) - -END() +SRCS( + partition_stats_ut.cpp +) + +END() diff --git a/ydb/core/sys_view/partition_stats/ya.make b/ydb/core/sys_view/partition_stats/ya.make index 48c79c52e0..d24bf5c89d 100644 --- a/ydb/core/sys_view/partition_stats/ya.make +++ b/ydb/core/sys_view/partition_stats/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - partition_stats.h - partition_stats.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + partition_stats.h + partition_stats.cpp +) + +PEERDIR( library/cpp/actors/core ydb/core/base ydb/core/kqp/runtime ydb/core/sys_view/common -) - +) + YQL_LAST_ABI_VERSION() END() diff --git a/ydb/core/sys_view/processor/db_counters.cpp b/ydb/core/sys_view/processor/db_counters.cpp index 573f529660..42bfbd6269 100644 --- a/ydb/core/sys_view/processor/db_counters.cpp +++ b/ydb/core/sys_view/processor/db_counters.cpp @@ -1,483 +1,483 @@ -#include "processor_impl.h" - +#include "processor_impl.h" + #include <ydb/core/base/counters.h> #include <ydb/core/base/path.h> #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/kqp/counters/kqp_counters.h> #include <ydb/core/tablet/tablet_counters_aggregator.h> #include <ydb/core/tablet_flat/flat_executor_counters.h> - -namespace NKikimr { -namespace NSysView { - -template <bool IsMax> -struct TAggregateCumulative { - static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { - auto cumulativeSize = src.GetCumulativeCount(); - auto histogramSize = src.HistogramSize(); - - if (dst->CumulativeSize() < cumulativeSize) { - dst->MutableCumulative()->Resize(cumulativeSize, 0); - } - if (dst->HistogramSize() < histogramSize) { - auto missing = histogramSize - dst->HistogramSize(); - for (; missing > 0; --missing) { - dst->AddHistogram(); - } - } - - const auto& from = src.GetCumulative(); - auto* to = dst->MutableCumulative(); - auto doubleDiffSize = from.size() / 2 * 2; - for (int i = 0; i < doubleDiffSize; ) { - auto index = from[i++]; - auto value = from[i++]; - if (index >= cumulativeSize) { - continue; - } - if constexpr (!IsMax) { - (*to)[index] += value; - } else { - (*to)[index] = std::max(value, (*to)[index]); - } - } - for (size_t i = 0; i < histogramSize; ++i) { - const auto& histogram = src.GetHistogram(i); - const auto& from = histogram.GetBuckets(); - auto* to = dst->MutableHistogram(i)->MutableBuckets(); - auto bucketCount = histogram.GetBucketsCount(); - if (to->size() < (int)bucketCount) { - to->Resize(bucketCount, 0); - } - auto doubleDiffSize = from.size(); - for (int b = 0; b < doubleDiffSize; ) { - auto index = from[b++]; - auto value = from[b++]; - if (index >= bucketCount) { - continue; - } - if constexpr (!IsMax) { - (*to)[index] += value; - } else { - (*to)[index] = std::max(value, (*to)[index]); - } - } - } - } -}; - -template <bool IsMax> -struct TAggregateSimple { - static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { - auto simpleSize = src.SimpleSize(); - if (dst->SimpleSize() < simpleSize) { - dst->MutableSimple()->Resize(simpleSize, 0); - } - const auto& from = src.GetSimple(); - auto* to = dst->MutableSimple(); - for (size_t i = 0; i < simpleSize; ++i) { - if constexpr (!IsMax) { - (*to)[i] += from[i]; - } else { - (*to)[i] = std::max(from[i], (*to)[i]); - } - } - } -}; - -struct TAggregateIncrementalSum { - static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { - TAggregateCumulative<false>::Apply(dst, src); - } -}; - -struct TAggregateIncrementalMax { - static void Apply(NKikimrSysView::TDbCounters* /*dst*/, const NKikimrSysView::TDbCounters& /*src*/) { - } -}; - -struct TAggregateStatefulSum { - static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { - TAggregateSimple<false>::Apply(dst, src); - } -}; - -struct TAggregateStatefulMax { - static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { - TAggregateSimple<true>::Apply(dst, src); - TAggregateCumulative<true>::Apply(dst, src); - } -}; - -static void SwapSimpleCounters(NKikimrSysView::TDbCounters* dst, NKikimrSysView::TDbCounters& src) { - dst->MutableSimple()->Swap(src.MutableSimple()); -}; - -static void SwapMaxCounters(NKikimrSysView::TDbCounters* dst, NKikimrSysView::TDbCounters& src) { - SwapSimpleCounters(dst, src); - dst->MutableCumulative()->Swap(src.MutableCumulative()); - dst->SetCumulativeCount(src.GetCumulativeCount()); -}; - -static void ResetSimpleCounters(NKikimrSysView::TDbCounters* dst) { - auto simpleSize = dst->SimpleSize(); - auto* to = dst->MutableSimple(); - for (size_t i = 0; i < simpleSize; ++i) { - (*to)[i] = 0; - } -} - -static void ResetMaxCounters(NKikimrSysView::TDbCounters* dst) { - ResetSimpleCounters(dst); - auto cumulativeSize = dst->CumulativeSize(); - auto* to = dst->MutableCumulative(); - for (size_t i = 0; i < cumulativeSize; ++i) { - (*to)[i] = 0; - } -} - -template <typename TAggrSum, typename TAggrMax> -static void AggregateCounters(NKikimr::NSysView::TDbServiceCounters* dst, - const NKikimrSysView::TDbServiceCounters& src) -{ - if (src.HasMain()) { - TAggrSum::Apply(dst->Proto().MutableMain(), src.GetMain()); - } - - for (const auto& srcTablet : src.GetTabletCounters()) { - auto* dstTablet = dst->FindOrAddTabletCounters(srcTablet.GetType()); - TAggrSum::Apply(dstTablet->MutableExecutorCounters(), srcTablet.GetExecutorCounters()); - TAggrSum::Apply(dstTablet->MutableAppCounters(), srcTablet.GetAppCounters()); - TAggrMax::Apply(dstTablet->MutableMaxExecutorCounters(), srcTablet.GetMaxExecutorCounters()); - TAggrMax::Apply(dstTablet->MutableMaxAppCounters(), srcTablet.GetMaxAppCounters()); - } - - for (const auto& srcReq : src.GetGRpcCounters()) { - auto* dstReq = dst->FindOrAddGRpcCounters(srcReq.GetGRpcService(), srcReq.GetGRpcRequest()); - TAggrSum::Apply(dstReq->MutableRequestCounters(), srcReq.GetRequestCounters()); - } -} - -static void AggregateIncrementalCounters(NKikimr::NSysView::TDbServiceCounters* dst, - const NKikimrSysView::TDbServiceCounters& src) -{ - AggregateCounters<TAggregateIncrementalSum, TAggregateIncrementalMax>(dst, src); -} - -static void AggregateStatefulCounters(NKikimr::NSysView::TDbServiceCounters* dst, - const NKikimrSysView::TDbServiceCounters& src) -{ - AggregateCounters<TAggregateStatefulSum, TAggregateStatefulMax>(dst, src); -} - -static void SwapStatefulCounters(NKikimr::NSysView::TDbServiceCounters* dst, - NKikimrSysView::TDbServiceCounters& src) -{ - if (src.HasMain()) { - SwapSimpleCounters(dst->Proto().MutableMain(), *src.MutableMain()); - } - - for (auto& srcTablet : *src.MutableTabletCounters()) { - auto* dstTablet = dst->FindOrAddTabletCounters(srcTablet.GetType()); - SwapSimpleCounters(dstTablet->MutableExecutorCounters(), *srcTablet.MutableExecutorCounters()); - SwapSimpleCounters(dstTablet->MutableAppCounters(), *srcTablet.MutableAppCounters()); - SwapMaxCounters(dstTablet->MutableMaxExecutorCounters(), *srcTablet.MutableMaxExecutorCounters()); - SwapMaxCounters(dstTablet->MutableMaxAppCounters(), *srcTablet.MutableMaxAppCounters()); - } - - for (auto& srcReq : *src.MutableGRpcCounters()) { - auto* dstReq = dst->FindOrAddGRpcCounters(srcReq.GetGRpcService(), srcReq.GetGRpcRequest()); - SwapSimpleCounters(dstReq->MutableRequestCounters(), *srcReq.MutableRequestCounters()); - } -} - -static void ResetStatefulCounters(NKikimrSysView::TDbServiceCounters* dst) { - if (dst->HasMain()) { - ResetSimpleCounters(dst->MutableMain()); - } - for (auto& dstTablet : *dst->MutableTabletCounters()) { - ResetSimpleCounters(dstTablet.MutableExecutorCounters()); - ResetSimpleCounters(dstTablet.MutableAppCounters()); - ResetMaxCounters(dstTablet.MutableMaxExecutorCounters()); - ResetMaxCounters(dstTablet.MutableMaxAppCounters()); - } - for (auto& dstReq : *dst->MutableGRpcCounters()) { - ResetSimpleCounters(dstReq.MutableRequestCounters()); - } -} - -void TSysViewProcessor::SendNavigate() { - if (!Database) { - ScheduleSendNavigate(); - return; - } - - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - auto request = MakeHolder<TNavigate>(); - request->ResultSet.push_back({}); - - auto& entry = request->ResultSet.back(); - entry.Path = SplitPath(Database); - entry.Operation = TNavigate::EOp::OpPath; - entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; - entry.RedirectRequired = false; - - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); -} - -TIntrusivePtr<IDbCounters> TSysViewProcessor::CreateCountersForService( - NKikimrSysView::EDbCountersService service) -{ - TIntrusivePtr<IDbCounters> result; - switch (service) { - case NKikimrSysView::KQP: - result = MakeIntrusive<NKqp::TKqpDbCounters>(ExternalGroup, InternalGroup); - break; - case NKikimrSysView::TABLETS: { - THolder<TTabletCountersBase> executorCounters(new NTabletFlatExecutor::TExecutorCounters); - result = CreateTabletDbCounters(ExternalGroup, InternalGroup, std::move(executorCounters)); - break; - } - case NKikimrSysView::GRPC: - result = NGRpcService::CreateGRpcDbCounters(ExternalGroup, InternalGroup); - break; - default: - break; - } - - if (result) { - Counters[service] = result; - } - return result; -} - -void TSysViewProcessor::AttachExternalCounters() { - if (!Database) { - return; - } - - GetServiceCounters(AppData()->Counters, "ydb_serverless", false) - ->GetSubgroup("database", Database) - ->GetSubgroup("cloud_id", CloudId) - ->GetSubgroup("folder_id", FolderId) - ->GetSubgroup("database_id", DatabaseId) - ->RegisterSubgroup("host", "", ExternalGroup); -} - -void TSysViewProcessor::AttachInternalCounters() { - if (!Database) { - return; - } - - GetServiceCounters(AppData()->Counters, "db", false) - ->GetSubgroup("database", Database) - ->RegisterSubgroup("host", "", InternalGroup); -} - -void TSysViewProcessor::DetachExternalCounters() { - if (!Database) { - return; - } - - GetServiceCounters(AppData()->Counters, "ydb_serverless", false) - ->RemoveSubgroup("database", Database); -} - -void TSysViewProcessor::DetachInternalCounters() { - if (!Database) { - return; - } - - GetServiceCounters(AppData()->Counters, "db", false) - ->RemoveSubgroup("database", Database); -} - -void TSysViewProcessor::Handle(TEvSysView::TEvSendDbCountersRequest::TPtr& ev) { - if (!AppData()->FeatureFlags.GetEnableDbCounters()) { - return; - } - - auto& record = ev->Get()->Record; - auto nodeId = record.GetNodeId(); - - auto& state = NodeCountersStates[nodeId]; - state.FreshCount = 0; - - if (state.Generation == record.GetGeneration()) { - SVLOG_D("[" << TabletID() << "] TEvSendDbCountersRequest, known generation: " - << "node id# " << nodeId - << ", generation# " << record.GetGeneration()); - - auto response = MakeHolder<TEvSysView::TEvSendDbCountersResponse>(); - response->Record.SetDatabase(Database); - response->Record.SetGeneration(state.Generation); - Send(ev->Sender, std::move(response)); - return; - } - - state.Generation = record.GetGeneration(); - - std::unordered_set<NKikimrSysView::EDbCountersService> incomingServicesSet; - - for (auto& serviceCounters : *record.MutableServiceCounters()) { - auto service = serviceCounters.GetService(); - incomingServicesSet.insert(service); - - auto& simpleState = state.Simple[service]; - SwapStatefulCounters(&simpleState, *serviceCounters.MutableCounters()); - - auto& aggrState = AggregatedCountersState[service]; - AggregateIncrementalCounters(&aggrState, serviceCounters.GetCounters()); - } - - for (auto it = state.Simple.begin(); it != state.Simple.end(); ) { - if (incomingServicesSet.find(it->first) == incomingServicesSet.end()) { - it = state.Simple.erase(it); - } else { - ++it; - } - } - - SVLOG_D("[" << TabletID() << "] TEvSendDbCountersRequest: " - << "node id# " << nodeId - << ", generation# " << state.Generation - << ", services count# " << incomingServicesSet.size() - << ", request size# " << record.ByteSize()); - - auto response = MakeHolder<TEvSysView::TEvSendDbCountersResponse>(); - response->Record.SetDatabase(Database); - response->Record.SetGeneration(state.Generation); - Send(ev->Sender, std::move(response)); -} - -void TSysViewProcessor::Handle(TEvPrivate::TEvApplyCounters::TPtr&) { - for (auto& [_, counters] : AggregatedCountersState) { - ResetStatefulCounters(&counters.Proto()); - } - - for (auto it = NodeCountersStates.begin(); it != NodeCountersStates.end(); ) { - auto& state = it->second; - if (state.FreshCount > 1) { - it = NodeCountersStates.erase(it); - continue; - } - ++state.FreshCount; - for (const auto& [service, counters] : state.Simple) { - AggregateStatefulCounters(&AggregatedCountersState[service], counters.Proto()); - } - ++it; - } - - for (auto& [service, aggrCounters] : AggregatedCountersState) { - TIntrusivePtr<IDbCounters> counters; - if (auto it = Counters.find(service); it != Counters.end()) { - counters = it->second; - } else { - counters = CreateCountersForService(service); - } - if (!counters) { - continue; - } - counters->FromProto(aggrCounters); - } - - SVLOG_D("[" << TabletID() << "] TEvApplyCounters: " - << "services count# " << AggregatedCountersState.size()); - - ScheduleApplyCounters(); -} - -void TSysViewProcessor::Handle(TEvPrivate::TEvSendNavigate::TPtr&) { - SendNavigate(); -} - -void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - - THolder<TNavigate> request(ev->Get()->Request.Release()); - if (request->ResultSet.size() != 1) { - SVLOG_CRIT("[" << TabletID() << "] TEvNavigateKeySetResult failed: " - << "result set size# " << request->ResultSet.size()); - ScheduleSendNavigate(); - return; - } - - auto& entry = request->ResultSet.back(); - - if (entry.Status != TNavigate::EStatus::Ok) { - SVLOG_W("[" << TabletID() << "] TEvNavigateKeySetResult failed: " - << "status# " << entry.Status); - ScheduleSendNavigate(); - return; - } - - for (const auto& [key, value] : entry.Attributes) { - if (key == "cloud_id") { - CloudId = value; - } else if (key == "folder_id") { - FolderId = value; - } else if (key == "database_id") { - DatabaseId = value; - } - } - - AttachExternalCounters(); - AttachInternalCounters(); - - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(entry.TableId.PathId)); - - SVLOG_D("[" << TabletID() << "] TEvNavigateKeySetResult: " - << "database# " << Database - << ", pathId# " << entry.TableId.PathId - << ", cloud_id# " << CloudId - << ", folder_id# " << FolderId - << ", database_id# " << DatabaseId); -} - -void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev) { - const auto& describeResult = ev->Get()->Result; - - TString path = describeResult->GetPath(); - if (path != Database) { - SVLOG_W("[" << TabletID() << "] TEvWatchNotifyUpdated, invalid path: " - << "database# " << Database - << ", path# " << path); - return; - } - - const auto& pathDescription = describeResult->GetPathDescription(); - const auto& userAttrs = pathDescription.GetUserAttributes(); - - TString cloudId, folderId, databaseId; - for (const auto& attr : userAttrs) { - if (attr.GetKey() == "cloud_id") { - cloudId = attr.GetValue(); - } else if (attr.GetKey() == "folder_id") { - folderId = attr.GetValue(); - } else if (attr.GetKey() == "database_id") { - databaseId = attr.GetValue(); - } - } - - SVLOG_D("[" << TabletID() << "] TEvWatchNotifyUpdated: " - << "database# " << Database - << ", cloud_id# " << cloudId - << ", folder_id# " << folderId - << ", database_id# " << databaseId); - - if (cloudId != CloudId || folderId != FolderId || databaseId != DatabaseId) { - DetachExternalCounters(); - CloudId = cloudId; - FolderId = folderId; - DatabaseId = databaseId; - AttachExternalCounters(); - } -} - -void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev) { - Y_UNUSED(ev); -} - -} // NSysView -} // NKikimr - + +namespace NKikimr { +namespace NSysView { + +template <bool IsMax> +struct TAggregateCumulative { + static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { + auto cumulativeSize = src.GetCumulativeCount(); + auto histogramSize = src.HistogramSize(); + + if (dst->CumulativeSize() < cumulativeSize) { + dst->MutableCumulative()->Resize(cumulativeSize, 0); + } + if (dst->HistogramSize() < histogramSize) { + auto missing = histogramSize - dst->HistogramSize(); + for (; missing > 0; --missing) { + dst->AddHistogram(); + } + } + + const auto& from = src.GetCumulative(); + auto* to = dst->MutableCumulative(); + auto doubleDiffSize = from.size() / 2 * 2; + for (int i = 0; i < doubleDiffSize; ) { + auto index = from[i++]; + auto value = from[i++]; + if (index >= cumulativeSize) { + continue; + } + if constexpr (!IsMax) { + (*to)[index] += value; + } else { + (*to)[index] = std::max(value, (*to)[index]); + } + } + for (size_t i = 0; i < histogramSize; ++i) { + const auto& histogram = src.GetHistogram(i); + const auto& from = histogram.GetBuckets(); + auto* to = dst->MutableHistogram(i)->MutableBuckets(); + auto bucketCount = histogram.GetBucketsCount(); + if (to->size() < (int)bucketCount) { + to->Resize(bucketCount, 0); + } + auto doubleDiffSize = from.size(); + for (int b = 0; b < doubleDiffSize; ) { + auto index = from[b++]; + auto value = from[b++]; + if (index >= bucketCount) { + continue; + } + if constexpr (!IsMax) { + (*to)[index] += value; + } else { + (*to)[index] = std::max(value, (*to)[index]); + } + } + } + } +}; + +template <bool IsMax> +struct TAggregateSimple { + static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { + auto simpleSize = src.SimpleSize(); + if (dst->SimpleSize() < simpleSize) { + dst->MutableSimple()->Resize(simpleSize, 0); + } + const auto& from = src.GetSimple(); + auto* to = dst->MutableSimple(); + for (size_t i = 0; i < simpleSize; ++i) { + if constexpr (!IsMax) { + (*to)[i] += from[i]; + } else { + (*to)[i] = std::max(from[i], (*to)[i]); + } + } + } +}; + +struct TAggregateIncrementalSum { + static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { + TAggregateCumulative<false>::Apply(dst, src); + } +}; + +struct TAggregateIncrementalMax { + static void Apply(NKikimrSysView::TDbCounters* /*dst*/, const NKikimrSysView::TDbCounters& /*src*/) { + } +}; + +struct TAggregateStatefulSum { + static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { + TAggregateSimple<false>::Apply(dst, src); + } +}; + +struct TAggregateStatefulMax { + static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) { + TAggregateSimple<true>::Apply(dst, src); + TAggregateCumulative<true>::Apply(dst, src); + } +}; + +static void SwapSimpleCounters(NKikimrSysView::TDbCounters* dst, NKikimrSysView::TDbCounters& src) { + dst->MutableSimple()->Swap(src.MutableSimple()); +}; + +static void SwapMaxCounters(NKikimrSysView::TDbCounters* dst, NKikimrSysView::TDbCounters& src) { + SwapSimpleCounters(dst, src); + dst->MutableCumulative()->Swap(src.MutableCumulative()); + dst->SetCumulativeCount(src.GetCumulativeCount()); +}; + +static void ResetSimpleCounters(NKikimrSysView::TDbCounters* dst) { + auto simpleSize = dst->SimpleSize(); + auto* to = dst->MutableSimple(); + for (size_t i = 0; i < simpleSize; ++i) { + (*to)[i] = 0; + } +} + +static void ResetMaxCounters(NKikimrSysView::TDbCounters* dst) { + ResetSimpleCounters(dst); + auto cumulativeSize = dst->CumulativeSize(); + auto* to = dst->MutableCumulative(); + for (size_t i = 0; i < cumulativeSize; ++i) { + (*to)[i] = 0; + } +} + +template <typename TAggrSum, typename TAggrMax> +static void AggregateCounters(NKikimr::NSysView::TDbServiceCounters* dst, + const NKikimrSysView::TDbServiceCounters& src) +{ + if (src.HasMain()) { + TAggrSum::Apply(dst->Proto().MutableMain(), src.GetMain()); + } + + for (const auto& srcTablet : src.GetTabletCounters()) { + auto* dstTablet = dst->FindOrAddTabletCounters(srcTablet.GetType()); + TAggrSum::Apply(dstTablet->MutableExecutorCounters(), srcTablet.GetExecutorCounters()); + TAggrSum::Apply(dstTablet->MutableAppCounters(), srcTablet.GetAppCounters()); + TAggrMax::Apply(dstTablet->MutableMaxExecutorCounters(), srcTablet.GetMaxExecutorCounters()); + TAggrMax::Apply(dstTablet->MutableMaxAppCounters(), srcTablet.GetMaxAppCounters()); + } + + for (const auto& srcReq : src.GetGRpcCounters()) { + auto* dstReq = dst->FindOrAddGRpcCounters(srcReq.GetGRpcService(), srcReq.GetGRpcRequest()); + TAggrSum::Apply(dstReq->MutableRequestCounters(), srcReq.GetRequestCounters()); + } +} + +static void AggregateIncrementalCounters(NKikimr::NSysView::TDbServiceCounters* dst, + const NKikimrSysView::TDbServiceCounters& src) +{ + AggregateCounters<TAggregateIncrementalSum, TAggregateIncrementalMax>(dst, src); +} + +static void AggregateStatefulCounters(NKikimr::NSysView::TDbServiceCounters* dst, + const NKikimrSysView::TDbServiceCounters& src) +{ + AggregateCounters<TAggregateStatefulSum, TAggregateStatefulMax>(dst, src); +} + +static void SwapStatefulCounters(NKikimr::NSysView::TDbServiceCounters* dst, + NKikimrSysView::TDbServiceCounters& src) +{ + if (src.HasMain()) { + SwapSimpleCounters(dst->Proto().MutableMain(), *src.MutableMain()); + } + + for (auto& srcTablet : *src.MutableTabletCounters()) { + auto* dstTablet = dst->FindOrAddTabletCounters(srcTablet.GetType()); + SwapSimpleCounters(dstTablet->MutableExecutorCounters(), *srcTablet.MutableExecutorCounters()); + SwapSimpleCounters(dstTablet->MutableAppCounters(), *srcTablet.MutableAppCounters()); + SwapMaxCounters(dstTablet->MutableMaxExecutorCounters(), *srcTablet.MutableMaxExecutorCounters()); + SwapMaxCounters(dstTablet->MutableMaxAppCounters(), *srcTablet.MutableMaxAppCounters()); + } + + for (auto& srcReq : *src.MutableGRpcCounters()) { + auto* dstReq = dst->FindOrAddGRpcCounters(srcReq.GetGRpcService(), srcReq.GetGRpcRequest()); + SwapSimpleCounters(dstReq->MutableRequestCounters(), *srcReq.MutableRequestCounters()); + } +} + +static void ResetStatefulCounters(NKikimrSysView::TDbServiceCounters* dst) { + if (dst->HasMain()) { + ResetSimpleCounters(dst->MutableMain()); + } + for (auto& dstTablet : *dst->MutableTabletCounters()) { + ResetSimpleCounters(dstTablet.MutableExecutorCounters()); + ResetSimpleCounters(dstTablet.MutableAppCounters()); + ResetMaxCounters(dstTablet.MutableMaxExecutorCounters()); + ResetMaxCounters(dstTablet.MutableMaxAppCounters()); + } + for (auto& dstReq : *dst->MutableGRpcCounters()) { + ResetSimpleCounters(dstReq.MutableRequestCounters()); + } +} + +void TSysViewProcessor::SendNavigate() { + if (!Database) { + ScheduleSendNavigate(); + return; + } + + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + auto request = MakeHolder<TNavigate>(); + request->ResultSet.push_back({}); + + auto& entry = request->ResultSet.back(); + entry.Path = SplitPath(Database); + entry.Operation = TNavigate::EOp::OpPath; + entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; + entry.RedirectRequired = false; + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); +} + +TIntrusivePtr<IDbCounters> TSysViewProcessor::CreateCountersForService( + NKikimrSysView::EDbCountersService service) +{ + TIntrusivePtr<IDbCounters> result; + switch (service) { + case NKikimrSysView::KQP: + result = MakeIntrusive<NKqp::TKqpDbCounters>(ExternalGroup, InternalGroup); + break; + case NKikimrSysView::TABLETS: { + THolder<TTabletCountersBase> executorCounters(new NTabletFlatExecutor::TExecutorCounters); + result = CreateTabletDbCounters(ExternalGroup, InternalGroup, std::move(executorCounters)); + break; + } + case NKikimrSysView::GRPC: + result = NGRpcService::CreateGRpcDbCounters(ExternalGroup, InternalGroup); + break; + default: + break; + } + + if (result) { + Counters[service] = result; + } + return result; +} + +void TSysViewProcessor::AttachExternalCounters() { + if (!Database) { + return; + } + + GetServiceCounters(AppData()->Counters, "ydb_serverless", false) + ->GetSubgroup("database", Database) + ->GetSubgroup("cloud_id", CloudId) + ->GetSubgroup("folder_id", FolderId) + ->GetSubgroup("database_id", DatabaseId) + ->RegisterSubgroup("host", "", ExternalGroup); +} + +void TSysViewProcessor::AttachInternalCounters() { + if (!Database) { + return; + } + + GetServiceCounters(AppData()->Counters, "db", false) + ->GetSubgroup("database", Database) + ->RegisterSubgroup("host", "", InternalGroup); +} + +void TSysViewProcessor::DetachExternalCounters() { + if (!Database) { + return; + } + + GetServiceCounters(AppData()->Counters, "ydb_serverless", false) + ->RemoveSubgroup("database", Database); +} + +void TSysViewProcessor::DetachInternalCounters() { + if (!Database) { + return; + } + + GetServiceCounters(AppData()->Counters, "db", false) + ->RemoveSubgroup("database", Database); +} + +void TSysViewProcessor::Handle(TEvSysView::TEvSendDbCountersRequest::TPtr& ev) { + if (!AppData()->FeatureFlags.GetEnableDbCounters()) { + return; + } + + auto& record = ev->Get()->Record; + auto nodeId = record.GetNodeId(); + + auto& state = NodeCountersStates[nodeId]; + state.FreshCount = 0; + + if (state.Generation == record.GetGeneration()) { + SVLOG_D("[" << TabletID() << "] TEvSendDbCountersRequest, known generation: " + << "node id# " << nodeId + << ", generation# " << record.GetGeneration()); + + auto response = MakeHolder<TEvSysView::TEvSendDbCountersResponse>(); + response->Record.SetDatabase(Database); + response->Record.SetGeneration(state.Generation); + Send(ev->Sender, std::move(response)); + return; + } + + state.Generation = record.GetGeneration(); + + std::unordered_set<NKikimrSysView::EDbCountersService> incomingServicesSet; + + for (auto& serviceCounters : *record.MutableServiceCounters()) { + auto service = serviceCounters.GetService(); + incomingServicesSet.insert(service); + + auto& simpleState = state.Simple[service]; + SwapStatefulCounters(&simpleState, *serviceCounters.MutableCounters()); + + auto& aggrState = AggregatedCountersState[service]; + AggregateIncrementalCounters(&aggrState, serviceCounters.GetCounters()); + } + + for (auto it = state.Simple.begin(); it != state.Simple.end(); ) { + if (incomingServicesSet.find(it->first) == incomingServicesSet.end()) { + it = state.Simple.erase(it); + } else { + ++it; + } + } + + SVLOG_D("[" << TabletID() << "] TEvSendDbCountersRequest: " + << "node id# " << nodeId + << ", generation# " << state.Generation + << ", services count# " << incomingServicesSet.size() + << ", request size# " << record.ByteSize()); + + auto response = MakeHolder<TEvSysView::TEvSendDbCountersResponse>(); + response->Record.SetDatabase(Database); + response->Record.SetGeneration(state.Generation); + Send(ev->Sender, std::move(response)); +} + +void TSysViewProcessor::Handle(TEvPrivate::TEvApplyCounters::TPtr&) { + for (auto& [_, counters] : AggregatedCountersState) { + ResetStatefulCounters(&counters.Proto()); + } + + for (auto it = NodeCountersStates.begin(); it != NodeCountersStates.end(); ) { + auto& state = it->second; + if (state.FreshCount > 1) { + it = NodeCountersStates.erase(it); + continue; + } + ++state.FreshCount; + for (const auto& [service, counters] : state.Simple) { + AggregateStatefulCounters(&AggregatedCountersState[service], counters.Proto()); + } + ++it; + } + + for (auto& [service, aggrCounters] : AggregatedCountersState) { + TIntrusivePtr<IDbCounters> counters; + if (auto it = Counters.find(service); it != Counters.end()) { + counters = it->second; + } else { + counters = CreateCountersForService(service); + } + if (!counters) { + continue; + } + counters->FromProto(aggrCounters); + } + + SVLOG_D("[" << TabletID() << "] TEvApplyCounters: " + << "services count# " << AggregatedCountersState.size()); + + ScheduleApplyCounters(); +} + +void TSysViewProcessor::Handle(TEvPrivate::TEvSendNavigate::TPtr&) { + SendNavigate(); +} + +void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + + THolder<TNavigate> request(ev->Get()->Request.Release()); + if (request->ResultSet.size() != 1) { + SVLOG_CRIT("[" << TabletID() << "] TEvNavigateKeySetResult failed: " + << "result set size# " << request->ResultSet.size()); + ScheduleSendNavigate(); + return; + } + + auto& entry = request->ResultSet.back(); + + if (entry.Status != TNavigate::EStatus::Ok) { + SVLOG_W("[" << TabletID() << "] TEvNavigateKeySetResult failed: " + << "status# " << entry.Status); + ScheduleSendNavigate(); + return; + } + + for (const auto& [key, value] : entry.Attributes) { + if (key == "cloud_id") { + CloudId = value; + } else if (key == "folder_id") { + FolderId = value; + } else if (key == "database_id") { + DatabaseId = value; + } + } + + AttachExternalCounters(); + AttachInternalCounters(); + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(entry.TableId.PathId)); + + SVLOG_D("[" << TabletID() << "] TEvNavigateKeySetResult: " + << "database# " << Database + << ", pathId# " << entry.TableId.PathId + << ", cloud_id# " << CloudId + << ", folder_id# " << FolderId + << ", database_id# " << DatabaseId); +} + +void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev) { + const auto& describeResult = ev->Get()->Result; + + TString path = describeResult->GetPath(); + if (path != Database) { + SVLOG_W("[" << TabletID() << "] TEvWatchNotifyUpdated, invalid path: " + << "database# " << Database + << ", path# " << path); + return; + } + + const auto& pathDescription = describeResult->GetPathDescription(); + const auto& userAttrs = pathDescription.GetUserAttributes(); + + TString cloudId, folderId, databaseId; + for (const auto& attr : userAttrs) { + if (attr.GetKey() == "cloud_id") { + cloudId = attr.GetValue(); + } else if (attr.GetKey() == "folder_id") { + folderId = attr.GetValue(); + } else if (attr.GetKey() == "database_id") { + databaseId = attr.GetValue(); + } + } + + SVLOG_D("[" << TabletID() << "] TEvWatchNotifyUpdated: " + << "database# " << Database + << ", cloud_id# " << cloudId + << ", folder_id# " << folderId + << ", database_id# " << databaseId); + + if (cloudId != CloudId || folderId != FolderId || databaseId != DatabaseId) { + DetachExternalCounters(); + CloudId = cloudId; + FolderId = folderId; + DatabaseId = databaseId; + AttachExternalCounters(); + } +} + +void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev) { + Y_UNUSED(ev); +} + +} // NSysView +} // NKikimr + diff --git a/ydb/core/sys_view/processor/processor.cpp b/ydb/core/sys_view/processor/processor.cpp index 64e715ea3b..d0c09f8e7b 100644 --- a/ydb/core/sys_view/processor/processor.cpp +++ b/ydb/core/sys_view/processor/processor.cpp @@ -1,18 +1,18 @@ -#include "processor.h" - -#include "processor_impl.h" - -namespace NKikimr { -namespace NSysView { - +#include "processor.h" + +#include "processor_impl.h" + +namespace NKikimr { +namespace NSysView { + IActor* CreateSysViewProcessor(const TActorId& tablet, TTabletStorageInfo* info) { - return new TSysViewProcessor(tablet, info, EProcessorMode::MINUTE); -} - + return new TSysViewProcessor(tablet, info, EProcessorMode::MINUTE); +} + IActor* CreateSysViewProcessorForTests(const TActorId& tablet, TTabletStorageInfo* info) { - return new TSysViewProcessor(tablet, info, EProcessorMode::FAST); -} - -} // NSysView -} // NKikimr - + return new TSysViewProcessor(tablet, info, EProcessorMode::FAST); +} + +} // NSysView +} // NKikimr + diff --git a/ydb/core/sys_view/processor/processor.h b/ydb/core/sys_view/processor/processor.h index 10a47fee99..ad08696597 100644 --- a/ydb/core/sys_view/processor/processor.h +++ b/ydb/core/sys_view/processor/processor.h @@ -1,13 +1,13 @@ -#pragma once - -#include <library/cpp/actors/core/actor.h> +#pragma once + +#include <library/cpp/actors/core/actor.h> #include <ydb/core/base/blobstorage.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + IActor* CreateSysViewProcessor(const TActorId& tablet, TTabletStorageInfo* info); IActor* CreateSysViewProcessorForTests(const TActorId& tablet, TTabletStorageInfo* info); - -} // NSysView -} // NKikimr + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/processor/processor_impl.cpp b/ydb/core/sys_view/processor/processor_impl.cpp index 49187f555e..86ddbfd423 100644 --- a/ydb/core/sys_view/processor/processor_impl.cpp +++ b/ydb/core/sys_view/processor/processor_impl.cpp @@ -1,685 +1,685 @@ -#include "processor_impl.h" - +#include "processor_impl.h" + #include <ydb/core/sys_view/service/sysview_service.h> #include <ydb/core/engine/minikql/flat_local_tx_factory.h> - -#include <library/cpp/monlib/service/pages/templates.h> - -namespace NKikimr { -namespace NSysView { - + +#include <library/cpp/monlib/service/pages/templates.h> + +namespace NKikimr { +namespace NSysView { + TSysViewProcessor::TSysViewProcessor(const TActorId& tablet, TTabletStorageInfo* info, EProcessorMode processorMode) - : TActor(&TThis::StateInit) - , TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory) - , TotalInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 6 : 60)) - , CollectInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 3 : 30)) - , ExternalGroup(new NMonitoring::TDynamicCounters) - , InternalGroup(new NMonitoring::TDynamicCounters) -{} - -void TSysViewProcessor::OnDetach(const TActorContext& ctx) { - DetachExternalCounters(); - DetachInternalCounters(); - - Die(ctx); -} - -void TSysViewProcessor::OnTabletDead(TEvTablet::TEvTabletDead::TPtr&, const TActorContext& ctx) { - DetachExternalCounters(); - DetachInternalCounters(); - - Die(ctx); -} - -void TSysViewProcessor::OnActivateExecutor(const TActorContext& ctx) { - SVLOG_I("[" << TabletID() << "] OnActivateExecutor"); - - // TODO: tablet counters - Execute(CreateTxInitSchema(), ctx); -} - -void TSysViewProcessor::DefaultSignalTabletActive(const TActorContext& ctx) { - Y_UNUSED(ctx); -} - -void TSysViewProcessor::Handle(TEvPrivate::TEvSendRequests::TPtr&) { - SVLOG_D("[" << TabletID() << "] Handle TEvPrivate::TEvSendRequests"); - SendRequests(); -} - -void TSysViewProcessor::PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value) { - db.Table<Schema::SysParams>().Key(id).Update( - NIceDb::TUpdate<Schema::SysParams::Value>(value)); -} - -void TSysViewProcessor::PersistDatabase(NIceDb::TNiceDb& db) { - PersistSysParam(db, Schema::SysParam_Database, Database); -} - -void TSysViewProcessor::PersistStage(NIceDb::TNiceDb& db) { - ui64 stage = static_cast<ui64>(CurrentStage); - PersistSysParam(db, Schema::SysParam_CurrentStage, ToString(stage)); -} - -void TSysViewProcessor::PersistIntervalEnd(NIceDb::TNiceDb& db) { - ui64 intervalEndUs = IntervalEnd.MicroSeconds(); - PersistSysParam(db, Schema::SysParam_IntervalEnd, ToString(intervalEndUs)); -} - -template <typename TSchema> -void TSysViewProcessor::PersistTopResults(NIceDb::TNiceDb& db, - TTop& top, TResultStatsMap& results, TInstant intervalEnd) -{ - ui64 intervalEndUs = intervalEnd.MicroSeconds(); - ui32 rank = 0; - - std::sort(top.begin(), top.end(), TopQueryCompare); - - for (const auto& entry : top) { - if (entry.Stats) { - auto key = std::make_pair(intervalEndUs, ++rank); - auto& resultStats = results[key]; - resultStats = *entry.Stats; - - TString text; - TString serialized; - resultStats.MutableQueryText()->swap(text); + : TActor(&TThis::StateInit) + , TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory) + , TotalInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 6 : 60)) + , CollectInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 3 : 30)) + , ExternalGroup(new NMonitoring::TDynamicCounters) + , InternalGroup(new NMonitoring::TDynamicCounters) +{} + +void TSysViewProcessor::OnDetach(const TActorContext& ctx) { + DetachExternalCounters(); + DetachInternalCounters(); + + Die(ctx); +} + +void TSysViewProcessor::OnTabletDead(TEvTablet::TEvTabletDead::TPtr&, const TActorContext& ctx) { + DetachExternalCounters(); + DetachInternalCounters(); + + Die(ctx); +} + +void TSysViewProcessor::OnActivateExecutor(const TActorContext& ctx) { + SVLOG_I("[" << TabletID() << "] OnActivateExecutor"); + + // TODO: tablet counters + Execute(CreateTxInitSchema(), ctx); +} + +void TSysViewProcessor::DefaultSignalTabletActive(const TActorContext& ctx) { + Y_UNUSED(ctx); +} + +void TSysViewProcessor::Handle(TEvPrivate::TEvSendRequests::TPtr&) { + SVLOG_D("[" << TabletID() << "] Handle TEvPrivate::TEvSendRequests"); + SendRequests(); +} + +void TSysViewProcessor::PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value) { + db.Table<Schema::SysParams>().Key(id).Update( + NIceDb::TUpdate<Schema::SysParams::Value>(value)); +} + +void TSysViewProcessor::PersistDatabase(NIceDb::TNiceDb& db) { + PersistSysParam(db, Schema::SysParam_Database, Database); +} + +void TSysViewProcessor::PersistStage(NIceDb::TNiceDb& db) { + ui64 stage = static_cast<ui64>(CurrentStage); + PersistSysParam(db, Schema::SysParam_CurrentStage, ToString(stage)); +} + +void TSysViewProcessor::PersistIntervalEnd(NIceDb::TNiceDb& db) { + ui64 intervalEndUs = IntervalEnd.MicroSeconds(); + PersistSysParam(db, Schema::SysParam_IntervalEnd, ToString(intervalEndUs)); +} + +template <typename TSchema> +void TSysViewProcessor::PersistTopResults(NIceDb::TNiceDb& db, + TTop& top, TResultStatsMap& results, TInstant intervalEnd) +{ + ui64 intervalEndUs = intervalEnd.MicroSeconds(); + ui32 rank = 0; + + std::sort(top.begin(), top.end(), TopQueryCompare); + + for (const auto& entry : top) { + if (entry.Stats) { + auto key = std::make_pair(intervalEndUs, ++rank); + auto& resultStats = results[key]; + resultStats = *entry.Stats; + + TString text; + TString serialized; + resultStats.MutableQueryText()->swap(text); Y_PROTOBUF_SUPPRESS_NODISCARD resultStats.SerializeToString(&serialized); - db.Table<TSchema>().Key(key).Update( - NIceDb::TUpdate<typename TSchema::Text>(text), - NIceDb::TUpdate<typename TSchema::Data>(serialized)); - resultStats.MutableQueryText()->swap(text); - } - } - - SVLOG_D("[" << TabletID() << "] PersistTopResults: " - << "table id# " << TSchema::TableId - << ", interval end# " << intervalEnd - << ", query count# " << top.size() - << ", persisted# " << rank); -} - -void TSysViewProcessor::PersistResults(NIceDb::TNiceDb& db) { - std::vector<std::pair<ui64, TQueryHash>> sorted; - sorted.reserve(QueryMetrics.size()); - for (const auto& [queryHash, metrics] : QueryMetrics) { - sorted.emplace_back(metrics.Metrics.GetCpuTimeUs().GetSum(), queryHash); - } - std::sort(sorted.begin(), sorted.end(), [] (auto& l, auto& r) { return l.first > r.first; }); - - ui64 intervalEndUs = IntervalEnd.MicroSeconds(); - ui32 rank = 0; - - for (const auto& entry : sorted) { - auto key = std::make_pair(intervalEndUs, ++rank); - - auto& queryMetrics = QueryMetrics[entry.second]; - auto& resultMetrics = MetricsOneMinute[key]; - resultMetrics.Text = queryMetrics.Text; - resultMetrics.Metrics = queryMetrics.Metrics; - - TString serialized; + db.Table<TSchema>().Key(key).Update( + NIceDb::TUpdate<typename TSchema::Text>(text), + NIceDb::TUpdate<typename TSchema::Data>(serialized)); + resultStats.MutableQueryText()->swap(text); + } + } + + SVLOG_D("[" << TabletID() << "] PersistTopResults: " + << "table id# " << TSchema::TableId + << ", interval end# " << intervalEnd + << ", query count# " << top.size() + << ", persisted# " << rank); +} + +void TSysViewProcessor::PersistResults(NIceDb::TNiceDb& db) { + std::vector<std::pair<ui64, TQueryHash>> sorted; + sorted.reserve(QueryMetrics.size()); + for (const auto& [queryHash, metrics] : QueryMetrics) { + sorted.emplace_back(metrics.Metrics.GetCpuTimeUs().GetSum(), queryHash); + } + std::sort(sorted.begin(), sorted.end(), [] (auto& l, auto& r) { return l.first > r.first; }); + + ui64 intervalEndUs = IntervalEnd.MicroSeconds(); + ui32 rank = 0; + + for (const auto& entry : sorted) { + auto key = std::make_pair(intervalEndUs, ++rank); + + auto& queryMetrics = QueryMetrics[entry.second]; + auto& resultMetrics = MetricsOneMinute[key]; + resultMetrics.Text = queryMetrics.Text; + resultMetrics.Metrics = queryMetrics.Metrics; + + TString serialized; Y_PROTOBUF_SUPPRESS_NODISCARD resultMetrics.Metrics.SerializeToString(&serialized); - db.Table<Schema::MetricsOneMinute>().Key(key).Update( - NIceDb::TUpdate<Schema::MetricsOneMinute::Text>(resultMetrics.Text), - NIceDb::TUpdate<Schema::MetricsOneMinute::Data>(serialized)); - } - - SVLOG_D("[" << TabletID() << "] PersistResults: " - << "interval end# " << IntervalEnd - << ", query count# " << sorted.size()); - - // TODO: metrics one hour? - - PersistTopResults<Schema::TopByDurationOneMinute>( - db, ByDurationMinute, TopByDurationOneMinute, IntervalEnd); - PersistTopResults<Schema::TopByReadBytesOneMinute>( - db, ByReadBytesMinute, TopByReadBytesOneMinute, IntervalEnd); - PersistTopResults<Schema::TopByCpuTimeOneMinute>( - db, ByCpuTimeMinute, TopByCpuTimeOneMinute, IntervalEnd); - PersistTopResults<Schema::TopByRequestUnitsOneMinute>( - db, ByRequestUnitsMinute, TopByRequestUnitsOneMinute, IntervalEnd); - - auto hourEnd = EndOfHourInterval(IntervalEnd); - - PersistTopResults<Schema::TopByDurationOneHour>( - db, ByDurationHour, TopByDurationOneHour, hourEnd); - PersistTopResults<Schema::TopByReadBytesOneHour>( - db, ByReadBytesHour, TopByReadBytesOneHour, hourEnd); - PersistTopResults<Schema::TopByCpuTimeOneHour>( - db, ByCpuTimeHour, TopByCpuTimeOneHour, hourEnd); - PersistTopResults<Schema::TopByRequestUnitsOneHour>( - db, ByRequestUnitsHour, TopByRequestUnitsOneHour, hourEnd); -} - -void TSysViewProcessor::ScheduleAggregate() { - auto rangeUs = RandomNumber<ui64>(TotalInterval.MicroSeconds() / 12); - auto deadline = IntervalEnd + CollectInterval + TDuration::MicroSeconds(rangeUs); - Schedule(deadline, new TEvPrivate::TEvAggregate); -} - -void TSysViewProcessor::ScheduleCollect() { - auto rangeUs = RandomNumber<ui64>(TotalInterval.MicroSeconds() / 12); - auto deadline = IntervalEnd + TotalInterval + TDuration::MicroSeconds(rangeUs); - Schedule(deadline, new TEvPrivate::TEvCollect); -} - -void TSysViewProcessor::ScheduleSendRequests() { - auto intervalUs = TotalInterval.MicroSeconds() / 12; - auto rangeUs = RandomNumber<ui64>(intervalUs); - auto deadline = IntervalEnd + CollectInterval + TDuration::MicroSeconds(intervalUs + rangeUs); - Schedule(deadline, new TEvPrivate::TEvSendRequests); -} - -void TSysViewProcessor::ScheduleApplyCounters() { - Schedule(ProcessCountersInterval, new TEvPrivate::TEvApplyCounters); -} - -void TSysViewProcessor::ScheduleSendNavigate() { - Schedule(SendNavigateInterval, new TEvPrivate::TEvSendNavigate); -} - -template <typename TSchema, typename TEntry> -void TSysViewProcessor::CutHistory(NIceDb::TNiceDb& db, - TResultMap<TEntry>& metricsMap, TDuration historySize) -{ - auto past = IntervalEnd - historySize; - auto key = std::make_pair(past.MicroSeconds(), 0); - - auto bound = metricsMap.lower_bound(key); - for (auto it = metricsMap.begin(); it != bound; ++it) { - db.Table<TSchema>().Key(it->first).Delete(); - } - metricsMap.erase(metricsMap.begin(), bound); -} - -TInstant TSysViewProcessor::EndOfHourInterval(TInstant intervalEnd) { - auto hourUs = ONE_HOUR_BUCKET_SIZE.MicroSeconds(); - auto hourEndUs = intervalEnd.MicroSeconds() / hourUs * hourUs; - if (hourEndUs != intervalEnd.MicroSeconds()) { - hourEndUs += hourUs; - } - return TInstant::MicroSeconds(hourEndUs); -} - -void TSysViewProcessor::ClearIntervalSummaries(NIceDb::TNiceDb& db) { - for (const auto& [queryHash, query] : Queries) { - ui32 count = query.Nodes.size(); - for (ui32 i = 0; i < count; ++i) { - db.Table<Schema::IntervalSummaries>().Key(queryHash, i).Delete(); - } - } - Queries.clear(); - ByCpu.clear(); - SummaryNodes.clear(); -} - -void TSysViewProcessor::Reset(NIceDb::TNiceDb& db, const TActorContext& ctx) { - // TODO: efficient delete? - - ClearIntervalSummaries(db); - - for (const auto& [queryHash, _] : QueryMetrics) { - db.Table<Schema::IntervalMetrics>().Key(queryHash).Delete(); - } - QueryMetrics.clear(); - - for (const auto& node : NodesToRequest) { - db.Table<Schema::NodesToRequest>().Key(node.NodeId).Delete(); - } - NodesToRequest.clear(); - NodesInFlight.clear(); - - auto clearTop = [&] (NKikimrSysView::EStatsType type, TTop& top) { - for (const auto& query : top) { - db.Table<Schema::IntervalTops>().Key((ui32)type, query.Hash).Delete(); - } - top.clear(); - }; - - clearTop(NKikimrSysView::TOP_DURATION_ONE_MINUTE, ByDurationMinute); - clearTop(NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, ByReadBytesMinute); - clearTop(NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, ByCpuTimeMinute); - clearTop(NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, ByRequestUnitsMinute); - - CurrentStage = COLLECT; - PersistStage(db); - - auto oldHourEnd = EndOfHourInterval(IntervalEnd); - - auto now = ctx.Now(); - auto intervalSize = TotalInterval.MicroSeconds(); - auto rounded = now.MicroSeconds() / intervalSize * intervalSize; - IntervalEnd = TInstant::MicroSeconds(rounded); - PersistIntervalEnd(db); - - auto newHourEnd = EndOfHourInterval(IntervalEnd); - - if (oldHourEnd != newHourEnd) { - clearTop(NKikimrSysView::TOP_DURATION_ONE_HOUR, ByDurationHour); - clearTop(NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, ByReadBytesHour); - clearTop(NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, ByCpuTimeHour); - clearTop(NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, ByRequestUnitsHour); - } - - SVLOG_D("[" << TabletID() << "] Reset: interval end# " << IntervalEnd); - - const auto minuteHistorySize = TotalInterval * ONE_MINUTE_BUCKET_COUNT; - const auto hourHistorySize = ONE_HOUR_BUCKET_SIZE * ONE_HOUR_BUCKET_COUNT; - - CutHistory<Schema::MetricsOneMinute, TQueryToMetrics>(db, MetricsOneMinute, minuteHistorySize); - CutHistory<Schema::MetricsOneHour, TQueryToMetrics>(db, MetricsOneHour, hourHistorySize); - - using TStats = NKikimrSysView::TQueryStats; - CutHistory<Schema::TopByDurationOneMinute, TStats>(db, TopByDurationOneMinute, minuteHistorySize); - CutHistory<Schema::TopByDurationOneHour, TStats>(db, TopByDurationOneHour, hourHistorySize); - CutHistory<Schema::TopByReadBytesOneMinute, TStats>(db, TopByReadBytesOneMinute, minuteHistorySize); - CutHistory<Schema::TopByReadBytesOneHour, TStats>(db, TopByReadBytesOneHour, hourHistorySize); - CutHistory<Schema::TopByCpuTimeOneMinute, TStats>(db, TopByCpuTimeOneMinute, minuteHistorySize); - CutHistory<Schema::TopByCpuTimeOneHour, TStats>(db, TopByCpuTimeOneHour, hourHistorySize); - CutHistory<Schema::TopByRequestUnitsOneMinute, TStats>(db, TopByRequestUnitsOneMinute, minuteHistorySize); - CutHistory<Schema::TopByRequestUnitsOneHour, TStats>(db, TopByRequestUnitsOneHour, hourHistorySize); -} - -void TSysViewProcessor::SendRequests() { - while (!NodesToRequest.empty() && NodesInFlight.size() < MaxInFlightRequests) { - auto& req = NodesToRequest.back(); - - auto request = MakeHolder<TEvSysView::TEvGetIntervalMetricsRequest>(); - auto& record = request->Record; - record.SetIntervalEndUs(IntervalEnd.MicroSeconds()); - record.SetDatabase(Database); - - auto fillHashes = [&] (const THashVector& hashes, NProtoBuf::RepeatedField<ui64>& result) { - result.Reserve(hashes.size()); - for (auto queryHash : hashes) { - result.Add(queryHash); - } - }; - - fillHashes(req.Hashes, *record.MutableMetrics()); - fillHashes(req.TextsToGet, *record.MutableQueryTextsToGet()); - fillHashes(req.ByDuration, *record.MutableTopByDuration()); - fillHashes(req.ByReadBytes, *record.MutableTopByReadBytes()); - fillHashes(req.ByCpuTime, *record.MutableTopByCpuTime()); - fillHashes(req.ByRequestUnits, *record.MutableTopByRequestUnits()); - - SVLOG_D("[" << TabletID() << "] Send TEvGetIntervalMetricsRequest: " - << "node id# " << req.NodeId - << ", hashes# " << req.Hashes.size() - << ", texts# " << req.TextsToGet.size() - << ", by duration# " << req.ByDuration.size() - << ", by read bytes# " << req.ByReadBytes.size() - << ", by cpu time# " << req.ByCpuTime.size() - << ", by request units# " << req.ByRequestUnits.size()); - - Send(MakeSysViewServiceID(req.NodeId), - std::move(request), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, - req.NodeId); - - NodesInFlight[req.NodeId] = std::move(req); - NodesToRequest.pop_back(); - } -} - -void TSysViewProcessor::IgnoreFailure(TNodeId nodeId) { - NodesInFlight.erase(nodeId); -} - -void TSysViewProcessor::Handle(TEvents::TEvPoisonPill::TPtr&) { - Become(&TThis::StateBroken); - Send(Tablet(), new TEvents::TEvPoisonPill); -} - -void TSysViewProcessor::Handle(TEvents::TEvUndelivered::TPtr& ev) { - auto nodeId = (TNodeId)ev.Get()->Cookie; - SVLOG_W("[" << TabletID() << "] TEvUndelivered: node id# " << nodeId); - IgnoreFailure(nodeId); -} - -void TSysViewProcessor::Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { - auto nodeId = ev->Get()->NodeId; - SVLOG_W("[" << TabletID() << "] TEvNodeDisconnected: node id# " << nodeId); - IgnoreFailure(nodeId); -} - -void TSysViewProcessor::Handle(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) { - const auto& record = ev->Get()->Record; - auto type = record.GetType(); - - if (PendingRequests.size() >= PendingRequestsLimit) { - if (type == NKikimrSysView::METRICS_ONE_MINUTE || type == NKikimrSysView::METRICS_ONE_HOUR) { - ReplyOverloaded<TEvSysView::TEvGetQueryMetricsResponse>(ev); - } else { - ReplyOverloaded<TEvSysView::TEvGetQueryStatsResponse>(ev); - } - return; - } - - PendingRequests.push(std::move(ev)); - - if (!ProcessInFly) { - Send(SelfId(), new TEvPrivate::TEvProcess()); - ProcessInFly = true; - } -} - -void TSysViewProcessor::Handle(TEvPrivate::TEvProcess::TPtr&) { - ProcessInFly = false; - - if (PendingRequests.empty()) { - return; - } - - TEvSysView::TEvGetQueryMetricsRequest::TPtr request = std::move(PendingRequests.front()); - PendingRequests.pop(); - - if (!PendingRequests.empty()) { - Send(SelfId(), new TEvPrivate::TEvProcess); - ProcessInFly = true; - } - - const auto& record = request->Get()->Record; - auto type = record.GetType(); - - if (type == NKikimrSysView::METRICS_ONE_MINUTE || type == NKikimrSysView::METRICS_ONE_HOUR) { - Reply<TQueryToMetrics, TEvSysView::TEvGetQueryMetricsResponse>(request); - } else { - Reply<NKikimrSysView::TQueryStats, TEvSysView::TEvGetQueryStatsResponse>(request); - } -} - -template <typename TResponse> -void TSysViewProcessor::ReplyOverloaded(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) { - auto response = MakeHolder<TResponse>(); - response->Record.SetOverloaded(true); - Send(ev->Sender, std::move(response)); -} - -template <typename TEntry, typename TResponse> -void TSysViewProcessor::Reply(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) { - const auto& record = ev->Get()->Record; - auto response = MakeHolder<TResponse>(); - response->Record.SetLastBatch(true); - - TResultMap<TEntry>* entries = nullptr; - if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) { - switch (record.GetType()) { - case NKikimrSysView::METRICS_ONE_MINUTE: - entries = &MetricsOneMinute; - break; - case NKikimrSysView::METRICS_ONE_HOUR: - entries = &MetricsOneHour; - break; - default: - SVLOG_CRIT("[" << TabletID() << "] unexpected stats type: " << (size_t)record.GetType()); - Send(ev->Sender, std::move(response)); - return; - } - } else { - switch (record.GetType()) { - case NKikimrSysView::TOP_DURATION_ONE_MINUTE: - entries = &TopByDurationOneMinute; - break; - case NKikimrSysView::TOP_DURATION_ONE_HOUR: - entries = &TopByDurationOneHour; - break; - case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE: - entries = &TopByReadBytesOneMinute; - break; - case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR: - entries = &TopByReadBytesOneHour; - break; - case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE: - entries = &TopByCpuTimeOneMinute; - break; - case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR: - entries = &TopByCpuTimeOneHour; - break; - case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE: - entries = &TopByRequestUnitsOneMinute; - break; - case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR: - entries = &TopByRequestUnitsOneHour; - break; - default: - SVLOG_CRIT("[" << TabletID() << "] unexpected stats type: " << (size_t)record.GetType()); - Send(ev->Sender, std::move(response)); - return; - } - } - - Y_VERIFY(entries); - - auto from = entries->begin(); - auto to = entries->end(); - - TString fromStr("[]"); - if (record.HasFrom()) { - auto key = std::make_pair(record.GetFrom().GetIntervalEndUs(), record.GetFrom().GetRank()); - if (!record.HasInclusiveFrom() || record.GetInclusiveFrom()) { - from = entries->lower_bound(key); - } else { - from = entries->upper_bound(key); - } - TStringBuilder str; - str << "[" << record.GetFrom().GetIntervalEndUs() - << ", " << record.GetFrom().GetRank() - << ", " << (record.GetInclusiveFrom() ? "inc]" : "exc]"); - fromStr = str; - } - - TString toStr("[]"); - if (record.HasTo()) { - auto key = std::make_pair(record.GetTo().GetIntervalEndUs(), record.GetTo().GetRank()); - if (!record.HasInclusiveTo() || !record.GetInclusiveTo()) { - to = entries->lower_bound(key); - } else { - to = entries->upper_bound(key); - } - TStringBuilder str; - str << "[" << record.GetTo().GetIntervalEndUs() - << ", " << record.GetTo().GetRank() - << ", " << (record.GetInclusiveTo() ? "inc]" : "exc]"); - toStr = str; - } - - TString nextStr("[]"); - size_t size = 0; - size_t count = 0; - for (auto it = from; it != to; ++it) { - const auto& key = it->first; - - auto& entry = *response->Record.AddEntries(); - - auto& entryKey = *entry.MutableKey(); - entryKey.SetIntervalEndUs(key.first); - entryKey.SetRank(key.second); - - if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) { - const auto& metrics = it->second; - entry.MutableMetrics()->CopyFrom(metrics.Metrics); - entry.SetQueryText(metrics.Text); - } else { - const auto& stats = it->second; - entry.MutableStats()->CopyFrom(stats); - } - - size += entry.ByteSizeLong(); - ++count; - - if (size >= BatchSizeLimit) { - auto* next = response->Record.MutableNext(); - next->SetIntervalEndUs(key.first); - next->SetRank(key.second + 1); - response->Record.SetLastBatch(false); - - TStringBuilder str; - str << "[" << next->GetIntervalEndUs() - << ", " << next->GetRank() - << "]"; - nextStr = str; - break; - } - } - - SVLOG_D("[" << TabletID() << "] Reply to TEvGetQueryMetricsRequest: " - << "from# " << fromStr - << ", to# " << toStr - << ", rows# " << count - << ", bytes# " << size - << ", next# " << nextStr); - - Send(ev->Sender, std::move(response)); -} - -bool TSysViewProcessor::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, - const TActorContext& ctx) -{ - if (!ev) { - return true; - } - - TStringStream str; - HTML(str) { - PRE() { - str << "---- SysViewProcessor ----" << Endl << Endl; - str << "Database: " << Database << Endl; - str << "IntervalEnd: " << IntervalEnd << Endl; - str << "CurrentStage: " << (CurrentStage == COLLECT ? "Collect" : "Aggregate") - << Endl << Endl; - { - str << "IntervalSummaries" << Endl; - str << " QueryCount: " << Queries.size() << Endl; - - auto it = ByCpu.rbegin(); - static constexpr size_t queriesLimit = 32; - for (size_t q = 0; - it != ByCpu.rend() && q < queriesLimit; - ++it, ++q) - { - const auto queryHash = it->second; - auto queryIt = Queries.find(queryHash); - Y_VERIFY(queryIt != Queries.end()); - const auto& query = queryIt->second; - - str << " Hash: " << queryHash - << ", Cpu: " << query.Cpu - << ", NodeCount: " << query.Nodes.size() - << ", Nodes: "; - - static constexpr size_t nodesLimit = 4; - auto nodeIt = query.Nodes.begin(); - for (size_t n = 0; - nodeIt != query.Nodes.end() && n < nodesLimit; - ++nodeIt, ++n) - { - str << "{ " << nodeIt->first << ", Cpu: " << nodeIt->second << " } "; - } - if (nodeIt != query.Nodes.end()) { - str << "..."; - } - str << Endl; - } - if (it != ByCpu.rend()) { - str << " ..." << Endl; - } - str << Endl; - } - { - str << "IntervalMetrics" << Endl; - for (const auto& [queryHash, metrics] : QueryMetrics) { - str << " Hash: " << queryHash - << ", Count: " << metrics.Metrics.GetCount() - << ", SumCpuTime: " << metrics.Metrics.GetCpuTimeUs().GetSum() << Endl; - } - str << Endl; - } - { - auto dumpNode = [&str] (const TNodeToQueries& node) { - str << " NodeId: " << node.NodeId - << ", Hashes: " << node.Hashes.size() - << ", TextsToGet: " << node.TextsToGet.size() - << ", ByDuration: " << node.ByDuration.size() - << ", ByReadBytes: " << node.ByReadBytes.size() - << ", ByCpuTime: " << node.ByCpuTime.size() - << ", ByRequestUnits: " << node.ByRequestUnits.size() - << Endl; - }; - str << "NodesToRequest" << Endl; - for (const auto& node : NodesToRequest) { - dumpNode(node); - } - str << Endl; - str << "NodesInFlight" << Endl; - for (const auto& [_, node] : NodesInFlight) { - dumpNode(node); - } - str << Endl; - } - { - auto printTop = [&str] (const TTop& top) { - for (const auto& query : top) { - str << " Hash: " << query.Hash - << ", Value: " << query.Value - << ", NodeId: " << query.NodeId << Endl; - } - }; - str << "ByDurationMinute" << Endl; - printTop(ByDurationMinute); - str << Endl; - str << "ByDurationHour" << Endl; - printTop(ByDurationHour); - str << Endl; - str << "ByReadBytesMinute" << Endl; - printTop(ByReadBytesMinute); - str << Endl; - str << "ByReadBytesHour" << Endl; - printTop(ByReadBytesHour); - str << Endl; - str << "ByCpuTimeMinute" << Endl; - printTop(ByCpuTimeMinute); - str << Endl; - str << "ByCpuTimeHour" << Endl; - printTop(ByCpuTimeHour); - str << Endl; - str << "ByRequestUnitsMinute" << Endl; - printTop(ByRequestUnitsMinute); - str << Endl; - str << "ByRequestUnitsHour" << Endl; - printTop(ByRequestUnitsHour); - str << Endl; - } - { - str << "MetricsOneMinute" << Endl - << " Count: " << MetricsOneMinute.size() << Endl << Endl; - str << "MetricsOneHour" << Endl - << " Count: " << MetricsOneHour.size() << Endl << Endl; - str << "TopByDurationOneMinute" << Endl - << " Count: " << TopByDurationOneMinute.size() << Endl << Endl; - str << "TopByDurationOneHour" << Endl - << " Count: " << TopByDurationOneHour.size() << Endl << Endl; - str << "TopByReadBytesOneMinute" << Endl - << " Count: " << TopByReadBytesOneMinute.size() << Endl << Endl; - str << "TopByReadBytesOneHour" << Endl - << " Count: " << TopByReadBytesOneHour.size() << Endl << Endl; - str << "TopByCpuTimeOneMinute" << Endl - << " Count: " << TopByCpuTimeOneMinute.size() << Endl << Endl; - str << "TopByCpuTimeOneHour" << Endl - << " Count: " << TopByCpuTimeOneHour.size() << Endl << Endl; - str << "TopByRequestUnitsOneMinute" << Endl - << " Count: " << TopByRequestUnitsOneMinute.size() << Endl << Endl; - str << "TopByRequestUnitsOneHour" << Endl - << " Count: " << TopByRequestUnitsOneHour.size() << Endl << Endl; - } - } - } - - ctx.Send(ev->Sender, new NMon::TEvRemoteHttpInfoRes(str.Str())); - return true; -} - -} // NSysView -} // NKikimr - + db.Table<Schema::MetricsOneMinute>().Key(key).Update( + NIceDb::TUpdate<Schema::MetricsOneMinute::Text>(resultMetrics.Text), + NIceDb::TUpdate<Schema::MetricsOneMinute::Data>(serialized)); + } + + SVLOG_D("[" << TabletID() << "] PersistResults: " + << "interval end# " << IntervalEnd + << ", query count# " << sorted.size()); + + // TODO: metrics one hour? + + PersistTopResults<Schema::TopByDurationOneMinute>( + db, ByDurationMinute, TopByDurationOneMinute, IntervalEnd); + PersistTopResults<Schema::TopByReadBytesOneMinute>( + db, ByReadBytesMinute, TopByReadBytesOneMinute, IntervalEnd); + PersistTopResults<Schema::TopByCpuTimeOneMinute>( + db, ByCpuTimeMinute, TopByCpuTimeOneMinute, IntervalEnd); + PersistTopResults<Schema::TopByRequestUnitsOneMinute>( + db, ByRequestUnitsMinute, TopByRequestUnitsOneMinute, IntervalEnd); + + auto hourEnd = EndOfHourInterval(IntervalEnd); + + PersistTopResults<Schema::TopByDurationOneHour>( + db, ByDurationHour, TopByDurationOneHour, hourEnd); + PersistTopResults<Schema::TopByReadBytesOneHour>( + db, ByReadBytesHour, TopByReadBytesOneHour, hourEnd); + PersistTopResults<Schema::TopByCpuTimeOneHour>( + db, ByCpuTimeHour, TopByCpuTimeOneHour, hourEnd); + PersistTopResults<Schema::TopByRequestUnitsOneHour>( + db, ByRequestUnitsHour, TopByRequestUnitsOneHour, hourEnd); +} + +void TSysViewProcessor::ScheduleAggregate() { + auto rangeUs = RandomNumber<ui64>(TotalInterval.MicroSeconds() / 12); + auto deadline = IntervalEnd + CollectInterval + TDuration::MicroSeconds(rangeUs); + Schedule(deadline, new TEvPrivate::TEvAggregate); +} + +void TSysViewProcessor::ScheduleCollect() { + auto rangeUs = RandomNumber<ui64>(TotalInterval.MicroSeconds() / 12); + auto deadline = IntervalEnd + TotalInterval + TDuration::MicroSeconds(rangeUs); + Schedule(deadline, new TEvPrivate::TEvCollect); +} + +void TSysViewProcessor::ScheduleSendRequests() { + auto intervalUs = TotalInterval.MicroSeconds() / 12; + auto rangeUs = RandomNumber<ui64>(intervalUs); + auto deadline = IntervalEnd + CollectInterval + TDuration::MicroSeconds(intervalUs + rangeUs); + Schedule(deadline, new TEvPrivate::TEvSendRequests); +} + +void TSysViewProcessor::ScheduleApplyCounters() { + Schedule(ProcessCountersInterval, new TEvPrivate::TEvApplyCounters); +} + +void TSysViewProcessor::ScheduleSendNavigate() { + Schedule(SendNavigateInterval, new TEvPrivate::TEvSendNavigate); +} + +template <typename TSchema, typename TEntry> +void TSysViewProcessor::CutHistory(NIceDb::TNiceDb& db, + TResultMap<TEntry>& metricsMap, TDuration historySize) +{ + auto past = IntervalEnd - historySize; + auto key = std::make_pair(past.MicroSeconds(), 0); + + auto bound = metricsMap.lower_bound(key); + for (auto it = metricsMap.begin(); it != bound; ++it) { + db.Table<TSchema>().Key(it->first).Delete(); + } + metricsMap.erase(metricsMap.begin(), bound); +} + +TInstant TSysViewProcessor::EndOfHourInterval(TInstant intervalEnd) { + auto hourUs = ONE_HOUR_BUCKET_SIZE.MicroSeconds(); + auto hourEndUs = intervalEnd.MicroSeconds() / hourUs * hourUs; + if (hourEndUs != intervalEnd.MicroSeconds()) { + hourEndUs += hourUs; + } + return TInstant::MicroSeconds(hourEndUs); +} + +void TSysViewProcessor::ClearIntervalSummaries(NIceDb::TNiceDb& db) { + for (const auto& [queryHash, query] : Queries) { + ui32 count = query.Nodes.size(); + for (ui32 i = 0; i < count; ++i) { + db.Table<Schema::IntervalSummaries>().Key(queryHash, i).Delete(); + } + } + Queries.clear(); + ByCpu.clear(); + SummaryNodes.clear(); +} + +void TSysViewProcessor::Reset(NIceDb::TNiceDb& db, const TActorContext& ctx) { + // TODO: efficient delete? + + ClearIntervalSummaries(db); + + for (const auto& [queryHash, _] : QueryMetrics) { + db.Table<Schema::IntervalMetrics>().Key(queryHash).Delete(); + } + QueryMetrics.clear(); + + for (const auto& node : NodesToRequest) { + db.Table<Schema::NodesToRequest>().Key(node.NodeId).Delete(); + } + NodesToRequest.clear(); + NodesInFlight.clear(); + + auto clearTop = [&] (NKikimrSysView::EStatsType type, TTop& top) { + for (const auto& query : top) { + db.Table<Schema::IntervalTops>().Key((ui32)type, query.Hash).Delete(); + } + top.clear(); + }; + + clearTop(NKikimrSysView::TOP_DURATION_ONE_MINUTE, ByDurationMinute); + clearTop(NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, ByReadBytesMinute); + clearTop(NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, ByCpuTimeMinute); + clearTop(NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, ByRequestUnitsMinute); + + CurrentStage = COLLECT; + PersistStage(db); + + auto oldHourEnd = EndOfHourInterval(IntervalEnd); + + auto now = ctx.Now(); + auto intervalSize = TotalInterval.MicroSeconds(); + auto rounded = now.MicroSeconds() / intervalSize * intervalSize; + IntervalEnd = TInstant::MicroSeconds(rounded); + PersistIntervalEnd(db); + + auto newHourEnd = EndOfHourInterval(IntervalEnd); + + if (oldHourEnd != newHourEnd) { + clearTop(NKikimrSysView::TOP_DURATION_ONE_HOUR, ByDurationHour); + clearTop(NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, ByReadBytesHour); + clearTop(NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, ByCpuTimeHour); + clearTop(NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, ByRequestUnitsHour); + } + + SVLOG_D("[" << TabletID() << "] Reset: interval end# " << IntervalEnd); + + const auto minuteHistorySize = TotalInterval * ONE_MINUTE_BUCKET_COUNT; + const auto hourHistorySize = ONE_HOUR_BUCKET_SIZE * ONE_HOUR_BUCKET_COUNT; + + CutHistory<Schema::MetricsOneMinute, TQueryToMetrics>(db, MetricsOneMinute, minuteHistorySize); + CutHistory<Schema::MetricsOneHour, TQueryToMetrics>(db, MetricsOneHour, hourHistorySize); + + using TStats = NKikimrSysView::TQueryStats; + CutHistory<Schema::TopByDurationOneMinute, TStats>(db, TopByDurationOneMinute, minuteHistorySize); + CutHistory<Schema::TopByDurationOneHour, TStats>(db, TopByDurationOneHour, hourHistorySize); + CutHistory<Schema::TopByReadBytesOneMinute, TStats>(db, TopByReadBytesOneMinute, minuteHistorySize); + CutHistory<Schema::TopByReadBytesOneHour, TStats>(db, TopByReadBytesOneHour, hourHistorySize); + CutHistory<Schema::TopByCpuTimeOneMinute, TStats>(db, TopByCpuTimeOneMinute, minuteHistorySize); + CutHistory<Schema::TopByCpuTimeOneHour, TStats>(db, TopByCpuTimeOneHour, hourHistorySize); + CutHistory<Schema::TopByRequestUnitsOneMinute, TStats>(db, TopByRequestUnitsOneMinute, minuteHistorySize); + CutHistory<Schema::TopByRequestUnitsOneHour, TStats>(db, TopByRequestUnitsOneHour, hourHistorySize); +} + +void TSysViewProcessor::SendRequests() { + while (!NodesToRequest.empty() && NodesInFlight.size() < MaxInFlightRequests) { + auto& req = NodesToRequest.back(); + + auto request = MakeHolder<TEvSysView::TEvGetIntervalMetricsRequest>(); + auto& record = request->Record; + record.SetIntervalEndUs(IntervalEnd.MicroSeconds()); + record.SetDatabase(Database); + + auto fillHashes = [&] (const THashVector& hashes, NProtoBuf::RepeatedField<ui64>& result) { + result.Reserve(hashes.size()); + for (auto queryHash : hashes) { + result.Add(queryHash); + } + }; + + fillHashes(req.Hashes, *record.MutableMetrics()); + fillHashes(req.TextsToGet, *record.MutableQueryTextsToGet()); + fillHashes(req.ByDuration, *record.MutableTopByDuration()); + fillHashes(req.ByReadBytes, *record.MutableTopByReadBytes()); + fillHashes(req.ByCpuTime, *record.MutableTopByCpuTime()); + fillHashes(req.ByRequestUnits, *record.MutableTopByRequestUnits()); + + SVLOG_D("[" << TabletID() << "] Send TEvGetIntervalMetricsRequest: " + << "node id# " << req.NodeId + << ", hashes# " << req.Hashes.size() + << ", texts# " << req.TextsToGet.size() + << ", by duration# " << req.ByDuration.size() + << ", by read bytes# " << req.ByReadBytes.size() + << ", by cpu time# " << req.ByCpuTime.size() + << ", by request units# " << req.ByRequestUnits.size()); + + Send(MakeSysViewServiceID(req.NodeId), + std::move(request), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, + req.NodeId); + + NodesInFlight[req.NodeId] = std::move(req); + NodesToRequest.pop_back(); + } +} + +void TSysViewProcessor::IgnoreFailure(TNodeId nodeId) { + NodesInFlight.erase(nodeId); +} + +void TSysViewProcessor::Handle(TEvents::TEvPoisonPill::TPtr&) { + Become(&TThis::StateBroken); + Send(Tablet(), new TEvents::TEvPoisonPill); +} + +void TSysViewProcessor::Handle(TEvents::TEvUndelivered::TPtr& ev) { + auto nodeId = (TNodeId)ev.Get()->Cookie; + SVLOG_W("[" << TabletID() << "] TEvUndelivered: node id# " << nodeId); + IgnoreFailure(nodeId); +} + +void TSysViewProcessor::Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { + auto nodeId = ev->Get()->NodeId; + SVLOG_W("[" << TabletID() << "] TEvNodeDisconnected: node id# " << nodeId); + IgnoreFailure(nodeId); +} + +void TSysViewProcessor::Handle(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) { + const auto& record = ev->Get()->Record; + auto type = record.GetType(); + + if (PendingRequests.size() >= PendingRequestsLimit) { + if (type == NKikimrSysView::METRICS_ONE_MINUTE || type == NKikimrSysView::METRICS_ONE_HOUR) { + ReplyOverloaded<TEvSysView::TEvGetQueryMetricsResponse>(ev); + } else { + ReplyOverloaded<TEvSysView::TEvGetQueryStatsResponse>(ev); + } + return; + } + + PendingRequests.push(std::move(ev)); + + if (!ProcessInFly) { + Send(SelfId(), new TEvPrivate::TEvProcess()); + ProcessInFly = true; + } +} + +void TSysViewProcessor::Handle(TEvPrivate::TEvProcess::TPtr&) { + ProcessInFly = false; + + if (PendingRequests.empty()) { + return; + } + + TEvSysView::TEvGetQueryMetricsRequest::TPtr request = std::move(PendingRequests.front()); + PendingRequests.pop(); + + if (!PendingRequests.empty()) { + Send(SelfId(), new TEvPrivate::TEvProcess); + ProcessInFly = true; + } + + const auto& record = request->Get()->Record; + auto type = record.GetType(); + + if (type == NKikimrSysView::METRICS_ONE_MINUTE || type == NKikimrSysView::METRICS_ONE_HOUR) { + Reply<TQueryToMetrics, TEvSysView::TEvGetQueryMetricsResponse>(request); + } else { + Reply<NKikimrSysView::TQueryStats, TEvSysView::TEvGetQueryStatsResponse>(request); + } +} + +template <typename TResponse> +void TSysViewProcessor::ReplyOverloaded(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) { + auto response = MakeHolder<TResponse>(); + response->Record.SetOverloaded(true); + Send(ev->Sender, std::move(response)); +} + +template <typename TEntry, typename TResponse> +void TSysViewProcessor::Reply(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) { + const auto& record = ev->Get()->Record; + auto response = MakeHolder<TResponse>(); + response->Record.SetLastBatch(true); + + TResultMap<TEntry>* entries = nullptr; + if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) { + switch (record.GetType()) { + case NKikimrSysView::METRICS_ONE_MINUTE: + entries = &MetricsOneMinute; + break; + case NKikimrSysView::METRICS_ONE_HOUR: + entries = &MetricsOneHour; + break; + default: + SVLOG_CRIT("[" << TabletID() << "] unexpected stats type: " << (size_t)record.GetType()); + Send(ev->Sender, std::move(response)); + return; + } + } else { + switch (record.GetType()) { + case NKikimrSysView::TOP_DURATION_ONE_MINUTE: + entries = &TopByDurationOneMinute; + break; + case NKikimrSysView::TOP_DURATION_ONE_HOUR: + entries = &TopByDurationOneHour; + break; + case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE: + entries = &TopByReadBytesOneMinute; + break; + case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR: + entries = &TopByReadBytesOneHour; + break; + case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE: + entries = &TopByCpuTimeOneMinute; + break; + case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR: + entries = &TopByCpuTimeOneHour; + break; + case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE: + entries = &TopByRequestUnitsOneMinute; + break; + case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR: + entries = &TopByRequestUnitsOneHour; + break; + default: + SVLOG_CRIT("[" << TabletID() << "] unexpected stats type: " << (size_t)record.GetType()); + Send(ev->Sender, std::move(response)); + return; + } + } + + Y_VERIFY(entries); + + auto from = entries->begin(); + auto to = entries->end(); + + TString fromStr("[]"); + if (record.HasFrom()) { + auto key = std::make_pair(record.GetFrom().GetIntervalEndUs(), record.GetFrom().GetRank()); + if (!record.HasInclusiveFrom() || record.GetInclusiveFrom()) { + from = entries->lower_bound(key); + } else { + from = entries->upper_bound(key); + } + TStringBuilder str; + str << "[" << record.GetFrom().GetIntervalEndUs() + << ", " << record.GetFrom().GetRank() + << ", " << (record.GetInclusiveFrom() ? "inc]" : "exc]"); + fromStr = str; + } + + TString toStr("[]"); + if (record.HasTo()) { + auto key = std::make_pair(record.GetTo().GetIntervalEndUs(), record.GetTo().GetRank()); + if (!record.HasInclusiveTo() || !record.GetInclusiveTo()) { + to = entries->lower_bound(key); + } else { + to = entries->upper_bound(key); + } + TStringBuilder str; + str << "[" << record.GetTo().GetIntervalEndUs() + << ", " << record.GetTo().GetRank() + << ", " << (record.GetInclusiveTo() ? "inc]" : "exc]"); + toStr = str; + } + + TString nextStr("[]"); + size_t size = 0; + size_t count = 0; + for (auto it = from; it != to; ++it) { + const auto& key = it->first; + + auto& entry = *response->Record.AddEntries(); + + auto& entryKey = *entry.MutableKey(); + entryKey.SetIntervalEndUs(key.first); + entryKey.SetRank(key.second); + + if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) { + const auto& metrics = it->second; + entry.MutableMetrics()->CopyFrom(metrics.Metrics); + entry.SetQueryText(metrics.Text); + } else { + const auto& stats = it->second; + entry.MutableStats()->CopyFrom(stats); + } + + size += entry.ByteSizeLong(); + ++count; + + if (size >= BatchSizeLimit) { + auto* next = response->Record.MutableNext(); + next->SetIntervalEndUs(key.first); + next->SetRank(key.second + 1); + response->Record.SetLastBatch(false); + + TStringBuilder str; + str << "[" << next->GetIntervalEndUs() + << ", " << next->GetRank() + << "]"; + nextStr = str; + break; + } + } + + SVLOG_D("[" << TabletID() << "] Reply to TEvGetQueryMetricsRequest: " + << "from# " << fromStr + << ", to# " << toStr + << ", rows# " << count + << ", bytes# " << size + << ", next# " << nextStr); + + Send(ev->Sender, std::move(response)); +} + +bool TSysViewProcessor::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, + const TActorContext& ctx) +{ + if (!ev) { + return true; + } + + TStringStream str; + HTML(str) { + PRE() { + str << "---- SysViewProcessor ----" << Endl << Endl; + str << "Database: " << Database << Endl; + str << "IntervalEnd: " << IntervalEnd << Endl; + str << "CurrentStage: " << (CurrentStage == COLLECT ? "Collect" : "Aggregate") + << Endl << Endl; + { + str << "IntervalSummaries" << Endl; + str << " QueryCount: " << Queries.size() << Endl; + + auto it = ByCpu.rbegin(); + static constexpr size_t queriesLimit = 32; + for (size_t q = 0; + it != ByCpu.rend() && q < queriesLimit; + ++it, ++q) + { + const auto queryHash = it->second; + auto queryIt = Queries.find(queryHash); + Y_VERIFY(queryIt != Queries.end()); + const auto& query = queryIt->second; + + str << " Hash: " << queryHash + << ", Cpu: " << query.Cpu + << ", NodeCount: " << query.Nodes.size() + << ", Nodes: "; + + static constexpr size_t nodesLimit = 4; + auto nodeIt = query.Nodes.begin(); + for (size_t n = 0; + nodeIt != query.Nodes.end() && n < nodesLimit; + ++nodeIt, ++n) + { + str << "{ " << nodeIt->first << ", Cpu: " << nodeIt->second << " } "; + } + if (nodeIt != query.Nodes.end()) { + str << "..."; + } + str << Endl; + } + if (it != ByCpu.rend()) { + str << " ..." << Endl; + } + str << Endl; + } + { + str << "IntervalMetrics" << Endl; + for (const auto& [queryHash, metrics] : QueryMetrics) { + str << " Hash: " << queryHash + << ", Count: " << metrics.Metrics.GetCount() + << ", SumCpuTime: " << metrics.Metrics.GetCpuTimeUs().GetSum() << Endl; + } + str << Endl; + } + { + auto dumpNode = [&str] (const TNodeToQueries& node) { + str << " NodeId: " << node.NodeId + << ", Hashes: " << node.Hashes.size() + << ", TextsToGet: " << node.TextsToGet.size() + << ", ByDuration: " << node.ByDuration.size() + << ", ByReadBytes: " << node.ByReadBytes.size() + << ", ByCpuTime: " << node.ByCpuTime.size() + << ", ByRequestUnits: " << node.ByRequestUnits.size() + << Endl; + }; + str << "NodesToRequest" << Endl; + for (const auto& node : NodesToRequest) { + dumpNode(node); + } + str << Endl; + str << "NodesInFlight" << Endl; + for (const auto& [_, node] : NodesInFlight) { + dumpNode(node); + } + str << Endl; + } + { + auto printTop = [&str] (const TTop& top) { + for (const auto& query : top) { + str << " Hash: " << query.Hash + << ", Value: " << query.Value + << ", NodeId: " << query.NodeId << Endl; + } + }; + str << "ByDurationMinute" << Endl; + printTop(ByDurationMinute); + str << Endl; + str << "ByDurationHour" << Endl; + printTop(ByDurationHour); + str << Endl; + str << "ByReadBytesMinute" << Endl; + printTop(ByReadBytesMinute); + str << Endl; + str << "ByReadBytesHour" << Endl; + printTop(ByReadBytesHour); + str << Endl; + str << "ByCpuTimeMinute" << Endl; + printTop(ByCpuTimeMinute); + str << Endl; + str << "ByCpuTimeHour" << Endl; + printTop(ByCpuTimeHour); + str << Endl; + str << "ByRequestUnitsMinute" << Endl; + printTop(ByRequestUnitsMinute); + str << Endl; + str << "ByRequestUnitsHour" << Endl; + printTop(ByRequestUnitsHour); + str << Endl; + } + { + str << "MetricsOneMinute" << Endl + << " Count: " << MetricsOneMinute.size() << Endl << Endl; + str << "MetricsOneHour" << Endl + << " Count: " << MetricsOneHour.size() << Endl << Endl; + str << "TopByDurationOneMinute" << Endl + << " Count: " << TopByDurationOneMinute.size() << Endl << Endl; + str << "TopByDurationOneHour" << Endl + << " Count: " << TopByDurationOneHour.size() << Endl << Endl; + str << "TopByReadBytesOneMinute" << Endl + << " Count: " << TopByReadBytesOneMinute.size() << Endl << Endl; + str << "TopByReadBytesOneHour" << Endl + << " Count: " << TopByReadBytesOneHour.size() << Endl << Endl; + str << "TopByCpuTimeOneMinute" << Endl + << " Count: " << TopByCpuTimeOneMinute.size() << Endl << Endl; + str << "TopByCpuTimeOneHour" << Endl + << " Count: " << TopByCpuTimeOneHour.size() << Endl << Endl; + str << "TopByRequestUnitsOneMinute" << Endl + << " Count: " << TopByRequestUnitsOneMinute.size() << Endl << Endl; + str << "TopByRequestUnitsOneHour" << Endl + << " Count: " << TopByRequestUnitsOneHour.size() << Endl << Endl; + } + } + } + + ctx.Send(ev->Sender, new NMon::TEvRemoteHttpInfoRes(str.Str())); + return true; +} + +} // NSysView +} // NKikimr + diff --git a/ydb/core/sys_view/processor/processor_impl.h b/ydb/core/sys_view/processor/processor_impl.h index 78e8ba7856..4b49ecbb93 100644 --- a/ydb/core/sys_view/processor/processor_impl.h +++ b/ydb/core/sys_view/processor/processor_impl.h @@ -1,10 +1,10 @@ -#pragma once - -#include "schema.h" - +#pragma once + +#include "schema.h" + #include <ydb/core/protos/sys_view.pb.h> #include <ydb/core/protos/counters_sysview_processor.pb.h> - + #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/sys_view/common/common.h> #include <ydb/core/sys_view/common/events.h> @@ -13,327 +13,327 @@ #include <ydb/core/tablet_flat/tablet_flat_executed.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/tx/tx.h> - -#include <library/cpp/actors/core/interconnect.h> -#include <library/cpp/actors/core/memory_track.h> - -namespace NKikimr { -namespace NSysView { - -static constexpr char MemoryLabelResults[] = "SysViewProcessor/Results"; - -class TSysViewProcessor : public TActor<TSysViewProcessor>, public NTabletFlatExecutor::TTabletExecutedFlat { -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::SYSTEM_VIEW_PROCESSOR; - } - + +#include <library/cpp/actors/core/interconnect.h> +#include <library/cpp/actors/core/memory_track.h> + +namespace NKikimr { +namespace NSysView { + +static constexpr char MemoryLabelResults[] = "SysViewProcessor/Results"; + +class TSysViewProcessor : public TActor<TSysViewProcessor>, public NTabletFlatExecutor::TTabletExecutedFlat { +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::SYSTEM_VIEW_PROCESSOR; + } + TSysViewProcessor(const TActorId& tablet, TTabletStorageInfo* info, EProcessorMode processorMode); - -private: - using Schema = TProcessorSchema; - using TTxBase = NTabletFlatExecutor::TTransactionBase<TSysViewProcessor>; - - struct TTxInitSchema; - struct TTxInit; - struct TTxConfigure; - struct TTxCollect; - struct TTxAggregate; - struct TTxIntervalSummary; - struct TTxIntervalMetrics; - - struct TEvPrivate { - enum EEv { - EvCollect = EventSpaceBegin(TEvents::ES_PRIVATE), - EvAggregate, - EvSendRequests, - EvProcess, - EvApplyCounters, - EvSendNavigate, - EvEnd - }; - - struct TEvCollect : public TEventLocal<TEvCollect, EvCollect> {}; - - struct TEvAggregate : public TEventLocal<TEvAggregate, EvAggregate> {}; - - struct TEvSendRequests : public TEventLocal<TEvSendRequests, EvSendRequests> {}; - - struct TEvProcess : public TEventLocal<TEvProcess, EvProcess> {}; - - struct TEvApplyCounters : public TEventLocal<TEvApplyCounters, EvApplyCounters> {}; - - struct TEvSendNavigate : public TEventLocal<TEvSendNavigate, EvSendNavigate> {}; - }; - - struct TTopQuery { - TQueryHash Hash; - ui64 Value; - TNodeId NodeId; - THolder<NKikimrSysView::TQueryStats> Stats; - }; - using TTop = std::vector<TTopQuery>; - - using TResultKey = std::pair<ui64, ui32>; - - template <typename TEntry> - using TResultMap = std::map<TResultKey, TEntry, std::less<TResultKey>, - NActors::NMemory::TAlloc<std::pair<const TResultKey, TEntry>, MemoryLabelResults>>; - using TResultStatsMap = TResultMap<NKikimrSysView::TQueryStats>; - -private: - static bool TopQueryCompare(const TTopQuery& l, const TTopQuery& r) { - return l.Value == r.Value ? l.Hash > r.Hash : l.Value > r.Value; - } - - void OnDetach(const TActorContext& ctx) override; - void OnTabletDead(TEvTablet::TEvTabletDead::TPtr& ev, const TActorContext& ctx) override; - void OnActivateExecutor(const TActorContext& ctx) override; - void DefaultSignalTabletActive(const TActorContext& ctx) override; - bool OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext &ctx) override; - - NTabletFlatExecutor::ITransaction* CreateTxInitSchema(); - NTabletFlatExecutor::ITransaction* CreateTxInit(); - - void Handle(TEvSysView::TEvConfigureProcessor::TPtr& ev); - void Handle(TEvPrivate::TEvCollect::TPtr& ev); - void Handle(TEvPrivate::TEvAggregate::TPtr& ev); - void Handle(TEvPrivate::TEvSendRequests::TPtr& ev); - void Handle(TEvPrivate::TEvProcess::TPtr& ev); - void Handle(TEvSysView::TEvIntervalQuerySummary::TPtr& ev); - void Handle(TEvSysView::TEvGetIntervalMetricsResponse::TPtr& ev); - void Handle(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev); - - void Handle(TEvSysView::TEvSendDbCountersRequest::TPtr& ev); - void Handle(TEvPrivate::TEvApplyCounters::TPtr& ev); - void Handle(TEvPrivate::TEvSendNavigate::TPtr& ev); - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev); - void Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev); - void Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev); - - void Handle(TEvents::TEvPoisonPill::TPtr& ev); - void Handle(TEvents::TEvUndelivered::TPtr& ev); - void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev); - - void PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value); - void PersistDatabase(NIceDb::TNiceDb& db); - void PersistStage(NIceDb::TNiceDb& db); - void PersistIntervalEnd(NIceDb::TNiceDb& db); - - template <typename TSchema> - void PersistTopResults(NIceDb::TNiceDb& db, - TTop& top, TResultStatsMap& results, TInstant intervalEnd); - - void PersistResults(NIceDb::TNiceDb& db); - - void ScheduleAggregate(); - void ScheduleCollect(); - void ScheduleSendRequests(); - void ScheduleApplyCounters(); - void ScheduleSendNavigate(); - - template <typename TSchema, typename TEntry> - void CutHistory(NIceDb::TNiceDb& db, TResultMap<TEntry>& results, - TDuration historySize); - - static TInstant EndOfHourInterval(TInstant intervalEnd); - - void ClearIntervalSummaries(NIceDb::TNiceDb& db); - - void Reset(NIceDb::TNiceDb& db, const TActorContext& ctx); - - void SendRequests(); - void IgnoreFailure(TNodeId nodeId); - - template <typename TResponse> - void ReplyOverloaded(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev); - - template <typename TEntry, typename TResponse> - void Reply(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev); - - TIntrusivePtr<IDbCounters> CreateCountersForService(NKikimrSysView::EDbCountersService service); - void AttachExternalCounters(); - void AttachInternalCounters(); - void DetachExternalCounters(); - void DetachInternalCounters(); - void SendNavigate(); - - STFUNC(StateInit) { - switch(ev->GetTypeRewrite()) { - hFunc(TEvSysView::TEvConfigureProcessor, Handle); - hFunc(TEvents::TEvPoisonPill, Handle); - IgnoreFunc(TEvSysView::TEvIntervalQuerySummary); - IgnoreFunc(TEvSysView::TEvGetIntervalMetricsResponse); - IgnoreFunc(TEvSysView::TEvGetQueryMetricsRequest); - IgnoreFunc(TEvSysView::TEvSendDbCountersRequest); - default: - if (!HandleDefaultEvents(ev, ctx)) { - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "TSysViewProcessor StateInit unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - } - - STFUNC(StateOffline) { - switch(ev->GetTypeRewrite()) { - hFunc(TEvents::TEvPoisonPill, Handle); - hFunc(TEvSysView::TEvConfigureProcessor, Handle); - IgnoreFunc(TEvSysView::TEvIntervalQuerySummary); - IgnoreFunc(TEvSysView::TEvGetIntervalMetricsResponse); - IgnoreFunc(TEvSysView::TEvGetQueryMetricsRequest); - IgnoreFunc(TEvSysView::TEvSendDbCountersRequest); - default: - if (!HandleDefaultEvents(ev, ctx)) { - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "TSysViewProcessor StateOffline unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - } - - STFUNC(StateWork) { - switch(ev->GetTypeRewrite()) { - hFunc(TEvSysView::TEvConfigureProcessor, Handle); - hFunc(TEvPrivate::TEvCollect, Handle); - hFunc(TEvPrivate::TEvAggregate, Handle); - hFunc(TEvPrivate::TEvSendRequests, Handle); - hFunc(TEvPrivate::TEvProcess, Handle); - hFunc(TEvSysView::TEvIntervalQuerySummary, Handle); - hFunc(TEvSysView::TEvGetIntervalMetricsResponse, Handle); - hFunc(TEvSysView::TEvGetQueryMetricsRequest, Handle); - hFunc(TEvSysView::TEvSendDbCountersRequest, Handle); - hFunc(TEvPrivate::TEvApplyCounters, Handle); - hFunc(TEvPrivate::TEvSendNavigate, Handle); - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - hFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated, Handle); - hFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted, Handle); - hFunc(TEvents::TEvPoisonPill, Handle); - hFunc(TEvents::TEvUndelivered, Handle); - hFunc(TEvInterconnect::TEvNodeDisconnected, Handle); - IgnoreFunc(TEvInterconnect::TEvNodeConnected); - IgnoreFunc(TEvTabletPipe::TEvServerConnected); - IgnoreFunc(TEvTabletPipe::TEvServerDisconnected); - default: - if (!HandleDefaultEvents(ev, ctx)) { - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "TSysViewProcessor StateWork unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - } - - STFUNC(StateBroken) { - HandleDefaultEvents(ev, ctx); - } - -private: - // limit on number of distinct queries when gathering summaries - static constexpr size_t DistinctQueriesLimit = 1024; - // limit on number of queries to aggregate metrics - static constexpr size_t TopCountLimit = 256; - // limit on number of concurrent metrics requests from services - static constexpr size_t MaxInFlightRequests = 16; - // limit on scan batch size - static constexpr size_t BatchSizeLimit = 4 << 20; - // interval of db counters processing - static constexpr TDuration ProcessCountersInterval = TDuration::Seconds(5); - // interval of sending next navigate request - static constexpr TDuration SendNavigateInterval = TDuration::Seconds(5); - - const TDuration TotalInterval; - const TDuration CollectInterval; - - TString Database; - - TInstant IntervalEnd; - - enum EStage { - COLLECT, - AGGREGATE - }; - EStage CurrentStage = COLLECT; - - // IntervalSummaries - struct TQueryToNodes { - ui64 Cpu = 0; - std::vector<std::pair<TNodeId, ui64>> Nodes; // nodeId, cpu - }; - std::unordered_map<TQueryHash, TQueryToNodes> Queries; - std::multimap<ui64, TQueryHash> ByCpu; - std::unordered_set<TNodeId> SummaryNodes; - - // IntervalMetrics - struct TQueryToMetrics { - NKikimrSysView::TQueryMetrics Metrics; - TString Text; - }; - std::unordered_map<TQueryHash, TQueryToMetrics> QueryMetrics; - - // NodesToRequest - using THashVector = std::vector<TQueryHash>; - struct TNodeToQueries { - TNodeId NodeId = 0; - THashVector Hashes; - THashVector TextsToGet; - THashVector ByDuration; - THashVector ByReadBytes; - THashVector ByCpuTime; - THashVector ByRequestUnits; - }; - std::vector<TNodeToQueries> NodesToRequest; - std::unordered_map<TNodeId, TNodeToQueries> NodesInFlight; - - // IntervalTops - TTop ByDurationMinute; - TTop ByReadBytesMinute; - TTop ByCpuTimeMinute; - TTop ByRequestUnitsMinute; - TTop ByDurationHour; - TTop ByReadBytesHour; - TTop ByCpuTimeHour; - TTop ByRequestUnitsHour; - - // Metrics... - using TResultMetricsMap = TResultMap<TQueryToMetrics>; - - TResultMetricsMap MetricsOneMinute; - TResultMetricsMap MetricsOneHour; - - // TopBy... - TResultStatsMap TopByDurationOneMinute; - TResultStatsMap TopByDurationOneHour; - TResultStatsMap TopByReadBytesOneMinute; - TResultStatsMap TopByReadBytesOneHour; - TResultStatsMap TopByCpuTimeOneMinute; - TResultStatsMap TopByCpuTimeOneHour; - TResultStatsMap TopByRequestUnitsOneMinute; - TResultStatsMap TopByRequestUnitsOneHour; - - // limited queue of user requests - static constexpr size_t PendingRequestsLimit = 5; - std::queue<TEvSysView::TEvGetQueryMetricsRequest::TPtr> PendingRequests; - bool ProcessInFly = false; - - // db counters - TString CloudId; - TString FolderId; - TString DatabaseId; - - NMonitoring::TDynamicCounterPtr ExternalGroup; - NMonitoring::TDynamicCounterPtr InternalGroup; - - using TDbCountersServiceMap = std::unordered_map<NKikimrSysView::EDbCountersService, - NKikimr::NSysView::TDbServiceCounters>; - - struct TNodeCountersState { - TDbCountersServiceMap Simple; // only simple counters - ui64 Generation = 0; - size_t FreshCount = 0; - }; - std::unordered_map<TNodeId, TNodeCountersState> NodeCountersStates; - TDbCountersServiceMap AggregatedCountersState; - - std::unordered_map<NKikimrSysView::EDbCountersService, TIntrusivePtr<IDbCounters>> Counters; -}; - -} // NSysView -} // NKikimr - + +private: + using Schema = TProcessorSchema; + using TTxBase = NTabletFlatExecutor::TTransactionBase<TSysViewProcessor>; + + struct TTxInitSchema; + struct TTxInit; + struct TTxConfigure; + struct TTxCollect; + struct TTxAggregate; + struct TTxIntervalSummary; + struct TTxIntervalMetrics; + + struct TEvPrivate { + enum EEv { + EvCollect = EventSpaceBegin(TEvents::ES_PRIVATE), + EvAggregate, + EvSendRequests, + EvProcess, + EvApplyCounters, + EvSendNavigate, + EvEnd + }; + + struct TEvCollect : public TEventLocal<TEvCollect, EvCollect> {}; + + struct TEvAggregate : public TEventLocal<TEvAggregate, EvAggregate> {}; + + struct TEvSendRequests : public TEventLocal<TEvSendRequests, EvSendRequests> {}; + + struct TEvProcess : public TEventLocal<TEvProcess, EvProcess> {}; + + struct TEvApplyCounters : public TEventLocal<TEvApplyCounters, EvApplyCounters> {}; + + struct TEvSendNavigate : public TEventLocal<TEvSendNavigate, EvSendNavigate> {}; + }; + + struct TTopQuery { + TQueryHash Hash; + ui64 Value; + TNodeId NodeId; + THolder<NKikimrSysView::TQueryStats> Stats; + }; + using TTop = std::vector<TTopQuery>; + + using TResultKey = std::pair<ui64, ui32>; + + template <typename TEntry> + using TResultMap = std::map<TResultKey, TEntry, std::less<TResultKey>, + NActors::NMemory::TAlloc<std::pair<const TResultKey, TEntry>, MemoryLabelResults>>; + using TResultStatsMap = TResultMap<NKikimrSysView::TQueryStats>; + +private: + static bool TopQueryCompare(const TTopQuery& l, const TTopQuery& r) { + return l.Value == r.Value ? l.Hash > r.Hash : l.Value > r.Value; + } + + void OnDetach(const TActorContext& ctx) override; + void OnTabletDead(TEvTablet::TEvTabletDead::TPtr& ev, const TActorContext& ctx) override; + void OnActivateExecutor(const TActorContext& ctx) override; + void DefaultSignalTabletActive(const TActorContext& ctx) override; + bool OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext &ctx) override; + + NTabletFlatExecutor::ITransaction* CreateTxInitSchema(); + NTabletFlatExecutor::ITransaction* CreateTxInit(); + + void Handle(TEvSysView::TEvConfigureProcessor::TPtr& ev); + void Handle(TEvPrivate::TEvCollect::TPtr& ev); + void Handle(TEvPrivate::TEvAggregate::TPtr& ev); + void Handle(TEvPrivate::TEvSendRequests::TPtr& ev); + void Handle(TEvPrivate::TEvProcess::TPtr& ev); + void Handle(TEvSysView::TEvIntervalQuerySummary::TPtr& ev); + void Handle(TEvSysView::TEvGetIntervalMetricsResponse::TPtr& ev); + void Handle(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev); + + void Handle(TEvSysView::TEvSendDbCountersRequest::TPtr& ev); + void Handle(TEvPrivate::TEvApplyCounters::TPtr& ev); + void Handle(TEvPrivate::TEvSendNavigate::TPtr& ev); + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev); + void Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev); + void Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev); + + void Handle(TEvents::TEvPoisonPill::TPtr& ev); + void Handle(TEvents::TEvUndelivered::TPtr& ev); + void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev); + + void PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value); + void PersistDatabase(NIceDb::TNiceDb& db); + void PersistStage(NIceDb::TNiceDb& db); + void PersistIntervalEnd(NIceDb::TNiceDb& db); + + template <typename TSchema> + void PersistTopResults(NIceDb::TNiceDb& db, + TTop& top, TResultStatsMap& results, TInstant intervalEnd); + + void PersistResults(NIceDb::TNiceDb& db); + + void ScheduleAggregate(); + void ScheduleCollect(); + void ScheduleSendRequests(); + void ScheduleApplyCounters(); + void ScheduleSendNavigate(); + + template <typename TSchema, typename TEntry> + void CutHistory(NIceDb::TNiceDb& db, TResultMap<TEntry>& results, + TDuration historySize); + + static TInstant EndOfHourInterval(TInstant intervalEnd); + + void ClearIntervalSummaries(NIceDb::TNiceDb& db); + + void Reset(NIceDb::TNiceDb& db, const TActorContext& ctx); + + void SendRequests(); + void IgnoreFailure(TNodeId nodeId); + + template <typename TResponse> + void ReplyOverloaded(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev); + + template <typename TEntry, typename TResponse> + void Reply(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev); + + TIntrusivePtr<IDbCounters> CreateCountersForService(NKikimrSysView::EDbCountersService service); + void AttachExternalCounters(); + void AttachInternalCounters(); + void DetachExternalCounters(); + void DetachInternalCounters(); + void SendNavigate(); + + STFUNC(StateInit) { + switch(ev->GetTypeRewrite()) { + hFunc(TEvSysView::TEvConfigureProcessor, Handle); + hFunc(TEvents::TEvPoisonPill, Handle); + IgnoreFunc(TEvSysView::TEvIntervalQuerySummary); + IgnoreFunc(TEvSysView::TEvGetIntervalMetricsResponse); + IgnoreFunc(TEvSysView::TEvGetQueryMetricsRequest); + IgnoreFunc(TEvSysView::TEvSendDbCountersRequest); + default: + if (!HandleDefaultEvents(ev, ctx)) { + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "TSysViewProcessor StateInit unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + } + + STFUNC(StateOffline) { + switch(ev->GetTypeRewrite()) { + hFunc(TEvents::TEvPoisonPill, Handle); + hFunc(TEvSysView::TEvConfigureProcessor, Handle); + IgnoreFunc(TEvSysView::TEvIntervalQuerySummary); + IgnoreFunc(TEvSysView::TEvGetIntervalMetricsResponse); + IgnoreFunc(TEvSysView::TEvGetQueryMetricsRequest); + IgnoreFunc(TEvSysView::TEvSendDbCountersRequest); + default: + if (!HandleDefaultEvents(ev, ctx)) { + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "TSysViewProcessor StateOffline unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + } + + STFUNC(StateWork) { + switch(ev->GetTypeRewrite()) { + hFunc(TEvSysView::TEvConfigureProcessor, Handle); + hFunc(TEvPrivate::TEvCollect, Handle); + hFunc(TEvPrivate::TEvAggregate, Handle); + hFunc(TEvPrivate::TEvSendRequests, Handle); + hFunc(TEvPrivate::TEvProcess, Handle); + hFunc(TEvSysView::TEvIntervalQuerySummary, Handle); + hFunc(TEvSysView::TEvGetIntervalMetricsResponse, Handle); + hFunc(TEvSysView::TEvGetQueryMetricsRequest, Handle); + hFunc(TEvSysView::TEvSendDbCountersRequest, Handle); + hFunc(TEvPrivate::TEvApplyCounters, Handle); + hFunc(TEvPrivate::TEvSendNavigate, Handle); + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + hFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated, Handle); + hFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted, Handle); + hFunc(TEvents::TEvPoisonPill, Handle); + hFunc(TEvents::TEvUndelivered, Handle); + hFunc(TEvInterconnect::TEvNodeDisconnected, Handle); + IgnoreFunc(TEvInterconnect::TEvNodeConnected); + IgnoreFunc(TEvTabletPipe::TEvServerConnected); + IgnoreFunc(TEvTabletPipe::TEvServerDisconnected); + default: + if (!HandleDefaultEvents(ev, ctx)) { + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "TSysViewProcessor StateWork unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + } + + STFUNC(StateBroken) { + HandleDefaultEvents(ev, ctx); + } + +private: + // limit on number of distinct queries when gathering summaries + static constexpr size_t DistinctQueriesLimit = 1024; + // limit on number of queries to aggregate metrics + static constexpr size_t TopCountLimit = 256; + // limit on number of concurrent metrics requests from services + static constexpr size_t MaxInFlightRequests = 16; + // limit on scan batch size + static constexpr size_t BatchSizeLimit = 4 << 20; + // interval of db counters processing + static constexpr TDuration ProcessCountersInterval = TDuration::Seconds(5); + // interval of sending next navigate request + static constexpr TDuration SendNavigateInterval = TDuration::Seconds(5); + + const TDuration TotalInterval; + const TDuration CollectInterval; + + TString Database; + + TInstant IntervalEnd; + + enum EStage { + COLLECT, + AGGREGATE + }; + EStage CurrentStage = COLLECT; + + // IntervalSummaries + struct TQueryToNodes { + ui64 Cpu = 0; + std::vector<std::pair<TNodeId, ui64>> Nodes; // nodeId, cpu + }; + std::unordered_map<TQueryHash, TQueryToNodes> Queries; + std::multimap<ui64, TQueryHash> ByCpu; + std::unordered_set<TNodeId> SummaryNodes; + + // IntervalMetrics + struct TQueryToMetrics { + NKikimrSysView::TQueryMetrics Metrics; + TString Text; + }; + std::unordered_map<TQueryHash, TQueryToMetrics> QueryMetrics; + + // NodesToRequest + using THashVector = std::vector<TQueryHash>; + struct TNodeToQueries { + TNodeId NodeId = 0; + THashVector Hashes; + THashVector TextsToGet; + THashVector ByDuration; + THashVector ByReadBytes; + THashVector ByCpuTime; + THashVector ByRequestUnits; + }; + std::vector<TNodeToQueries> NodesToRequest; + std::unordered_map<TNodeId, TNodeToQueries> NodesInFlight; + + // IntervalTops + TTop ByDurationMinute; + TTop ByReadBytesMinute; + TTop ByCpuTimeMinute; + TTop ByRequestUnitsMinute; + TTop ByDurationHour; + TTop ByReadBytesHour; + TTop ByCpuTimeHour; + TTop ByRequestUnitsHour; + + // Metrics... + using TResultMetricsMap = TResultMap<TQueryToMetrics>; + + TResultMetricsMap MetricsOneMinute; + TResultMetricsMap MetricsOneHour; + + // TopBy... + TResultStatsMap TopByDurationOneMinute; + TResultStatsMap TopByDurationOneHour; + TResultStatsMap TopByReadBytesOneMinute; + TResultStatsMap TopByReadBytesOneHour; + TResultStatsMap TopByCpuTimeOneMinute; + TResultStatsMap TopByCpuTimeOneHour; + TResultStatsMap TopByRequestUnitsOneMinute; + TResultStatsMap TopByRequestUnitsOneHour; + + // limited queue of user requests + static constexpr size_t PendingRequestsLimit = 5; + std::queue<TEvSysView::TEvGetQueryMetricsRequest::TPtr> PendingRequests; + bool ProcessInFly = false; + + // db counters + TString CloudId; + TString FolderId; + TString DatabaseId; + + NMonitoring::TDynamicCounterPtr ExternalGroup; + NMonitoring::TDynamicCounterPtr InternalGroup; + + using TDbCountersServiceMap = std::unordered_map<NKikimrSysView::EDbCountersService, + NKikimr::NSysView::TDbServiceCounters>; + + struct TNodeCountersState { + TDbCountersServiceMap Simple; // only simple counters + ui64 Generation = 0; + size_t FreshCount = 0; + }; + std::unordered_map<TNodeId, TNodeCountersState> NodeCountersStates; + TDbCountersServiceMap AggregatedCountersState; + + std::unordered_map<NKikimrSysView::EDbCountersService, TIntrusivePtr<IDbCounters>> Counters; +}; + +} // NSysView +} // NKikimr + diff --git a/ydb/core/sys_view/processor/schema.cpp b/ydb/core/sys_view/processor/schema.cpp index 5239455e0c..e9fd222405 100644 --- a/ydb/core/sys_view/processor/schema.cpp +++ b/ydb/core/sys_view/processor/schema.cpp @@ -1 +1 @@ -#include "schema.h" +#include "schema.h" diff --git a/ydb/core/sys_view/processor/schema.h b/ydb/core/sys_view/processor/schema.h index b9ae04dff1..159ad84f58 100644 --- a/ydb/core/sys_view/processor/schema.h +++ b/ydb/core/sys_view/processor/schema.h @@ -1,116 +1,116 @@ -#pragma once - +#pragma once + #include <ydb/core/tablet_flat/flat_cxx_database.h> - -namespace NKikimr { -namespace NSysView { - -struct TProcessorSchema : NIceDb::Schema { - struct SysParams : Table<1> { - struct Id : Column<1, NScheme::NTypeIds::Uint64> {}; - struct Value : Column<2, NScheme::NTypeIds::Utf8> {}; - - using TKey = TableKey<Id>; - using TColumns = TableColumns<Id, Value>; - }; - - struct IntervalSummaries : Table<2> { - struct QueryHash : Column<1, NScheme::NTypeIds::Uint64> {}; - struct Index : Column<2, NScheme::NTypeIds::Uint32> {}; - struct CPU : Column<3, NScheme::NTypeIds::Uint64> {}; - struct NodeId : Column<4, NScheme::NTypeIds::Uint32> {}; - - using TKey = TableKey<QueryHash, Index>; - using TColumns = TableColumns<QueryHash, Index, CPU, NodeId>; - }; - - struct IntervalMetrics : Table<3> { - struct QueryHash : Column<1, NScheme::NTypeIds::Uint64> {}; - struct Metrics : Column<2, NScheme::NTypeIds::String> {}; - struct Text : Column<3, NScheme::NTypeIds::Utf8> {}; - - using TKey = TableKey<QueryHash>; - using TColumns = TableColumns<QueryHash, Metrics, Text>; - }; - - struct IntervalTops : Table<4> { - struct TypeCol : Column<1, NScheme::NTypeIds::Uint32> { - static TString GetColumnName(const TString&) { return "Type"; } - }; - struct QueryHash : Column<2, NScheme::NTypeIds::Uint64> {}; - struct Value : Column<3, NScheme::NTypeIds::Uint64> {}; - struct NodeId : Column<4, NScheme::NTypeIds::Uint32> {}; - struct Stats : Column<5, NScheme::NTypeIds::String> {}; - - using TKey = TableKey<TypeCol, QueryHash>; - using TColumns = TableColumns<TypeCol, QueryHash, Value, NodeId, Stats>; - }; - - struct NodesToRequest : Table<5> { - struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; - struct QueryHashes : Column<2, NScheme::NTypeIds::String> {}; - struct TextsToGet : Column<3, NScheme::NTypeIds::String> {}; - struct ByDuration : Column<4, NScheme::NTypeIds::String> {}; - struct ByReadBytes : Column<5, NScheme::NTypeIds::String> {}; - struct ByCpuTime : Column<6, NScheme::NTypeIds::String> {}; - struct ByRequestUnits : Column<7, NScheme::NTypeIds::String> {}; - - using TKey = TableKey<NodeId>; - using TColumns = TableColumns<NodeId, QueryHashes, TextsToGet, - ByDuration, ByReadBytes, ByCpuTime, ByRequestUnits>; - }; - -#define RESULT_TABLE(TableName, TableID) \ - struct TableName : Table<TableID> { \ - struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; \ - struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; \ - struct Text : Column<3, NScheme::NTypeIds::Utf8> {}; \ - struct Data : Column<4, NScheme::NTypeIds::String> {}; \ - \ - using TKey = TableKey<IntervalEnd, Rank>; \ - using TColumns = TableColumns<IntervalEnd, Rank, Text, Data>; \ - }; - - RESULT_TABLE(MetricsOneMinute, 6) - RESULT_TABLE(MetricsOneHour, 7) - RESULT_TABLE(TopByDurationOneMinute, 8) - RESULT_TABLE(TopByDurationOneHour, 9) - RESULT_TABLE(TopByReadBytesOneMinute, 10) - RESULT_TABLE(TopByReadBytesOneHour, 11) - RESULT_TABLE(TopByCpuTimeOneMinute, 12) - RESULT_TABLE(TopByCpuTimeOneHour, 13) - RESULT_TABLE(TopByRequestUnitsOneMinute, 14) - RESULT_TABLE(TopByRequestUnitsOneHour, 15) - -#undef RESULT_TABLE - - using TTables = SchemaTables< - SysParams, - IntervalSummaries, - IntervalMetrics, - NodesToRequest, - IntervalTops, - MetricsOneMinute, - MetricsOneHour, - TopByDurationOneMinute, - TopByDurationOneHour, - TopByReadBytesOneMinute, - TopByReadBytesOneHour, - TopByCpuTimeOneMinute, - TopByCpuTimeOneHour, - TopByRequestUnitsOneMinute, - TopByRequestUnitsOneHour - >; - - using TSettings = SchemaSettings< - ExecutorLogBatching<true>, - ExecutorLogFlushPeriod<512> - >; - - static constexpr ui64 SysParam_Database = 1; - static constexpr ui64 SysParam_CurrentStage = 2; - static constexpr ui64 SysParam_IntervalEnd = 3; -}; - -} -} + +namespace NKikimr { +namespace NSysView { + +struct TProcessorSchema : NIceDb::Schema { + struct SysParams : Table<1> { + struct Id : Column<1, NScheme::NTypeIds::Uint64> {}; + struct Value : Column<2, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey<Id>; + using TColumns = TableColumns<Id, Value>; + }; + + struct IntervalSummaries : Table<2> { + struct QueryHash : Column<1, NScheme::NTypeIds::Uint64> {}; + struct Index : Column<2, NScheme::NTypeIds::Uint32> {}; + struct CPU : Column<3, NScheme::NTypeIds::Uint64> {}; + struct NodeId : Column<4, NScheme::NTypeIds::Uint32> {}; + + using TKey = TableKey<QueryHash, Index>; + using TColumns = TableColumns<QueryHash, Index, CPU, NodeId>; + }; + + struct IntervalMetrics : Table<3> { + struct QueryHash : Column<1, NScheme::NTypeIds::Uint64> {}; + struct Metrics : Column<2, NScheme::NTypeIds::String> {}; + struct Text : Column<3, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey<QueryHash>; + using TColumns = TableColumns<QueryHash, Metrics, Text>; + }; + + struct IntervalTops : Table<4> { + struct TypeCol : Column<1, NScheme::NTypeIds::Uint32> { + static TString GetColumnName(const TString&) { return "Type"; } + }; + struct QueryHash : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Value : Column<3, NScheme::NTypeIds::Uint64> {}; + struct NodeId : Column<4, NScheme::NTypeIds::Uint32> {}; + struct Stats : Column<5, NScheme::NTypeIds::String> {}; + + using TKey = TableKey<TypeCol, QueryHash>; + using TColumns = TableColumns<TypeCol, QueryHash, Value, NodeId, Stats>; + }; + + struct NodesToRequest : Table<5> { + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct QueryHashes : Column<2, NScheme::NTypeIds::String> {}; + struct TextsToGet : Column<3, NScheme::NTypeIds::String> {}; + struct ByDuration : Column<4, NScheme::NTypeIds::String> {}; + struct ByReadBytes : Column<5, NScheme::NTypeIds::String> {}; + struct ByCpuTime : Column<6, NScheme::NTypeIds::String> {}; + struct ByRequestUnits : Column<7, NScheme::NTypeIds::String> {}; + + using TKey = TableKey<NodeId>; + using TColumns = TableColumns<NodeId, QueryHashes, TextsToGet, + ByDuration, ByReadBytes, ByCpuTime, ByRequestUnits>; + }; + +#define RESULT_TABLE(TableName, TableID) \ + struct TableName : Table<TableID> { \ + struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; \ + struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; \ + struct Text : Column<3, NScheme::NTypeIds::Utf8> {}; \ + struct Data : Column<4, NScheme::NTypeIds::String> {}; \ + \ + using TKey = TableKey<IntervalEnd, Rank>; \ + using TColumns = TableColumns<IntervalEnd, Rank, Text, Data>; \ + }; + + RESULT_TABLE(MetricsOneMinute, 6) + RESULT_TABLE(MetricsOneHour, 7) + RESULT_TABLE(TopByDurationOneMinute, 8) + RESULT_TABLE(TopByDurationOneHour, 9) + RESULT_TABLE(TopByReadBytesOneMinute, 10) + RESULT_TABLE(TopByReadBytesOneHour, 11) + RESULT_TABLE(TopByCpuTimeOneMinute, 12) + RESULT_TABLE(TopByCpuTimeOneHour, 13) + RESULT_TABLE(TopByRequestUnitsOneMinute, 14) + RESULT_TABLE(TopByRequestUnitsOneHour, 15) + +#undef RESULT_TABLE + + using TTables = SchemaTables< + SysParams, + IntervalSummaries, + IntervalMetrics, + NodesToRequest, + IntervalTops, + MetricsOneMinute, + MetricsOneHour, + TopByDurationOneMinute, + TopByDurationOneHour, + TopByReadBytesOneMinute, + TopByReadBytesOneHour, + TopByCpuTimeOneMinute, + TopByCpuTimeOneHour, + TopByRequestUnitsOneMinute, + TopByRequestUnitsOneHour + >; + + using TSettings = SchemaSettings< + ExecutorLogBatching<true>, + ExecutorLogFlushPeriod<512> + >; + + static constexpr ui64 SysParam_Database = 1; + static constexpr ui64 SysParam_CurrentStage = 2; + static constexpr ui64 SysParam_IntervalEnd = 3; +}; + +} +} diff --git a/ydb/core/sys_view/processor/tx_aggregate.cpp b/ydb/core/sys_view/processor/tx_aggregate.cpp index fe351733cc..654b47a689 100644 --- a/ydb/core/sys_view/processor/tx_aggregate.cpp +++ b/ydb/core/sys_view/processor/tx_aggregate.cpp @@ -1,128 +1,128 @@ -#include "processor_impl.h" - -namespace NKikimr { -namespace NSysView { - -struct TSysViewProcessor::TTxAggregate : public TTxBase { - explicit TTxAggregate(TSelf* self) - : TTxBase(self) - {} - - TTxType GetTxType() const override { return TXTYPE_AGGREGATE; } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - SVLOG_D("[" << Self->TabletID() << "] TTxAggregate::Execute"); - - NIceDb::TNiceDb db(txc.DB); - - auto deadline = Self->IntervalEnd + Self->TotalInterval; - if (ctx.Now() >= deadline) { - Self->Reset(db, ctx); - return true; - } - - std::unordered_map<TNodeId, TNodeToQueries> nodesToRequest; - size_t count = 0; - for (auto it = Self->ByCpu.rbegin(); - it != Self->ByCpu.rend() && count < Self->TopCountLimit; - ++it, ++count) - { - auto queryHash = it->second; - const auto& nodes = Self->Queries[queryHash].Nodes; - for (const auto& node : nodes) { - nodesToRequest[node.first].Hashes.emplace_back(queryHash); - } - - static constexpr size_t textReqsCount = 3; - const auto nodesCount = nodes.size(); - if (nodesCount <= textReqsCount) { - for (const auto& node : nodes) { - nodesToRequest[node.first].TextsToGet.emplace_back(queryHash); - } - } else { - std::unordered_set<TNodeId> used; - for (size_t count = 0; count < textReqsCount; ) { - auto nodeId = nodes[RandomNumber<ui64>(nodesCount)].first; - if (used.find(nodeId) != used.end()) { - continue; - } - nodesToRequest[nodeId].TextsToGet.emplace_back(queryHash); - used.insert(nodeId); - ++count; - } - } - } - - for (auto& entry : Self->ByDurationMinute) { - nodesToRequest[entry.NodeId].ByDuration.emplace_back(entry.Hash); - } - for (auto& entry : Self->ByReadBytesMinute) { - nodesToRequest[entry.NodeId].ByReadBytes.emplace_back(entry.Hash); - } - for (auto& entry : Self->ByCpuTimeMinute) { - nodesToRequest[entry.NodeId].ByCpuTime.emplace_back(entry.Hash); - } - for (auto& entry : Self->ByRequestUnitsMinute) { - nodesToRequest[entry.NodeId].ByRequestUnits.emplace_back(entry.Hash); - } - - Self->NodesToRequest.reserve(nodesToRequest.size()); - - for (auto& [nodeId, queries] : nodesToRequest) { - queries.NodeId = nodeId; - auto& hashes = queries.Hashes; - auto& texts = queries.TextsToGet; - auto& byDuration = queries.ByDuration; - auto& byReadBytes = queries.ByReadBytes; - auto& byCpuTime = queries.ByCpuTime; - auto& byRequestUnits = queries.ByRequestUnits; - - db.Table<Schema::NodesToRequest>().Key(nodeId).Update( - NIceDb::TUpdate<Schema::NodesToRequest::QueryHashes>( - TString((char*)hashes.data(), hashes.size() * sizeof(TQueryHash))), - NIceDb::TUpdate<Schema::NodesToRequest::TextsToGet>( - TString((char*)texts.data(), texts.size() * sizeof(TQueryHash))), - NIceDb::TUpdate<Schema::NodesToRequest::ByDuration>( - TString((char*)byDuration.data(), byDuration.size() * sizeof(TQueryHash))), - NIceDb::TUpdate<Schema::NodesToRequest::ByReadBytes>( - TString((char*)byReadBytes.data(), byReadBytes.size() * sizeof(TQueryHash))), - NIceDb::TUpdate<Schema::NodesToRequest::ByCpuTime>( - TString((char*)byCpuTime.data(), byCpuTime.size() * sizeof(TQueryHash))), - NIceDb::TUpdate<Schema::NodesToRequest::ByRequestUnits>( - TString((char*)byRequestUnits.data(), byRequestUnits.size() * sizeof(TQueryHash)))); - - Self->NodesToRequest.emplace_back(std::move(queries)); - } - - Self->ClearIntervalSummaries(db); - - if (Self->NodesToRequest.empty()) { - Self->PersistResults(db); - } - - Self->CurrentStage = AGGREGATE; - Self->PersistStage(db); - - return true; - } - - void Complete(const TActorContext&) override { - SVLOG_D("[" << Self->TabletID() << "] TTxAggregate::Complete"); - - if (Self->CurrentStage == COLLECT) { - Self->ScheduleAggregate(); - } else { - Self->ScheduleCollect(); - if (!Self->NodesToRequest.empty()) { - Self->ScheduleSendRequests(); - } - } - } -}; - -void TSysViewProcessor::Handle(TEvPrivate::TEvAggregate::TPtr&) { - Execute(new TTxAggregate(this), TActivationContext::AsActorContext()); -} - -} // NSysView -} // NKikimr +#include "processor_impl.h" + +namespace NKikimr { +namespace NSysView { + +struct TSysViewProcessor::TTxAggregate : public TTxBase { + explicit TTxAggregate(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_AGGREGATE; } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + SVLOG_D("[" << Self->TabletID() << "] TTxAggregate::Execute"); + + NIceDb::TNiceDb db(txc.DB); + + auto deadline = Self->IntervalEnd + Self->TotalInterval; + if (ctx.Now() >= deadline) { + Self->Reset(db, ctx); + return true; + } + + std::unordered_map<TNodeId, TNodeToQueries> nodesToRequest; + size_t count = 0; + for (auto it = Self->ByCpu.rbegin(); + it != Self->ByCpu.rend() && count < Self->TopCountLimit; + ++it, ++count) + { + auto queryHash = it->second; + const auto& nodes = Self->Queries[queryHash].Nodes; + for (const auto& node : nodes) { + nodesToRequest[node.first].Hashes.emplace_back(queryHash); + } + + static constexpr size_t textReqsCount = 3; + const auto nodesCount = nodes.size(); + if (nodesCount <= textReqsCount) { + for (const auto& node : nodes) { + nodesToRequest[node.first].TextsToGet.emplace_back(queryHash); + } + } else { + std::unordered_set<TNodeId> used; + for (size_t count = 0; count < textReqsCount; ) { + auto nodeId = nodes[RandomNumber<ui64>(nodesCount)].first; + if (used.find(nodeId) != used.end()) { + continue; + } + nodesToRequest[nodeId].TextsToGet.emplace_back(queryHash); + used.insert(nodeId); + ++count; + } + } + } + + for (auto& entry : Self->ByDurationMinute) { + nodesToRequest[entry.NodeId].ByDuration.emplace_back(entry.Hash); + } + for (auto& entry : Self->ByReadBytesMinute) { + nodesToRequest[entry.NodeId].ByReadBytes.emplace_back(entry.Hash); + } + for (auto& entry : Self->ByCpuTimeMinute) { + nodesToRequest[entry.NodeId].ByCpuTime.emplace_back(entry.Hash); + } + for (auto& entry : Self->ByRequestUnitsMinute) { + nodesToRequest[entry.NodeId].ByRequestUnits.emplace_back(entry.Hash); + } + + Self->NodesToRequest.reserve(nodesToRequest.size()); + + for (auto& [nodeId, queries] : nodesToRequest) { + queries.NodeId = nodeId; + auto& hashes = queries.Hashes; + auto& texts = queries.TextsToGet; + auto& byDuration = queries.ByDuration; + auto& byReadBytes = queries.ByReadBytes; + auto& byCpuTime = queries.ByCpuTime; + auto& byRequestUnits = queries.ByRequestUnits; + + db.Table<Schema::NodesToRequest>().Key(nodeId).Update( + NIceDb::TUpdate<Schema::NodesToRequest::QueryHashes>( + TString((char*)hashes.data(), hashes.size() * sizeof(TQueryHash))), + NIceDb::TUpdate<Schema::NodesToRequest::TextsToGet>( + TString((char*)texts.data(), texts.size() * sizeof(TQueryHash))), + NIceDb::TUpdate<Schema::NodesToRequest::ByDuration>( + TString((char*)byDuration.data(), byDuration.size() * sizeof(TQueryHash))), + NIceDb::TUpdate<Schema::NodesToRequest::ByReadBytes>( + TString((char*)byReadBytes.data(), byReadBytes.size() * sizeof(TQueryHash))), + NIceDb::TUpdate<Schema::NodesToRequest::ByCpuTime>( + TString((char*)byCpuTime.data(), byCpuTime.size() * sizeof(TQueryHash))), + NIceDb::TUpdate<Schema::NodesToRequest::ByRequestUnits>( + TString((char*)byRequestUnits.data(), byRequestUnits.size() * sizeof(TQueryHash)))); + + Self->NodesToRequest.emplace_back(std::move(queries)); + } + + Self->ClearIntervalSummaries(db); + + if (Self->NodesToRequest.empty()) { + Self->PersistResults(db); + } + + Self->CurrentStage = AGGREGATE; + Self->PersistStage(db); + + return true; + } + + void Complete(const TActorContext&) override { + SVLOG_D("[" << Self->TabletID() << "] TTxAggregate::Complete"); + + if (Self->CurrentStage == COLLECT) { + Self->ScheduleAggregate(); + } else { + Self->ScheduleCollect(); + if (!Self->NodesToRequest.empty()) { + Self->ScheduleSendRequests(); + } + } + } +}; + +void TSysViewProcessor::Handle(TEvPrivate::TEvAggregate::TPtr&) { + Execute(new TTxAggregate(this), TActivationContext::AsActorContext()); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/processor/tx_collect.cpp b/ydb/core/sys_view/processor/tx_collect.cpp index fd282e18e9..67a197c366 100644 --- a/ydb/core/sys_view/processor/tx_collect.cpp +++ b/ydb/core/sys_view/processor/tx_collect.cpp @@ -1,39 +1,39 @@ -#include "processor_impl.h" - -namespace NKikimr { -namespace NSysView { - -struct TSysViewProcessor::TTxCollect : public TTxBase { - explicit TTxCollect(TSelf* self) - : TTxBase(self) - {} - - TTxType GetTxType() const override { return TXTYPE_COLLECT; } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - SVLOG_D("[" << Self->TabletID() << "] TTxCollect::Execute"); - - NIceDb::TNiceDb db(txc.DB); - - if (!Self->NodesInFlight.empty() || !Self->NodesToRequest.empty()) { - Self->PersistResults(db); - } - - Self->Reset(db, ctx); - - return true; - } - - void Complete(const TActorContext&) override { - SVLOG_D("[" << Self->TabletID() << "] TTxCollect::Complete"); - - Self->ScheduleAggregate(); - } -}; - -void TSysViewProcessor::Handle(TEvPrivate::TEvCollect::TPtr&) { - Execute(new TTxCollect(this), TActivationContext::AsActorContext()); -} - -} // NSysView -} // NKikimr +#include "processor_impl.h" + +namespace NKikimr { +namespace NSysView { + +struct TSysViewProcessor::TTxCollect : public TTxBase { + explicit TTxCollect(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_COLLECT; } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + SVLOG_D("[" << Self->TabletID() << "] TTxCollect::Execute"); + + NIceDb::TNiceDb db(txc.DB); + + if (!Self->NodesInFlight.empty() || !Self->NodesToRequest.empty()) { + Self->PersistResults(db); + } + + Self->Reset(db, ctx); + + return true; + } + + void Complete(const TActorContext&) override { + SVLOG_D("[" << Self->TabletID() << "] TTxCollect::Complete"); + + Self->ScheduleAggregate(); + } +}; + +void TSysViewProcessor::Handle(TEvPrivate::TEvCollect::TPtr&) { + Execute(new TTxCollect(this), TActivationContext::AsActorContext()); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/processor/tx_configure.cpp b/ydb/core/sys_view/processor/tx_configure.cpp index 4ddc8c6572..68a2a133e5 100644 --- a/ydb/core/sys_view/processor/tx_configure.cpp +++ b/ydb/core/sys_view/processor/tx_configure.cpp @@ -1,44 +1,44 @@ -#include "processor_impl.h" - -namespace NKikimr { -namespace NSysView { - -struct TSysViewProcessor::TTxConfigure : public TTxBase { - NKikimrSysView::TEvConfigureProcessor Record; +#include "processor_impl.h" + +namespace NKikimr { +namespace NSysView { + +struct TSysViewProcessor::TTxConfigure : public TTxBase { + NKikimrSysView::TEvConfigureProcessor Record; TActorId Sender; - + TTxConfigure(TSelf* self, NKikimrSysView::TEvConfigureProcessor&& record, const TActorId& sender) - : TTxBase(self) - , Record(std::move(record)) - , Sender(sender) - {} - - TTxType GetTxType() const override { return TXTYPE_CONFIGURE; } - - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SVLOG_D("[" << Self->TabletID() << "] TTxConfigure::Execute: " - << "database# " << Record.GetDatabase()); - - NIceDb::TNiceDb db(txc.DB); - - Self->Database = Record.GetDatabase(); - Self->PersistDatabase(db); - return true; - } - - void Complete(const TActorContext& ctx) override { - SVLOG_D("[" << Self->TabletID() << "] TTxConfigure::Complete"); - - ctx.Send(Sender, new TEvSubDomain::TEvConfigureStatus( - NKikimrTx::TEvSubDomainConfigurationAck::SUCCESS, Self->TabletID())); - } -}; - -void TSysViewProcessor::Handle(TEvSysView::TEvConfigureProcessor::TPtr& ev) { - auto& record = ev->Get()->Record; - Execute(new TTxConfigure(this, std::move(record), ev->Sender), - TActivationContext::AsActorContext()); -} - -} // NSysView -} // NKikimr + : TTxBase(self) + , Record(std::move(record)) + , Sender(sender) + {} + + TTxType GetTxType() const override { return TXTYPE_CONFIGURE; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SVLOG_D("[" << Self->TabletID() << "] TTxConfigure::Execute: " + << "database# " << Record.GetDatabase()); + + NIceDb::TNiceDb db(txc.DB); + + Self->Database = Record.GetDatabase(); + Self->PersistDatabase(db); + return true; + } + + void Complete(const TActorContext& ctx) override { + SVLOG_D("[" << Self->TabletID() << "] TTxConfigure::Complete"); + + ctx.Send(Sender, new TEvSubDomain::TEvConfigureStatus( + NKikimrTx::TEvSubDomainConfigurationAck::SUCCESS, Self->TabletID())); + } +}; + +void TSysViewProcessor::Handle(TEvSysView::TEvConfigureProcessor::TPtr& ev) { + auto& record = ev->Get()->Record; + Execute(new TTxConfigure(this, std::move(record), ev->Sender), + TActivationContext::AsActorContext()); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/processor/tx_init.cpp b/ydb/core/sys_view/processor/tx_init.cpp index 7c78d3da53..89c7fd637f 100644 --- a/ydb/core/sys_view/processor/tx_init.cpp +++ b/ydb/core/sys_view/processor/tx_init.cpp @@ -1,382 +1,382 @@ -#include "processor_impl.h" - -namespace NKikimr { -namespace NSysView { - -struct TSysViewProcessor::TTxInit : public TTxBase { - explicit TTxInit(TSelf* self) - : TTxBase(self) - {} - - TTxType GetTxType() const override { return TXTYPE_INIT; } - - template <typename TSchema, typename TEntry> - bool LoadResults(NIceDb::TNiceDb& db, TResultMap<TEntry>& results) { - results.clear(); - - auto rowset = db.Table<TSchema>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - ui64 intervalEnd = rowset.template GetValue<typename TSchema::IntervalEnd>(); - ui32 rank = rowset.template GetValue<typename TSchema::Rank>(); - TString text = rowset.template GetValue<typename TSchema::Text>(); - TString data = rowset.template GetValue<typename TSchema::Data>(); - - auto key = std::make_pair(intervalEnd, rank); - auto& result = results[key]; - if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) { - result.Text = std::move(text); - if (data) { +#include "processor_impl.h" + +namespace NKikimr { +namespace NSysView { + +struct TSysViewProcessor::TTxInit : public TTxBase { + explicit TTxInit(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_INIT; } + + template <typename TSchema, typename TEntry> + bool LoadResults(NIceDb::TNiceDb& db, TResultMap<TEntry>& results) { + results.clear(); + + auto rowset = db.Table<TSchema>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + ui64 intervalEnd = rowset.template GetValue<typename TSchema::IntervalEnd>(); + ui32 rank = rowset.template GetValue<typename TSchema::Rank>(); + TString text = rowset.template GetValue<typename TSchema::Text>(); + TString data = rowset.template GetValue<typename TSchema::Data>(); + + auto key = std::make_pair(intervalEnd, rank); + auto& result = results[key]; + if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) { + result.Text = std::move(text); + if (data) { Y_PROTOBUF_SUPPRESS_NODISCARD result.Metrics.ParseFromString(data); - } - } else { - if (data) { + } + } else { + if (data) { Y_PROTOBUF_SUPPRESS_NODISCARD result.ParseFromString(data); - } - result.SetQueryText(std::move(text)); - } - - if (!rowset.Next()) { - return false; - } - } - - SVLOG_D("[" << Self->TabletID() << "] Loading results: " - << "table# " << TSchema::TableId - << ", results count# " << results.size()); - - return true; - }; - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - SVLOG_D("[" << Self->TabletID() << "] TTxInit::Execute"); - - NIceDb::TNiceDb db(txc.DB); - - { // precharge - auto sysParamsRowset = db.Table<Schema::SysParams>().Range().Select(); - auto intervalSummariesRowset = db.Table<Schema::IntervalSummaries>().Range().Select(); - auto intervalMetricsRowset = db.Table<Schema::IntervalMetrics>().Range().Select(); - auto intervalTopsRowset = db.Table<Schema::IntervalTops>().Range().Select(); - auto nodesToRequestRowset = db.Table<Schema::NodesToRequest>().Range().Select(); - auto metricsOneMinuteRowset = db.Table<Schema::MetricsOneMinute>().Range().Select(); - auto metricsOneHourRowset = db.Table<Schema::MetricsOneHour>().Range().Select(); - auto durationOneMinuteRowset = db.Table<Schema::TopByDurationOneMinute>().Range().Select(); - auto durationOneHourRowset = db.Table<Schema::TopByDurationOneHour>().Range().Select(); - auto readBytesOneMinuteRowset = db.Table<Schema::TopByDurationOneMinute>().Range().Select(); - auto readBytesOneHourRowset = db.Table<Schema::TopByReadBytesOneHour>().Range().Select(); - auto cpuTimeOneMinuteRowset = db.Table<Schema::TopByCpuTimeOneMinute>().Range().Select(); - auto cpuTimeOneHourRowset = db.Table<Schema::TopByCpuTimeOneHour>().Range().Select(); - auto reqUnitsOneMinuteRowset = db.Table<Schema::TopByRequestUnitsOneMinute>().Range().Select(); - auto reqUnitsOneHourRowset = db.Table<Schema::TopByRequestUnitsOneHour>().Range().Select(); - - if (!sysParamsRowset.IsReady() || - !intervalSummariesRowset.IsReady() || - !intervalMetricsRowset.IsReady() || - !intervalTopsRowset.IsReady() || - !nodesToRequestRowset.IsReady() || - !metricsOneMinuteRowset.IsReady() || - !metricsOneHourRowset.IsReady() || - !durationOneMinuteRowset.IsReady() || - !durationOneHourRowset.IsReady() || - !readBytesOneMinuteRowset.IsReady() || - !readBytesOneHourRowset.IsReady() || - !cpuTimeOneMinuteRowset.IsReady() || - !cpuTimeOneHourRowset.IsReady() || - !reqUnitsOneMinuteRowset.IsReady() || - !reqUnitsOneHourRowset.IsReady()) - { - return false; - } - } - - // SysParams - { - auto rowset = db.Table<Schema::SysParams>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - ui64 id = rowset.GetValue<Schema::SysParams::Id>(); - TString value = rowset.GetValue<Schema::SysParams::Value>(); - - switch (id) { - case Schema::SysParam_Database: - Self->Database = value; - SVLOG_D("[" << Self->TabletID() << "] Loading database: " << Self->Database); - break; - case Schema::SysParam_CurrentStage: { - auto stage = FromString<ui64>(value); - Self->CurrentStage = static_cast<EStage>(stage); - SVLOG_D("[" << Self->TabletID() << "] Loading stage: " << stage); - break; - } - case Schema::SysParam_IntervalEnd: - Self->IntervalEnd = TInstant::MicroSeconds(FromString<ui64>(value)); - SVLOG_D("[" << Self->TabletID() << "] Loading interval end: " << Self->IntervalEnd); - break; - default: - SVLOG_CRIT("[" << Self->TabletID() << "] Unexpected SysParam id: " << id); - } - - if (!rowset.Next()) { - return false; - } - } - } - - // IntervalSummaries - { - Self->Queries.clear(); - Self->ByCpu.clear(); - Self->SummaryNodes.clear(); - - auto rowset = db.Table<Schema::IntervalSummaries>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - size_t totalNodeIdsCount = 0; - while (!rowset.EndOfSet()) { - TQueryHash queryHash = rowset.GetValue<Schema::IntervalSummaries::QueryHash>(); - ui64 cpu = rowset.GetValue<Schema::IntervalSummaries::CPU>(); - TNodeId nodeId = rowset.GetValue<Schema::IntervalSummaries::NodeId>(); - - auto& query = Self->Queries[queryHash]; - query.Cpu += cpu; - query.Nodes.emplace_back(nodeId, cpu); - ++totalNodeIdsCount; - - Self->SummaryNodes.insert(nodeId); - - if (!rowset.Next()) { - return false; - } - } - - for (const auto& [queryHash, query] : Self->Queries) { - Self->ByCpu.emplace(query.Cpu, queryHash); - } - - SVLOG_D("[" << Self->TabletID() << "] Loading interval summaries: " - << "query count# " << Self->Queries.size() - << ", node ids count# " << Self->SummaryNodes.size() - << ", total count# " << totalNodeIdsCount); - } - - // IntervalMetrics - { - Self->QueryMetrics.clear(); - - auto rowset = db.Table<Schema::IntervalMetrics>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - while (!rowset.EndOfSet()) { - TQueryHash queryHash = rowset.GetValue<Schema::IntervalMetrics::QueryHash>(); - TString metrics = rowset.GetValue<Schema::IntervalMetrics::Metrics>(); - TString text = rowset.GetValue<Schema::IntervalMetrics::Text>(); - - auto& queryMetrics = Self->QueryMetrics[queryHash]; - queryMetrics.Text = text; - if (metrics) { + } + result.SetQueryText(std::move(text)); + } + + if (!rowset.Next()) { + return false; + } + } + + SVLOG_D("[" << Self->TabletID() << "] Loading results: " + << "table# " << TSchema::TableId + << ", results count# " << results.size()); + + return true; + }; + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + SVLOG_D("[" << Self->TabletID() << "] TTxInit::Execute"); + + NIceDb::TNiceDb db(txc.DB); + + { // precharge + auto sysParamsRowset = db.Table<Schema::SysParams>().Range().Select(); + auto intervalSummariesRowset = db.Table<Schema::IntervalSummaries>().Range().Select(); + auto intervalMetricsRowset = db.Table<Schema::IntervalMetrics>().Range().Select(); + auto intervalTopsRowset = db.Table<Schema::IntervalTops>().Range().Select(); + auto nodesToRequestRowset = db.Table<Schema::NodesToRequest>().Range().Select(); + auto metricsOneMinuteRowset = db.Table<Schema::MetricsOneMinute>().Range().Select(); + auto metricsOneHourRowset = db.Table<Schema::MetricsOneHour>().Range().Select(); + auto durationOneMinuteRowset = db.Table<Schema::TopByDurationOneMinute>().Range().Select(); + auto durationOneHourRowset = db.Table<Schema::TopByDurationOneHour>().Range().Select(); + auto readBytesOneMinuteRowset = db.Table<Schema::TopByDurationOneMinute>().Range().Select(); + auto readBytesOneHourRowset = db.Table<Schema::TopByReadBytesOneHour>().Range().Select(); + auto cpuTimeOneMinuteRowset = db.Table<Schema::TopByCpuTimeOneMinute>().Range().Select(); + auto cpuTimeOneHourRowset = db.Table<Schema::TopByCpuTimeOneHour>().Range().Select(); + auto reqUnitsOneMinuteRowset = db.Table<Schema::TopByRequestUnitsOneMinute>().Range().Select(); + auto reqUnitsOneHourRowset = db.Table<Schema::TopByRequestUnitsOneHour>().Range().Select(); + + if (!sysParamsRowset.IsReady() || + !intervalSummariesRowset.IsReady() || + !intervalMetricsRowset.IsReady() || + !intervalTopsRowset.IsReady() || + !nodesToRequestRowset.IsReady() || + !metricsOneMinuteRowset.IsReady() || + !metricsOneHourRowset.IsReady() || + !durationOneMinuteRowset.IsReady() || + !durationOneHourRowset.IsReady() || + !readBytesOneMinuteRowset.IsReady() || + !readBytesOneHourRowset.IsReady() || + !cpuTimeOneMinuteRowset.IsReady() || + !cpuTimeOneHourRowset.IsReady() || + !reqUnitsOneMinuteRowset.IsReady() || + !reqUnitsOneHourRowset.IsReady()) + { + return false; + } + } + + // SysParams + { + auto rowset = db.Table<Schema::SysParams>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + ui64 id = rowset.GetValue<Schema::SysParams::Id>(); + TString value = rowset.GetValue<Schema::SysParams::Value>(); + + switch (id) { + case Schema::SysParam_Database: + Self->Database = value; + SVLOG_D("[" << Self->TabletID() << "] Loading database: " << Self->Database); + break; + case Schema::SysParam_CurrentStage: { + auto stage = FromString<ui64>(value); + Self->CurrentStage = static_cast<EStage>(stage); + SVLOG_D("[" << Self->TabletID() << "] Loading stage: " << stage); + break; + } + case Schema::SysParam_IntervalEnd: + Self->IntervalEnd = TInstant::MicroSeconds(FromString<ui64>(value)); + SVLOG_D("[" << Self->TabletID() << "] Loading interval end: " << Self->IntervalEnd); + break; + default: + SVLOG_CRIT("[" << Self->TabletID() << "] Unexpected SysParam id: " << id); + } + + if (!rowset.Next()) { + return false; + } + } + } + + // IntervalSummaries + { + Self->Queries.clear(); + Self->ByCpu.clear(); + Self->SummaryNodes.clear(); + + auto rowset = db.Table<Schema::IntervalSummaries>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + size_t totalNodeIdsCount = 0; + while (!rowset.EndOfSet()) { + TQueryHash queryHash = rowset.GetValue<Schema::IntervalSummaries::QueryHash>(); + ui64 cpu = rowset.GetValue<Schema::IntervalSummaries::CPU>(); + TNodeId nodeId = rowset.GetValue<Schema::IntervalSummaries::NodeId>(); + + auto& query = Self->Queries[queryHash]; + query.Cpu += cpu; + query.Nodes.emplace_back(nodeId, cpu); + ++totalNodeIdsCount; + + Self->SummaryNodes.insert(nodeId); + + if (!rowset.Next()) { + return false; + } + } + + for (const auto& [queryHash, query] : Self->Queries) { + Self->ByCpu.emplace(query.Cpu, queryHash); + } + + SVLOG_D("[" << Self->TabletID() << "] Loading interval summaries: " + << "query count# " << Self->Queries.size() + << ", node ids count# " << Self->SummaryNodes.size() + << ", total count# " << totalNodeIdsCount); + } + + // IntervalMetrics + { + Self->QueryMetrics.clear(); + + auto rowset = db.Table<Schema::IntervalMetrics>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + while (!rowset.EndOfSet()) { + TQueryHash queryHash = rowset.GetValue<Schema::IntervalMetrics::QueryHash>(); + TString metrics = rowset.GetValue<Schema::IntervalMetrics::Metrics>(); + TString text = rowset.GetValue<Schema::IntervalMetrics::Text>(); + + auto& queryMetrics = Self->QueryMetrics[queryHash]; + queryMetrics.Text = text; + if (metrics) { Y_PROTOBUF_SUPPRESS_NODISCARD queryMetrics.Metrics.ParseFromString(metrics); - } - - if (!rowset.Next()) { - return false; - } - } - SVLOG_D("[" << Self->TabletID() << "] Loading interval metrics: " - << "query count# " << Self->QueryMetrics.size()); - } - - // IntervalTops - { - Self->ByDurationMinute.clear(); - Self->ByDurationHour.clear(); - Self->ByReadBytesMinute.clear(); - Self->ByReadBytesHour.clear(); - Self->ByCpuTimeMinute.clear(); - Self->ByCpuTimeHour.clear(); - Self->ByRequestUnitsMinute.clear(); - Self->ByRequestUnitsHour.clear(); - - auto rowset = db.Table<Schema::IntervalTops>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - size_t queryCount = 0; - while (!rowset.EndOfSet()) { - ui32 type = rowset.GetValue<Schema::IntervalTops::TypeCol>(); - TQueryHash queryHash = rowset.GetValue<Schema::IntervalTops::QueryHash>(); - ui64 value = rowset.GetValue<Schema::IntervalTops::Value>(); - TNodeId nodeId = rowset.GetValue<Schema::IntervalTops::NodeId>(); - TString stats = rowset.GetValue<Schema::IntervalTops::Stats>(); - - TTopQuery query{queryHash, value, nodeId, {}}; - if (stats) { - query.Stats = MakeHolder<NKikimrSysView::TQueryStats>(); + } + + if (!rowset.Next()) { + return false; + } + } + SVLOG_D("[" << Self->TabletID() << "] Loading interval metrics: " + << "query count# " << Self->QueryMetrics.size()); + } + + // IntervalTops + { + Self->ByDurationMinute.clear(); + Self->ByDurationHour.clear(); + Self->ByReadBytesMinute.clear(); + Self->ByReadBytesHour.clear(); + Self->ByCpuTimeMinute.clear(); + Self->ByCpuTimeHour.clear(); + Self->ByRequestUnitsMinute.clear(); + Self->ByRequestUnitsHour.clear(); + + auto rowset = db.Table<Schema::IntervalTops>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + size_t queryCount = 0; + while (!rowset.EndOfSet()) { + ui32 type = rowset.GetValue<Schema::IntervalTops::TypeCol>(); + TQueryHash queryHash = rowset.GetValue<Schema::IntervalTops::QueryHash>(); + ui64 value = rowset.GetValue<Schema::IntervalTops::Value>(); + TNodeId nodeId = rowset.GetValue<Schema::IntervalTops::NodeId>(); + TString stats = rowset.GetValue<Schema::IntervalTops::Stats>(); + + TTopQuery query{queryHash, value, nodeId, {}}; + if (stats) { + query.Stats = MakeHolder<NKikimrSysView::TQueryStats>(); Y_PROTOBUF_SUPPRESS_NODISCARD query.Stats->ParseFromString(stats); - } - - switch ((NKikimrSysView::EStatsType)type) { - case NKikimrSysView::TOP_DURATION_ONE_MINUTE: - Self->ByDurationMinute.emplace_back(std::move(query)); - break; - case NKikimrSysView::TOP_DURATION_ONE_HOUR: - Self->ByDurationHour.emplace_back(std::move(query)); - break; - case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE: - Self->ByReadBytesMinute.emplace_back(std::move(query)); - break; - case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR: - Self->ByReadBytesHour.emplace_back(std::move(query)); - break; - case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE: - Self->ByCpuTimeMinute.emplace_back(std::move(query)); - break; - case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR: - Self->ByCpuTimeHour.emplace_back(std::move(query)); - break; - case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE: - Self->ByRequestUnitsMinute.emplace_back(std::move(query)); - break; - case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR: - Self->ByRequestUnitsHour.emplace_back(std::move(query)); - break; - default: - SVLOG_CRIT("[" << Self->TabletID() << "] ignoring unexpected stats type: " << type); - } - - ++queryCount; - if (!rowset.Next()) { - return false; - } - } - - std::sort(Self->ByDurationMinute.begin(), Self->ByDurationMinute.end(), TopQueryCompare); - std::sort(Self->ByDurationHour.begin(), Self->ByDurationHour.end(), TopQueryCompare); - std::sort(Self->ByReadBytesMinute.begin(), Self->ByReadBytesMinute.end(), TopQueryCompare); - std::sort(Self->ByReadBytesHour.begin(), Self->ByReadBytesHour.end(), TopQueryCompare); - std::sort(Self->ByCpuTimeMinute.begin(), Self->ByCpuTimeMinute.end(), TopQueryCompare); - std::sort(Self->ByCpuTimeHour.begin(), Self->ByCpuTimeHour.end(), TopQueryCompare); - std::sort(Self->ByRequestUnitsMinute.begin(), Self->ByRequestUnitsMinute.end(), TopQueryCompare); - std::sort(Self->ByRequestUnitsHour.begin(), Self->ByRequestUnitsHour.end(), TopQueryCompare); - - SVLOG_D("[" << Self->TabletID() << "] Loading interval tops: " - << "total query count# " << queryCount); - } - - // NodesToRequest - { - Self->NodesToRequest.clear(); - Self->NodesInFlight.clear(); - - auto rowset = db.Table<Schema::NodesToRequest>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - size_t totalHashesCount = 0; - while (!rowset.EndOfSet()) { - TNodeId nodeId = rowset.GetValue<Schema::NodesToRequest::NodeId>(); - TString hashes = rowset.GetValue<Schema::NodesToRequest::QueryHashes>(); - TString textsToGet = rowset.GetValue<Schema::NodesToRequest::TextsToGet>(); - TString byDuration = rowset.GetValue<Schema::NodesToRequest::ByDuration>(); - TString byReadBytes = rowset.GetValue<Schema::NodesToRequest::ByReadBytes>(); - TString byCpuTime = rowset.GetValue<Schema::NodesToRequest::ByCpuTime>(); - TString byRequestUnits = rowset.GetValue<Schema::NodesToRequest::ByRequestUnits>(); - - auto loadHashes = [&totalHashesCount] (const TString& from, THashVector& to) { - auto size = from.size() / sizeof(TQueryHash); - to.resize(size); - std::memcpy(to.data(), from.data(), size * sizeof(TQueryHash)); - totalHashesCount += size; - }; - - TNodeToQueries entry; - entry.NodeId = nodeId; - loadHashes(hashes, entry.Hashes); - loadHashes(textsToGet, entry.TextsToGet); - loadHashes(byDuration, entry.ByDuration); - loadHashes(byReadBytes, entry.ByReadBytes); - loadHashes(byCpuTime, entry.ByCpuTime); - loadHashes(byRequestUnits, entry.ByRequestUnits); - Self->NodesToRequest.emplace_back(std::move(entry)); - - if (!rowset.Next()) { - return false; - } - } - SVLOG_D("[" << Self->TabletID() << "] Loading nodes to request: " - << "nodes count# " << Self->NodesToRequest.size() - << ", hashes count# " << totalHashesCount); - } - - // Metrics... - if (!LoadResults<Schema::MetricsOneMinute, TQueryToMetrics>(db, Self->MetricsOneMinute)) - return false; - if (!LoadResults<Schema::MetricsOneHour, TQueryToMetrics>(db, Self->MetricsOneHour)) - return false; - - // TopBy... - using TStats = NKikimrSysView::TQueryStats; - if (!LoadResults<Schema::TopByDurationOneMinute, TStats>(db, Self->TopByDurationOneMinute)) - return false; - if (!LoadResults<Schema::TopByDurationOneHour, TStats>(db, Self->TopByDurationOneHour)) - return false; - if (!LoadResults<Schema::TopByReadBytesOneMinute, TStats>(db, Self->TopByReadBytesOneMinute)) - return false; - if (!LoadResults<Schema::TopByReadBytesOneHour, TStats>(db, Self->TopByReadBytesOneHour)) - return false; - if (!LoadResults<Schema::TopByCpuTimeOneMinute, TStats>(db, Self->TopByCpuTimeOneMinute)) - return false; - if (!LoadResults<Schema::TopByCpuTimeOneHour, TStats>(db, Self->TopByCpuTimeOneHour)) - return false; - if (!LoadResults<Schema::TopByRequestUnitsOneMinute, TStats>(db, Self->TopByRequestUnitsOneMinute)) - return false; - if (!LoadResults<Schema::TopByRequestUnitsOneHour, TStats>(db, Self->TopByRequestUnitsOneHour)) - return false; - - auto deadline = Self->IntervalEnd + Self->TotalInterval; - if (ctx.Now() >= deadline) { - Self->Reset(db, ctx); - } - - return true; - } - - void Complete(const TActorContext& ctx) override { - SVLOG_D("[" << Self->TabletID() << "] TTxInit::Complete"); - - if (Self->CurrentStage == COLLECT) { - Self->ScheduleAggregate(); - } else { - Self->ScheduleCollect(); - if (!Self->NodesToRequest.empty()) { - Self->ScheduleSendRequests(); - } - } - - if (AppData()->FeatureFlags.GetEnableDbCounters()) { - Self->ScheduleApplyCounters(); - Self->SendNavigate(); - } - - Self->SignalTabletActive(ctx); - Self->Become(&TThis::StateWork); - } -}; - -NTabletFlatExecutor::ITransaction* TSysViewProcessor::CreateTxInit() { - return new TTxInit(this); -} - -} // NSysView -} // NKikimr + } + + switch ((NKikimrSysView::EStatsType)type) { + case NKikimrSysView::TOP_DURATION_ONE_MINUTE: + Self->ByDurationMinute.emplace_back(std::move(query)); + break; + case NKikimrSysView::TOP_DURATION_ONE_HOUR: + Self->ByDurationHour.emplace_back(std::move(query)); + break; + case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE: + Self->ByReadBytesMinute.emplace_back(std::move(query)); + break; + case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR: + Self->ByReadBytesHour.emplace_back(std::move(query)); + break; + case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE: + Self->ByCpuTimeMinute.emplace_back(std::move(query)); + break; + case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR: + Self->ByCpuTimeHour.emplace_back(std::move(query)); + break; + case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE: + Self->ByRequestUnitsMinute.emplace_back(std::move(query)); + break; + case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR: + Self->ByRequestUnitsHour.emplace_back(std::move(query)); + break; + default: + SVLOG_CRIT("[" << Self->TabletID() << "] ignoring unexpected stats type: " << type); + } + + ++queryCount; + if (!rowset.Next()) { + return false; + } + } + + std::sort(Self->ByDurationMinute.begin(), Self->ByDurationMinute.end(), TopQueryCompare); + std::sort(Self->ByDurationHour.begin(), Self->ByDurationHour.end(), TopQueryCompare); + std::sort(Self->ByReadBytesMinute.begin(), Self->ByReadBytesMinute.end(), TopQueryCompare); + std::sort(Self->ByReadBytesHour.begin(), Self->ByReadBytesHour.end(), TopQueryCompare); + std::sort(Self->ByCpuTimeMinute.begin(), Self->ByCpuTimeMinute.end(), TopQueryCompare); + std::sort(Self->ByCpuTimeHour.begin(), Self->ByCpuTimeHour.end(), TopQueryCompare); + std::sort(Self->ByRequestUnitsMinute.begin(), Self->ByRequestUnitsMinute.end(), TopQueryCompare); + std::sort(Self->ByRequestUnitsHour.begin(), Self->ByRequestUnitsHour.end(), TopQueryCompare); + + SVLOG_D("[" << Self->TabletID() << "] Loading interval tops: " + << "total query count# " << queryCount); + } + + // NodesToRequest + { + Self->NodesToRequest.clear(); + Self->NodesInFlight.clear(); + + auto rowset = db.Table<Schema::NodesToRequest>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + size_t totalHashesCount = 0; + while (!rowset.EndOfSet()) { + TNodeId nodeId = rowset.GetValue<Schema::NodesToRequest::NodeId>(); + TString hashes = rowset.GetValue<Schema::NodesToRequest::QueryHashes>(); + TString textsToGet = rowset.GetValue<Schema::NodesToRequest::TextsToGet>(); + TString byDuration = rowset.GetValue<Schema::NodesToRequest::ByDuration>(); + TString byReadBytes = rowset.GetValue<Schema::NodesToRequest::ByReadBytes>(); + TString byCpuTime = rowset.GetValue<Schema::NodesToRequest::ByCpuTime>(); + TString byRequestUnits = rowset.GetValue<Schema::NodesToRequest::ByRequestUnits>(); + + auto loadHashes = [&totalHashesCount] (const TString& from, THashVector& to) { + auto size = from.size() / sizeof(TQueryHash); + to.resize(size); + std::memcpy(to.data(), from.data(), size * sizeof(TQueryHash)); + totalHashesCount += size; + }; + + TNodeToQueries entry; + entry.NodeId = nodeId; + loadHashes(hashes, entry.Hashes); + loadHashes(textsToGet, entry.TextsToGet); + loadHashes(byDuration, entry.ByDuration); + loadHashes(byReadBytes, entry.ByReadBytes); + loadHashes(byCpuTime, entry.ByCpuTime); + loadHashes(byRequestUnits, entry.ByRequestUnits); + Self->NodesToRequest.emplace_back(std::move(entry)); + + if (!rowset.Next()) { + return false; + } + } + SVLOG_D("[" << Self->TabletID() << "] Loading nodes to request: " + << "nodes count# " << Self->NodesToRequest.size() + << ", hashes count# " << totalHashesCount); + } + + // Metrics... + if (!LoadResults<Schema::MetricsOneMinute, TQueryToMetrics>(db, Self->MetricsOneMinute)) + return false; + if (!LoadResults<Schema::MetricsOneHour, TQueryToMetrics>(db, Self->MetricsOneHour)) + return false; + + // TopBy... + using TStats = NKikimrSysView::TQueryStats; + if (!LoadResults<Schema::TopByDurationOneMinute, TStats>(db, Self->TopByDurationOneMinute)) + return false; + if (!LoadResults<Schema::TopByDurationOneHour, TStats>(db, Self->TopByDurationOneHour)) + return false; + if (!LoadResults<Schema::TopByReadBytesOneMinute, TStats>(db, Self->TopByReadBytesOneMinute)) + return false; + if (!LoadResults<Schema::TopByReadBytesOneHour, TStats>(db, Self->TopByReadBytesOneHour)) + return false; + if (!LoadResults<Schema::TopByCpuTimeOneMinute, TStats>(db, Self->TopByCpuTimeOneMinute)) + return false; + if (!LoadResults<Schema::TopByCpuTimeOneHour, TStats>(db, Self->TopByCpuTimeOneHour)) + return false; + if (!LoadResults<Schema::TopByRequestUnitsOneMinute, TStats>(db, Self->TopByRequestUnitsOneMinute)) + return false; + if (!LoadResults<Schema::TopByRequestUnitsOneHour, TStats>(db, Self->TopByRequestUnitsOneHour)) + return false; + + auto deadline = Self->IntervalEnd + Self->TotalInterval; + if (ctx.Now() >= deadline) { + Self->Reset(db, ctx); + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + SVLOG_D("[" << Self->TabletID() << "] TTxInit::Complete"); + + if (Self->CurrentStage == COLLECT) { + Self->ScheduleAggregate(); + } else { + Self->ScheduleCollect(); + if (!Self->NodesToRequest.empty()) { + Self->ScheduleSendRequests(); + } + } + + if (AppData()->FeatureFlags.GetEnableDbCounters()) { + Self->ScheduleApplyCounters(); + Self->SendNavigate(); + } + + Self->SignalTabletActive(ctx); + Self->Become(&TThis::StateWork); + } +}; + +NTabletFlatExecutor::ITransaction* TSysViewProcessor::CreateTxInit() { + return new TTxInit(this); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/processor/tx_init_schema.cpp b/ydb/core/sys_view/processor/tx_init_schema.cpp index b7701f4167..e0485d741f 100644 --- a/ydb/core/sys_view/processor/tx_init_schema.cpp +++ b/ydb/core/sys_view/processor/tx_init_schema.cpp @@ -1,57 +1,57 @@ -#include "processor_impl.h" - -namespace NKikimr { -namespace NSysView { - -struct TSysViewProcessor::TTxInitSchema : public TTxBase { - explicit TTxInitSchema(TSelf* self) - : TTxBase(self) - {} - - TTxType GetTxType() const override { return TXTYPE_INIT_SCHEMA; } - - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SVLOG_D("[" << Self->TabletID() << "] TTxInitSchema::Execute"); - - NIceDb::TNiceDb(txc.DB).Materialize<Schema>(); - - static constexpr NIceDb::TTableId resultTableIds[] = { - Schema::MetricsOneMinute::TableId, - Schema::MetricsOneHour::TableId, - Schema::TopByDurationOneMinute::TableId, - Schema::TopByDurationOneHour::TableId, - Schema::TopByReadBytesOneMinute::TableId, - Schema::TopByReadBytesOneHour::TableId, - Schema::TopByCpuTimeOneMinute::TableId, - Schema::TopByCpuTimeOneHour::TableId, - Schema::TopByRequestUnitsOneMinute::TableId, - Schema::TopByRequestUnitsOneHour::TableId - }; - - for (auto id : resultTableIds) { - txc.DB.Alter().SetCompactionPolicy(id, *NLocalDb::CreateDefaultUserTablePolicy()); - } - - return true; - } - - void Complete(const TActorContext& ctx) override { - SVLOG_D("[" << Self->TabletID() << "] TTxInitSchema::Complete"); - +#include "processor_impl.h" + +namespace NKikimr { +namespace NSysView { + +struct TSysViewProcessor::TTxInitSchema : public TTxBase { + explicit TTxInitSchema(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_INIT_SCHEMA; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SVLOG_D("[" << Self->TabletID() << "] TTxInitSchema::Execute"); + + NIceDb::TNiceDb(txc.DB).Materialize<Schema>(); + + static constexpr NIceDb::TTableId resultTableIds[] = { + Schema::MetricsOneMinute::TableId, + Schema::MetricsOneHour::TableId, + Schema::TopByDurationOneMinute::TableId, + Schema::TopByDurationOneHour::TableId, + Schema::TopByReadBytesOneMinute::TableId, + Schema::TopByReadBytesOneHour::TableId, + Schema::TopByCpuTimeOneMinute::TableId, + Schema::TopByCpuTimeOneHour::TableId, + Schema::TopByRequestUnitsOneMinute::TableId, + Schema::TopByRequestUnitsOneHour::TableId + }; + + for (auto id : resultTableIds) { + txc.DB.Alter().SetCompactionPolicy(id, *NLocalDb::CreateDefaultUserTablePolicy()); + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + SVLOG_D("[" << Self->TabletID() << "] TTxInitSchema::Complete"); + if (!AppData()->FeatureFlags.GetEnablePersistentQueryStats()) { - SVLOG_D("[" << Self->TabletID() << "] tablet is offline"); - Self->SignalTabletActive(ctx); - Self->Become(&TThis::StateOffline); - return; - } - - Self->Execute(Self->CreateTxInit(), ctx); - } -}; - -NTabletFlatExecutor::ITransaction* TSysViewProcessor::CreateTxInitSchema() { - return new TTxInitSchema(this); -} - -} // NSysView -} // NKikimr + SVLOG_D("[" << Self->TabletID() << "] tablet is offline"); + Self->SignalTabletActive(ctx); + Self->Become(&TThis::StateOffline); + return; + } + + Self->Execute(Self->CreateTxInit(), ctx); + } +}; + +NTabletFlatExecutor::ITransaction* TSysViewProcessor::CreateTxInitSchema() { + return new TTxInitSchema(this); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/processor/tx_interval_metrics.cpp b/ydb/core/sys_view/processor/tx_interval_metrics.cpp index e19ebd96fa..4c6d992487 100644 --- a/ydb/core/sys_view/processor/tx_interval_metrics.cpp +++ b/ydb/core/sys_view/processor/tx_interval_metrics.cpp @@ -1,140 +1,140 @@ -#include "processor_impl.h" - +#include "processor_impl.h" + #include <ydb/core/sys_view/service/query_interval.h> - -namespace NKikimr { -namespace NSysView { - -struct TSysViewProcessor::TTxIntervalMetrics : public TTxBase { - TNodeId NodeId; - NKikimrSysView::TEvGetIntervalMetricsResponse Record; - - TTxIntervalMetrics(TSelf* self, TNodeId nodeId, - NKikimrSysView::TEvGetIntervalMetricsResponse&& record) - : TTxBase(self) - , NodeId(nodeId) - , Record(std::move(record)) - {} - - TTxType GetTxType() const override { return TXTYPE_INTERVAL_METRICS; } - - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SVLOG_D("[" << Self->TabletID() << "] TTxIntervalMetrics::Execute: " - << "node id# " << NodeId - << ", metrics count# " << Record.MetricsSize() - << ", texts count# " << Record.QueryTextsSize()); - - NIceDb::TNiceDb db(txc.DB); - - for (auto& queryText : *Record.MutableQueryTexts()) { - auto queryHash = queryText.GetHash(); - auto& text = *queryText.MutableText(); - - db.Table<Schema::IntervalMetrics>().Key(queryHash).Update( - NIceDb::TUpdate<Schema::IntervalMetrics::Text>(text)); - - Self->QueryMetrics[queryHash].Text = std::move(text); - } - - for (auto& metrics : *Record.MutableMetrics()) { - auto queryHash = metrics.GetQueryTextHash(); - auto& newMetrics = Self->QueryMetrics[queryHash].Metrics; - - if (!newMetrics.GetCount()) { - newMetrics.Swap(&metrics); - } else { - Aggregate(newMetrics, metrics); - } - - TString serialized; + +namespace NKikimr { +namespace NSysView { + +struct TSysViewProcessor::TTxIntervalMetrics : public TTxBase { + TNodeId NodeId; + NKikimrSysView::TEvGetIntervalMetricsResponse Record; + + TTxIntervalMetrics(TSelf* self, TNodeId nodeId, + NKikimrSysView::TEvGetIntervalMetricsResponse&& record) + : TTxBase(self) + , NodeId(nodeId) + , Record(std::move(record)) + {} + + TTxType GetTxType() const override { return TXTYPE_INTERVAL_METRICS; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SVLOG_D("[" << Self->TabletID() << "] TTxIntervalMetrics::Execute: " + << "node id# " << NodeId + << ", metrics count# " << Record.MetricsSize() + << ", texts count# " << Record.QueryTextsSize()); + + NIceDb::TNiceDb db(txc.DB); + + for (auto& queryText : *Record.MutableQueryTexts()) { + auto queryHash = queryText.GetHash(); + auto& text = *queryText.MutableText(); + + db.Table<Schema::IntervalMetrics>().Key(queryHash).Update( + NIceDb::TUpdate<Schema::IntervalMetrics::Text>(text)); + + Self->QueryMetrics[queryHash].Text = std::move(text); + } + + for (auto& metrics : *Record.MutableMetrics()) { + auto queryHash = metrics.GetQueryTextHash(); + auto& newMetrics = Self->QueryMetrics[queryHash].Metrics; + + if (!newMetrics.GetCount()) { + newMetrics.Swap(&metrics); + } else { + Aggregate(newMetrics, metrics); + } + + TString serialized; Y_PROTOBUF_SUPPRESS_NODISCARD newMetrics.SerializeToString(&serialized); - db.Table<Schema::IntervalMetrics>().Key(queryHash).Update( - NIceDb::TUpdate<Schema::IntervalMetrics::Metrics>(serialized)); - } - - auto fillTops = [&] (TTop& minuteTop, TTop& hourTop, - NKikimrSysView::EStatsType minuteType, NKikimrSysView::EStatsType hourType, - const NProtoBuf::RepeatedPtrField<NKikimrSysView::TQueryStats>& queryStats) - { - for (auto& stats : queryStats) { - auto queryHash = stats.GetQueryTextHash(); - TString serialized; - for (auto& query : minuteTop) { - if (query.Hash == queryHash) { - query.Stats = MakeHolder<NKikimrSysView::TQueryStats>(); - query.Stats->CopyFrom(stats); + db.Table<Schema::IntervalMetrics>().Key(queryHash).Update( + NIceDb::TUpdate<Schema::IntervalMetrics::Metrics>(serialized)); + } + + auto fillTops = [&] (TTop& minuteTop, TTop& hourTop, + NKikimrSysView::EStatsType minuteType, NKikimrSysView::EStatsType hourType, + const NProtoBuf::RepeatedPtrField<NKikimrSysView::TQueryStats>& queryStats) + { + for (auto& stats : queryStats) { + auto queryHash = stats.GetQueryTextHash(); + TString serialized; + for (auto& query : minuteTop) { + if (query.Hash == queryHash) { + query.Stats = MakeHolder<NKikimrSysView::TQueryStats>(); + query.Stats->CopyFrom(stats); Y_PROTOBUF_SUPPRESS_NODISCARD query.Stats->SerializeToString(&serialized); - db.Table<Schema::IntervalTops>().Key((ui32)minuteType, queryHash).Update( - NIceDb::TUpdate<Schema::IntervalTops::Stats>(serialized)); - break; - } - } - for (auto& query : hourTop) { - if (!query.Stats && query.Hash == queryHash) { - query.Stats = MakeHolder<NKikimrSysView::TQueryStats>(); - query.Stats->CopyFrom(stats); - // hash must be in a minute top as well - db.Table<Schema::IntervalTops>().Key((ui32)hourType, queryHash).Update( - NIceDb::TUpdate<Schema::IntervalTops::Stats>(serialized)); - break; - } - } - } - }; - - fillTops(Self->ByDurationMinute, Self->ByDurationHour, - NKikimrSysView::TOP_DURATION_ONE_MINUTE, NKikimrSysView::TOP_DURATION_ONE_HOUR, - *Record.MutableTopByDuration()); - - fillTops(Self->ByReadBytesMinute, Self->ByReadBytesHour, - NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, - *Record.MutableTopByReadBytes()); - - fillTops(Self->ByCpuTimeMinute, Self->ByCpuTimeHour, - NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, - *Record.MutableTopByCpuTime()); - - fillTops(Self->ByRequestUnitsMinute, Self->ByRequestUnitsHour, - NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, - *Record.MutableTopByRequestUnits()); - - Self->NodesInFlight.erase(NodeId); - db.Table<Schema::NodesToRequest>().Key(NodeId).Delete(); - - if (Self->NodesInFlight.empty() && Self->NodesToRequest.empty()) { - Self->PersistResults(db); - } else { - Self->SendRequests(); - } - return true; - } - - void Complete(const TActorContext&) override { - SVLOG_D("[" << Self->TabletID() << "] TTxIntervalMetrics::Complete"); - } -}; - -void TSysViewProcessor::Handle(TEvSysView::TEvGetIntervalMetricsResponse::TPtr& ev) { - auto& record = ev->Get()->Record; - TNodeId nodeId = ev.Get()->Cookie; - - if (CurrentStage != AGGREGATE) { - SVLOG_W("[" << TabletID() << "] TEvGetIntervalMetricsResponse, wrong stage: " - << "node id# " << nodeId); - return; - } - - if (record.GetIntervalEndUs() != IntervalEnd.MicroSeconds()) { - SVLOG_W("[" << TabletID() << "] TEvGetIntervalMetricsResponse, time mismatch: " - << "node id# " << nodeId - << "interval end# " << IntervalEnd - << "event interval end# " << TInstant::MicroSeconds(record.GetIntervalEndUs())); - return; - } - - Execute(new TTxIntervalMetrics(this, nodeId, std::move(record)), - TActivationContext::AsActorContext()); -} - -} // NSysView -} // NKikimr + db.Table<Schema::IntervalTops>().Key((ui32)minuteType, queryHash).Update( + NIceDb::TUpdate<Schema::IntervalTops::Stats>(serialized)); + break; + } + } + for (auto& query : hourTop) { + if (!query.Stats && query.Hash == queryHash) { + query.Stats = MakeHolder<NKikimrSysView::TQueryStats>(); + query.Stats->CopyFrom(stats); + // hash must be in a minute top as well + db.Table<Schema::IntervalTops>().Key((ui32)hourType, queryHash).Update( + NIceDb::TUpdate<Schema::IntervalTops::Stats>(serialized)); + break; + } + } + } + }; + + fillTops(Self->ByDurationMinute, Self->ByDurationHour, + NKikimrSysView::TOP_DURATION_ONE_MINUTE, NKikimrSysView::TOP_DURATION_ONE_HOUR, + *Record.MutableTopByDuration()); + + fillTops(Self->ByReadBytesMinute, Self->ByReadBytesHour, + NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, + *Record.MutableTopByReadBytes()); + + fillTops(Self->ByCpuTimeMinute, Self->ByCpuTimeHour, + NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, + *Record.MutableTopByCpuTime()); + + fillTops(Self->ByRequestUnitsMinute, Self->ByRequestUnitsHour, + NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, + *Record.MutableTopByRequestUnits()); + + Self->NodesInFlight.erase(NodeId); + db.Table<Schema::NodesToRequest>().Key(NodeId).Delete(); + + if (Self->NodesInFlight.empty() && Self->NodesToRequest.empty()) { + Self->PersistResults(db); + } else { + Self->SendRequests(); + } + return true; + } + + void Complete(const TActorContext&) override { + SVLOG_D("[" << Self->TabletID() << "] TTxIntervalMetrics::Complete"); + } +}; + +void TSysViewProcessor::Handle(TEvSysView::TEvGetIntervalMetricsResponse::TPtr& ev) { + auto& record = ev->Get()->Record; + TNodeId nodeId = ev.Get()->Cookie; + + if (CurrentStage != AGGREGATE) { + SVLOG_W("[" << TabletID() << "] TEvGetIntervalMetricsResponse, wrong stage: " + << "node id# " << nodeId); + return; + } + + if (record.GetIntervalEndUs() != IntervalEnd.MicroSeconds()) { + SVLOG_W("[" << TabletID() << "] TEvGetIntervalMetricsResponse, time mismatch: " + << "node id# " << nodeId + << "interval end# " << IntervalEnd + << "event interval end# " << TInstant::MicroSeconds(record.GetIntervalEndUs())); + return; + } + + Execute(new TTxIntervalMetrics(this, nodeId, std::move(record)), + TActivationContext::AsActorContext()); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/processor/tx_interval_summary.cpp b/ydb/core/sys_view/processor/tx_interval_summary.cpp index 4124a0aa77..35fe2789e1 100644 --- a/ydb/core/sys_view/processor/tx_interval_summary.cpp +++ b/ydb/core/sys_view/processor/tx_interval_summary.cpp @@ -1,228 +1,228 @@ -#include "processor_impl.h" - -namespace NKikimr { -namespace NSysView { - -struct TSysViewProcessor::TTxIntervalSummary : public TTxBase { - NKikimrSysView::TEvIntervalQuerySummary Record; - - TTxIntervalSummary(TSelf* self, NKikimrSysView::TEvIntervalQuerySummary&& record) - : TTxBase(self) - , Record(std::move(record)) - {} - - TTxType GetTxType() const override { return TXTYPE_INTERVAL_SUMMARY; } - - void AddSummary(NIceDb::TNiceDb& db, ui64 queryHash, ui64 cpu, TNodeId nodeId) { - if (auto queryIt = Self->Queries.find(queryHash); queryIt != Self->Queries.end()) { - auto& query = queryIt->second; - - auto range = Self->ByCpu.equal_range(query.Cpu); - auto it = range.first; - for (; it != range.second; ++it) { - if (it->second == queryHash) { - break; - } - } - Y_VERIFY(it != range.second); - Self->ByCpu.erase(it); - - query.Cpu += cpu; - Self->ByCpu.emplace(query.Cpu, queryHash); - - auto& nodes = query.Nodes; - size_t nodeIndex = nodes.size(); - nodes.emplace_back(nodeId, cpu); - - db.Table<Schema::IntervalSummaries>().Key(queryHash, nodeIndex).Update( - NIceDb::TUpdate<Schema::IntervalSummaries::CPU>(cpu), - NIceDb::TUpdate<Schema::IntervalSummaries::NodeId>(nodeId)); - - } else { - if (Self->ByCpu.size() == DistinctQueriesLimit) { - auto it = Self->ByCpu.begin(); - if (it->first >= cpu) { - return; - } - auto removeHash = it->second; - const auto& removeQuery = Self->Queries[removeHash]; - for (ui32 i = 0; i < removeQuery.Nodes.size(); ++i) { - db.Table<Schema::IntervalSummaries>().Key(removeHash, i).Delete(); - } - Self->Queries.erase(removeHash); - Self->ByCpu.erase(it); - } - - TQueryToNodes query{cpu, {{nodeId, cpu}}}; - - Self->Queries.emplace(queryHash, std::move(query)); - Self->ByCpu.emplace(cpu, queryHash); - - db.Table<Schema::IntervalSummaries>().Key(queryHash, 0).Update( - NIceDb::TUpdate<Schema::IntervalSummaries::CPU>(cpu), - NIceDb::TUpdate<Schema::IntervalSummaries::NodeId>(nodeId)); - } - } - - void ProcessTop(NIceDb::TNiceDb& db, - TNodeId nodeId, - NKikimrSysView::EStatsType statsType, - const NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries, - TTop& top) - { - TTop result; - std::unordered_set<TQueryHash> seenHashes; - size_t queryIndex = 0; - auto topIt = top.begin(); - - auto copyNewQuery = [&] () { - auto queryHash = queries.GetHashes(queryIndex); - auto value = queries.GetValues(queryIndex); - - TTopQuery topQuery{queryHash, value, nodeId, {}}; - result.emplace_back(std::move(topQuery)); - - db.Table<Schema::IntervalTops>().Key((ui32)statsType, queryHash).Update( - NIceDb::TUpdate<Schema::IntervalTops::Value>(value), - NIceDb::TUpdate<Schema::IntervalTops::NodeId>(nodeId)); - - seenHashes.insert(queryHash); - ++queryIndex; - }; - - while (result.size() < TOP_QUERIES_COUNT) { - if (topIt == top.end()) { - if (queryIndex == queries.HashesSize()) { - break; - } - auto queryHash = queries.GetHashes(queryIndex); - if (seenHashes.find(queryHash) != seenHashes.end()) { - ++queryIndex; - continue; - } - copyNewQuery(); - } else { - auto topHash = topIt->Hash; - if (seenHashes.find(topHash) != seenHashes.end()) { - ++topIt; - continue; - } - if (queryIndex == queries.HashesSize()) { - result.emplace_back(std::move(*topIt++)); - seenHashes.insert(topHash); - continue; - } - auto queryHash = queries.GetHashes(queryIndex); - if (seenHashes.find(queryHash) != seenHashes.end()) { - ++queryIndex; - continue; - } - if (topIt->Value >= queries.GetValues(queryIndex)) { - result.emplace_back(std::move(*topIt++)); - seenHashes.insert(topHash); - } else { - copyNewQuery(); - } - } - } - - for (; topIt != top.end(); ++topIt) { - auto topHash = topIt->Hash; - if (seenHashes.find(topHash) != seenHashes.end()) { - continue; - } - db.Table<Schema::IntervalTops>().Key((ui32)statsType, topHash).Delete(); - } - - top.swap(result); - } - - bool Execute(TTransactionContext& txc, const TActorContext&) override { - auto nodeId = Record.GetNodeId(); - - if (Record.GetMetrics().ValuesSize() != Record.GetMetrics().HashesSize() || - Record.GetTopByDuration().ValuesSize() != Record.GetTopByDuration().HashesSize() || - Record.GetTopByReadBytes().ValuesSize() != Record.GetTopByReadBytes().HashesSize() || - Record.GetTopByCpuTime().ValuesSize() != Record.GetTopByCpuTime().HashesSize() || - Record.GetTopByRequestUnits().ValuesSize() != Record.GetTopByRequestUnits().HashesSize()) - { - SVLOG_W("[" << Self->TabletID() << "] TTxIntervalSummary::Execute, malformed summary: " - << "node id# " << nodeId); - return true; - } - - if (Self->SummaryNodes.find(nodeId) != Self->SummaryNodes.end()) { - SVLOG_W("[" << Self->TabletID() << "] TTxIntervalSummary::Execute, duplicate summary: " - << "node id# " << nodeId); - return true; - } - Self->SummaryNodes.insert(nodeId); - - const auto& metrics = Record.GetMetrics(); - auto count = metrics.HashesSize(); - - SVLOG_D("[" << Self->TabletID() << "] TTxIntervalSummary::Execute: " - << "node id# " << nodeId - << ", query count# " << count); - - NIceDb::TNiceDb db(txc.DB); - for (size_t i = 0; i < count; ++i) { - AddSummary(db, metrics.GetHashes(i), metrics.GetValues(i), nodeId); - } - - ProcessTop(db, nodeId, NKikimrSysView::TOP_DURATION_ONE_MINUTE, - Record.GetTopByDuration(), Self->ByDurationMinute); - ProcessTop(db, nodeId, NKikimrSysView::TOP_DURATION_ONE_HOUR, - Record.GetTopByDuration(), Self->ByDurationHour); - ProcessTop(db, nodeId, NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, - Record.GetTopByReadBytes(), Self->ByReadBytesMinute); - ProcessTop(db, nodeId, NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, - Record.GetTopByReadBytes(), Self->ByReadBytesHour); - ProcessTop(db, nodeId, NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, - Record.GetTopByCpuTime(), Self->ByCpuTimeMinute); - ProcessTop(db, nodeId, NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, - Record.GetTopByCpuTime(), Self->ByCpuTimeHour); - ProcessTop(db, nodeId, NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, - Record.GetTopByRequestUnits(), Self->ByRequestUnitsMinute); - ProcessTop(db, nodeId, NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, - Record.GetTopByRequestUnits(), Self->ByRequestUnitsHour); - - return true; - } - - void Complete(const TActorContext&) override { - SVLOG_D("[" << Self->TabletID() << "] TTxIntervalSummary::Complete"); - } -}; - -void TSysViewProcessor::Handle(TEvSysView::TEvIntervalQuerySummary::TPtr& ev) { - auto& record = ev->Get()->Record; - auto nodeId = record.GetNodeId(); - - if (CurrentStage != COLLECT) { - SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, wrong stage: " - << "node id# " << nodeId); - return; - } - - if (record.GetIntervalEndUs() != IntervalEnd.MicroSeconds()) { - SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, time mismath: " - << "node id# " << nodeId - << ", interval end# " << IntervalEnd - << ", event interval end# " << TInstant::MicroSeconds(record.GetIntervalEndUs())); - return; - } - - if (record.GetDatabase() != Database) { - SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, db mismatch: " - << "node id# " << nodeId - << ", database# " << Database - << ", event database# " << record.GetDatabase()); - return; - } - - Execute(new TTxIntervalSummary(this, std::move(record)), TActivationContext::AsActorContext()); -} - -} // NSysView -} // NKikimr +#include "processor_impl.h" + +namespace NKikimr { +namespace NSysView { + +struct TSysViewProcessor::TTxIntervalSummary : public TTxBase { + NKikimrSysView::TEvIntervalQuerySummary Record; + + TTxIntervalSummary(TSelf* self, NKikimrSysView::TEvIntervalQuerySummary&& record) + : TTxBase(self) + , Record(std::move(record)) + {} + + TTxType GetTxType() const override { return TXTYPE_INTERVAL_SUMMARY; } + + void AddSummary(NIceDb::TNiceDb& db, ui64 queryHash, ui64 cpu, TNodeId nodeId) { + if (auto queryIt = Self->Queries.find(queryHash); queryIt != Self->Queries.end()) { + auto& query = queryIt->second; + + auto range = Self->ByCpu.equal_range(query.Cpu); + auto it = range.first; + for (; it != range.second; ++it) { + if (it->second == queryHash) { + break; + } + } + Y_VERIFY(it != range.second); + Self->ByCpu.erase(it); + + query.Cpu += cpu; + Self->ByCpu.emplace(query.Cpu, queryHash); + + auto& nodes = query.Nodes; + size_t nodeIndex = nodes.size(); + nodes.emplace_back(nodeId, cpu); + + db.Table<Schema::IntervalSummaries>().Key(queryHash, nodeIndex).Update( + NIceDb::TUpdate<Schema::IntervalSummaries::CPU>(cpu), + NIceDb::TUpdate<Schema::IntervalSummaries::NodeId>(nodeId)); + + } else { + if (Self->ByCpu.size() == DistinctQueriesLimit) { + auto it = Self->ByCpu.begin(); + if (it->first >= cpu) { + return; + } + auto removeHash = it->second; + const auto& removeQuery = Self->Queries[removeHash]; + for (ui32 i = 0; i < removeQuery.Nodes.size(); ++i) { + db.Table<Schema::IntervalSummaries>().Key(removeHash, i).Delete(); + } + Self->Queries.erase(removeHash); + Self->ByCpu.erase(it); + } + + TQueryToNodes query{cpu, {{nodeId, cpu}}}; + + Self->Queries.emplace(queryHash, std::move(query)); + Self->ByCpu.emplace(cpu, queryHash); + + db.Table<Schema::IntervalSummaries>().Key(queryHash, 0).Update( + NIceDb::TUpdate<Schema::IntervalSummaries::CPU>(cpu), + NIceDb::TUpdate<Schema::IntervalSummaries::NodeId>(nodeId)); + } + } + + void ProcessTop(NIceDb::TNiceDb& db, + TNodeId nodeId, + NKikimrSysView::EStatsType statsType, + const NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries, + TTop& top) + { + TTop result; + std::unordered_set<TQueryHash> seenHashes; + size_t queryIndex = 0; + auto topIt = top.begin(); + + auto copyNewQuery = [&] () { + auto queryHash = queries.GetHashes(queryIndex); + auto value = queries.GetValues(queryIndex); + + TTopQuery topQuery{queryHash, value, nodeId, {}}; + result.emplace_back(std::move(topQuery)); + + db.Table<Schema::IntervalTops>().Key((ui32)statsType, queryHash).Update( + NIceDb::TUpdate<Schema::IntervalTops::Value>(value), + NIceDb::TUpdate<Schema::IntervalTops::NodeId>(nodeId)); + + seenHashes.insert(queryHash); + ++queryIndex; + }; + + while (result.size() < TOP_QUERIES_COUNT) { + if (topIt == top.end()) { + if (queryIndex == queries.HashesSize()) { + break; + } + auto queryHash = queries.GetHashes(queryIndex); + if (seenHashes.find(queryHash) != seenHashes.end()) { + ++queryIndex; + continue; + } + copyNewQuery(); + } else { + auto topHash = topIt->Hash; + if (seenHashes.find(topHash) != seenHashes.end()) { + ++topIt; + continue; + } + if (queryIndex == queries.HashesSize()) { + result.emplace_back(std::move(*topIt++)); + seenHashes.insert(topHash); + continue; + } + auto queryHash = queries.GetHashes(queryIndex); + if (seenHashes.find(queryHash) != seenHashes.end()) { + ++queryIndex; + continue; + } + if (topIt->Value >= queries.GetValues(queryIndex)) { + result.emplace_back(std::move(*topIt++)); + seenHashes.insert(topHash); + } else { + copyNewQuery(); + } + } + } + + for (; topIt != top.end(); ++topIt) { + auto topHash = topIt->Hash; + if (seenHashes.find(topHash) != seenHashes.end()) { + continue; + } + db.Table<Schema::IntervalTops>().Key((ui32)statsType, topHash).Delete(); + } + + top.swap(result); + } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + auto nodeId = Record.GetNodeId(); + + if (Record.GetMetrics().ValuesSize() != Record.GetMetrics().HashesSize() || + Record.GetTopByDuration().ValuesSize() != Record.GetTopByDuration().HashesSize() || + Record.GetTopByReadBytes().ValuesSize() != Record.GetTopByReadBytes().HashesSize() || + Record.GetTopByCpuTime().ValuesSize() != Record.GetTopByCpuTime().HashesSize() || + Record.GetTopByRequestUnits().ValuesSize() != Record.GetTopByRequestUnits().HashesSize()) + { + SVLOG_W("[" << Self->TabletID() << "] TTxIntervalSummary::Execute, malformed summary: " + << "node id# " << nodeId); + return true; + } + + if (Self->SummaryNodes.find(nodeId) != Self->SummaryNodes.end()) { + SVLOG_W("[" << Self->TabletID() << "] TTxIntervalSummary::Execute, duplicate summary: " + << "node id# " << nodeId); + return true; + } + Self->SummaryNodes.insert(nodeId); + + const auto& metrics = Record.GetMetrics(); + auto count = metrics.HashesSize(); + + SVLOG_D("[" << Self->TabletID() << "] TTxIntervalSummary::Execute: " + << "node id# " << nodeId + << ", query count# " << count); + + NIceDb::TNiceDb db(txc.DB); + for (size_t i = 0; i < count; ++i) { + AddSummary(db, metrics.GetHashes(i), metrics.GetValues(i), nodeId); + } + + ProcessTop(db, nodeId, NKikimrSysView::TOP_DURATION_ONE_MINUTE, + Record.GetTopByDuration(), Self->ByDurationMinute); + ProcessTop(db, nodeId, NKikimrSysView::TOP_DURATION_ONE_HOUR, + Record.GetTopByDuration(), Self->ByDurationHour); + ProcessTop(db, nodeId, NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, + Record.GetTopByReadBytes(), Self->ByReadBytesMinute); + ProcessTop(db, nodeId, NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, + Record.GetTopByReadBytes(), Self->ByReadBytesHour); + ProcessTop(db, nodeId, NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, + Record.GetTopByCpuTime(), Self->ByCpuTimeMinute); + ProcessTop(db, nodeId, NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, + Record.GetTopByCpuTime(), Self->ByCpuTimeHour); + ProcessTop(db, nodeId, NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, + Record.GetTopByRequestUnits(), Self->ByRequestUnitsMinute); + ProcessTop(db, nodeId, NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, + Record.GetTopByRequestUnits(), Self->ByRequestUnitsHour); + + return true; + } + + void Complete(const TActorContext&) override { + SVLOG_D("[" << Self->TabletID() << "] TTxIntervalSummary::Complete"); + } +}; + +void TSysViewProcessor::Handle(TEvSysView::TEvIntervalQuerySummary::TPtr& ev) { + auto& record = ev->Get()->Record; + auto nodeId = record.GetNodeId(); + + if (CurrentStage != COLLECT) { + SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, wrong stage: " + << "node id# " << nodeId); + return; + } + + if (record.GetIntervalEndUs() != IntervalEnd.MicroSeconds()) { + SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, time mismath: " + << "node id# " << nodeId + << ", interval end# " << IntervalEnd + << ", event interval end# " << TInstant::MicroSeconds(record.GetIntervalEndUs())); + return; + } + + if (record.GetDatabase() != Database) { + SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, db mismatch: " + << "node id# " << nodeId + << ", database# " << Database + << ", event database# " << record.GetDatabase()); + return; + } + + Execute(new TTxIntervalSummary(this, std::move(record)), TActivationContext::AsActorContext()); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/processor/ya.make b/ydb/core/sys_view/processor/ya.make index 16bda2c8be..ddf4664cfa 100644 --- a/ydb/core/sys_view/processor/ya.make +++ b/ydb/core/sys_view/processor/ya.make @@ -1,28 +1,28 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - processor.h - processor.cpp - processor_impl.h - processor_impl.cpp - schema.h - schema.cpp - db_counters.cpp - tx_init.cpp - tx_init_schema.cpp - tx_configure.cpp - tx_collect.cpp - tx_aggregate.cpp - tx_interval_summary.cpp - tx_interval_metrics.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + processor.h + processor.cpp + processor_impl.h + processor_impl.cpp + schema.h + schema.cpp + db_counters.cpp + tx_init.cpp + tx_init_schema.cpp + tx_configure.cpp + tx_collect.cpp + tx_aggregate.cpp + tx_interval_summary.cpp + tx_interval_metrics.cpp +) + +PEERDIR( ydb/core/base ydb/core/engine/minikql ydb/core/grpc_services/counters @@ -31,8 +31,8 @@ PEERDIR( ydb/core/tablet ydb/core/tablet_flat ydb/core/tx/scheme_cache -) - +) + YQL_LAST_ABI_VERSION() - -END() + +END() diff --git a/ydb/core/sys_view/query_stats/query_metrics.cpp b/ydb/core/sys_view/query_stats/query_metrics.cpp index 79af63e46f..5f2acb5c23 100644 --- a/ydb/core/sys_view/query_stats/query_metrics.cpp +++ b/ydb/core/sys_view/query_stats/query_metrics.cpp @@ -1,173 +1,173 @@ -#include "query_metrics.h" - +#include "query_metrics.h" + #include <ydb/core/sys_view/common/common.h> #include <ydb/core/sys_view/common/events.h> #include <ydb/core/sys_view/common/keys.h> #include <ydb/core/sys_view/common/schema.h> #include <ydb/core/sys_view/common/scan_actor_base_impl.h> - + #include <ydb/core/base/tablet_pipecache.h> - -#include <library/cpp/actors/core/hfunc.h> - -namespace NKikimr { -namespace NSysView { - -using namespace NActors; - -template <> -void SetField<0>(NKikimrSysView::TQueryMetricsKey& key, ui64 value) { - key.SetIntervalEndUs(value); -} - -template <> -void SetField<1>(NKikimrSysView::TQueryMetricsKey& key, ui32 value) { - key.SetRank(value); -} - -class TQueryMetricsScan : public TScanActorBase<TQueryMetricsScan> { -public: - using TBase = TScanActorBase<TQueryMetricsScan>; - + +#include <library/cpp/actors/core/hfunc.h> + +namespace NKikimr { +namespace NSysView { + +using namespace NActors; + +template <> +void SetField<0>(NKikimrSysView::TQueryMetricsKey& key, ui64 value) { + key.SetIntervalEndUs(value); +} + +template <> +void SetField<1>(NKikimrSysView::TQueryMetricsKey& key, ui32 value) { + key.SetRank(value); +} + +class TQueryMetricsScan : public TScanActorBase<TQueryMetricsScan> { +public: + using TBase = TScanActorBase<TQueryMetricsScan>; + static constexpr auto ActorActivityType() { - return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; - } - + return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; + } + TQueryMetricsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) - : TBase(ownerId, scanId, tableId, tableRange, columns) - { - ConvertKeyRange<NKikimrSysView::TEvGetQueryMetricsRequest, ui64, ui32>(Request, TableRange); - Request.SetType(NKikimrSysView::METRICS_ONE_MINUTE); - } - - STFUNC(StateScan) { - switch (ev->GetTypeRewrite()) { - hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); - hFunc(TEvSysView::TEvGetQueryMetricsResponse, Handle); - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); - hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution); - cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); - cFunc(TEvents::TEvPoison::EventType, PassAway); - default: - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "NSysView::TQueryMetricsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - -private: - void ProceedToScan() override { - Become(&TThis::StateScan); - if (AckReceived) { - RequestBatch(); - } - } - - void RequestBatch() { - if (!SysViewProcessorId) { - SVLOG_W("No sysview processor for database " << TenantName - << ", sending empty response"); - ReplyEmptyAndDie(); - return; - } - - if (BatchRequestInFlight) { - return; - } - - auto request = MakeHolder<TEvSysView::TEvGetQueryMetricsRequest>(); - request->Record.CopyFrom(Request); - - Send(MakePipePeNodeCacheID(false), - new TEvPipeCache::TEvForward(request.Release(), SysViewProcessorId, true), - IEventHandle::FlagTrackDelivery); - - BatchRequestInFlight = true; - } - - void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { - RequestBatch(); - } - - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { - ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in query metrics scan"); - } - - void Handle(TEvSysView::TEvGetQueryMetricsResponse::TPtr& ev) { - using TEntry = NKikimrSysView::TQueryMetricsEntry; - using TExtractor = std::function<TCell(const TEntry&)>; - using TSchema = Schema::QueryMetrics; - - struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { - TExtractorsMap() { - insert({TSchema::IntervalEnd::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetKey().GetIntervalEndUs()); - }}); - insert({TSchema::Rank::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui32>(entry.GetKey().GetRank()); - }}); - insert({TSchema::QueryText::ColumnId, [] (const TEntry& entry) { - const auto& text = entry.GetQueryText(); - return TCell(text.data(), text.size()); - }}); - insert({TSchema::Count::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetMetrics().GetCount()); - }}); - -#define ADD_METRICS(FieldName, MetricsName) \ - insert({TSchema::Sum ## FieldName::ColumnId, [] (const TEntry& entry) { \ - return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetSum()); \ - }}); \ - insert({TSchema::Min ## FieldName::ColumnId, [] (const TEntry& entry) { \ - return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetMin()); \ - }}); \ - insert({TSchema::Max ## FieldName::ColumnId, [] (const TEntry& entry) { \ - return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetMax()); \ - }}); - - ADD_METRICS(CPUTime, CpuTimeUs); - ADD_METRICS(Duration, DurationUs); - ADD_METRICS(ReadRows, ReadRows); - ADD_METRICS(ReadBytes, ReadBytes); - ADD_METRICS(UpdateRows, UpdateRows); - ADD_METRICS(UpdateBytes, UpdateBytes); - ADD_METRICS(DeleteRows, DeleteRows); - ADD_METRICS(RequestUnits, RequestUnits); -#undef ADD_METRICS - } - }; - - const auto& record = ev->Get()->Record; - if (record.HasOverloaded() && record.GetOverloaded()) { - ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "SysViewProcessor is overloaded"); - return; - } - - ReplyBatch<TEvSysView::TEvGetQueryMetricsResponse, TEntry, TExtractorsMap, true>(ev); - - if (!record.GetLastBatch()) { - Y_VERIFY(record.HasNext()); - Request.MutableFrom()->CopyFrom(record.GetNext()); - Request.SetInclusiveFrom(true); - } - - BatchRequestInFlight = false; - } - - void PassAway() override { - Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); - TBase::PassAway(); - } - -private: - NKikimrSysView::TEvGetQueryMetricsRequest Request; -}; - + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) + : TBase(ownerId, scanId, tableId, tableRange, columns) + { + ConvertKeyRange<NKikimrSysView::TEvGetQueryMetricsRequest, ui64, ui32>(Request, TableRange); + Request.SetType(NKikimrSysView::METRICS_ONE_MINUTE); + } + + STFUNC(StateScan) { + switch (ev->GetTypeRewrite()) { + hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); + hFunc(TEvSysView::TEvGetQueryMetricsResponse, Handle); + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution); + cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); + cFunc(TEvents::TEvPoison::EventType, PassAway); + default: + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "NSysView::TQueryMetricsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + +private: + void ProceedToScan() override { + Become(&TThis::StateScan); + if (AckReceived) { + RequestBatch(); + } + } + + void RequestBatch() { + if (!SysViewProcessorId) { + SVLOG_W("No sysview processor for database " << TenantName + << ", sending empty response"); + ReplyEmptyAndDie(); + return; + } + + if (BatchRequestInFlight) { + return; + } + + auto request = MakeHolder<TEvSysView::TEvGetQueryMetricsRequest>(); + request->Record.CopyFrom(Request); + + Send(MakePipePeNodeCacheID(false), + new TEvPipeCache::TEvForward(request.Release(), SysViewProcessorId, true), + IEventHandle::FlagTrackDelivery); + + BatchRequestInFlight = true; + } + + void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { + RequestBatch(); + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { + ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in query metrics scan"); + } + + void Handle(TEvSysView::TEvGetQueryMetricsResponse::TPtr& ev) { + using TEntry = NKikimrSysView::TQueryMetricsEntry; + using TExtractor = std::function<TCell(const TEntry&)>; + using TSchema = Schema::QueryMetrics; + + struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { + TExtractorsMap() { + insert({TSchema::IntervalEnd::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetKey().GetIntervalEndUs()); + }}); + insert({TSchema::Rank::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui32>(entry.GetKey().GetRank()); + }}); + insert({TSchema::QueryText::ColumnId, [] (const TEntry& entry) { + const auto& text = entry.GetQueryText(); + return TCell(text.data(), text.size()); + }}); + insert({TSchema::Count::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetMetrics().GetCount()); + }}); + +#define ADD_METRICS(FieldName, MetricsName) \ + insert({TSchema::Sum ## FieldName::ColumnId, [] (const TEntry& entry) { \ + return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetSum()); \ + }}); \ + insert({TSchema::Min ## FieldName::ColumnId, [] (const TEntry& entry) { \ + return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetMin()); \ + }}); \ + insert({TSchema::Max ## FieldName::ColumnId, [] (const TEntry& entry) { \ + return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetMax()); \ + }}); + + ADD_METRICS(CPUTime, CpuTimeUs); + ADD_METRICS(Duration, DurationUs); + ADD_METRICS(ReadRows, ReadRows); + ADD_METRICS(ReadBytes, ReadBytes); + ADD_METRICS(UpdateRows, UpdateRows); + ADD_METRICS(UpdateBytes, UpdateBytes); + ADD_METRICS(DeleteRows, DeleteRows); + ADD_METRICS(RequestUnits, RequestUnits); +#undef ADD_METRICS + } + }; + + const auto& record = ev->Get()->Record; + if (record.HasOverloaded() && record.GetOverloaded()) { + ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "SysViewProcessor is overloaded"); + return; + } + + ReplyBatch<TEvSysView::TEvGetQueryMetricsResponse, TEntry, TExtractorsMap, true>(ev); + + if (!record.GetLastBatch()) { + Y_VERIFY(record.HasNext()); + Request.MutableFrom()->CopyFrom(record.GetNext()); + Request.SetInclusiveFrom(true); + } + + BatchRequestInFlight = false; + } + + void PassAway() override { + Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + TBase::PassAway(); + } + +private: + NKikimrSysView::TEvGetQueryMetricsRequest Request; +}; + THolder<IActor> CreateQueryMetricsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ return MakeHolder<TQueryMetricsScan>(ownerId, scanId, tableId, tableRange, columns); -} - -} // NSysView -} // NKikimr +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/query_stats/query_metrics.h b/ydb/core/sys_view/query_stats/query_metrics.h index 7e3997ab0f..e374081126 100644 --- a/ydb/core/sys_view/query_stats/query_metrics.h +++ b/ydb/core/sys_view/query_stats/query_metrics.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + THolder<IActor> CreateQueryMetricsScan(const TActorId& ownerId, ui32 scanId, - const TTableId& tableId, const TTableRange& tableRange, - const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableId& tableId, const TTableRange& tableRange, + const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/query_stats/query_stats.cpp b/ydb/core/sys_view/query_stats/query_stats.cpp index b608b7545d..918c9b781e 100644 --- a/ydb/core/sys_view/query_stats/query_stats.cpp +++ b/ydb/core/sys_view/query_stats/query_stats.cpp @@ -1,5 +1,5 @@ -#include "query_stats.h" - +#include "query_stats.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/sys_view/common/common.h> #include <ydb/core/sys_view/common/events.h> @@ -8,548 +8,548 @@ #include <ydb/core/sys_view/common/scan_actor_base_impl.h> #include <ydb/core/sys_view/service/query_history.h> #include <ydb/core/sys_view/service/sysview_service.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/log.h> - -namespace NKikimr { -namespace NSysView { - -using namespace NActors; - -TQueryStatsBucketRange::TQueryStatsBucketRange(const TSerializedTableRange& range, const TDuration& bucketSize) { - const ui64 bucketSizeUs = bucketSize.MicroSeconds(); - - const auto& cellsFrom = range.From.GetCells(); - const auto& cellsTo = range.To.GetCells(); - - if (cellsFrom.size() > 0 && !cellsFrom[0].IsNull()) { - auto fromUs = cellsFrom[0].AsValue<ui64>(); - FromBucket = fromUs / bucketSizeUs; - bool exactBound = (FromBucket * bucketSizeUs == fromUs); - - TMaybe<ui32> rank; - if (cellsFrom.size() > 1 && !cellsFrom[1].IsNull()) { - rank = cellsFrom[1].AsValue<ui32>(); - if (!range.FromInclusive && *rank < std::numeric_limits<ui32>::max()) { - ++*rank; - } - } - - if (FromBucket) { - if (exactBound && (range.FromInclusive || rank)) { - --FromBucket; - if (rank) { - FromRank = rank; - } - } - } - } - - if (cellsTo.size() > 0 && !cellsTo[0].IsNull()) { - auto toUs = cellsTo[0].template AsValue<ui64>(); - ToBucket = toUs / bucketSizeUs; - bool exactBound = (ToBucket * bucketSizeUs == toUs); - - TMaybe<ui32> rank; - if (cellsTo.size() > 1 && !cellsTo[1].IsNull()) { - rank = cellsTo[1].template AsValue<ui32>(); - if (!range.ToInclusive && *rank) { - --*rank; - } - } - - if (ToBucket) { - --ToBucket; - if (exactBound) { - if (rank) { - ToRank = rank; - } else if (!range.ToInclusive) { - if (!ToBucket) { - IsEmpty = true; - } else { - --ToBucket; - } - } - } - } else { - IsEmpty = true; - } - } - - if (FromBucket > ToBucket) { - IsEmpty = true; - } -} - -template <typename TGreater> -class TQueryStatsScan : public TScanActorBase<TQueryStatsScan<TGreater>> { -public: - using TBase = TScanActorBase<TQueryStatsScan<TGreater>>; - + +namespace NKikimr { +namespace NSysView { + +using namespace NActors; + +TQueryStatsBucketRange::TQueryStatsBucketRange(const TSerializedTableRange& range, const TDuration& bucketSize) { + const ui64 bucketSizeUs = bucketSize.MicroSeconds(); + + const auto& cellsFrom = range.From.GetCells(); + const auto& cellsTo = range.To.GetCells(); + + if (cellsFrom.size() > 0 && !cellsFrom[0].IsNull()) { + auto fromUs = cellsFrom[0].AsValue<ui64>(); + FromBucket = fromUs / bucketSizeUs; + bool exactBound = (FromBucket * bucketSizeUs == fromUs); + + TMaybe<ui32> rank; + if (cellsFrom.size() > 1 && !cellsFrom[1].IsNull()) { + rank = cellsFrom[1].AsValue<ui32>(); + if (!range.FromInclusive && *rank < std::numeric_limits<ui32>::max()) { + ++*rank; + } + } + + if (FromBucket) { + if (exactBound && (range.FromInclusive || rank)) { + --FromBucket; + if (rank) { + FromRank = rank; + } + } + } + } + + if (cellsTo.size() > 0 && !cellsTo[0].IsNull()) { + auto toUs = cellsTo[0].template AsValue<ui64>(); + ToBucket = toUs / bucketSizeUs; + bool exactBound = (ToBucket * bucketSizeUs == toUs); + + TMaybe<ui32> rank; + if (cellsTo.size() > 1 && !cellsTo[1].IsNull()) { + rank = cellsTo[1].template AsValue<ui32>(); + if (!range.ToInclusive && *rank) { + --*rank; + } + } + + if (ToBucket) { + --ToBucket; + if (exactBound) { + if (rank) { + ToRank = rank; + } else if (!range.ToInclusive) { + if (!ToBucket) { + IsEmpty = true; + } else { + --ToBucket; + } + } + } + } else { + IsEmpty = true; + } + } + + if (FromBucket > ToBucket) { + IsEmpty = true; + } +} + +template <typename TGreater> +class TQueryStatsScan : public TScanActorBase<TQueryStatsScan<TGreater>> { +public: + using TBase = TScanActorBase<TQueryStatsScan<TGreater>>; + static constexpr auto ActorActivityType() { - return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; - } - + return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; + } + TQueryStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns, - NKikimrSysView::EStatsType statsType, - ui64 bucketCount, const TDuration& bucketSize) - : TBase(ownerId, scanId, tableId, tableRange, columns) - , StatsType(statsType) - , BucketRange(this->TableRange, bucketSize) - { + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns, + NKikimrSysView::EStatsType statsType, + ui64 bucketCount, const TDuration& bucketSize) + : TBase(ownerId, scanId, tableId, tableRange, columns) + , StatsType(statsType) + , BucketRange(this->TableRange, bucketSize) + { auto now = TAppData::TimeProvider->Now(); - History = MakeHolder<TScanQueryHistory<TGreater>>(bucketCount, bucketSize, now); - - ConvertKeyRange<NKikimrSysView::TEvGetQueryMetricsRequest, ui64, ui32>(Request, this->TableRange); - Request.SetType(statsType); - } - - STFUNC(StateScan) { - switch (ev->GetTypeRewrite()) { - hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); - hFunc(TEvSysView::TEvGetQueryStatsResult, Handle); - hFunc(TEvents::TEvUndelivered, Undelivered); - hFunc(TEvInterconnect::TEvNodeConnected, Connected); - hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected); - hFunc(TEvPrivate::TEvRetryNode, RetryNode); - hFunc(TEvSysView::TEvGetQueryStatsResponse, Handle); - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + History = MakeHolder<TScanQueryHistory<TGreater>>(bucketCount, bucketSize, now); + + ConvertKeyRange<NKikimrSysView::TEvGetQueryMetricsRequest, ui64, ui32>(Request, this->TableRange); + Request.SetType(statsType); + } + + STFUNC(StateScan) { + switch (ev->GetTypeRewrite()) { + hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); + hFunc(TEvSysView::TEvGetQueryStatsResult, Handle); + hFunc(TEvents::TEvUndelivered, Undelivered); + hFunc(TEvInterconnect::TEvNodeConnected, Connected); + hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected); + hFunc(TEvPrivate::TEvRetryNode, RetryNode); + hFunc(TEvSysView::TEvGetQueryStatsResponse, Handle); + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); hFunc(NKqp::TEvKqp::TEvAbortExecution, TBase::HandleAbortExecution); - cFunc(TEvents::TEvWakeup::EventType, TBase::HandleTimeout); - cFunc(TEvents::TEvPoison::EventType, PassAway); - default: - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "NSysView::TQueryStatsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - -private: - struct TEvPrivate { - enum EEv { - EvRetryNode = EventSpaceBegin(TEvents::ES_PRIVATE), - - EvEnd - }; - - struct TEvRetryNode : public TEventLocal<TEvRetryNode, EvRetryNode> { - ui32 NodeId; - - explicit TEvRetryNode(ui32 nodeId) - : NodeId(nodeId) - {} - }; - }; - - void ProceedToScan() override { - this->Become(&TQueryStatsScan<TGreater>::StateScan); - if (this->AckReceived) { - StartScan(); - } - } - - void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { - StartScan(); - } - - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { - this->ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in query stats scan"); - } - - void RequestBatch() { - if (this->BatchRequestInFlight) { - return; - } - - this->UseProcessor = true; - - auto request = MakeHolder<TEvSysView::TEvGetQueryMetricsRequest>(); - request->Record.CopyFrom(Request); - - this->Send(MakePipePeNodeCacheID(false), - new TEvPipeCache::TEvForward(request.Release(), this->SysViewProcessorId, true), - IEventHandle::FlagTrackDelivery); - - this->BatchRequestInFlight = true; - } - - void StartScan() { + cFunc(TEvents::TEvWakeup::EventType, TBase::HandleTimeout); + cFunc(TEvents::TEvPoison::EventType, PassAway); + default: + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "NSysView::TQueryStatsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + +private: + struct TEvPrivate { + enum EEv { + EvRetryNode = EventSpaceBegin(TEvents::ES_PRIVATE), + + EvEnd + }; + + struct TEvRetryNode : public TEventLocal<TEvRetryNode, EvRetryNode> { + ui32 NodeId; + + explicit TEvRetryNode(ui32 nodeId) + : NodeId(nodeId) + {} + }; + }; + + void ProceedToScan() override { + this->Become(&TQueryStatsScan<TGreater>::StateScan); + if (this->AckReceived) { + StartScan(); + } + } + + void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { + StartScan(); + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { + this->ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in query stats scan"); + } + + void RequestBatch() { + if (this->BatchRequestInFlight) { + return; + } + + this->UseProcessor = true; + + auto request = MakeHolder<TEvSysView::TEvGetQueryMetricsRequest>(); + request->Record.CopyFrom(Request); + + this->Send(MakePipePeNodeCacheID(false), + new TEvPipeCache::TEvForward(request.Release(), this->SysViewProcessorId, true), + IEventHandle::FlagTrackDelivery); + + this->BatchRequestInFlight = true; + } + + void StartScan() { if (AppData()->FeatureFlags.GetEnablePersistentQueryStats() && this->SysViewProcessorId) { - RequestBatch(); - } else { - StartOldScan(); - } - } - - void StartOldScan() { - if (BucketRange.IsEmpty || this->TenantNodes.empty()) { - this->ReplyEmptyAndDie(); - return; - } - - NodesToRequest.reserve(this->TenantNodes.size()); - for (const auto& nodeId : this->TenantNodes) { - Nodes[nodeId] = TRetryState{false, 0, StartRetryInterval}; - NodesToRequest.push_back(nodeId); - } - - SendRequests(); - } - - void ScheduleRetry(ui32 nodeId) { - if (NodesReplied.find(nodeId) != NodesReplied.end()) { - return; - } - - auto& retryState = Nodes[nodeId]; - if (++retryState.Attempt == MaxRetryAttempts) { - NodesReplied.insert(nodeId); // TODO: warnings (answer is partial) - } else { - this->Schedule(retryState.Interval, new typename TEvPrivate::TEvRetryNode(nodeId)); - retryState.Interval = TDuration::MilliSeconds((ui64)(retryState.Interval.MilliSeconds() * RetryMultiplier)); - } - - SendRequests(); - } - - void Handle(TEvSysView::TEvGetQueryStatsResult::TPtr& ev) { - ui32 nodeId = ev.Get()->Cookie; - - History->Merge(std::move(ev->Get()->Record)); - NodesReplied.insert(nodeId); - - auto& retryState = Nodes[nodeId]; - if (retryState.InFly) { - --RequestsInFly; - retryState.InFly = false; - } - - SendRequests(); - } - - void RetryNode(typename TEvPrivate::TEvRetryNode::TPtr& ev) { - auto nodeId = ev->Get()->NodeId; - NodesToRequest.push_back(nodeId); - SendRequests(); - } - - void Undelivered(TEvents::TEvUndelivered::TPtr& ev) { - ui32 nodeId = ev.Get()->Cookie; - auto& retryState = Nodes[nodeId]; - - if (retryState.InFly) { - retryState.InFly = false; - --RequestsInFly; - ScheduleRetry(nodeId); - } - } - - void Connected(TEvInterconnect::TEvNodeConnected::TPtr&) { - } - - void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { - ui32 nodeId = ev->Get()->NodeId; - auto& retryState = Nodes[nodeId]; - - if (retryState.InFly) { - --RequestsInFly; - retryState.InFly = false; - ScheduleRetry(nodeId); - } - } - - void SendRequests() { - while (!NodesToRequest.empty() && RequestsInFly < MaxRequestsInFly) { - auto nodeId = NodesToRequest.back(); - NodesToRequest.pop_back(); - - auto sysViewServiceID = MakeSysViewServiceID(nodeId); - - auto request = MakeHolder<TEvSysView::TEvGetQueryStats>(); - request->Record.SetStatsType(StatsType); - request->Record.SetStartBucket(std::max(History->GetStartBucket(), BucketRange.FromBucket)); - request->Record.SetTenantName(this->TenantName); - - this->Send(sysViewServiceID, std::move(request), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); - - Nodes[nodeId].InFly = true; - ++RequestsInFly; - } - - if (NodesReplied.size() == Nodes.size()) { - ReplyAndDie(); - } - } - - using TEntry = NKikimrSysView::TQueryStatsEntry; - using TExtractor = std::function<TCell(const TEntry&)>; - using TSchema = Schema::QueryStats; - - struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { - TExtractorsMap() { - insert({TSchema::IntervalEnd::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetKey().GetIntervalEndUs()); - }}); - insert({TSchema::Rank::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui32>(entry.GetKey().GetRank()); - }}); - insert({TSchema::QueryText::ColumnId, [] (const TEntry& entry) { - const auto& text = entry.GetStats().GetQueryText(); - return TCell(text.data(), text.size()); - }}); - insert({TSchema::Duration::ColumnId, [] (const TEntry& entry) { - return TCell::Make<i64>(entry.GetStats().GetDurationMs() * 1000); - }}); - insert({TSchema::EndTime::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetEndTimeMs() * 1000); - }}); - insert({TSchema::ReadRows::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetStats().GetReadRows()); - }}); - insert({TSchema::ReadBytes::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetStats().GetReadBytes()); - }}); - insert({TSchema::UpdateRows::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetStats().GetUpdateRows()); - }}); - insert({TSchema::UpdateBytes::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetStats().GetUpdateBytes()); - }}); - insert({TSchema::DeleteRows::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetStats().GetDeleteRows()); - }}); - insert({TSchema::DeleteBytes::ColumnId, [] (const TEntry&) { - return TCell::Make<ui64>(0); // deprecated - }}); - insert({TSchema::Partitions::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetStats().GetPartitionCount()); - }}); - insert({TSchema::UserSID::ColumnId, [] (const TEntry& entry) { - const auto& stats = entry.GetStats(); - if (!stats.HasUserSID()) { - return TCell(); - } - const auto& sid = stats.GetUserSID(); - return TCell(sid.data(), sid.size()); - }}); - insert({TSchema::ParametersSize::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetParametersSize()); - }}); - insert({TSchema::CompileDuration::ColumnId, [] (const TEntry& entry) { - const auto& stats = entry.GetStats(); - return stats.HasCompileDurationMs() ? TCell::Make<ui64>(stats.GetCompileDurationMs() * 1000) : TCell(); - }}); - insert({TSchema::FromQueryCache::ColumnId, [] (const TEntry& entry) { - const auto& stats = entry.GetStats(); - return stats.HasFromQueryCache() ? TCell::Make<bool>(stats.GetFromQueryCache()) : TCell(); - }}); - insert({TSchema::CPUTime::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetTotalCpuTimeUs()); - }}); - insert({TSchema::ShardCount::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetCnt()); - }}); - insert({TSchema::SumShardCPUTime::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetSum()); - }}); - insert({TSchema::MinShardCPUTime::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetMin()); - }}); - insert({TSchema::MaxShardCPUTime::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetMax()); - }}); - insert({TSchema::ComputeNodesCount::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetCnt()); - }}); - insert({TSchema::SumComputeCPUTime::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetSum()); - }}); - insert({TSchema::MinComputeCPUTime::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetMin()); - }}); - insert({TSchema::MaxComputeCPUTime::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetMax()); - }}); - insert({TSchema::CompileCPUTime::ColumnId, [] (const TEntry& entry) { - const auto& stats = entry.GetStats(); - return stats.HasCompileCpuTimeUs() ? TCell::Make<ui64>(stats.GetCompileCpuTimeUs()) : TCell(); - }}); - insert({TSchema::ProcessCPUTime::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetProcessCpuTimeUs()); - }}); - insert({TSchema::TypeCol::ColumnId, [] (const TEntry& entry) { - const auto& type = entry.GetStats().GetType(); - return TCell(type.data(), type.size()); - }}); - insert({TSchema::RequestUnits::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetStats().GetRequestUnits()); - }}); - } - }; - - void Handle(TEvSysView::TEvGetQueryStatsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - if (record.HasOverloaded() && record.GetOverloaded()) { - this->ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "SysViewProcessor is overloaded"); - return; - } - - this->template ReplyBatch<TEvSysView::TEvGetQueryStatsResponse, TEntry, TExtractorsMap, true>(ev); - - if (!record.GetLastBatch()) { - Y_VERIFY(record.HasNext()); - Request.MutableFrom()->CopyFrom(record.GetNext()); - Request.SetInclusiveFrom(true); - } - - this->BatchRequestInFlight = false; - } - - void ReplyAndDie() { - static TExtractorsMap extractors; - - auto fromBucket = std::max(History->GetStartBucket(), BucketRange.FromBucket); - auto toBucket = std::min(History->GetLastBucket(), BucketRange.ToBucket); - - if (fromBucket > toBucket) { - this->ReplyEmptyAndDie(); - } - - auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(this->ScanId); - batch->Finished = true; - - TEntry entry; - auto& key = *entry.MutableKey(); - key.SetIntervalEndUs((fromBucket + 1) * History->GetBucketSizeMs() * 1000); - - for (ui64 bucket = fromBucket; bucket <= toBucket; ++bucket) { - const auto& top = History->GetBucketTop(bucket); - - ui32 fromRank = 1; - if (BucketRange.FromRank && bucket == BucketRange.FromBucket) { - fromRank = std::max(fromRank, *BucketRange.FromRank); - } - - ui32 toRank = top.StatsSize(); - if (BucketRange.ToRank && bucket == BucketRange.ToBucket) { - toRank = std::min(toRank, *BucketRange.ToRank); - } - - TVector<TCell> cells; - for (ui32 r = fromRank; r <= toRank; ++r) { - key.SetRank(r); - entry.MutableStats()->CopyFrom(top.GetStats(r - 1)); - - for (auto column : this->Columns) { - auto extractor = extractors.find(column.Tag); - if (extractor == extractors.end()) { - cells.push_back(TCell()); - } else { - cells.push_back(extractor->second(entry)); - } - } - - TArrayRef<const TCell> ref(cells); + RequestBatch(); + } else { + StartOldScan(); + } + } + + void StartOldScan() { + if (BucketRange.IsEmpty || this->TenantNodes.empty()) { + this->ReplyEmptyAndDie(); + return; + } + + NodesToRequest.reserve(this->TenantNodes.size()); + for (const auto& nodeId : this->TenantNodes) { + Nodes[nodeId] = TRetryState{false, 0, StartRetryInterval}; + NodesToRequest.push_back(nodeId); + } + + SendRequests(); + } + + void ScheduleRetry(ui32 nodeId) { + if (NodesReplied.find(nodeId) != NodesReplied.end()) { + return; + } + + auto& retryState = Nodes[nodeId]; + if (++retryState.Attempt == MaxRetryAttempts) { + NodesReplied.insert(nodeId); // TODO: warnings (answer is partial) + } else { + this->Schedule(retryState.Interval, new typename TEvPrivate::TEvRetryNode(nodeId)); + retryState.Interval = TDuration::MilliSeconds((ui64)(retryState.Interval.MilliSeconds() * RetryMultiplier)); + } + + SendRequests(); + } + + void Handle(TEvSysView::TEvGetQueryStatsResult::TPtr& ev) { + ui32 nodeId = ev.Get()->Cookie; + + History->Merge(std::move(ev->Get()->Record)); + NodesReplied.insert(nodeId); + + auto& retryState = Nodes[nodeId]; + if (retryState.InFly) { + --RequestsInFly; + retryState.InFly = false; + } + + SendRequests(); + } + + void RetryNode(typename TEvPrivate::TEvRetryNode::TPtr& ev) { + auto nodeId = ev->Get()->NodeId; + NodesToRequest.push_back(nodeId); + SendRequests(); + } + + void Undelivered(TEvents::TEvUndelivered::TPtr& ev) { + ui32 nodeId = ev.Get()->Cookie; + auto& retryState = Nodes[nodeId]; + + if (retryState.InFly) { + retryState.InFly = false; + --RequestsInFly; + ScheduleRetry(nodeId); + } + } + + void Connected(TEvInterconnect::TEvNodeConnected::TPtr&) { + } + + void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { + ui32 nodeId = ev->Get()->NodeId; + auto& retryState = Nodes[nodeId]; + + if (retryState.InFly) { + --RequestsInFly; + retryState.InFly = false; + ScheduleRetry(nodeId); + } + } + + void SendRequests() { + while (!NodesToRequest.empty() && RequestsInFly < MaxRequestsInFly) { + auto nodeId = NodesToRequest.back(); + NodesToRequest.pop_back(); + + auto sysViewServiceID = MakeSysViewServiceID(nodeId); + + auto request = MakeHolder<TEvSysView::TEvGetQueryStats>(); + request->Record.SetStatsType(StatsType); + request->Record.SetStartBucket(std::max(History->GetStartBucket(), BucketRange.FromBucket)); + request->Record.SetTenantName(this->TenantName); + + this->Send(sysViewServiceID, std::move(request), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); + + Nodes[nodeId].InFly = true; + ++RequestsInFly; + } + + if (NodesReplied.size() == Nodes.size()) { + ReplyAndDie(); + } + } + + using TEntry = NKikimrSysView::TQueryStatsEntry; + using TExtractor = std::function<TCell(const TEntry&)>; + using TSchema = Schema::QueryStats; + + struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { + TExtractorsMap() { + insert({TSchema::IntervalEnd::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetKey().GetIntervalEndUs()); + }}); + insert({TSchema::Rank::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui32>(entry.GetKey().GetRank()); + }}); + insert({TSchema::QueryText::ColumnId, [] (const TEntry& entry) { + const auto& text = entry.GetStats().GetQueryText(); + return TCell(text.data(), text.size()); + }}); + insert({TSchema::Duration::ColumnId, [] (const TEntry& entry) { + return TCell::Make<i64>(entry.GetStats().GetDurationMs() * 1000); + }}); + insert({TSchema::EndTime::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetEndTimeMs() * 1000); + }}); + insert({TSchema::ReadRows::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetStats().GetReadRows()); + }}); + insert({TSchema::ReadBytes::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetStats().GetReadBytes()); + }}); + insert({TSchema::UpdateRows::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetStats().GetUpdateRows()); + }}); + insert({TSchema::UpdateBytes::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetStats().GetUpdateBytes()); + }}); + insert({TSchema::DeleteRows::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetStats().GetDeleteRows()); + }}); + insert({TSchema::DeleteBytes::ColumnId, [] (const TEntry&) { + return TCell::Make<ui64>(0); // deprecated + }}); + insert({TSchema::Partitions::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetStats().GetPartitionCount()); + }}); + insert({TSchema::UserSID::ColumnId, [] (const TEntry& entry) { + const auto& stats = entry.GetStats(); + if (!stats.HasUserSID()) { + return TCell(); + } + const auto& sid = stats.GetUserSID(); + return TCell(sid.data(), sid.size()); + }}); + insert({TSchema::ParametersSize::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetParametersSize()); + }}); + insert({TSchema::CompileDuration::ColumnId, [] (const TEntry& entry) { + const auto& stats = entry.GetStats(); + return stats.HasCompileDurationMs() ? TCell::Make<ui64>(stats.GetCompileDurationMs() * 1000) : TCell(); + }}); + insert({TSchema::FromQueryCache::ColumnId, [] (const TEntry& entry) { + const auto& stats = entry.GetStats(); + return stats.HasFromQueryCache() ? TCell::Make<bool>(stats.GetFromQueryCache()) : TCell(); + }}); + insert({TSchema::CPUTime::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetTotalCpuTimeUs()); + }}); + insert({TSchema::ShardCount::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetCnt()); + }}); + insert({TSchema::SumShardCPUTime::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetSum()); + }}); + insert({TSchema::MinShardCPUTime::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetMin()); + }}); + insert({TSchema::MaxShardCPUTime::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetMax()); + }}); + insert({TSchema::ComputeNodesCount::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetCnt()); + }}); + insert({TSchema::SumComputeCPUTime::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetSum()); + }}); + insert({TSchema::MinComputeCPUTime::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetMin()); + }}); + insert({TSchema::MaxComputeCPUTime::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetMax()); + }}); + insert({TSchema::CompileCPUTime::ColumnId, [] (const TEntry& entry) { + const auto& stats = entry.GetStats(); + return stats.HasCompileCpuTimeUs() ? TCell::Make<ui64>(stats.GetCompileCpuTimeUs()) : TCell(); + }}); + insert({TSchema::ProcessCPUTime::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetProcessCpuTimeUs()); + }}); + insert({TSchema::TypeCol::ColumnId, [] (const TEntry& entry) { + const auto& type = entry.GetStats().GetType(); + return TCell(type.data(), type.size()); + }}); + insert({TSchema::RequestUnits::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetStats().GetRequestUnits()); + }}); + } + }; + + void Handle(TEvSysView::TEvGetQueryStatsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + if (record.HasOverloaded() && record.GetOverloaded()) { + this->ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "SysViewProcessor is overloaded"); + return; + } + + this->template ReplyBatch<TEvSysView::TEvGetQueryStatsResponse, TEntry, TExtractorsMap, true>(ev); + + if (!record.GetLastBatch()) { + Y_VERIFY(record.HasNext()); + Request.MutableFrom()->CopyFrom(record.GetNext()); + Request.SetInclusiveFrom(true); + } + + this->BatchRequestInFlight = false; + } + + void ReplyAndDie() { + static TExtractorsMap extractors; + + auto fromBucket = std::max(History->GetStartBucket(), BucketRange.FromBucket); + auto toBucket = std::min(History->GetLastBucket(), BucketRange.ToBucket); + + if (fromBucket > toBucket) { + this->ReplyEmptyAndDie(); + } + + auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(this->ScanId); + batch->Finished = true; + + TEntry entry; + auto& key = *entry.MutableKey(); + key.SetIntervalEndUs((fromBucket + 1) * History->GetBucketSizeMs() * 1000); + + for (ui64 bucket = fromBucket; bucket <= toBucket; ++bucket) { + const auto& top = History->GetBucketTop(bucket); + + ui32 fromRank = 1; + if (BucketRange.FromRank && bucket == BucketRange.FromBucket) { + fromRank = std::max(fromRank, *BucketRange.FromRank); + } + + ui32 toRank = top.StatsSize(); + if (BucketRange.ToRank && bucket == BucketRange.ToBucket) { + toRank = std::min(toRank, *BucketRange.ToRank); + } + + TVector<TCell> cells; + for (ui32 r = fromRank; r <= toRank; ++r) { + key.SetRank(r); + entry.MutableStats()->CopyFrom(top.GetStats(r - 1)); + + for (auto column : this->Columns) { + auto extractor = extractors.find(column.Tag); + if (extractor == extractors.end()) { + cells.push_back(TCell()); + } else { + cells.push_back(extractor->second(entry)); + } + } + + TArrayRef<const TCell> ref(cells); batch->Rows.emplace_back(TOwnedCellVec::Make(ref)); - cells.clear(); - } - - key.SetIntervalEndUs(key.GetIntervalEndUs() + History->GetBucketSizeMs() * 1000); - } - - TBase::SendBatch(std::move(batch)); - } - - void PassAway() override { - if (UseProcessor) { - this->Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); - } else { - for (auto& [nodeId, _] : Nodes) { - this->Send(TActorContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe); - } - } - TBase::PassAway(); - } - -private: - static constexpr size_t MaxRequestsInFly = 10; - - static constexpr size_t MaxRetryAttempts = 5; - static constexpr TDuration StartRetryInterval = TDuration::MilliSeconds(500); - static constexpr double RetryMultiplier = 2.0; - - const NKikimrSysView::EStatsType StatsType; - - TQueryStatsBucketRange BucketRange; - - struct TRetryState { - bool InFly = false; - ui32 Attempt = 0; - TDuration Interval; - }; - - THashMap<ui32, TRetryState> Nodes; - THashSet<ui32> NodesReplied; - - TVector<ui32> NodesToRequest; - size_t RequestsInFly = 0; - - THolder<TScanQueryHistory<TGreater>> History; - - bool UseProcessor = false; - NKikimrSysView::TEvGetQueryMetricsRequest Request; -}; - + cells.clear(); + } + + key.SetIntervalEndUs(key.GetIntervalEndUs() + History->GetBucketSizeMs() * 1000); + } + + TBase::SendBatch(std::move(batch)); + } + + void PassAway() override { + if (UseProcessor) { + this->Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + } else { + for (auto& [nodeId, _] : Nodes) { + this->Send(TActorContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe); + } + } + TBase::PassAway(); + } + +private: + static constexpr size_t MaxRequestsInFly = 10; + + static constexpr size_t MaxRetryAttempts = 5; + static constexpr TDuration StartRetryInterval = TDuration::MilliSeconds(500); + static constexpr double RetryMultiplier = 2.0; + + const NKikimrSysView::EStatsType StatsType; + + TQueryStatsBucketRange BucketRange; + + struct TRetryState { + bool InFly = false; + ui32 Attempt = 0; + TDuration Interval; + }; + + THashMap<ui32, TRetryState> Nodes; + THashSet<ui32> NodesReplied; + + TVector<ui32> NodesToRequest; + size_t RequestsInFly = 0; + + THolder<TScanQueryHistory<TGreater>> History; + + bool UseProcessor = false; + NKikimrSysView::TEvGetQueryMetricsRequest Request; +}; + THolder<IActor> CreateQueryStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TStringBuf viewName, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ - if (viewName == TopQueriesByDuration1MinuteName) { + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ + if (viewName == TopQueriesByDuration1MinuteName) { return MakeHolder<TQueryStatsScan<TDurationGreater>>(ownerId, scanId, tableId, tableRange, columns, - NKikimrSysView::TOP_DURATION_ONE_MINUTE, - ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE); - - } else if (viewName == TopQueriesByDuration1HourName) { + NKikimrSysView::TOP_DURATION_ONE_MINUTE, + ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE); + + } else if (viewName == TopQueriesByDuration1HourName) { return MakeHolder<TQueryStatsScan<TDurationGreater>>(ownerId, scanId, tableId, tableRange, columns, - NKikimrSysView::TOP_DURATION_ONE_HOUR, - ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE); - - } else if (viewName == TopQueriesByReadBytes1MinuteName) { + NKikimrSysView::TOP_DURATION_ONE_HOUR, + ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE); + + } else if (viewName == TopQueriesByReadBytes1MinuteName) { return MakeHolder<TQueryStatsScan<TReadBytesGreater>>(ownerId, scanId, tableId, tableRange, columns, - NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, - ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE); - - } else if (viewName == TopQueriesByReadBytes1HourName) { + NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, + ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE); + + } else if (viewName == TopQueriesByReadBytes1HourName) { return MakeHolder<TQueryStatsScan<TReadBytesGreater>>(ownerId, scanId, tableId, tableRange, columns, - NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, - ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE); - - } else if (viewName == TopQueriesByCpuTime1MinuteName) { + NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, + ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE); + + } else if (viewName == TopQueriesByCpuTime1MinuteName) { return MakeHolder<TQueryStatsScan<TCpuTimeGreater>>(ownerId, scanId, tableId, tableRange, columns, - NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, - ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE); - - } else if (viewName == TopQueriesByCpuTime1HourName) { + NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, + ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE); + + } else if (viewName == TopQueriesByCpuTime1HourName) { return MakeHolder<TQueryStatsScan<TCpuTimeGreater>>(ownerId, scanId, tableId, tableRange, columns, - NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, - ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE); - } else if (viewName == TopQueriesByRequestUnits1MinuteName) { - return MakeHolder<TQueryStatsScan<TRequestUnitsGreater>>(ownerId, scanId, tableId, tableRange, columns, - NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, - ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE); - - } else if (viewName == TopQueriesByRequestUnits1HourName) { - return MakeHolder<TQueryStatsScan<TRequestUnitsGreater>>(ownerId, scanId, tableId, tableRange, columns, - NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, - ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE); - } - return {}; -} - -} // NSysView -} // NKikimr + NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, + ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE); + } else if (viewName == TopQueriesByRequestUnits1MinuteName) { + return MakeHolder<TQueryStatsScan<TRequestUnitsGreater>>(ownerId, scanId, tableId, tableRange, columns, + NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, + ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE); + + } else if (viewName == TopQueriesByRequestUnits1HourName) { + return MakeHolder<TQueryStatsScan<TRequestUnitsGreater>>(ownerId, scanId, tableId, tableRange, columns, + NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, + ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE); + } + return {}; +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/query_stats/query_stats.h b/ydb/core/sys_view/query_stats/query_stats.h index 47786c7bf5..b9fe4b7ecc 100644 --- a/ydb/core/sys_view/query_stats/query_stats.h +++ b/ydb/core/sys_view/query_stats/query_stats.h @@ -1,25 +1,25 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - -struct TQueryStatsBucketRange { - ui64 FromBucket = 0; - TMaybe<ui32> FromRank; - - ui64 ToBucket = std::numeric_limits<ui64>::max(); - TMaybe<ui32> ToRank; - - bool IsEmpty = false; - - explicit TQueryStatsBucketRange(const TSerializedTableRange& range, const TDuration& bucketSize); -}; - + +namespace NKikimr { +namespace NSysView { + +struct TQueryStatsBucketRange { + ui64 FromBucket = 0; + TMaybe<ui32> FromRank; + + ui64 ToBucket = std::numeric_limits<ui64>::max(); + TMaybe<ui32> ToRank; + + bool IsEmpty = false; + + explicit TQueryStatsBucketRange(const TSerializedTableRange& range, const TDuration& bucketSize); +}; + THolder<IActor> CreateQueryStatsScan(const TActorId& ownerId, ui32 scanId, - const TTableId& tableId, const TStringBuf viewName, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableId& tableId, const TStringBuf viewName, + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/query_stats/query_stats_ut.cpp b/ydb/core/sys_view/query_stats/query_stats_ut.cpp index f8859a7ddc..4b8c5141cb 100644 --- a/ydb/core/sys_view/query_stats/query_stats_ut.cpp +++ b/ydb/core/sys_view/query_stats/query_stats_ut.cpp @@ -1,101 +1,101 @@ -#include "query_stats.h" - +#include "query_stats.h" + #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { -namespace NSysView { - -Y_UNIT_TEST_SUITE(QueryStats) { - Y_UNIT_TEST(Ranges) { - auto test = [] (TMaybe<ui64> fromInstant, TMaybe<ui32> fromRank, bool inclusiveFrom, - TMaybe<ui64> toInstant, TMaybe<ui32> toRank, bool inclusiveTo, - ui64 fromBucketExpected, TMaybe<ui32> fromRankExpected, - ui64 toBucketExpected, TMaybe<ui32> toRankExpected, - bool isEmptyExpected) - { - TVector<TCell> from; - if (fromInstant) { - from.push_back(TCell::Make<ui64>(*fromInstant)); - if (fromRank) { - from.push_back(TCell::Make<ui32>(*fromRank)); - } - } - TVector<TCell> to; - if (toInstant) { - to.push_back(TCell::Make<ui64>(*toInstant)); - if (toRank) { - to.push_back(TCell::Make<ui32>(*toRank)); - } - } - TSerializedTableRange tableRange(from, inclusiveFrom, to, inclusiveTo); - - TQueryStatsBucketRange range(tableRange, TDuration::MicroSeconds(10)); - - UNIT_ASSERT_VALUES_EQUAL(range.FromBucket, fromBucketExpected); - UNIT_ASSERT_VALUES_EQUAL(range.FromRank, fromRankExpected); - UNIT_ASSERT_VALUES_EQUAL(range.ToBucket, toBucketExpected); - UNIT_ASSERT_VALUES_EQUAL(range.ToRank, toRankExpected); - UNIT_ASSERT_VALUES_EQUAL(range.IsEmpty, isEmptyExpected); - }; - - auto maxUi64 = std::numeric_limits<ui64>::max(); - - test({}, {}, false, {}, {}, false, - 0, {}, maxUi64, {}, false); - test({}, {}, true, {}, {}, true, - 0, {}, maxUi64, {}, false); - - test(0, {}, false, 0, {}, false, - 0, {}, 0, {}, true); - test(0, {}, true, 0, {}, true, - 0, {}, 0, {}, true); - - test(5, {}, false, 5, {}, false, - 0, {}, 0, {}, true); - test(5, {}, true, 5, {}, true, - 0, {}, 0, {}, true); - - test(10, {}, false, 10, {}, false, - 1, {}, 0, {}, true); - test(10, {}, true, 10, {}, true, - 0, {}, 0, {}, false); - - test(15, {}, false, 15, {}, false, - 1, {}, 0, {}, true); - test(15, {}, true, 15, {}, true, - 1, {}, 0, {}, true); - - test(20, {}, false, 20, {}, false, - 2, {}, 0, {}, true); - test(20, {}, true, 20, {}, true, - 1, {}, 1, {}, false); - - test(0, 3, false, 0, 7, false, - 0, {}, 0, {}, true); - test(0, 3, true, 0, 3, true, - 0, {}, 0, {}, true); - - test(5, 3, false, 5, 7, false, - 0, {}, 0, {}, true); - test(5, 3, true, 5, 7, true, - 0, {}, 0, {}, true); - - test(10, 3, false, 10, 7, false, - 0, 4, 0, 6, false); - test(10, 3, true, 10, 7, true, - 0, 3, 0, 7, false); - - test(15, 3, false, 15, 7, false, - 1, {}, 0, {}, true); - test(15, 3, true, 15, 7, true, - 1, {}, 0, {}, true); - - test(20, 3, false, 20, 7, false, - 1, 4, 1, 6, false); - test(20, 3, true, 20, 7, true, - 1, 3, 1, 7, false); - } -} - -} // NSysView -} // NKikimr + +namespace NKikimr { +namespace NSysView { + +Y_UNIT_TEST_SUITE(QueryStats) { + Y_UNIT_TEST(Ranges) { + auto test = [] (TMaybe<ui64> fromInstant, TMaybe<ui32> fromRank, bool inclusiveFrom, + TMaybe<ui64> toInstant, TMaybe<ui32> toRank, bool inclusiveTo, + ui64 fromBucketExpected, TMaybe<ui32> fromRankExpected, + ui64 toBucketExpected, TMaybe<ui32> toRankExpected, + bool isEmptyExpected) + { + TVector<TCell> from; + if (fromInstant) { + from.push_back(TCell::Make<ui64>(*fromInstant)); + if (fromRank) { + from.push_back(TCell::Make<ui32>(*fromRank)); + } + } + TVector<TCell> to; + if (toInstant) { + to.push_back(TCell::Make<ui64>(*toInstant)); + if (toRank) { + to.push_back(TCell::Make<ui32>(*toRank)); + } + } + TSerializedTableRange tableRange(from, inclusiveFrom, to, inclusiveTo); + + TQueryStatsBucketRange range(tableRange, TDuration::MicroSeconds(10)); + + UNIT_ASSERT_VALUES_EQUAL(range.FromBucket, fromBucketExpected); + UNIT_ASSERT_VALUES_EQUAL(range.FromRank, fromRankExpected); + UNIT_ASSERT_VALUES_EQUAL(range.ToBucket, toBucketExpected); + UNIT_ASSERT_VALUES_EQUAL(range.ToRank, toRankExpected); + UNIT_ASSERT_VALUES_EQUAL(range.IsEmpty, isEmptyExpected); + }; + + auto maxUi64 = std::numeric_limits<ui64>::max(); + + test({}, {}, false, {}, {}, false, + 0, {}, maxUi64, {}, false); + test({}, {}, true, {}, {}, true, + 0, {}, maxUi64, {}, false); + + test(0, {}, false, 0, {}, false, + 0, {}, 0, {}, true); + test(0, {}, true, 0, {}, true, + 0, {}, 0, {}, true); + + test(5, {}, false, 5, {}, false, + 0, {}, 0, {}, true); + test(5, {}, true, 5, {}, true, + 0, {}, 0, {}, true); + + test(10, {}, false, 10, {}, false, + 1, {}, 0, {}, true); + test(10, {}, true, 10, {}, true, + 0, {}, 0, {}, false); + + test(15, {}, false, 15, {}, false, + 1, {}, 0, {}, true); + test(15, {}, true, 15, {}, true, + 1, {}, 0, {}, true); + + test(20, {}, false, 20, {}, false, + 2, {}, 0, {}, true); + test(20, {}, true, 20, {}, true, + 1, {}, 1, {}, false); + + test(0, 3, false, 0, 7, false, + 0, {}, 0, {}, true); + test(0, 3, true, 0, 3, true, + 0, {}, 0, {}, true); + + test(5, 3, false, 5, 7, false, + 0, {}, 0, {}, true); + test(5, 3, true, 5, 7, true, + 0, {}, 0, {}, true); + + test(10, 3, false, 10, 7, false, + 0, 4, 0, 6, false); + test(10, 3, true, 10, 7, true, + 0, 3, 0, 7, false); + + test(15, 3, false, 15, 7, false, + 1, {}, 0, {}, true); + test(15, 3, true, 15, 7, true, + 1, {}, 0, {}, true); + + test(20, 3, false, 20, 7, false, + 1, 4, 1, 6, false); + test(20, 3, true, 20, 7, true, + 1, 3, 1, 7, false); + } +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/query_stats/ut/ya.make b/ydb/core/sys_view/query_stats/ut/ya.make index 0674a86a62..89561102ce 100644 --- a/ydb/core/sys_view/query_stats/ut/ya.make +++ b/ydb/core/sys_view/query_stats/ut/ya.make @@ -1,25 +1,25 @@ UNITTEST_FOR(ydb/core/sys_view/query_stats) - -OWNER( - monster - g:kikimr -) - -FORK_SUBTESTS() - -SIZE(MEDIUM) - -TIMEOUT(600) - -PEERDIR( + +OWNER( + monster + g:kikimr +) + +FORK_SUBTESTS() + +SIZE(MEDIUM) + +TIMEOUT(600) + +PEERDIR( library/cpp/testing/unittest ydb/core/testlib -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - query_stats_ut.cpp -) - -END() +SRCS( + query_stats_ut.cpp +) + +END() diff --git a/ydb/core/sys_view/query_stats/ya.make b/ydb/core/sys_view/query_stats/ya.make index 7683ef2088..8a572dbeae 100644 --- a/ydb/core/sys_view/query_stats/ya.make +++ b/ydb/core/sys_view/query_stats/ya.make @@ -1,25 +1,25 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - query_metrics.h - query_metrics.cpp - query_stats.h - query_stats.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + query_metrics.h + query_metrics.cpp + query_stats.h + query_stats.cpp +) + +PEERDIR( library/cpp/actors/core ydb/core/base ydb/core/kqp/runtime ydb/core/sys_view/common ydb/core/sys_view/service -) - +) + YQL_LAST_ABI_VERSION() END() diff --git a/ydb/core/sys_view/scan.cpp b/ydb/core/sys_view/scan.cpp index 97a169b675..f337cb6ba7 100644 --- a/ydb/core/sys_view/scan.cpp +++ b/ydb/core/sys_view/scan.cpp @@ -1,5 +1,5 @@ -#include "scan.h" - +#include "scan.h" + #include <ydb/core/sys_view/common/schema.h> #include <ydb/core/sys_view/partition_stats/partition_stats.h> #include <ydb/core/sys_view/nodes/nodes.h> @@ -11,63 +11,63 @@ #include <ydb/core/sys_view/storage/storage_pools.h> #include <ydb/core/sys_view/storage/storage_stats.h> #include <ydb/core/sys_view/tablets/tablets.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + THolder<IActor> CreateSystemViewScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ - if (tableId.SysViewInfo == PartitionStatsName) { - return CreatePartitionStatsScan(ownerId, scanId, tableId, tableRange, columns); - } + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ + if (tableId.SysViewInfo == PartitionStatsName) { + return CreatePartitionStatsScan(ownerId, scanId, tableId, tableRange, columns); + } - if (tableId.SysViewInfo == NodesName) { - return CreateNodesScan(ownerId, scanId, tableId, tableRange, columns); - } + if (tableId.SysViewInfo == NodesName) { + return CreateNodesScan(ownerId, scanId, tableId, tableRange, columns); + } - if (tableId.SysViewInfo == TopQueriesByDuration1MinuteName || - tableId.SysViewInfo == TopQueriesByDuration1HourName || - tableId.SysViewInfo == TopQueriesByReadBytes1MinuteName || - tableId.SysViewInfo == TopQueriesByReadBytes1HourName || - tableId.SysViewInfo == TopQueriesByCpuTime1MinuteName || - tableId.SysViewInfo == TopQueriesByCpuTime1HourName || - tableId.SysViewInfo == TopQueriesByRequestUnits1MinuteName || - tableId.SysViewInfo == TopQueriesByRequestUnits1HourName) - { - return CreateQueryStatsScan(ownerId, scanId, tableId, tableId.SysViewInfo, tableRange, columns); - } - - if (tableId.SysViewInfo == PDisksName) { - return CreatePDisksScan(ownerId, scanId, tableId, tableRange, columns); - } + if (tableId.SysViewInfo == TopQueriesByDuration1MinuteName || + tableId.SysViewInfo == TopQueriesByDuration1HourName || + tableId.SysViewInfo == TopQueriesByReadBytes1MinuteName || + tableId.SysViewInfo == TopQueriesByReadBytes1HourName || + tableId.SysViewInfo == TopQueriesByCpuTime1MinuteName || + tableId.SysViewInfo == TopQueriesByCpuTime1HourName || + tableId.SysViewInfo == TopQueriesByRequestUnits1MinuteName || + tableId.SysViewInfo == TopQueriesByRequestUnits1HourName) + { + return CreateQueryStatsScan(ownerId, scanId, tableId, tableId.SysViewInfo, tableRange, columns); + } - if (tableId.SysViewInfo == VSlotsName) { - return CreateVSlotsScan(ownerId, scanId, tableId, tableRange, columns); - } + if (tableId.SysViewInfo == PDisksName) { + return CreatePDisksScan(ownerId, scanId, tableId, tableRange, columns); + } - if (tableId.SysViewInfo == GroupsName) { - return CreateGroupsScan(ownerId, scanId, tableId, tableRange, columns); - } + if (tableId.SysViewInfo == VSlotsName) { + return CreateVSlotsScan(ownerId, scanId, tableId, tableRange, columns); + } + + if (tableId.SysViewInfo == GroupsName) { + return CreateGroupsScan(ownerId, scanId, tableId, tableRange, columns); + } + + if (tableId.SysViewInfo == StoragePoolsName) { + return CreateStoragePoolsScan(ownerId, scanId, tableId, tableRange, columns); + } - if (tableId.SysViewInfo == StoragePoolsName) { - return CreateStoragePoolsScan(ownerId, scanId, tableId, tableRange, columns); - } - if (tableId.SysViewInfo == StorageStatsName) { return CreateStorageStatsScan(ownerId, scanId, tableId, tableRange, columns); } - if (tableId.SysViewInfo == TabletsName) { - return CreateTabletsScan(ownerId, scanId, tableId, tableRange, columns); - } - - if (tableId.SysViewInfo == QueryMetricsName) { - return CreateQueryMetricsScan(ownerId, scanId, tableId, tableRange, columns); - } - - return nullptr; -} - -} // NSysView -} // NKikimr + if (tableId.SysViewInfo == TabletsName) { + return CreateTabletsScan(ownerId, scanId, tableId, tableRange, columns); + } + + if (tableId.SysViewInfo == QueryMetricsName) { + return CreateQueryMetricsScan(ownerId, scanId, tableId, tableRange, columns); + } + + return nullptr; +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/scan.h b/ydb/core/sys_view/scan.h index 19034c317d..bfac578bfa 100644 --- a/ydb/core/sys_view/scan.h +++ b/ydb/core/sys_view/scan.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + THolder<IActor> CreateSystemViewScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/service/db_counters.cpp b/ydb/core/sys_view/service/db_counters.cpp index 3395137237..cc3a16d16a 100644 --- a/ydb/core/sys_view/service/db_counters.cpp +++ b/ydb/core/sys_view/service/db_counters.cpp @@ -1,117 +1,117 @@ -#include "db_counters.h" - +#include "db_counters.h" + #include <ydb/core/base/path.h> #include <ydb/core/sys_view/common/events.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> - -#include <library/cpp/actors/core/log.h> - -namespace NKikimr { -namespace NSysView { - -class TDbWatcherActor - : public TActor<TDbWatcherActor> -{ - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - - TIntrusivePtr<TDbWatcherCallback> Callback; - - THashMap<TString, TPathId> DatabaseToPathId; - THashMap<TPathId, TString> PathIdToDatabase; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::DB_WATCHER_ACTOR; - } - - explicit TDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback) - : TActor(&TThis::StateFunc) - , Callback(callback) - { - } - -private: - void Handle(NSysView::TEvSysView::TEvWatchDatabase::TPtr& ev) { - auto database = ev->Get()->Database; - auto pathId = ev->Get()->PathId; - - if (!database) { - if (!pathId || PathIdToDatabase.FindPtr(pathId)) { - return; - } - PathIdToDatabase.emplace(pathId, ""); - - auto key = pathId.Hash(); - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(pathId, key)); - return; - } - - if (DatabaseToPathId.FindPtr(database)) { - return; - } - DatabaseToPathId.emplace(database, TPathId()); - - auto request = MakeHolder<TNavigate>(); - request->ResultSet.push_back({}); - - auto& entry = request->ResultSet.back(); - entry.Path = SplitPath(database); - entry.Operation = TNavigate::EOp::OpPath; - entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; - entry.RedirectRequired = false; - - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - auto* request = ev->Get()->Request.Get(); - if (request->ResultSet.size() != 1) { - return; - } - auto& entry = request->ResultSet.back(); - auto database = CanonizePath(entry.Path); - if (entry.Status != TNavigate::EStatus::Ok) { - DatabaseToPathId.erase(database); - Callback->OnDatabaseRemoved(database, TPathId()); - return; - } - auto pathId = entry.DomainInfo->DomainKey; - DatabaseToPathId[database] = pathId; - PathIdToDatabase[pathId] = database; - - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(pathId)); - } - - void Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev) { - auto pathId = ev->Get()->PathId; - - if (auto* db = PathIdToDatabase.FindPtr(pathId)) { - auto database = *db; - if (database) { - DatabaseToPathId.erase(database); - } - PathIdToDatabase.erase(pathId); - - Callback->OnDatabaseRemoved(database, pathId); - - auto key = pathId.Hash(); - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchRemove(key)); - } - } - - STRICT_STFUNC(StateFunc, { - hFunc(NSysView::TEvSysView::TEvWatchDatabase, Handle) - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated); - IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyUnavailable); - hFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted, Handle); - }) -}; - -IActor* CreateDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback) { - return new TDbWatcherActor(callback); -} - -} -} - + +#include <library/cpp/actors/core/log.h> + +namespace NKikimr { +namespace NSysView { + +class TDbWatcherActor + : public TActor<TDbWatcherActor> +{ + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + + TIntrusivePtr<TDbWatcherCallback> Callback; + + THashMap<TString, TPathId> DatabaseToPathId; + THashMap<TPathId, TString> PathIdToDatabase; + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::DB_WATCHER_ACTOR; + } + + explicit TDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback) + : TActor(&TThis::StateFunc) + , Callback(callback) + { + } + +private: + void Handle(NSysView::TEvSysView::TEvWatchDatabase::TPtr& ev) { + auto database = ev->Get()->Database; + auto pathId = ev->Get()->PathId; + + if (!database) { + if (!pathId || PathIdToDatabase.FindPtr(pathId)) { + return; + } + PathIdToDatabase.emplace(pathId, ""); + + auto key = pathId.Hash(); + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(pathId, key)); + return; + } + + if (DatabaseToPathId.FindPtr(database)) { + return; + } + DatabaseToPathId.emplace(database, TPathId()); + + auto request = MakeHolder<TNavigate>(); + request->ResultSet.push_back({}); + + auto& entry = request->ResultSet.back(); + entry.Path = SplitPath(database); + entry.Operation = TNavigate::EOp::OpPath; + entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; + entry.RedirectRequired = false; + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + auto* request = ev->Get()->Request.Get(); + if (request->ResultSet.size() != 1) { + return; + } + auto& entry = request->ResultSet.back(); + auto database = CanonizePath(entry.Path); + if (entry.Status != TNavigate::EStatus::Ok) { + DatabaseToPathId.erase(database); + Callback->OnDatabaseRemoved(database, TPathId()); + return; + } + auto pathId = entry.DomainInfo->DomainKey; + DatabaseToPathId[database] = pathId; + PathIdToDatabase[pathId] = database; + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(pathId)); + } + + void Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev) { + auto pathId = ev->Get()->PathId; + + if (auto* db = PathIdToDatabase.FindPtr(pathId)) { + auto database = *db; + if (database) { + DatabaseToPathId.erase(database); + } + PathIdToDatabase.erase(pathId); + + Callback->OnDatabaseRemoved(database, pathId); + + auto key = pathId.Hash(); + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchRemove(key)); + } + } + + STRICT_STFUNC(StateFunc, { + hFunc(NSysView::TEvSysView::TEvWatchDatabase, Handle) + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated); + IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyUnavailable); + hFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted, Handle); + }) +}; + +IActor* CreateDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback) { + return new TDbWatcherActor(callback); +} + +} +} + diff --git a/ydb/core/sys_view/service/db_counters.h b/ydb/core/sys_view/service/db_counters.h index 9943fa01e7..92e15e51e2 100644 --- a/ydb/core/sys_view/service/db_counters.h +++ b/ydb/core/sys_view/service/db_counters.h @@ -1,15 +1,15 @@ -#pragma once - +#pragma once + #include <ydb/core/scheme/scheme_tabledefs.h> - -namespace NKikimr { -namespace NSysView { - -struct TDbWatcherCallback : public TThrRefBase { - virtual void OnDatabaseRemoved(const TString& database, TPathId pathId) = 0; -}; - -IActor* CreateDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback); - -} -} + +namespace NKikimr { +namespace NSysView { + +struct TDbWatcherCallback : public TThrRefBase { + virtual void OnDatabaseRemoved(const TString& database, TPathId pathId) = 0; +}; + +IActor* CreateDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback); + +} +} diff --git a/ydb/core/sys_view/service/ext_counters.cpp b/ydb/core/sys_view/service/ext_counters.cpp index ae97e7b179..bebbd04786 100644 --- a/ydb/core/sys_view/service/ext_counters.cpp +++ b/ydb/core/sys_view/service/ext_counters.cpp @@ -1,109 +1,109 @@ -#include "ext_counters.h" - +#include "ext_counters.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> - -namespace NKikimr { -namespace NSysView { - -class TExtCountersUpdaterActor - : public TActorBootstrapped<TExtCountersUpdaterActor> -{ - using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; - - const TExtCountersConfig Config; - - TCounterPtr MemoryUsedBytes; - TCounterPtr MemoryLimitBytes; - TVector<TCounterPtr> CpuUsedCorePercents; - TVector<TCounterPtr> CpuLimitCorePercents; - - TCounterPtr AnonRssSize; - TCounterPtr CGroupMemLimit; - TVector<TCounterPtr> PoolElapsedMicrosec; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::EXT_COUNTERS_UPDATER_ACTOR; - } - - explicit TExtCountersUpdaterActor(TExtCountersConfig&& config) - : Config(std::move(config)) - {} - - void Bootstrap() { - auto ydbGroup = GetServiceCounters(AppData()->Counters, "ydb"); - - MemoryUsedBytes = ydbGroup->GetNamedCounter("name", - "resources.memory.used_bytes", false); - MemoryLimitBytes = ydbGroup->GetNamedCounter("name", - "resources.memory.limit_bytes", false); - - auto poolCount = Config.Pools.size(); - CpuUsedCorePercents.resize(poolCount); - CpuLimitCorePercents.resize(poolCount); - - for (size_t i = 0; i < poolCount; ++i) { - auto name = to_lower(Config.Pools[i].Name); - CpuUsedCorePercents[i] = ydbGroup->GetSubgroup("pool", name)->GetNamedCounter("name", - "resources.cpu.used_core_percents", true); - CpuLimitCorePercents[i] = ydbGroup->GetSubgroup("pool", name)->GetNamedCounter("name", - "resources.cpu.limit_core_percents", false); - } - - Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup); - Become(&TThis::StateWork); - } - -private: - void Initialize() { - if (!AnonRssSize) { - auto utilsGroup = GetServiceCounters(AppData()->Counters, "utils"); - - AnonRssSize = utilsGroup->FindCounter("Process/AnonRssSize"); - CGroupMemLimit = utilsGroup->FindCounter("Process/CGroupMemLimit"); - - PoolElapsedMicrosec.resize(Config.Pools.size()); - for (size_t i = 0; i < Config.Pools.size(); ++i) { - auto poolGroup = utilsGroup->FindSubgroup("execpool", Config.Pools[i].Name); - if (poolGroup) { - PoolElapsedMicrosec[i] = poolGroup->FindCounter("ElapsedMicrosec"); - } - } - } - } - - void Transform() { - Initialize(); - - if (AnonRssSize) { - MemoryUsedBytes->Set(AnonRssSize->Val()); - } - if (CGroupMemLimit) { - MemoryLimitBytes->Set(CGroupMemLimit->Val()); - } - for (size_t i = 0; i < Config.Pools.size(); ++i) { - if (PoolElapsedMicrosec[i]) { - CpuUsedCorePercents[i]->Set(PoolElapsedMicrosec[i]->Val() / 10000.); - CpuLimitCorePercents[i]->Set(Config.Pools[i].ThreadCount * 100); - } - } - } - - void HandleWakeup() { - Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup); - Transform(); - } - - STRICT_STFUNC(StateWork, { - cFunc(TEvents::TSystem::Wakeup, HandleWakeup); - }) -}; - -IActor* CreateExtCountersUpdater(TExtCountersConfig&& config) { - return new TExtCountersUpdaterActor(std::move(config)); -} - -} -} - + +namespace NKikimr { +namespace NSysView { + +class TExtCountersUpdaterActor + : public TActorBootstrapped<TExtCountersUpdaterActor> +{ + using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; + + const TExtCountersConfig Config; + + TCounterPtr MemoryUsedBytes; + TCounterPtr MemoryLimitBytes; + TVector<TCounterPtr> CpuUsedCorePercents; + TVector<TCounterPtr> CpuLimitCorePercents; + + TCounterPtr AnonRssSize; + TCounterPtr CGroupMemLimit; + TVector<TCounterPtr> PoolElapsedMicrosec; + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::EXT_COUNTERS_UPDATER_ACTOR; + } + + explicit TExtCountersUpdaterActor(TExtCountersConfig&& config) + : Config(std::move(config)) + {} + + void Bootstrap() { + auto ydbGroup = GetServiceCounters(AppData()->Counters, "ydb"); + + MemoryUsedBytes = ydbGroup->GetNamedCounter("name", + "resources.memory.used_bytes", false); + MemoryLimitBytes = ydbGroup->GetNamedCounter("name", + "resources.memory.limit_bytes", false); + + auto poolCount = Config.Pools.size(); + CpuUsedCorePercents.resize(poolCount); + CpuLimitCorePercents.resize(poolCount); + + for (size_t i = 0; i < poolCount; ++i) { + auto name = to_lower(Config.Pools[i].Name); + CpuUsedCorePercents[i] = ydbGroup->GetSubgroup("pool", name)->GetNamedCounter("name", + "resources.cpu.used_core_percents", true); + CpuLimitCorePercents[i] = ydbGroup->GetSubgroup("pool", name)->GetNamedCounter("name", + "resources.cpu.limit_core_percents", false); + } + + Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup); + Become(&TThis::StateWork); + } + +private: + void Initialize() { + if (!AnonRssSize) { + auto utilsGroup = GetServiceCounters(AppData()->Counters, "utils"); + + AnonRssSize = utilsGroup->FindCounter("Process/AnonRssSize"); + CGroupMemLimit = utilsGroup->FindCounter("Process/CGroupMemLimit"); + + PoolElapsedMicrosec.resize(Config.Pools.size()); + for (size_t i = 0; i < Config.Pools.size(); ++i) { + auto poolGroup = utilsGroup->FindSubgroup("execpool", Config.Pools[i].Name); + if (poolGroup) { + PoolElapsedMicrosec[i] = poolGroup->FindCounter("ElapsedMicrosec"); + } + } + } + } + + void Transform() { + Initialize(); + + if (AnonRssSize) { + MemoryUsedBytes->Set(AnonRssSize->Val()); + } + if (CGroupMemLimit) { + MemoryLimitBytes->Set(CGroupMemLimit->Val()); + } + for (size_t i = 0; i < Config.Pools.size(); ++i) { + if (PoolElapsedMicrosec[i]) { + CpuUsedCorePercents[i]->Set(PoolElapsedMicrosec[i]->Val() / 10000.); + CpuLimitCorePercents[i]->Set(Config.Pools[i].ThreadCount * 100); + } + } + } + + void HandleWakeup() { + Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup); + Transform(); + } + + STRICT_STFUNC(StateWork, { + cFunc(TEvents::TSystem::Wakeup, HandleWakeup); + }) +}; + +IActor* CreateExtCountersUpdater(TExtCountersConfig&& config) { + return new TExtCountersUpdaterActor(std::move(config)); +} + +} +} + diff --git a/ydb/core/sys_view/service/ext_counters.h b/ydb/core/sys_view/service/ext_counters.h index 84f91dc9cd..0097591227 100644 --- a/ydb/core/sys_view/service/ext_counters.h +++ b/ydb/core/sys_view/service/ext_counters.h @@ -1,19 +1,19 @@ -#pragma once - -#include <library/cpp/actors/core/actor.h> - -namespace NKikimr { -namespace NSysView { - -struct TExtCountersConfig { - struct TPool { - TString Name; - ui32 ThreadCount = 0; - }; - TVector<TPool> Pools; -}; - -NActors::IActor* CreateExtCountersUpdater(TExtCountersConfig&& config); - -} -} +#pragma once + +#include <library/cpp/actors/core/actor.h> + +namespace NKikimr { +namespace NSysView { + +struct TExtCountersConfig { + struct TPool { + TString Name; + ui32 ThreadCount = 0; + }; + TVector<TPool> Pools; +}; + +NActors::IActor* CreateExtCountersUpdater(TExtCountersConfig&& config); + +} +} diff --git a/ydb/core/sys_view/service/query_history.h b/ydb/core/sys_view/service/query_history.h index 6d92d789a1..08dd23fc73 100644 --- a/ydb/core/sys_view/service/query_history.h +++ b/ydb/core/sys_view/service/query_history.h @@ -1,511 +1,511 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/sys_view.pb.h> - -#include <util/generic/vector.h> -#include <util/datetime/base.h> - -namespace NKikimr { -namespace NSysView { - -// TODO: configs -constexpr TDuration ONE_MINUTE_BUCKET_SIZE = TDuration::Minutes(1); -constexpr ui64 ONE_MINUTE_BUCKET_COUNT = 6 * 60; // 6 hours - -constexpr TDuration ONE_HOUR_BUCKET_SIZE = TDuration::Hours(1); -constexpr ui64 ONE_HOUR_BUCKET_COUNT = 24 * 7 * 2; // 2 weeks - -constexpr size_t TOP_QUERIES_COUNT = 5; - -using TNodeId = ui32; -using TQueryHash = ui64; - -using TQueryStatsPtr = std::shared_ptr<NKikimrSysView::TQueryStats>; - -using TQueryTextsByHash = std::unordered_map<TQueryHash, TString>; - -struct TReadBytesGreater { - using TValue = ui64; - - static TValue GetValue(const TQueryStatsPtr stats) { - return stats->GetStats().GetReadBytes(); - } - bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const { - return a->GetStats().GetReadBytes() == b->GetStats().GetReadBytes() - ? a->GetNodeId() > b->GetNodeId() - : a->GetStats().GetReadBytes() > b->GetStats().GetReadBytes(); - } - bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const { - return operator()(a.get(), b.get()); - } -}; - -struct TDurationGreater { - using TValue = ui64; - - static TValue GetValue(const TQueryStatsPtr stats) { - return stats->GetDurationMs(); - } - bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const { - return a->GetDurationMs() == b->GetDurationMs() - ? a->GetNodeId() > b->GetNodeId() - : a->GetDurationMs() > b->GetDurationMs(); - } - bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const { - return operator()(a.get(), b.get()); - } -}; - -struct TCpuTimeGreater { - using TValue = ui64; - - static TValue GetValue(const TQueryStatsPtr stats) { - return stats->GetTotalCpuTimeUs(); - } - bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const { - return a->GetTotalCpuTimeUs() == b->GetTotalCpuTimeUs() - ? a->GetNodeId() > b->GetNodeId() - : a->GetTotalCpuTimeUs() > b->GetTotalCpuTimeUs(); - } - bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const { - return operator()(a.get(), b.get()); - } -}; - -struct TRequestUnitsGreater { - using TValue = ui64; - - static TValue GetValue(const TQueryStatsPtr stats) { - return stats->GetRequestUnits(); - } - bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const { - return a->GetRequestUnits() == b->GetRequestUnits() - ? a->GetNodeId() > b->GetNodeId() - : a->GetRequestUnits() > b->GetRequestUnits(); - } - bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const { - return operator()(a.get(), b.get()); - } -}; - -inline void CopyQueryStatsNoText(NKikimrSysView::TQueryStats* to, const NKikimrSysView::TQueryStats& from) { - to->SetQueryTextHash(from.GetQueryTextHash()); - to->SetDurationMs(from.GetDurationMs()); - to->SetEndTimeMs(from.GetEndTimeMs()); - to->MutableStats()->CopyFrom(from.GetStats()); - if (from.HasUserSID()) { - to->SetUserSID(from.GetUserSID()); - } - to->SetParametersSize(from.GetParametersSize()); - if (from.HasCompileDurationMs()) { - to->SetCompileDurationMs(from.GetCompileDurationMs()); - } - if (from.HasFromQueryCache()) { - to->SetFromQueryCache(from.GetFromQueryCache()); - } - to->SetNodeId(from.GetNodeId()); - to->MutableShardsCpuTimeUs()->CopyFrom(from.GetShardsCpuTimeUs()); - to->MutableComputeCpuTimeUs()->CopyFrom(from.GetComputeCpuTimeUs()); - if (from.HasCompileCpuTimeUs()) { - to->SetCompileCpuTimeUs(from.GetCompileCpuTimeUs()); - } - to->SetProcessCpuTimeUs(from.GetProcessCpuTimeUs()); - to->SetTotalCpuTimeUs(from.GetTotalCpuTimeUs()); - to->SetType(from.GetType()); - to->SetRequestUnits(from.GetRequestUnits()); -} - -template <typename TGreater> -struct TQueryStatsBucket { - TVector<TQueryStatsPtr> Top; - - void Add(TQueryStatsPtr stats) { - auto it = std::lower_bound(Top.begin(), Top.end(), stats, TGreater()); - if (Top.size() == TOP_QUERIES_COUNT && it == Top.end()) { - return; - } - Top.insert(it, stats); - if (Top.size() > TOP_QUERIES_COUNT) { - Top.resize(TOP_QUERIES_COUNT); - } - } - - void Clear() { - Top.clear(); - } - - void Fill(NKikimrSysView::TTopQueryStats* bucket, TQueryTextsByHash& texts) const { - for (auto& queryStats : Top) { - CopyQueryStatsNoText(bucket->AddStats(), *queryStats); - texts.try_emplace(queryStats->GetQueryTextHash(), queryStats->GetQueryText()); - } - } -}; - -template <typename TGreater> -struct TQueryStatsDedupBucket { - using TValue = typename TGreater::TValue; - - std::unordered_map<TQueryHash, TQueryStatsPtr> Stats; // query hash -> stats - std::multimap<TValue, TQueryHash> Top; // top by characteristic -> query hash - - void Add(TQueryStatsPtr stats) { - if (!stats->HasQueryTextHash()) { - return; - } - - auto queryHash = stats->GetQueryTextHash(); - auto hashIt = Stats.find(queryHash); - - if (hashIt != Stats.end()) { - if (!TGreater()(stats, hashIt->second)) { - return; - } - - TValue old = TGreater::GetValue(hashIt->second); - const auto range = Top.equal_range(old); - auto it = range.first; - for (; it != range.second; ++it) { - if (it->second == queryHash) { - break; - } - } - - Y_VERIFY(it != range.second); - Top.erase(it); - - TValue current = TGreater::GetValue(stats); - Top.emplace(std::make_pair(current, queryHash)); - - hashIt->second = stats; - - } else { - if (Top.size() == TOP_QUERIES_COUNT) { - auto it = Top.begin(); - TValue value = TGreater::GetValue(stats); - if (value <= it->first) { - return; - } - Stats.erase(it->second); - Top.erase(it); - } - - TValue current = TGreater::GetValue(stats); - Top.emplace(std::make_pair(current, queryHash)); - Stats.emplace(std::make_pair(queryHash, stats)); - } - } - - void Clear() { - Stats.clear(); - Top.clear(); - } - - void Swap(TQueryStatsDedupBucket& other) { - Stats.swap(other.Stats); - Top.swap(other.Top); - } - - void Fill(NKikimrSysView::TTopQueryStats* bucket, TQueryTextsByHash& texts) const { - for (auto it = Top.rbegin(); it != Top.rend(); ++it) { - auto hashIt = Stats.find(it->second); - Y_VERIFY(hashIt != Stats.end()); - - auto* dstStats = bucket->AddStats(); - auto& srcStats = *hashIt->second; - CopyQueryStatsNoText(dstStats, srcStats); - texts.try_emplace(srcStats.GetQueryTextHash(), srcStats.GetQueryText()); - } - } - - void FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const { - for (auto it = Top.rbegin(); it != Top.rend(); ++it) { - queries.AddValues(it->first); - queries.AddHashes(it->second); - } - } - - void FillStats(const NProtoBuf::RepeatedField<ui64>& hashes, - NProtoBuf::RepeatedPtrField<NKikimrSysView::TQueryStats>& stats) const - { - for (auto queryHash : hashes) { - auto it = Stats.find(queryHash); - if (it == Stats.end()) { - continue; - } - stats.Add()->CopyFrom(*it->second); - } - } -}; - -template <typename TGreater> -struct TAggrQueryStatsBucket -{ - NKikimrSysView::TTopQueryStats Top; - - void Merge(NKikimrSysView::TTopQueryStats&& src, const TQueryTextsByHash& texts) { - auto& dst = Top; - NKikimrSysView::TTopQueryStats result; - - size_t dstIndex = 0; - size_t srcIndex = 0; - - for (size_t i = 0; i < TOP_QUERIES_COUNT; ++i) { - auto copySrc = [&result, &texts] (NKikimrSysView::TQueryStats* srcPtr) { - auto* stats = result.AddStats(); - stats->Swap(srcPtr); - if (auto it = texts.find(stats->GetQueryTextHash()); it != texts.end()) { - stats->SetQueryText(it->second); - } - }; - - if (dstIndex == dst.StatsSize()) { - if (srcIndex == src.StatsSize()) { - break; - } - copySrc(src.MutableStats(srcIndex++)); - } else { - if (srcIndex == src.StatsSize()) { - result.AddStats()->Swap(dst.MutableStats(dstIndex++)); - } else { - auto* dstPtr = dst.MutableStats(dstIndex); - auto* srcPtr = src.MutableStats(srcIndex); - if (TGreater()(dstPtr, srcPtr)) { - result.AddStats()->Swap(dstPtr); - ++dstIndex; - } else { - copySrc(srcPtr); - ++srcIndex; - } - } - } - } - - Top.Swap(&result); - } - - void Clear() { - Top.Clear(); - } -}; - -template <typename TGreater> -struct TAggrQueryStatsDedupBucket -{ - NKikimrSysView::TTopQueryStats Top; - - void Merge(NKikimrSysView::TTopQueryStats&& src, const TQueryTextsByHash& texts) { - auto& dst = Top; - NKikimrSysView::TTopQueryStats result; - - size_t dstIndex = 0; - size_t srcIndex = 0; - std::unordered_set<TQueryHash> seenHashes; - - while (result.StatsSize() < TOP_QUERIES_COUNT) { - auto copySrc = [&result, &texts] (NKikimrSysView::TQueryStats* srcPtr) { - auto* stats = result.AddStats(); - stats->Swap(srcPtr); - if (auto it = texts.find(stats->GetQueryTextHash()); it != texts.end()) { - stats->SetQueryText(it->second); - } - }; - - if (dstIndex == dst.StatsSize()) { - if (srcIndex == src.StatsSize()) { - break; - } - auto* srcPtr = src.MutableStats(srcIndex++); - auto srcHash = srcPtr->GetQueryTextHash(); - if (seenHashes.find(srcHash) != seenHashes.end()) { - continue; - } - copySrc(srcPtr); - seenHashes.insert(srcHash); - } else { - auto* dstPtr = dst.MutableStats(dstIndex); - auto dstHash = dstPtr->GetQueryTextHash(); - if (seenHashes.find(dstHash) != seenHashes.end()) { - ++dstIndex; - continue; - } - if (srcIndex == src.StatsSize()) { - result.AddStats()->Swap(dstPtr); - seenHashes.insert(dstHash); - ++dstIndex; - continue; - } - auto* srcPtr = src.MutableStats(srcIndex); - auto srcHash = srcPtr->GetQueryTextHash(); - if (seenHashes.find(srcHash) != seenHashes.end()) { - ++srcIndex; - continue; - } - if (TGreater()(dstPtr, srcPtr)) { - result.AddStats()->Swap(dstPtr); - seenHashes.insert(dstHash); - ++dstIndex; - } else { - copySrc(srcPtr); - seenHashes.insert(srcHash); - ++srcIndex; - } - } - } - - Top.Swap(&result); - } - - void Clear() { - Top.Clear(); - } -}; - -template <typename TBucket> -class TQueryHistoryBase -{ -public: - TQueryHistoryBase(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now) - : BucketCount(bucketCount) - , BucketSizeMs(bucketSize.MilliSeconds()) - { - Y_VERIFY(bucketCount > 0); - Y_VERIFY(BucketSizeMs > 0); - - Buckets.resize(bucketCount); - LastBucket = now.MilliSeconds() / BucketSizeMs; - } - - void Clear() { - for (auto& bucket : Buckets) { - bucket.Clear(); - } - } - - ui64 GetBucketCount() const { - return BucketCount; - } - - ui64 GetBucketSizeMs() const { - return BucketSizeMs; - } - - ui64 GetStartBucket() const { - return LastBucket > BucketCount - 1 ? LastBucket - BucketCount + 1 : 0; - } - - ui64 GetLastBucket() const { - return LastBucket; - } - - void RefreshBuckets(ui64 bucket) { - if (bucket > LastBucket) { - auto toClear = std::min(bucket, LastBucket + BucketCount); - for (auto b = LastBucket + 1; b <= toClear; ++b) { - Buckets[b % BucketCount].Clear(); - } - LastBucket = bucket; - } - } - -protected: - const ui64 BucketCount; - const ui64 BucketSizeMs; - - TVector<TBucket> Buckets; - ui64 LastBucket = 0; -}; - -template <typename TGreater> -class TServiceQueryHistory : public TQueryHistoryBase<TQueryStatsDedupBucket<TGreater>> -{ - using TBase = TQueryHistoryBase<TQueryStatsDedupBucket<TGreater>>; - -public: - TServiceQueryHistory(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now) - : TBase(bucketCount, bucketSize, now) - {} - - void Add(TQueryStatsPtr stats) { - ui64 timestampMs = stats->GetEndTimeMs(); - auto bucket = timestampMs / this->BucketSizeMs; - if (bucket < this->GetStartBucket()) { - return; // too old - } - this->RefreshBuckets(bucket); - this->Buckets[bucket % this->BucketCount].Add(stats); - } - - void ToProto(ui64 startBucket, NKikimrSysView::TEvGetQueryStatsResult& message) const { - ui64 fromBucket = std::max(startBucket, this->GetStartBucket()); - - TQueryTextsByHash texts; - - for (ui64 b = fromBucket; b <= this->LastBucket; ++b) { - const auto& bucket = this->Buckets[b % this->BucketCount]; - bucket.Fill(message.AddBuckets(), texts); - } - - for (const auto& [hash, text] : texts) { - auto* hashText = message.AddQueryTexts(); - hashText->SetHash(hash); - hashText->SetText(text); - } - - message.SetLastBucket(this->LastBucket); - message.SetBucketSizeMs(this->BucketSizeMs); - } -}; - -template <typename TGreater> -class TScanQueryHistory : public TQueryHistoryBase<TAggrQueryStatsDedupBucket<TGreater>> -{ - using TBase = TQueryHistoryBase<TAggrQueryStatsDedupBucket<TGreater>>; - -public: - TScanQueryHistory(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now) - : TBase(bucketCount, bucketSize, now) - {} - - void Merge(NKikimrSysView::TEvGetQueryStatsResult&& message) { - if (!message.HasLastBucket() || !message.HasBucketSizeMs()) { - return; - } - if (message.GetBucketSizeMs() != this->BucketSizeMs) { - return; - } - - TQueryTextsByHash texts; - for (const auto& hashedText : message.GetQueryTexts()) { - texts.emplace(std::make_pair(hashedText.GetHash(), hashedText.GetText())); - } - - ui64 lastBucket = message.GetLastBucket(); - this->RefreshBuckets(lastBucket); - - Y_VERIFY(lastBucket + 1 >= message.BucketsSize()); - - ui64 startBucket = lastBucket + 1 - message.BucketsSize(); - ui64 thisStartBucket = this->GetStartBucket(); - - ui64 shift = thisStartBucket > startBucket ? thisStartBucket - startBucket : 0; - for (ui64 b = shift; b < message.BucketsSize(); ++b) { - auto bucket = b + startBucket; - this->Buckets[bucket % this->BucketCount].Merge( - std::move(*message.MutableBuckets(b)), texts); - } - } - - ui64 GetStartBucketIntervalEndMs() const { - return (this->GetStartBucket() + 1) * this->BucketSizeMs; - } - - const NKikimrSysView::TTopQueryStats& GetBucketTop(ui64 bucket) const { - Y_VERIFY(bucket >= this->GetStartBucket()); - Y_VERIFY(bucket <= this->GetLastBucket()); - return this->Buckets[bucket % this->BucketCount].Top; - } -}; - -} // NSysView -} // NKikimr + +#include <util/generic/vector.h> +#include <util/datetime/base.h> + +namespace NKikimr { +namespace NSysView { + +// TODO: configs +constexpr TDuration ONE_MINUTE_BUCKET_SIZE = TDuration::Minutes(1); +constexpr ui64 ONE_MINUTE_BUCKET_COUNT = 6 * 60; // 6 hours + +constexpr TDuration ONE_HOUR_BUCKET_SIZE = TDuration::Hours(1); +constexpr ui64 ONE_HOUR_BUCKET_COUNT = 24 * 7 * 2; // 2 weeks + +constexpr size_t TOP_QUERIES_COUNT = 5; + +using TNodeId = ui32; +using TQueryHash = ui64; + +using TQueryStatsPtr = std::shared_ptr<NKikimrSysView::TQueryStats>; + +using TQueryTextsByHash = std::unordered_map<TQueryHash, TString>; + +struct TReadBytesGreater { + using TValue = ui64; + + static TValue GetValue(const TQueryStatsPtr stats) { + return stats->GetStats().GetReadBytes(); + } + bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const { + return a->GetStats().GetReadBytes() == b->GetStats().GetReadBytes() + ? a->GetNodeId() > b->GetNodeId() + : a->GetStats().GetReadBytes() > b->GetStats().GetReadBytes(); + } + bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const { + return operator()(a.get(), b.get()); + } +}; + +struct TDurationGreater { + using TValue = ui64; + + static TValue GetValue(const TQueryStatsPtr stats) { + return stats->GetDurationMs(); + } + bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const { + return a->GetDurationMs() == b->GetDurationMs() + ? a->GetNodeId() > b->GetNodeId() + : a->GetDurationMs() > b->GetDurationMs(); + } + bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const { + return operator()(a.get(), b.get()); + } +}; + +struct TCpuTimeGreater { + using TValue = ui64; + + static TValue GetValue(const TQueryStatsPtr stats) { + return stats->GetTotalCpuTimeUs(); + } + bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const { + return a->GetTotalCpuTimeUs() == b->GetTotalCpuTimeUs() + ? a->GetNodeId() > b->GetNodeId() + : a->GetTotalCpuTimeUs() > b->GetTotalCpuTimeUs(); + } + bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const { + return operator()(a.get(), b.get()); + } +}; + +struct TRequestUnitsGreater { + using TValue = ui64; + + static TValue GetValue(const TQueryStatsPtr stats) { + return stats->GetRequestUnits(); + } + bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const { + return a->GetRequestUnits() == b->GetRequestUnits() + ? a->GetNodeId() > b->GetNodeId() + : a->GetRequestUnits() > b->GetRequestUnits(); + } + bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const { + return operator()(a.get(), b.get()); + } +}; + +inline void CopyQueryStatsNoText(NKikimrSysView::TQueryStats* to, const NKikimrSysView::TQueryStats& from) { + to->SetQueryTextHash(from.GetQueryTextHash()); + to->SetDurationMs(from.GetDurationMs()); + to->SetEndTimeMs(from.GetEndTimeMs()); + to->MutableStats()->CopyFrom(from.GetStats()); + if (from.HasUserSID()) { + to->SetUserSID(from.GetUserSID()); + } + to->SetParametersSize(from.GetParametersSize()); + if (from.HasCompileDurationMs()) { + to->SetCompileDurationMs(from.GetCompileDurationMs()); + } + if (from.HasFromQueryCache()) { + to->SetFromQueryCache(from.GetFromQueryCache()); + } + to->SetNodeId(from.GetNodeId()); + to->MutableShardsCpuTimeUs()->CopyFrom(from.GetShardsCpuTimeUs()); + to->MutableComputeCpuTimeUs()->CopyFrom(from.GetComputeCpuTimeUs()); + if (from.HasCompileCpuTimeUs()) { + to->SetCompileCpuTimeUs(from.GetCompileCpuTimeUs()); + } + to->SetProcessCpuTimeUs(from.GetProcessCpuTimeUs()); + to->SetTotalCpuTimeUs(from.GetTotalCpuTimeUs()); + to->SetType(from.GetType()); + to->SetRequestUnits(from.GetRequestUnits()); +} + +template <typename TGreater> +struct TQueryStatsBucket { + TVector<TQueryStatsPtr> Top; + + void Add(TQueryStatsPtr stats) { + auto it = std::lower_bound(Top.begin(), Top.end(), stats, TGreater()); + if (Top.size() == TOP_QUERIES_COUNT && it == Top.end()) { + return; + } + Top.insert(it, stats); + if (Top.size() > TOP_QUERIES_COUNT) { + Top.resize(TOP_QUERIES_COUNT); + } + } + + void Clear() { + Top.clear(); + } + + void Fill(NKikimrSysView::TTopQueryStats* bucket, TQueryTextsByHash& texts) const { + for (auto& queryStats : Top) { + CopyQueryStatsNoText(bucket->AddStats(), *queryStats); + texts.try_emplace(queryStats->GetQueryTextHash(), queryStats->GetQueryText()); + } + } +}; + +template <typename TGreater> +struct TQueryStatsDedupBucket { + using TValue = typename TGreater::TValue; + + std::unordered_map<TQueryHash, TQueryStatsPtr> Stats; // query hash -> stats + std::multimap<TValue, TQueryHash> Top; // top by characteristic -> query hash + + void Add(TQueryStatsPtr stats) { + if (!stats->HasQueryTextHash()) { + return; + } + + auto queryHash = stats->GetQueryTextHash(); + auto hashIt = Stats.find(queryHash); + + if (hashIt != Stats.end()) { + if (!TGreater()(stats, hashIt->second)) { + return; + } + + TValue old = TGreater::GetValue(hashIt->second); + const auto range = Top.equal_range(old); + auto it = range.first; + for (; it != range.second; ++it) { + if (it->second == queryHash) { + break; + } + } + + Y_VERIFY(it != range.second); + Top.erase(it); + + TValue current = TGreater::GetValue(stats); + Top.emplace(std::make_pair(current, queryHash)); + + hashIt->second = stats; + + } else { + if (Top.size() == TOP_QUERIES_COUNT) { + auto it = Top.begin(); + TValue value = TGreater::GetValue(stats); + if (value <= it->first) { + return; + } + Stats.erase(it->second); + Top.erase(it); + } + + TValue current = TGreater::GetValue(stats); + Top.emplace(std::make_pair(current, queryHash)); + Stats.emplace(std::make_pair(queryHash, stats)); + } + } + + void Clear() { + Stats.clear(); + Top.clear(); + } + + void Swap(TQueryStatsDedupBucket& other) { + Stats.swap(other.Stats); + Top.swap(other.Top); + } + + void Fill(NKikimrSysView::TTopQueryStats* bucket, TQueryTextsByHash& texts) const { + for (auto it = Top.rbegin(); it != Top.rend(); ++it) { + auto hashIt = Stats.find(it->second); + Y_VERIFY(hashIt != Stats.end()); + + auto* dstStats = bucket->AddStats(); + auto& srcStats = *hashIt->second; + CopyQueryStatsNoText(dstStats, srcStats); + texts.try_emplace(srcStats.GetQueryTextHash(), srcStats.GetQueryText()); + } + } + + void FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const { + for (auto it = Top.rbegin(); it != Top.rend(); ++it) { + queries.AddValues(it->first); + queries.AddHashes(it->second); + } + } + + void FillStats(const NProtoBuf::RepeatedField<ui64>& hashes, + NProtoBuf::RepeatedPtrField<NKikimrSysView::TQueryStats>& stats) const + { + for (auto queryHash : hashes) { + auto it = Stats.find(queryHash); + if (it == Stats.end()) { + continue; + } + stats.Add()->CopyFrom(*it->second); + } + } +}; + +template <typename TGreater> +struct TAggrQueryStatsBucket +{ + NKikimrSysView::TTopQueryStats Top; + + void Merge(NKikimrSysView::TTopQueryStats&& src, const TQueryTextsByHash& texts) { + auto& dst = Top; + NKikimrSysView::TTopQueryStats result; + + size_t dstIndex = 0; + size_t srcIndex = 0; + + for (size_t i = 0; i < TOP_QUERIES_COUNT; ++i) { + auto copySrc = [&result, &texts] (NKikimrSysView::TQueryStats* srcPtr) { + auto* stats = result.AddStats(); + stats->Swap(srcPtr); + if (auto it = texts.find(stats->GetQueryTextHash()); it != texts.end()) { + stats->SetQueryText(it->second); + } + }; + + if (dstIndex == dst.StatsSize()) { + if (srcIndex == src.StatsSize()) { + break; + } + copySrc(src.MutableStats(srcIndex++)); + } else { + if (srcIndex == src.StatsSize()) { + result.AddStats()->Swap(dst.MutableStats(dstIndex++)); + } else { + auto* dstPtr = dst.MutableStats(dstIndex); + auto* srcPtr = src.MutableStats(srcIndex); + if (TGreater()(dstPtr, srcPtr)) { + result.AddStats()->Swap(dstPtr); + ++dstIndex; + } else { + copySrc(srcPtr); + ++srcIndex; + } + } + } + } + + Top.Swap(&result); + } + + void Clear() { + Top.Clear(); + } +}; + +template <typename TGreater> +struct TAggrQueryStatsDedupBucket +{ + NKikimrSysView::TTopQueryStats Top; + + void Merge(NKikimrSysView::TTopQueryStats&& src, const TQueryTextsByHash& texts) { + auto& dst = Top; + NKikimrSysView::TTopQueryStats result; + + size_t dstIndex = 0; + size_t srcIndex = 0; + std::unordered_set<TQueryHash> seenHashes; + + while (result.StatsSize() < TOP_QUERIES_COUNT) { + auto copySrc = [&result, &texts] (NKikimrSysView::TQueryStats* srcPtr) { + auto* stats = result.AddStats(); + stats->Swap(srcPtr); + if (auto it = texts.find(stats->GetQueryTextHash()); it != texts.end()) { + stats->SetQueryText(it->second); + } + }; + + if (dstIndex == dst.StatsSize()) { + if (srcIndex == src.StatsSize()) { + break; + } + auto* srcPtr = src.MutableStats(srcIndex++); + auto srcHash = srcPtr->GetQueryTextHash(); + if (seenHashes.find(srcHash) != seenHashes.end()) { + continue; + } + copySrc(srcPtr); + seenHashes.insert(srcHash); + } else { + auto* dstPtr = dst.MutableStats(dstIndex); + auto dstHash = dstPtr->GetQueryTextHash(); + if (seenHashes.find(dstHash) != seenHashes.end()) { + ++dstIndex; + continue; + } + if (srcIndex == src.StatsSize()) { + result.AddStats()->Swap(dstPtr); + seenHashes.insert(dstHash); + ++dstIndex; + continue; + } + auto* srcPtr = src.MutableStats(srcIndex); + auto srcHash = srcPtr->GetQueryTextHash(); + if (seenHashes.find(srcHash) != seenHashes.end()) { + ++srcIndex; + continue; + } + if (TGreater()(dstPtr, srcPtr)) { + result.AddStats()->Swap(dstPtr); + seenHashes.insert(dstHash); + ++dstIndex; + } else { + copySrc(srcPtr); + seenHashes.insert(srcHash); + ++srcIndex; + } + } + } + + Top.Swap(&result); + } + + void Clear() { + Top.Clear(); + } +}; + +template <typename TBucket> +class TQueryHistoryBase +{ +public: + TQueryHistoryBase(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now) + : BucketCount(bucketCount) + , BucketSizeMs(bucketSize.MilliSeconds()) + { + Y_VERIFY(bucketCount > 0); + Y_VERIFY(BucketSizeMs > 0); + + Buckets.resize(bucketCount); + LastBucket = now.MilliSeconds() / BucketSizeMs; + } + + void Clear() { + for (auto& bucket : Buckets) { + bucket.Clear(); + } + } + + ui64 GetBucketCount() const { + return BucketCount; + } + + ui64 GetBucketSizeMs() const { + return BucketSizeMs; + } + + ui64 GetStartBucket() const { + return LastBucket > BucketCount - 1 ? LastBucket - BucketCount + 1 : 0; + } + + ui64 GetLastBucket() const { + return LastBucket; + } + + void RefreshBuckets(ui64 bucket) { + if (bucket > LastBucket) { + auto toClear = std::min(bucket, LastBucket + BucketCount); + for (auto b = LastBucket + 1; b <= toClear; ++b) { + Buckets[b % BucketCount].Clear(); + } + LastBucket = bucket; + } + } + +protected: + const ui64 BucketCount; + const ui64 BucketSizeMs; + + TVector<TBucket> Buckets; + ui64 LastBucket = 0; +}; + +template <typename TGreater> +class TServiceQueryHistory : public TQueryHistoryBase<TQueryStatsDedupBucket<TGreater>> +{ + using TBase = TQueryHistoryBase<TQueryStatsDedupBucket<TGreater>>; + +public: + TServiceQueryHistory(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now) + : TBase(bucketCount, bucketSize, now) + {} + + void Add(TQueryStatsPtr stats) { + ui64 timestampMs = stats->GetEndTimeMs(); + auto bucket = timestampMs / this->BucketSizeMs; + if (bucket < this->GetStartBucket()) { + return; // too old + } + this->RefreshBuckets(bucket); + this->Buckets[bucket % this->BucketCount].Add(stats); + } + + void ToProto(ui64 startBucket, NKikimrSysView::TEvGetQueryStatsResult& message) const { + ui64 fromBucket = std::max(startBucket, this->GetStartBucket()); + + TQueryTextsByHash texts; + + for (ui64 b = fromBucket; b <= this->LastBucket; ++b) { + const auto& bucket = this->Buckets[b % this->BucketCount]; + bucket.Fill(message.AddBuckets(), texts); + } + + for (const auto& [hash, text] : texts) { + auto* hashText = message.AddQueryTexts(); + hashText->SetHash(hash); + hashText->SetText(text); + } + + message.SetLastBucket(this->LastBucket); + message.SetBucketSizeMs(this->BucketSizeMs); + } +}; + +template <typename TGreater> +class TScanQueryHistory : public TQueryHistoryBase<TAggrQueryStatsDedupBucket<TGreater>> +{ + using TBase = TQueryHistoryBase<TAggrQueryStatsDedupBucket<TGreater>>; + +public: + TScanQueryHistory(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now) + : TBase(bucketCount, bucketSize, now) + {} + + void Merge(NKikimrSysView::TEvGetQueryStatsResult&& message) { + if (!message.HasLastBucket() || !message.HasBucketSizeMs()) { + return; + } + if (message.GetBucketSizeMs() != this->BucketSizeMs) { + return; + } + + TQueryTextsByHash texts; + for (const auto& hashedText : message.GetQueryTexts()) { + texts.emplace(std::make_pair(hashedText.GetHash(), hashedText.GetText())); + } + + ui64 lastBucket = message.GetLastBucket(); + this->RefreshBuckets(lastBucket); + + Y_VERIFY(lastBucket + 1 >= message.BucketsSize()); + + ui64 startBucket = lastBucket + 1 - message.BucketsSize(); + ui64 thisStartBucket = this->GetStartBucket(); + + ui64 shift = thisStartBucket > startBucket ? thisStartBucket - startBucket : 0; + for (ui64 b = shift; b < message.BucketsSize(); ++b) { + auto bucket = b + startBucket; + this->Buckets[bucket % this->BucketCount].Merge( + std::move(*message.MutableBuckets(b)), texts); + } + } + + ui64 GetStartBucketIntervalEndMs() const { + return (this->GetStartBucket() + 1) * this->BucketSizeMs; + } + + const NKikimrSysView::TTopQueryStats& GetBucketTop(ui64 bucket) const { + Y_VERIFY(bucket >= this->GetStartBucket()); + Y_VERIFY(bucket <= this->GetLastBucket()); + return this->Buckets[bucket % this->BucketCount].Top; + } +}; + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/service/query_history_ut.cpp b/ydb/core/sys_view/service/query_history_ut.cpp index edbbd9007b..9c64e9a4f3 100644 --- a/ydb/core/sys_view/service/query_history_ut.cpp +++ b/ydb/core/sys_view/service/query_history_ut.cpp @@ -1,476 +1,476 @@ -#include "query_history.h" - +#include "query_history.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/xrange.h> -#include <util/digest/murmur.h> -#include <util/random/random.h> - -namespace NKikimr { -namespace NSysView { - -namespace { - -auto MakeStatsPtr() { - return std::make_shared<NKikimrSysView::TQueryStats>(); -} - -ui64 QueryHash(const TString& queryText) { - return MurmurHash<ui64>(queryText.data(), queryText.size()); -} - -void FillStatsWithNodeId(TQueryTextsByHash& texts, NKikimrSysView::TQueryStats* message, ui32 nodeId, - const TString& text, ui64 duration) -{ - message->SetNodeId(nodeId); - - auto hash = MurmurHash<ui64>(text.data(), text.size()); - texts[hash] = text; - message->SetQueryText(text); - message->SetQueryTextHash(hash); - - message->SetDurationMs(duration); -} - -void FillStats(TQueryTextsByHash& texts, NKikimrSysView::TQueryStats* message, - const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0) -{ - message->SetNodeId(1); - - auto hash = MurmurHash<ui64>(text.data(), text.size()); - texts[hash] = text; - message->SetQueryText(text); - message->SetQueryTextHash(hash); - - message->SetDurationMs(duration); - message->SetEndTimeMs(endTime); - - auto* stats = message->MutableStats(); - stats->SetReadBytes(readBytes); -} - -void FillStats(TQueryTextsByHash& texts, TQueryStatsPtr messagePtr, - const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0) -{ - FillStats(texts, messagePtr.get(), text, duration, endTime, readBytes); -} - -template <typename TBucket> -void AddStatsToBucket(TQueryTextsByHash& texts, TBucket& bucket, - const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0) -{ - auto stats = MakeStatsPtr(); - FillStats(texts, stats.get(), text, duration, endTime, readBytes); - bucket.Add(stats); -} - -using TResult = NKikimrSysView::TEvGetQueryStatsResult; -using THistory = TScanQueryHistory<TDurationGreater>; - -void FillStatsResult(TQueryTextsByHash& texts, TResult* message, ui64 bucketSizeMs, const TString& text, - ui64 lastBucket, const std::initializer_list<ui64>& bucketsStatsCount) -{ - message->SetLastBucket(lastBucket); - message->SetBucketSizeMs(bucketSizeMs); - - auto bucketIndex = lastBucket - bucketsStatsCount.size() + 1; - - for (auto count : bucketsStatsCount) { - auto* bucket = message->AddBuckets(); - for (size_t i : xrange(count)) { - auto* stats = bucket->AddStats(); - FillStats(texts, stats, text + ToString(i), i, bucketIndex * bucketSizeMs); - } - ++bucketIndex; - } -} - -void ValidateResultBuckets(const THistory& history, - ui64 lastBucket, const std::initializer_list<ui64>& bucketsStatsCount) -{ - UNIT_ASSERT_EQUAL(history.GetLastBucket(), lastBucket); - UNIT_ASSERT_EQUAL(history.GetBucketCount(), bucketsStatsCount.size()); - - size_t i = history.GetStartBucket(); - for (auto count : bucketsStatsCount) { - UNIT_ASSERT_EQUAL(history.GetBucketTop(i).StatsSize(), count); - ++i; - } -} - -void TestMerge( - ui64 lastBucket1, const std::initializer_list<ui64>& bucketsStatsCount1, - ui64 lastBucket2, const std::initializer_list<ui64>& bucketsStatsCount2, - ui64 lastBucketHistory, ui64 historySize, - ui64 lastBucketResult, const std::initializer_list<ui64>& bucketsStatsCountResult) -{ - constexpr ui64 bucketSizeMs = 10; - - TQueryTextsByHash texts; - - TResult r1; - FillStatsResult(texts, &r1, bucketSizeMs, "a", lastBucket1, bucketsStatsCount1); - - TResult r2; - FillStatsResult(texts, &r2, bucketSizeMs, "b", lastBucket2, bucketsStatsCount2); - - THistory history(historySize, TDuration::MilliSeconds(bucketSizeMs), - TInstant::MilliSeconds(lastBucketHistory * bucketSizeMs)); - - history.Merge(std::move(r1)); - history.Merge(std::move(r2)); - - ValidateResultBuckets(history, lastBucketResult, bucketsStatsCountResult); -} - -} // namespace - -Y_UNIT_TEST_SUITE(SysViewQueryHistory) { - - Y_UNIT_TEST(TopDurationAdd) { - TQueryStatsBucket<TDurationGreater> bucket; - - TQueryTextsByHash texts; - - AddStatsToBucket(texts, bucket, "a", 10); - AddStatsToBucket(texts, bucket, "b", 20); - AddStatsToBucket(texts, bucket, "c", 15); - - NKikimrSysView::TTopQueryStats proto; - bucket.Fill(&proto, texts); - - UNIT_ASSERT_EQUAL(proto.StatsSize(), 3); - - UNIT_ASSERT_EQUAL(proto.GetStats(0).GetDurationMs(), 20); - UNIT_ASSERT_EQUAL(proto.GetStats(1).GetDurationMs(), 15); - UNIT_ASSERT_EQUAL(proto.GetStats(2).GetDurationMs(), 10); - } - - Y_UNIT_TEST(TopReadBytesAdd) { - TQueryStatsBucket<TReadBytesGreater> bucket; - - TQueryTextsByHash texts; - - AddStatsToBucket(texts, bucket, "a", 10, 0, 7); - AddStatsToBucket(texts, bucket, "b", 20, 0, 20); - AddStatsToBucket(texts, bucket, "c", 15, 0, 100); - - NKikimrSysView::TTopQueryStats proto; - bucket.Fill(&proto, texts); - - UNIT_ASSERT_EQUAL(proto.StatsSize(), 3); - - UNIT_ASSERT_EQUAL(proto.GetStats(0).GetStats().GetReadBytes(), 100); - UNIT_ASSERT_EQUAL(proto.GetStats(1).GetStats().GetReadBytes(), 20); - UNIT_ASSERT_EQUAL(proto.GetStats(2).GetStats().GetReadBytes(), 7); - } - - Y_UNIT_TEST(AddDedup) { - TQueryStatsDedupBucket<TDurationGreater> bucket; - - TQueryTextsByHash texts; - - AddStatsToBucket(texts, bucket, "a", 10); - AddStatsToBucket(texts, bucket, "b", 20); - AddStatsToBucket(texts, bucket, "c", 15); - AddStatsToBucket(texts, bucket, "b", 5); - AddStatsToBucket(texts, bucket, "c", 25); - - NKikimrSysView::TTopQueryStats proto; - bucket.Fill(&proto, texts); - - UNIT_ASSERT_EQUAL(proto.StatsSize(), 3); - - UNIT_ASSERT_EQUAL(proto.GetStats(0).GetDurationMs(), 25); - UNIT_ASSERT_EQUAL(proto.GetStats(0).GetQueryTextHash(), QueryHash("c")); - UNIT_ASSERT_EQUAL(proto.GetStats(1).GetDurationMs(), 20); - UNIT_ASSERT_EQUAL(proto.GetStats(1).GetQueryTextHash(), QueryHash("b")); - UNIT_ASSERT_EQUAL(proto.GetStats(2).GetDurationMs(), 10); - UNIT_ASSERT_EQUAL(proto.GetStats(2).GetQueryTextHash(), QueryHash("a")); - } - - Y_UNIT_TEST(AddDedup2) { - TQueryStatsDedupBucket<TDurationGreater> bucket; - - TQueryTextsByHash texts; - - for (size_t i : xrange(TOP_QUERIES_COUNT)) { - AddStatsToBucket(texts, bucket, ToString(i), (i + 1) * 10); - } - AddStatsToBucket(texts, bucket, "a", 5); - AddStatsToBucket(texts, bucket, "b", 15); - - NKikimrSysView::TTopQueryStats proto; - bucket.Fill(&proto, texts); - - UNIT_ASSERT_EQUAL(proto.StatsSize(), TOP_QUERIES_COUNT); - auto last = TOP_QUERIES_COUNT - 1; - - UNIT_ASSERT_EQUAL(proto.GetStats(last).GetDurationMs(), 15); - UNIT_ASSERT_EQUAL(proto.GetStats(last).GetQueryTextHash(), QueryHash("b")); - } - - Y_UNIT_TEST(AddDedupRandom) { - TQueryStatsDedupBucket<TDurationGreater> bucket; - - TQueryTextsByHash texts; - - for (size_t i : xrange(10000)) { - Y_UNUSED(i); - AddStatsToBucket(texts, bucket, ToString(RandomNumber<ui32>(1000)), RandomNumber<ui64>(5000)); - } - - UNIT_ASSERT_EQUAL(bucket.Top.size(), TOP_QUERIES_COUNT); - UNIT_ASSERT_EQUAL(bucket.Stats.size(), TOP_QUERIES_COUNT); - - for (auto& top : bucket.Top) { - auto it = bucket.Stats.find(top.second); - UNIT_ASSERT(it != bucket.Stats.end()); - UNIT_ASSERT_EQUAL(it->second->GetDurationMs(), top.first); - } - } - - Y_UNIT_TEST(AggrMerge) { - TAggrQueryStatsBucket<TDurationGreater> bucket; - - TQueryTextsByHash texts; - - NKikimrSysView::TTopQueryStats s1; - FillStats(texts, s1.AddStats(), "a", 30); - FillStats(texts, s1.AddStats(), "b", 20); - FillStats(texts, s1.AddStats(), "c", 10); - - NKikimrSysView::TTopQueryStats s2; - FillStats(texts, s2.AddStats(), "x", 35); - FillStats(texts, s2.AddStats(), "y", 25); - FillStats(texts, s2.AddStats(), "z", 22); - - bucket.Merge(std::move(s1), texts); - bucket.Merge(std::move(s2), texts); - - UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 5); - - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetDurationMs(), 35); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetDurationMs(), 30); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetDurationMs(), 25); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetDurationMs(), 22); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetDurationMs(), 20); - } - - Y_UNIT_TEST(AggrMergeDedup) { - TAggrQueryStatsDedupBucket<TDurationGreater> bucket; - - TQueryTextsByHash texts; - - NKikimrSysView::TTopQueryStats s1; - FillStats(texts, s1.AddStats(), "a", 30); - FillStats(texts, s1.AddStats(), "b", 20); - FillStats(texts, s1.AddStats(), "c", 10); - FillStats(texts, s1.AddStats(), "d", 5); - - NKikimrSysView::TTopQueryStats s2; - FillStats(texts, s2.AddStats(), "c", 35); - FillStats(texts, s2.AddStats(), "a", 25); - FillStats(texts, s2.AddStats(), "b", 25); - FillStats(texts, s2.AddStats(), "e", 15); - - bucket.Merge(std::move(s1), texts); - bucket.Merge(std::move(s2), texts); - - UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 5); - - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetDurationMs(), 35); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "c"); - - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetDurationMs(), 30); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "a"); - - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetDurationMs(), 25); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "b"); - - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetDurationMs(), 15); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "e"); - - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetDurationMs(), 5); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetQueryText(), "d"); - } - - Y_UNIT_TEST(StableMerge) { - TAggrQueryStatsDedupBucket<TDurationGreater> bucket; - - TQueryTextsByHash texts; - - NKikimrSysView::TTopQueryStats s1; - FillStatsWithNodeId(texts, s1.AddStats(), 1, "a", 10); - FillStatsWithNodeId(texts, s1.AddStats(), 1, "b", 10); - - NKikimrSysView::TTopQueryStats s2; - FillStatsWithNodeId(texts, s2.AddStats(), 2, "x", 10); - FillStatsWithNodeId(texts, s2.AddStats(), 2, "y", 10); - - bucket.Merge(std::move(s1), texts); - - bucket.Merge(std::move(s2), texts); - - UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 4); - - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "x"); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "y"); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "a"); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "b"); - } - - Y_UNIT_TEST(StableMerge2) { - TAggrQueryStatsDedupBucket<TDurationGreater> bucket; - - TQueryTextsByHash texts; - - NKikimrSysView::TTopQueryStats s1; - FillStatsWithNodeId(texts, s1.AddStats(), 1, "a", 10); - FillStatsWithNodeId(texts, s1.AddStats(), 1, "b", 10); - - NKikimrSysView::TTopQueryStats s2; - FillStatsWithNodeId(texts, s2.AddStats(), 2, "x", 10); - FillStatsWithNodeId(texts, s2.AddStats(), 2, "y", 10); - - bucket.Merge(std::move(s2), texts); - - bucket.Merge(std::move(s1), texts); - - UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 4); - - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "x"); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "y"); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "a"); - UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "b"); - } - - Y_UNIT_TEST(ServiceQueryHistoryAdd) { - TServiceQueryHistory<TDurationGreater> history(5, TDuration::MilliSeconds(10), TInstant::MilliSeconds(75)); - - TQueryTextsByHash texts; - - auto s1 = MakeStatsPtr(); - FillStats(texts, s1, "a", 1, 25); - history.Add(s1); - - auto s2 = MakeStatsPtr(); - FillStats(texts, s2, "b", 5, 35); - history.Add(s2); - - auto s3 = MakeStatsPtr(); - FillStats(texts, s3, "c", 1, 50); - history.Add(s3); - - auto s4 = MakeStatsPtr(); - FillStats(texts, s4, "d", 10, 51); - history.Add(s4); - - NKikimrSysView::TEvGetQueryStatsResult result; - history.ToProto(3, result); - - UNIT_ASSERT_EQUAL(result.GetLastBucket(), 7); - UNIT_ASSERT_EQUAL(result.GetBucketSizeMs(), 10); - - UNIT_ASSERT_EQUAL(result.BucketsSize(), 5); - - UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 1); - UNIT_ASSERT_EQUAL(result.GetBuckets(0).GetStats(0).GetEndTimeMs(), 35); - UNIT_ASSERT_EQUAL(result.GetBuckets(0).GetStats(0).GetDurationMs(), 5); - - UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0); - - UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 2); - UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetEndTimeMs(), 51); - UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetDurationMs(), 10); - UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(1).GetEndTimeMs(), 50); - UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(1).GetDurationMs(), 1); - - UNIT_ASSERT_EQUAL(result.GetBuckets(3).StatsSize(), 0); - UNIT_ASSERT_EQUAL(result.GetBuckets(4).StatsSize(), 0); - - UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 3); - - auto s5 = MakeStatsPtr(); - FillStats(texts, s5, "e", 5, 93); - history.Add(s5); - - result.Clear(); - history.ToProto(5, result); - - UNIT_ASSERT_EQUAL(result.GetLastBucket(), 9); - - UNIT_ASSERT_EQUAL(result.BucketsSize(), 5); - - UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 2); - UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0); - UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 0); - UNIT_ASSERT_EQUAL(result.GetBuckets(3).StatsSize(), 0); - UNIT_ASSERT_EQUAL(result.GetBuckets(4).StatsSize(), 1); - - UNIT_ASSERT_EQUAL(result.GetBuckets(4).GetStats(0).GetQueryTextHash(), QueryHash("e")); - - UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 3); - - result.Clear(); - history.ToProto(7, result); - - UNIT_ASSERT_EQUAL(result.GetLastBucket(), 9); - UNIT_ASSERT_EQUAL(result.BucketsSize(), 3); - - UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 0); - UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0); - UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 1); - - UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetQueryTextHash(), QueryHash("e")); - - UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 1); - } - - Y_UNIT_TEST(ScanQueryHistoryMerge) { - TestMerge( - 100, {1, 2, 3}, - 100, {1, 2}, - 100, 3, - 100, {1, 3, 5}); - - TestMerge( - 100, {1, 2, 3}, - 99, {1, 2}, - 100, 3, - 100, {2, 4, 3}); - - TestMerge( - 100, {1, 2, 3}, - 98, {1, 2}, - 100, 3, - 100, {3, 2, 3}); - - TestMerge( - 100, {1, 2, 3}, - 100, {1, 2}, - 90, 5, - 100, {0, 0, 1, 3, 5}); - - TestMerge( - 100, {1, 2, 3, 4}, - 99, {1, 2}, - 101, 5, - 101, {1, 3, 5, 4, 0}); - - TestMerge( - 100, {1, 2, 3}, - 102, {1, 2, 3, 4}, - 90, 5, - 102, {1, 3, 5, 3, 4}); - - TestMerge( - 100, {1, 2, 3}, - 101, {2, 1, 2, 3}, - 90, 2, - 101, {5, 3}); - } -} - -} // NSysView -} // NKikimr + +#include <util/generic/xrange.h> +#include <util/digest/murmur.h> +#include <util/random/random.h> + +namespace NKikimr { +namespace NSysView { + +namespace { + +auto MakeStatsPtr() { + return std::make_shared<NKikimrSysView::TQueryStats>(); +} + +ui64 QueryHash(const TString& queryText) { + return MurmurHash<ui64>(queryText.data(), queryText.size()); +} + +void FillStatsWithNodeId(TQueryTextsByHash& texts, NKikimrSysView::TQueryStats* message, ui32 nodeId, + const TString& text, ui64 duration) +{ + message->SetNodeId(nodeId); + + auto hash = MurmurHash<ui64>(text.data(), text.size()); + texts[hash] = text; + message->SetQueryText(text); + message->SetQueryTextHash(hash); + + message->SetDurationMs(duration); +} + +void FillStats(TQueryTextsByHash& texts, NKikimrSysView::TQueryStats* message, + const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0) +{ + message->SetNodeId(1); + + auto hash = MurmurHash<ui64>(text.data(), text.size()); + texts[hash] = text; + message->SetQueryText(text); + message->SetQueryTextHash(hash); + + message->SetDurationMs(duration); + message->SetEndTimeMs(endTime); + + auto* stats = message->MutableStats(); + stats->SetReadBytes(readBytes); +} + +void FillStats(TQueryTextsByHash& texts, TQueryStatsPtr messagePtr, + const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0) +{ + FillStats(texts, messagePtr.get(), text, duration, endTime, readBytes); +} + +template <typename TBucket> +void AddStatsToBucket(TQueryTextsByHash& texts, TBucket& bucket, + const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0) +{ + auto stats = MakeStatsPtr(); + FillStats(texts, stats.get(), text, duration, endTime, readBytes); + bucket.Add(stats); +} + +using TResult = NKikimrSysView::TEvGetQueryStatsResult; +using THistory = TScanQueryHistory<TDurationGreater>; + +void FillStatsResult(TQueryTextsByHash& texts, TResult* message, ui64 bucketSizeMs, const TString& text, + ui64 lastBucket, const std::initializer_list<ui64>& bucketsStatsCount) +{ + message->SetLastBucket(lastBucket); + message->SetBucketSizeMs(bucketSizeMs); + + auto bucketIndex = lastBucket - bucketsStatsCount.size() + 1; + + for (auto count : bucketsStatsCount) { + auto* bucket = message->AddBuckets(); + for (size_t i : xrange(count)) { + auto* stats = bucket->AddStats(); + FillStats(texts, stats, text + ToString(i), i, bucketIndex * bucketSizeMs); + } + ++bucketIndex; + } +} + +void ValidateResultBuckets(const THistory& history, + ui64 lastBucket, const std::initializer_list<ui64>& bucketsStatsCount) +{ + UNIT_ASSERT_EQUAL(history.GetLastBucket(), lastBucket); + UNIT_ASSERT_EQUAL(history.GetBucketCount(), bucketsStatsCount.size()); + + size_t i = history.GetStartBucket(); + for (auto count : bucketsStatsCount) { + UNIT_ASSERT_EQUAL(history.GetBucketTop(i).StatsSize(), count); + ++i; + } +} + +void TestMerge( + ui64 lastBucket1, const std::initializer_list<ui64>& bucketsStatsCount1, + ui64 lastBucket2, const std::initializer_list<ui64>& bucketsStatsCount2, + ui64 lastBucketHistory, ui64 historySize, + ui64 lastBucketResult, const std::initializer_list<ui64>& bucketsStatsCountResult) +{ + constexpr ui64 bucketSizeMs = 10; + + TQueryTextsByHash texts; + + TResult r1; + FillStatsResult(texts, &r1, bucketSizeMs, "a", lastBucket1, bucketsStatsCount1); + + TResult r2; + FillStatsResult(texts, &r2, bucketSizeMs, "b", lastBucket2, bucketsStatsCount2); + + THistory history(historySize, TDuration::MilliSeconds(bucketSizeMs), + TInstant::MilliSeconds(lastBucketHistory * bucketSizeMs)); + + history.Merge(std::move(r1)); + history.Merge(std::move(r2)); + + ValidateResultBuckets(history, lastBucketResult, bucketsStatsCountResult); +} + +} // namespace + +Y_UNIT_TEST_SUITE(SysViewQueryHistory) { + + Y_UNIT_TEST(TopDurationAdd) { + TQueryStatsBucket<TDurationGreater> bucket; + + TQueryTextsByHash texts; + + AddStatsToBucket(texts, bucket, "a", 10); + AddStatsToBucket(texts, bucket, "b", 20); + AddStatsToBucket(texts, bucket, "c", 15); + + NKikimrSysView::TTopQueryStats proto; + bucket.Fill(&proto, texts); + + UNIT_ASSERT_EQUAL(proto.StatsSize(), 3); + + UNIT_ASSERT_EQUAL(proto.GetStats(0).GetDurationMs(), 20); + UNIT_ASSERT_EQUAL(proto.GetStats(1).GetDurationMs(), 15); + UNIT_ASSERT_EQUAL(proto.GetStats(2).GetDurationMs(), 10); + } + + Y_UNIT_TEST(TopReadBytesAdd) { + TQueryStatsBucket<TReadBytesGreater> bucket; + + TQueryTextsByHash texts; + + AddStatsToBucket(texts, bucket, "a", 10, 0, 7); + AddStatsToBucket(texts, bucket, "b", 20, 0, 20); + AddStatsToBucket(texts, bucket, "c", 15, 0, 100); + + NKikimrSysView::TTopQueryStats proto; + bucket.Fill(&proto, texts); + + UNIT_ASSERT_EQUAL(proto.StatsSize(), 3); + + UNIT_ASSERT_EQUAL(proto.GetStats(0).GetStats().GetReadBytes(), 100); + UNIT_ASSERT_EQUAL(proto.GetStats(1).GetStats().GetReadBytes(), 20); + UNIT_ASSERT_EQUAL(proto.GetStats(2).GetStats().GetReadBytes(), 7); + } + + Y_UNIT_TEST(AddDedup) { + TQueryStatsDedupBucket<TDurationGreater> bucket; + + TQueryTextsByHash texts; + + AddStatsToBucket(texts, bucket, "a", 10); + AddStatsToBucket(texts, bucket, "b", 20); + AddStatsToBucket(texts, bucket, "c", 15); + AddStatsToBucket(texts, bucket, "b", 5); + AddStatsToBucket(texts, bucket, "c", 25); + + NKikimrSysView::TTopQueryStats proto; + bucket.Fill(&proto, texts); + + UNIT_ASSERT_EQUAL(proto.StatsSize(), 3); + + UNIT_ASSERT_EQUAL(proto.GetStats(0).GetDurationMs(), 25); + UNIT_ASSERT_EQUAL(proto.GetStats(0).GetQueryTextHash(), QueryHash("c")); + UNIT_ASSERT_EQUAL(proto.GetStats(1).GetDurationMs(), 20); + UNIT_ASSERT_EQUAL(proto.GetStats(1).GetQueryTextHash(), QueryHash("b")); + UNIT_ASSERT_EQUAL(proto.GetStats(2).GetDurationMs(), 10); + UNIT_ASSERT_EQUAL(proto.GetStats(2).GetQueryTextHash(), QueryHash("a")); + } + + Y_UNIT_TEST(AddDedup2) { + TQueryStatsDedupBucket<TDurationGreater> bucket; + + TQueryTextsByHash texts; + + for (size_t i : xrange(TOP_QUERIES_COUNT)) { + AddStatsToBucket(texts, bucket, ToString(i), (i + 1) * 10); + } + AddStatsToBucket(texts, bucket, "a", 5); + AddStatsToBucket(texts, bucket, "b", 15); + + NKikimrSysView::TTopQueryStats proto; + bucket.Fill(&proto, texts); + + UNIT_ASSERT_EQUAL(proto.StatsSize(), TOP_QUERIES_COUNT); + auto last = TOP_QUERIES_COUNT - 1; + + UNIT_ASSERT_EQUAL(proto.GetStats(last).GetDurationMs(), 15); + UNIT_ASSERT_EQUAL(proto.GetStats(last).GetQueryTextHash(), QueryHash("b")); + } + + Y_UNIT_TEST(AddDedupRandom) { + TQueryStatsDedupBucket<TDurationGreater> bucket; + + TQueryTextsByHash texts; + + for (size_t i : xrange(10000)) { + Y_UNUSED(i); + AddStatsToBucket(texts, bucket, ToString(RandomNumber<ui32>(1000)), RandomNumber<ui64>(5000)); + } + + UNIT_ASSERT_EQUAL(bucket.Top.size(), TOP_QUERIES_COUNT); + UNIT_ASSERT_EQUAL(bucket.Stats.size(), TOP_QUERIES_COUNT); + + for (auto& top : bucket.Top) { + auto it = bucket.Stats.find(top.second); + UNIT_ASSERT(it != bucket.Stats.end()); + UNIT_ASSERT_EQUAL(it->second->GetDurationMs(), top.first); + } + } + + Y_UNIT_TEST(AggrMerge) { + TAggrQueryStatsBucket<TDurationGreater> bucket; + + TQueryTextsByHash texts; + + NKikimrSysView::TTopQueryStats s1; + FillStats(texts, s1.AddStats(), "a", 30); + FillStats(texts, s1.AddStats(), "b", 20); + FillStats(texts, s1.AddStats(), "c", 10); + + NKikimrSysView::TTopQueryStats s2; + FillStats(texts, s2.AddStats(), "x", 35); + FillStats(texts, s2.AddStats(), "y", 25); + FillStats(texts, s2.AddStats(), "z", 22); + + bucket.Merge(std::move(s1), texts); + bucket.Merge(std::move(s2), texts); + + UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 5); + + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetDurationMs(), 35); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetDurationMs(), 30); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetDurationMs(), 25); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetDurationMs(), 22); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetDurationMs(), 20); + } + + Y_UNIT_TEST(AggrMergeDedup) { + TAggrQueryStatsDedupBucket<TDurationGreater> bucket; + + TQueryTextsByHash texts; + + NKikimrSysView::TTopQueryStats s1; + FillStats(texts, s1.AddStats(), "a", 30); + FillStats(texts, s1.AddStats(), "b", 20); + FillStats(texts, s1.AddStats(), "c", 10); + FillStats(texts, s1.AddStats(), "d", 5); + + NKikimrSysView::TTopQueryStats s2; + FillStats(texts, s2.AddStats(), "c", 35); + FillStats(texts, s2.AddStats(), "a", 25); + FillStats(texts, s2.AddStats(), "b", 25); + FillStats(texts, s2.AddStats(), "e", 15); + + bucket.Merge(std::move(s1), texts); + bucket.Merge(std::move(s2), texts); + + UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 5); + + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetDurationMs(), 35); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "c"); + + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetDurationMs(), 30); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "a"); + + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetDurationMs(), 25); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "b"); + + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetDurationMs(), 15); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "e"); + + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetDurationMs(), 5); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetQueryText(), "d"); + } + + Y_UNIT_TEST(StableMerge) { + TAggrQueryStatsDedupBucket<TDurationGreater> bucket; + + TQueryTextsByHash texts; + + NKikimrSysView::TTopQueryStats s1; + FillStatsWithNodeId(texts, s1.AddStats(), 1, "a", 10); + FillStatsWithNodeId(texts, s1.AddStats(), 1, "b", 10); + + NKikimrSysView::TTopQueryStats s2; + FillStatsWithNodeId(texts, s2.AddStats(), 2, "x", 10); + FillStatsWithNodeId(texts, s2.AddStats(), 2, "y", 10); + + bucket.Merge(std::move(s1), texts); + + bucket.Merge(std::move(s2), texts); + + UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 4); + + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "x"); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "y"); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "a"); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "b"); + } + + Y_UNIT_TEST(StableMerge2) { + TAggrQueryStatsDedupBucket<TDurationGreater> bucket; + + TQueryTextsByHash texts; + + NKikimrSysView::TTopQueryStats s1; + FillStatsWithNodeId(texts, s1.AddStats(), 1, "a", 10); + FillStatsWithNodeId(texts, s1.AddStats(), 1, "b", 10); + + NKikimrSysView::TTopQueryStats s2; + FillStatsWithNodeId(texts, s2.AddStats(), 2, "x", 10); + FillStatsWithNodeId(texts, s2.AddStats(), 2, "y", 10); + + bucket.Merge(std::move(s2), texts); + + bucket.Merge(std::move(s1), texts); + + UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 4); + + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "x"); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "y"); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "a"); + UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "b"); + } + + Y_UNIT_TEST(ServiceQueryHistoryAdd) { + TServiceQueryHistory<TDurationGreater> history(5, TDuration::MilliSeconds(10), TInstant::MilliSeconds(75)); + + TQueryTextsByHash texts; + + auto s1 = MakeStatsPtr(); + FillStats(texts, s1, "a", 1, 25); + history.Add(s1); + + auto s2 = MakeStatsPtr(); + FillStats(texts, s2, "b", 5, 35); + history.Add(s2); + + auto s3 = MakeStatsPtr(); + FillStats(texts, s3, "c", 1, 50); + history.Add(s3); + + auto s4 = MakeStatsPtr(); + FillStats(texts, s4, "d", 10, 51); + history.Add(s4); + + NKikimrSysView::TEvGetQueryStatsResult result; + history.ToProto(3, result); + + UNIT_ASSERT_EQUAL(result.GetLastBucket(), 7); + UNIT_ASSERT_EQUAL(result.GetBucketSizeMs(), 10); + + UNIT_ASSERT_EQUAL(result.BucketsSize(), 5); + + UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 1); + UNIT_ASSERT_EQUAL(result.GetBuckets(0).GetStats(0).GetEndTimeMs(), 35); + UNIT_ASSERT_EQUAL(result.GetBuckets(0).GetStats(0).GetDurationMs(), 5); + + UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0); + + UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 2); + UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetEndTimeMs(), 51); + UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetDurationMs(), 10); + UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(1).GetEndTimeMs(), 50); + UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(1).GetDurationMs(), 1); + + UNIT_ASSERT_EQUAL(result.GetBuckets(3).StatsSize(), 0); + UNIT_ASSERT_EQUAL(result.GetBuckets(4).StatsSize(), 0); + + UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 3); + + auto s5 = MakeStatsPtr(); + FillStats(texts, s5, "e", 5, 93); + history.Add(s5); + + result.Clear(); + history.ToProto(5, result); + + UNIT_ASSERT_EQUAL(result.GetLastBucket(), 9); + + UNIT_ASSERT_EQUAL(result.BucketsSize(), 5); + + UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 2); + UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0); + UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 0); + UNIT_ASSERT_EQUAL(result.GetBuckets(3).StatsSize(), 0); + UNIT_ASSERT_EQUAL(result.GetBuckets(4).StatsSize(), 1); + + UNIT_ASSERT_EQUAL(result.GetBuckets(4).GetStats(0).GetQueryTextHash(), QueryHash("e")); + + UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 3); + + result.Clear(); + history.ToProto(7, result); + + UNIT_ASSERT_EQUAL(result.GetLastBucket(), 9); + UNIT_ASSERT_EQUAL(result.BucketsSize(), 3); + + UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 0); + UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0); + UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 1); + + UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetQueryTextHash(), QueryHash("e")); + + UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 1); + } + + Y_UNIT_TEST(ScanQueryHistoryMerge) { + TestMerge( + 100, {1, 2, 3}, + 100, {1, 2}, + 100, 3, + 100, {1, 3, 5}); + + TestMerge( + 100, {1, 2, 3}, + 99, {1, 2}, + 100, 3, + 100, {2, 4, 3}); + + TestMerge( + 100, {1, 2, 3}, + 98, {1, 2}, + 100, 3, + 100, {3, 2, 3}); + + TestMerge( + 100, {1, 2, 3}, + 100, {1, 2}, + 90, 5, + 100, {0, 0, 1, 3, 5}); + + TestMerge( + 100, {1, 2, 3, 4}, + 99, {1, 2}, + 101, 5, + 101, {1, 3, 5, 4, 0}); + + TestMerge( + 100, {1, 2, 3}, + 102, {1, 2, 3, 4}, + 90, 5, + 102, {1, 3, 5, 3, 4}); + + TestMerge( + 100, {1, 2, 3}, + 101, {2, 1, 2, 3}, + 90, 2, + 101, {5, 3}); + } +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/service/query_interval.cpp b/ydb/core/sys_view/service/query_interval.cpp index a19feff095..daa4241b88 100644 --- a/ydb/core/sys_view/service/query_interval.cpp +++ b/ydb/core/sys_view/service/query_interval.cpp @@ -1,155 +1,155 @@ -#include "query_interval.h" - -namespace NKikimr { -namespace NSysView { - -void Initialize(NKikimrSysView::TQueryMetrics& metrics) { - metrics.MutableDurationUs()->SetMin(Max<ui64>()); - metrics.MutableCpuTimeUs()->SetMin(Max<ui64>()); - metrics.MutableRequestUnits()->SetMin(Max<ui64>()); - metrics.MutableReadRows()->SetMin(Max<ui64>()); - metrics.MutableReadBytes()->SetMin(Max<ui64>()); - metrics.MutableUpdateRows()->SetMin(Max<ui64>()); - metrics.MutableUpdateBytes()->SetMin(Max<ui64>()); - metrics.MutableDeleteRows()->SetMin(Max<ui64>()); -} - -static void Aggregate(NKikimrSysView::TQueryMetrics::TMetrics& metrics, ui64 value) { - metrics.SetSum(metrics.GetSum() + value); - metrics.SetMin(std::min(metrics.GetMin(), value)); - metrics.SetMax(std::max(metrics.GetMax(), value)); -} - -static void Aggregate(NKikimrSysView::TQueryMetrics& metrics, TQueryStatsPtr stats) { - auto& dataStats = stats->GetStats(); - metrics.SetCount(metrics.GetCount() + 1); - - Aggregate(*metrics.MutableDurationUs(), stats->GetDurationMs() * 1000); - Aggregate(*metrics.MutableCpuTimeUs(), stats->GetTotalCpuTimeUs()); - Aggregate(*metrics.MutableRequestUnits(), stats->GetRequestUnits()); - - Aggregate(*metrics.MutableReadRows(), dataStats.GetReadRows()); - Aggregate(*metrics.MutableReadBytes(), dataStats.GetReadBytes()); - Aggregate(*metrics.MutableUpdateRows(), dataStats.GetUpdateRows()); - Aggregate(*metrics.MutableUpdateBytes(), dataStats.GetUpdateBytes()); - Aggregate(*metrics.MutableDeleteRows(), dataStats.GetDeleteRows()); -} - -static void Aggregate(NKikimrSysView::TQueryMetrics::TMetrics& metrics, - const NKikimrSysView::TQueryMetrics::TMetrics& from) -{ - metrics.SetSum(metrics.GetSum() + from.GetSum()); - metrics.SetMin(std::min(metrics.GetMin(), from.GetMin())); - metrics.SetMax(std::max(metrics.GetMax(), from.GetMax())); -} - -void Aggregate(NKikimrSysView::TQueryMetrics& metrics, - const NKikimrSysView::TQueryMetrics& from) -{ - metrics.SetCount(metrics.GetCount() + from.GetCount()); - - Aggregate(*metrics.MutableDurationUs(), from.GetDurationUs()); - Aggregate(*metrics.MutableCpuTimeUs(), from.GetCpuTimeUs()); - Aggregate(*metrics.MutableRequestUnits(), from.GetRequestUnits()); - Aggregate(*metrics.MutableReadRows(), from.GetReadRows()); - Aggregate(*metrics.MutableReadBytes(), from.GetReadBytes()); - Aggregate(*metrics.MutableUpdateRows(), from.GetUpdateRows()); - Aggregate(*metrics.MutableUpdateBytes(), from.GetUpdateBytes()); - Aggregate(*metrics.MutableDeleteRows(), from.GetDeleteRows()); -} - -bool TQueryInterval::Empty() const { - return Metrics.empty(); -} - -void TQueryInterval::Clear() { - Texts.clear(); - Metrics.clear(); - ByCpu.clear(); -} - -void TQueryInterval::Swap(TQueryInterval& other) { - Texts.swap(other.Texts); - Metrics.swap(other.Metrics); - ByCpu.swap(other.ByCpu); -} - -void TQueryInterval::Add(TQueryStatsPtr stats) { - if (!stats->HasQueryTextHash()) { - return; - } - auto queryHash = stats->GetQueryTextHash(); - - if (auto metricsIt = Metrics.find(queryHash); metricsIt != Metrics.end()) { - auto oldCpu = metricsIt->second.GetCpuTimeUs().GetSum(); - Aggregate(metricsIt->second, stats); - - auto range = ByCpu.equal_range(oldCpu); - auto it = range.first; - for (; it != range.second; ++it) { - if (it->second == queryHash) { - break; - } - } - Y_VERIFY(it != range.second); - ByCpu.erase(it); - - auto newCpu = metricsIt->second.GetCpuTimeUs().GetSum(); - ByCpu.emplace(newCpu, queryHash); - - } else { - auto cpu = stats->GetTotalCpuTimeUs(); - - if (ByCpu.size() == CountLimit) { - auto it = ByCpu.begin(); - if (it->first >= cpu) { - return; - } - Texts.erase(it->second); - Metrics.erase(it->second); - ByCpu.erase(it); - } - - NKikimrSysView::TQueryMetrics metrics; - Initialize(metrics); - metrics.SetQueryTextHash(queryHash); - Aggregate(metrics, stats); - - Texts.emplace(queryHash, stats->GetQueryText()); - Metrics.emplace(queryHash, std::move(metrics)); - ByCpu.emplace(cpu, queryHash); - } -} - -void TQueryInterval::FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const { - for (auto it = ByCpu.rbegin(); it != ByCpu.rend(); ++it) { - queries.AddValues(it->first); - queries.AddHashes(it->second); - } -} - -void TQueryInterval::FillMetrics(const NKikimrSysView::TEvGetIntervalMetricsRequest& request, - NKikimrSysView::TEvGetIntervalMetricsResponse& response) const -{ - for (auto queryHash : request.GetMetrics()) { - auto metricsIt = Metrics.find(queryHash); - if (metricsIt == Metrics.end()) { - continue; - } - response.AddMetrics()->CopyFrom(metricsIt->second); - } - - for (auto queryHash : request.GetQueryTextsToGet()) { - auto textsIt = Texts.find(queryHash); - if (textsIt == Texts.end()) { - continue; - } - auto& queryText = *response.AddQueryTexts(); - queryText.SetHash(queryHash); - queryText.SetText(textsIt->second); - } -} - -} // NSysView -} // NKikimr - +#include "query_interval.h" + +namespace NKikimr { +namespace NSysView { + +void Initialize(NKikimrSysView::TQueryMetrics& metrics) { + metrics.MutableDurationUs()->SetMin(Max<ui64>()); + metrics.MutableCpuTimeUs()->SetMin(Max<ui64>()); + metrics.MutableRequestUnits()->SetMin(Max<ui64>()); + metrics.MutableReadRows()->SetMin(Max<ui64>()); + metrics.MutableReadBytes()->SetMin(Max<ui64>()); + metrics.MutableUpdateRows()->SetMin(Max<ui64>()); + metrics.MutableUpdateBytes()->SetMin(Max<ui64>()); + metrics.MutableDeleteRows()->SetMin(Max<ui64>()); +} + +static void Aggregate(NKikimrSysView::TQueryMetrics::TMetrics& metrics, ui64 value) { + metrics.SetSum(metrics.GetSum() + value); + metrics.SetMin(std::min(metrics.GetMin(), value)); + metrics.SetMax(std::max(metrics.GetMax(), value)); +} + +static void Aggregate(NKikimrSysView::TQueryMetrics& metrics, TQueryStatsPtr stats) { + auto& dataStats = stats->GetStats(); + metrics.SetCount(metrics.GetCount() + 1); + + Aggregate(*metrics.MutableDurationUs(), stats->GetDurationMs() * 1000); + Aggregate(*metrics.MutableCpuTimeUs(), stats->GetTotalCpuTimeUs()); + Aggregate(*metrics.MutableRequestUnits(), stats->GetRequestUnits()); + + Aggregate(*metrics.MutableReadRows(), dataStats.GetReadRows()); + Aggregate(*metrics.MutableReadBytes(), dataStats.GetReadBytes()); + Aggregate(*metrics.MutableUpdateRows(), dataStats.GetUpdateRows()); + Aggregate(*metrics.MutableUpdateBytes(), dataStats.GetUpdateBytes()); + Aggregate(*metrics.MutableDeleteRows(), dataStats.GetDeleteRows()); +} + +static void Aggregate(NKikimrSysView::TQueryMetrics::TMetrics& metrics, + const NKikimrSysView::TQueryMetrics::TMetrics& from) +{ + metrics.SetSum(metrics.GetSum() + from.GetSum()); + metrics.SetMin(std::min(metrics.GetMin(), from.GetMin())); + metrics.SetMax(std::max(metrics.GetMax(), from.GetMax())); +} + +void Aggregate(NKikimrSysView::TQueryMetrics& metrics, + const NKikimrSysView::TQueryMetrics& from) +{ + metrics.SetCount(metrics.GetCount() + from.GetCount()); + + Aggregate(*metrics.MutableDurationUs(), from.GetDurationUs()); + Aggregate(*metrics.MutableCpuTimeUs(), from.GetCpuTimeUs()); + Aggregate(*metrics.MutableRequestUnits(), from.GetRequestUnits()); + Aggregate(*metrics.MutableReadRows(), from.GetReadRows()); + Aggregate(*metrics.MutableReadBytes(), from.GetReadBytes()); + Aggregate(*metrics.MutableUpdateRows(), from.GetUpdateRows()); + Aggregate(*metrics.MutableUpdateBytes(), from.GetUpdateBytes()); + Aggregate(*metrics.MutableDeleteRows(), from.GetDeleteRows()); +} + +bool TQueryInterval::Empty() const { + return Metrics.empty(); +} + +void TQueryInterval::Clear() { + Texts.clear(); + Metrics.clear(); + ByCpu.clear(); +} + +void TQueryInterval::Swap(TQueryInterval& other) { + Texts.swap(other.Texts); + Metrics.swap(other.Metrics); + ByCpu.swap(other.ByCpu); +} + +void TQueryInterval::Add(TQueryStatsPtr stats) { + if (!stats->HasQueryTextHash()) { + return; + } + auto queryHash = stats->GetQueryTextHash(); + + if (auto metricsIt = Metrics.find(queryHash); metricsIt != Metrics.end()) { + auto oldCpu = metricsIt->second.GetCpuTimeUs().GetSum(); + Aggregate(metricsIt->second, stats); + + auto range = ByCpu.equal_range(oldCpu); + auto it = range.first; + for (; it != range.second; ++it) { + if (it->second == queryHash) { + break; + } + } + Y_VERIFY(it != range.second); + ByCpu.erase(it); + + auto newCpu = metricsIt->second.GetCpuTimeUs().GetSum(); + ByCpu.emplace(newCpu, queryHash); + + } else { + auto cpu = stats->GetTotalCpuTimeUs(); + + if (ByCpu.size() == CountLimit) { + auto it = ByCpu.begin(); + if (it->first >= cpu) { + return; + } + Texts.erase(it->second); + Metrics.erase(it->second); + ByCpu.erase(it); + } + + NKikimrSysView::TQueryMetrics metrics; + Initialize(metrics); + metrics.SetQueryTextHash(queryHash); + Aggregate(metrics, stats); + + Texts.emplace(queryHash, stats->GetQueryText()); + Metrics.emplace(queryHash, std::move(metrics)); + ByCpu.emplace(cpu, queryHash); + } +} + +void TQueryInterval::FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const { + for (auto it = ByCpu.rbegin(); it != ByCpu.rend(); ++it) { + queries.AddValues(it->first); + queries.AddHashes(it->second); + } +} + +void TQueryInterval::FillMetrics(const NKikimrSysView::TEvGetIntervalMetricsRequest& request, + NKikimrSysView::TEvGetIntervalMetricsResponse& response) const +{ + for (auto queryHash : request.GetMetrics()) { + auto metricsIt = Metrics.find(queryHash); + if (metricsIt == Metrics.end()) { + continue; + } + response.AddMetrics()->CopyFrom(metricsIt->second); + } + + for (auto queryHash : request.GetQueryTextsToGet()) { + auto textsIt = Texts.find(queryHash); + if (textsIt == Texts.end()) { + continue; + } + auto& queryText = *response.AddQueryTexts(); + queryText.SetHash(queryHash); + queryText.SetText(textsIt->second); + } +} + +} // NSysView +} // NKikimr + diff --git a/ydb/core/sys_view/service/query_interval.h b/ydb/core/sys_view/service/query_interval.h index 8da12b82b6..126bd788ff 100644 --- a/ydb/core/sys_view/service/query_interval.h +++ b/ydb/core/sys_view/service/query_interval.h @@ -1,34 +1,34 @@ -#pragma once - -#include "query_history.h" - -namespace NKikimr { -namespace NSysView { - -void Initialize(NKikimrSysView::TQueryMetrics& metrics); - -void Aggregate(NKikimrSysView::TQueryMetrics& metrics, const NKikimrSysView::TQueryMetrics& from); - -class TQueryInterval { - std::unordered_map<TQueryHash, TString> Texts; // hash -> text - std::unordered_map<TQueryHash, NKikimrSysView::TQueryMetrics> Metrics; // hash -> metrics - std::multimap<ui64, TQueryHash> ByCpu; // cpu sum -> hash - - static constexpr size_t CountLimit = 1024; - -public: - bool Empty() const; - void Clear(); - void Swap(TQueryInterval& other); - - void Add(TQueryStatsPtr stats); - - void FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const; - - void FillMetrics(const NKikimrSysView::TEvGetIntervalMetricsRequest& request, - NKikimrSysView::TEvGetIntervalMetricsResponse& response) const; -}; - -} // NSysView -} // NKikimr - +#pragma once + +#include "query_history.h" + +namespace NKikimr { +namespace NSysView { + +void Initialize(NKikimrSysView::TQueryMetrics& metrics); + +void Aggregate(NKikimrSysView::TQueryMetrics& metrics, const NKikimrSysView::TQueryMetrics& from); + +class TQueryInterval { + std::unordered_map<TQueryHash, TString> Texts; // hash -> text + std::unordered_map<TQueryHash, NKikimrSysView::TQueryMetrics> Metrics; // hash -> metrics + std::multimap<ui64, TQueryHash> ByCpu; // cpu sum -> hash + + static constexpr size_t CountLimit = 1024; + +public: + bool Empty() const; + void Clear(); + void Swap(TQueryInterval& other); + + void Add(TQueryStatsPtr stats); + + void FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const; + + void FillMetrics(const NKikimrSysView::TEvGetIntervalMetricsRequest& request, + NKikimrSysView::TEvGetIntervalMetricsResponse& response) const; +}; + +} // NSysView +} // NKikimr + diff --git a/ydb/core/sys_view/service/sysview_service.cpp b/ydb/core/sys_view/service/sysview_service.cpp index b407cd39ba..1f56906599 100644 --- a/ydb/core/sys_view/service/sysview_service.cpp +++ b/ydb/core/sys_view/service/sysview_service.cpp @@ -1,1149 +1,1149 @@ -#include "db_counters.h" -#include "query_history.h" -#include "query_interval.h" -#include "sysview_service.h" - +#include "db_counters.h" +#include "query_history.h" +#include "query_interval.h" +#include "sysview_service.h" + #include <ydb/core/sys_view/common/common.h> #include <ydb/core/sys_view/common/events.h> #include <ydb/core/base/appdata.h> #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/mind/tenant_pool.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/log.h> - -using namespace NActors; - -namespace NKikimr { -namespace NSysView { - -constexpr size_t QUERY_TEXT_LIMIT = 4096; - -void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery* queryStats, - TDuration queryDuration, const TString& queryText, - const TString& userSID, ui64 parametersSize, const TString& database, - const NKikimrKqp::EQueryType type, ui64 requestUnits) -{ + +using namespace NActors; + +namespace NKikimr { +namespace NSysView { + +constexpr size_t QUERY_TEXT_LIMIT = 4096; + +void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery* queryStats, + TDuration queryDuration, const TString& queryText, + const TString& userSID, ui64 parametersSize, const TString& database, + const NKikimrKqp::EQueryType type, ui64 requestUnits) +{ if (!AppData()->FeatureFlags.GetEnableSystemViews()) { - return; - } - - auto collectEv = MakeHolder<NSysView::TEvSysView::TEvCollectQueryStats>(); - collectEv->Database = database; - - auto& stats = collectEv->QueryStats; - auto& dataStats = *stats.MutableStats(); - auto& shardsCpuTime = *stats.MutableShardsCpuTimeUs(); - auto& computeCpuTime = *stats.MutableComputeCpuTimeUs(); - - auto nodeId = ctx.SelfID.NodeId(); - stats.SetNodeId(nodeId); - - stats.SetEndTimeMs(TInstant::Now().MilliSeconds()); - stats.SetDurationMs(queryDuration.MilliSeconds()); - - stats.SetQueryTextHash(MurmurHash<ui64>(queryText.data(), queryText.size())); - if (queryText.size() > QUERY_TEXT_LIMIT) { - auto limitedText = queryText.substr(0, QUERY_TEXT_LIMIT); - stats.SetQueryText(limitedText); - } else { - stats.SetQueryText(queryText); - } - - if (userSID) { - stats.SetUserSID(userSID); - } - stats.SetParametersSize(parametersSize); - - TString strType; - switch (type) { - case NKikimrKqp::QUERY_TYPE_SQL_DML: - case NKikimrKqp::QUERY_TYPE_PREPARED_DML: - strType = "data"; - break; - case NKikimrKqp::QUERY_TYPE_SQL_SCAN: - strType = "scan"; - break; - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: + return; + } + + auto collectEv = MakeHolder<NSysView::TEvSysView::TEvCollectQueryStats>(); + collectEv->Database = database; + + auto& stats = collectEv->QueryStats; + auto& dataStats = *stats.MutableStats(); + auto& shardsCpuTime = *stats.MutableShardsCpuTimeUs(); + auto& computeCpuTime = *stats.MutableComputeCpuTimeUs(); + + auto nodeId = ctx.SelfID.NodeId(); + stats.SetNodeId(nodeId); + + stats.SetEndTimeMs(TInstant::Now().MilliSeconds()); + stats.SetDurationMs(queryDuration.MilliSeconds()); + + stats.SetQueryTextHash(MurmurHash<ui64>(queryText.data(), queryText.size())); + if (queryText.size() > QUERY_TEXT_LIMIT) { + auto limitedText = queryText.substr(0, QUERY_TEXT_LIMIT); + stats.SetQueryText(limitedText); + } else { + stats.SetQueryText(queryText); + } + + if (userSID) { + stats.SetUserSID(userSID); + } + stats.SetParametersSize(parametersSize); + + TString strType; + switch (type) { + case NKikimrKqp::QUERY_TYPE_SQL_DML: + case NKikimrKqp::QUERY_TYPE_PREPARED_DML: + strType = "data"; + break; + case NKikimrKqp::QUERY_TYPE_SQL_SCAN: + strType = "scan"; + break; + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: - strType = "script"; - break; - default: - break; - } - stats.SetType(strType); - - if (!queryStats || queryStats->GetExecutions().empty()) { - ctx.Send(NSysView::MakeSysViewServiceID(nodeId), std::move(collectEv)); - return; - } - - shardsCpuTime.SetMin(Max<ui64>()); - computeCpuTime.SetMin(Max<ui64>()); - + strType = "script"; + break; + default: + break; + } + stats.SetType(strType); + + if (!queryStats || queryStats->GetExecutions().empty()) { + ctx.Send(NSysView::MakeSysViewServiceID(nodeId), std::move(collectEv)); + return; + } + + shardsCpuTime.SetMin(Max<ui64>()); + computeCpuTime.SetMin(Max<ui64>()); + auto aggregate = [] (NKikimrSysView::TStatsAggr& to, const NYql::NDqProto::TDqStatsAggr& from) { - to.SetMin(std::min(to.GetMin(), from.GetMin())); - to.SetMax(std::max(to.GetMax(), from.GetMax())); - to.SetSum(to.GetSum() + from.GetSum()); - to.SetCnt(to.GetCnt() + from.GetCnt()); - }; - + to.SetMin(std::min(to.GetMin(), from.GetMin())); + to.SetMax(std::max(to.GetMax(), from.GetMax())); + to.SetSum(to.GetSum() + from.GetSum()); + to.SetCnt(to.GetCnt() + from.GetCnt()); + }; + for (const NYql::NDqProto::TDqExecutionStats& exec : queryStats->GetExecutions()) { NKqpProto::TKqpExecutionExtraStats execExtra; if (exec.HasExtra()) { bool ok = exec.GetExtra().UnpackTo(&execExtra); Y_UNUSED(ok); } - + dataStats.SetPartitionCount(dataStats.GetPartitionCount() + execExtra.GetAffectedShards()); - for (auto& table : exec.GetTables()) { - dataStats.SetReadRows(dataStats.GetReadRows() + table.GetReadRows()); - dataStats.SetReadBytes(dataStats.GetReadBytes() + table.GetReadBytes()); - dataStats.SetUpdateRows(dataStats.GetUpdateRows() + table.GetWriteRows()); - dataStats.SetUpdateBytes(dataStats.GetUpdateBytes() + table.GetWriteBytes()); - dataStats.SetDeleteRows(dataStats.GetDeleteRows() + table.GetEraseRows()); - } - + for (auto& table : exec.GetTables()) { + dataStats.SetReadRows(dataStats.GetReadRows() + table.GetReadRows()); + dataStats.SetReadBytes(dataStats.GetReadBytes() + table.GetReadBytes()); + dataStats.SetUpdateRows(dataStats.GetUpdateRows() + table.GetWriteRows()); + dataStats.SetUpdateBytes(dataStats.GetUpdateBytes() + table.GetWriteBytes()); + dataStats.SetDeleteRows(dataStats.GetDeleteRows() + table.GetEraseRows()); + } + aggregate(shardsCpuTime, execExtra.GetShardsCpuTimeUs()); aggregate(computeCpuTime, execExtra.GetComputeCpuTimeUs()); - } - - if (queryStats->HasCompilation()) { - auto& compileStats = queryStats->GetCompilation(); - stats.SetFromQueryCache(compileStats.GetFromCache()); - stats.SetCompileDurationMs(compileStats.GetDurationUs() / 1'000); - stats.SetCompileCpuTimeUs(compileStats.GetCpuTimeUs()); - } - - stats.SetProcessCpuTimeUs(queryStats->GetWorkerCpuTimeUs()); - stats.SetTotalCpuTimeUs( - stats.GetProcessCpuTimeUs() + - stats.GetCompileCpuTimeUs() + - stats.GetShardsCpuTimeUs().GetSum() + - stats.GetComputeCpuTimeUs().GetSum() - ); - - stats.SetRequestUnits(requestUnits); - - ctx.Send(NSysView::MakeSysViewServiceID(nodeId), std::move(collectEv)); -} - -static void CopyCounters(NKikimrSysView::TDbCounters* diff, - const NKikimrSysView::TDbCounters& current) -{ - auto simpleSize = current.SimpleSize(); - auto cumulativeSize = current.CumulativeSize(); - auto histogramSize = current.HistogramSize(); - - diff->MutableSimple()->Reserve(simpleSize); - diff->MutableCumulative()->Reserve(cumulativeSize); - diff->MutableHistogram()->Reserve(histogramSize); - - for (size_t i = 0; i < simpleSize; ++i) { - diff->AddSimple(current.GetSimple(i)); - } - - diff->SetCumulativeCount(cumulativeSize); - for (size_t i = 0; i < cumulativeSize; ++i) { - auto value = current.GetCumulative(i); - if (!value) { - continue; - } - diff->AddCumulative(i); - diff->AddCumulative(value); - } - - for (size_t i = 0; i < histogramSize; ++i) { - const auto& currentH = current.GetHistogram(i); - auto bucketCount = currentH.BucketsSize(); - - auto* histogram = diff->AddHistogram(); - histogram->MutableBuckets()->Reserve(bucketCount); - histogram->SetBucketsCount(bucketCount); - for (size_t b = 0; b < bucketCount; ++b) { - auto value = currentH.GetBuckets(b); - if (!value) { - continue; - } - histogram->AddBuckets(b); - histogram->AddBuckets(value); - } - } -} - -static void CalculateCountersDiff(NKikimrSysView::TDbCounters* diff, - const NKikimrSysView::TDbCounters& current, - NKikimrSysView::TDbCounters& prev) -{ - auto simpleSize = current.SimpleSize(); - auto cumulativeSize = current.CumulativeSize(); - auto histogramSize = current.HistogramSize(); - - if (prev.SimpleSize() != simpleSize) { - SVLOG_CRIT("CalculateCountersDiff: simple count mismatch, prev " - << prev.SimpleSize() << ", current " << simpleSize); - prev.MutableSimple()->Resize(simpleSize, 0); - } - if (prev.CumulativeSize() != cumulativeSize) { - SVLOG_CRIT("CalculateCountersDiff: cumulative count mismatch, prev " - << prev.CumulativeSize() << ", current " << cumulativeSize); - prev.MutableCumulative()->Resize(cumulativeSize, 0); - } - if (prev.HistogramSize() != histogramSize) { - SVLOG_CRIT("CalculateCountersDiff: histogram count mismatch, prev " - << prev.HistogramSize() << ", current " << histogramSize); - if (prev.HistogramSize() < histogramSize) { - auto missing = histogramSize - prev.HistogramSize(); - for (; missing > 0; --missing) { - prev.AddHistogram(); - } - } - } - - diff->MutableSimple()->Reserve(simpleSize); - diff->MutableCumulative()->Reserve(cumulativeSize); - diff->MutableHistogram()->Reserve(histogramSize); - - for (size_t i = 0; i < simpleSize; ++i) { - diff->AddSimple(current.GetSimple(i)); - } - - diff->SetCumulativeCount(cumulativeSize); - for (size_t i = 0; i < cumulativeSize; ++i) { - auto value = current.GetCumulative(i) - prev.GetCumulative(i); - if (!value) { - continue; - } - diff->AddCumulative(i); - diff->AddCumulative(value); - } - - for (size_t i = 0; i < histogramSize; ++i) { - const auto& currentH = current.GetHistogram(i); - auto& prevH = *prev.MutableHistogram(i); - auto bucketCount = currentH.BucketsSize(); - if (prevH.BucketsSize() != bucketCount) { - SVLOG_CRIT("CalculateCountersDiff: histogram buckets count mismatch, index " << i - << ", prev " << prevH.BucketsSize() << ", current " << bucketCount); - prevH.MutableBuckets()->Resize(bucketCount, 0); - } - auto* histogram = diff->AddHistogram(); - histogram->MutableBuckets()->Reserve(bucketCount); - histogram->SetBucketsCount(bucketCount); - for (size_t b = 0; b < bucketCount; ++b) { - auto value = currentH.GetBuckets(b) - prevH.GetBuckets(b); - if (!value) { - continue; - } - histogram->AddBuckets(b); - histogram->AddBuckets(value); - } - } -} - -static void CalculateCountersDiff(NKikimrSysView::TDbServiceCounters* diff, - const NKikimr::NSysView::TDbServiceCounters& current, - NKikimr::NSysView::TDbServiceCounters& prev) -{ - if (current.Proto().HasMain()) { - if (prev.Proto().HasMain()) { - CalculateCountersDiff(diff->MutableMain(), - current.Proto().GetMain(), *prev.Proto().MutableMain()); - } else { - CopyCounters(diff->MutableMain(), current.Proto().GetMain()); - } - } - - for (const auto& currentT : current.Proto().GetTabletCounters()) { - const auto type = currentT.GetType(); - auto* diffT = diff->AddTabletCounters(); - diffT->SetType(type); - - auto* prevT = prev.FindTabletCounters(type); - if (prevT) { - CalculateCountersDiff(diffT->MutableExecutorCounters(), - currentT.GetExecutorCounters(), *prevT->MutableExecutorCounters()); - CalculateCountersDiff(diffT->MutableAppCounters(), - currentT.GetAppCounters(), *prevT->MutableAppCounters()); - } else { - CopyCounters(diffT->MutableExecutorCounters(), currentT.GetExecutorCounters()); - CopyCounters(diffT->MutableAppCounters(), currentT.GetAppCounters()); - } - CopyCounters(diffT->MutableMaxExecutorCounters(), currentT.GetMaxExecutorCounters()); - CopyCounters(diffT->MutableMaxAppCounters(), currentT.GetMaxAppCounters()); - } - - for (const auto& currentR : current.Proto().GetGRpcCounters()) { - const auto grpcService = currentR.GetGRpcService(); - const auto grpcRequest = currentR.GetGRpcRequest(); - auto* diffR = diff->AddGRpcCounters(); - diffR->SetGRpcService(grpcService); - diffR->SetGRpcRequest(grpcRequest); - - auto* prevR = prev.FindGRpcCounters(grpcService, grpcRequest); - if (prevR) { - CalculateCountersDiff(diffR->MutableRequestCounters(), - currentR.GetRequestCounters(), *prevR->MutableRequestCounters()); - } else { - CopyCounters(diffR->MutableRequestCounters(), currentR.GetRequestCounters()); - } - } -} - -class TSysViewService : public TActorBootstrapped<TSysViewService> { -public: - using TBase = TActorBootstrapped<TSysViewService>; - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::SYSTEM_VIEW_SERVICE; - } - - explicit TSysViewService( - TExtCountersConfig&& config, - bool hasExternalCounters, - EProcessorMode processorMode) - : Config(std::move(config)) - , HasExternalCounters(hasExternalCounters) - , TotalInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 6 : 60)) - , CollectInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 3 : 30)) - , SendInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 2 : 20)) - {} - - void Bootstrap(const TActorContext &ctx) { - auto now = AppData()->TimeProvider->Now(); - - TopByDuration1Minute = MakeHolder<TServiceQueryHistory<TDurationGreater>>( - ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now); - TopByDuration1Hour = MakeHolder<TServiceQueryHistory<TDurationGreater>>( - ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now); - - TopByReadBytes1Minute = MakeHolder<TServiceQueryHistory<TReadBytesGreater>>( - ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now); - TopByReadBytes1Hour = MakeHolder<TServiceQueryHistory<TReadBytesGreater>>( - ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now); - - TopByCpuTime1Minute = MakeHolder<TServiceQueryHistory<TCpuTimeGreater>>( - ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now); - TopByCpuTime1Hour = MakeHolder<TServiceQueryHistory<TCpuTimeGreater>>( - ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now); - - TopByRequestUnits1Minute = MakeHolder<TServiceQueryHistory<TRequestUnitsGreater>>( - ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now); - TopByRequestUnits1Hour = MakeHolder<TServiceQueryHistory<TRequestUnitsGreater>>( - ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now); - - ctx.Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe()); - - ScanLimiter = MakeIntrusive<TScanLimiter>(ConcurrentScansLimit); - + } + + if (queryStats->HasCompilation()) { + auto& compileStats = queryStats->GetCompilation(); + stats.SetFromQueryCache(compileStats.GetFromCache()); + stats.SetCompileDurationMs(compileStats.GetDurationUs() / 1'000); + stats.SetCompileCpuTimeUs(compileStats.GetCpuTimeUs()); + } + + stats.SetProcessCpuTimeUs(queryStats->GetWorkerCpuTimeUs()); + stats.SetTotalCpuTimeUs( + stats.GetProcessCpuTimeUs() + + stats.GetCompileCpuTimeUs() + + stats.GetShardsCpuTimeUs().GetSum() + + stats.GetComputeCpuTimeUs().GetSum() + ); + + stats.SetRequestUnits(requestUnits); + + ctx.Send(NSysView::MakeSysViewServiceID(nodeId), std::move(collectEv)); +} + +static void CopyCounters(NKikimrSysView::TDbCounters* diff, + const NKikimrSysView::TDbCounters& current) +{ + auto simpleSize = current.SimpleSize(); + auto cumulativeSize = current.CumulativeSize(); + auto histogramSize = current.HistogramSize(); + + diff->MutableSimple()->Reserve(simpleSize); + diff->MutableCumulative()->Reserve(cumulativeSize); + diff->MutableHistogram()->Reserve(histogramSize); + + for (size_t i = 0; i < simpleSize; ++i) { + diff->AddSimple(current.GetSimple(i)); + } + + diff->SetCumulativeCount(cumulativeSize); + for (size_t i = 0; i < cumulativeSize; ++i) { + auto value = current.GetCumulative(i); + if (!value) { + continue; + } + diff->AddCumulative(i); + diff->AddCumulative(value); + } + + for (size_t i = 0; i < histogramSize; ++i) { + const auto& currentH = current.GetHistogram(i); + auto bucketCount = currentH.BucketsSize(); + + auto* histogram = diff->AddHistogram(); + histogram->MutableBuckets()->Reserve(bucketCount); + histogram->SetBucketsCount(bucketCount); + for (size_t b = 0; b < bucketCount; ++b) { + auto value = currentH.GetBuckets(b); + if (!value) { + continue; + } + histogram->AddBuckets(b); + histogram->AddBuckets(value); + } + } +} + +static void CalculateCountersDiff(NKikimrSysView::TDbCounters* diff, + const NKikimrSysView::TDbCounters& current, + NKikimrSysView::TDbCounters& prev) +{ + auto simpleSize = current.SimpleSize(); + auto cumulativeSize = current.CumulativeSize(); + auto histogramSize = current.HistogramSize(); + + if (prev.SimpleSize() != simpleSize) { + SVLOG_CRIT("CalculateCountersDiff: simple count mismatch, prev " + << prev.SimpleSize() << ", current " << simpleSize); + prev.MutableSimple()->Resize(simpleSize, 0); + } + if (prev.CumulativeSize() != cumulativeSize) { + SVLOG_CRIT("CalculateCountersDiff: cumulative count mismatch, prev " + << prev.CumulativeSize() << ", current " << cumulativeSize); + prev.MutableCumulative()->Resize(cumulativeSize, 0); + } + if (prev.HistogramSize() != histogramSize) { + SVLOG_CRIT("CalculateCountersDiff: histogram count mismatch, prev " + << prev.HistogramSize() << ", current " << histogramSize); + if (prev.HistogramSize() < histogramSize) { + auto missing = histogramSize - prev.HistogramSize(); + for (; missing > 0; --missing) { + prev.AddHistogram(); + } + } + } + + diff->MutableSimple()->Reserve(simpleSize); + diff->MutableCumulative()->Reserve(cumulativeSize); + diff->MutableHistogram()->Reserve(histogramSize); + + for (size_t i = 0; i < simpleSize; ++i) { + diff->AddSimple(current.GetSimple(i)); + } + + diff->SetCumulativeCount(cumulativeSize); + for (size_t i = 0; i < cumulativeSize; ++i) { + auto value = current.GetCumulative(i) - prev.GetCumulative(i); + if (!value) { + continue; + } + diff->AddCumulative(i); + diff->AddCumulative(value); + } + + for (size_t i = 0; i < histogramSize; ++i) { + const auto& currentH = current.GetHistogram(i); + auto& prevH = *prev.MutableHistogram(i); + auto bucketCount = currentH.BucketsSize(); + if (prevH.BucketsSize() != bucketCount) { + SVLOG_CRIT("CalculateCountersDiff: histogram buckets count mismatch, index " << i + << ", prev " << prevH.BucketsSize() << ", current " << bucketCount); + prevH.MutableBuckets()->Resize(bucketCount, 0); + } + auto* histogram = diff->AddHistogram(); + histogram->MutableBuckets()->Reserve(bucketCount); + histogram->SetBucketsCount(bucketCount); + for (size_t b = 0; b < bucketCount; ++b) { + auto value = currentH.GetBuckets(b) - prevH.GetBuckets(b); + if (!value) { + continue; + } + histogram->AddBuckets(b); + histogram->AddBuckets(value); + } + } +} + +static void CalculateCountersDiff(NKikimrSysView::TDbServiceCounters* diff, + const NKikimr::NSysView::TDbServiceCounters& current, + NKikimr::NSysView::TDbServiceCounters& prev) +{ + if (current.Proto().HasMain()) { + if (prev.Proto().HasMain()) { + CalculateCountersDiff(diff->MutableMain(), + current.Proto().GetMain(), *prev.Proto().MutableMain()); + } else { + CopyCounters(diff->MutableMain(), current.Proto().GetMain()); + } + } + + for (const auto& currentT : current.Proto().GetTabletCounters()) { + const auto type = currentT.GetType(); + auto* diffT = diff->AddTabletCounters(); + diffT->SetType(type); + + auto* prevT = prev.FindTabletCounters(type); + if (prevT) { + CalculateCountersDiff(diffT->MutableExecutorCounters(), + currentT.GetExecutorCounters(), *prevT->MutableExecutorCounters()); + CalculateCountersDiff(diffT->MutableAppCounters(), + currentT.GetAppCounters(), *prevT->MutableAppCounters()); + } else { + CopyCounters(diffT->MutableExecutorCounters(), currentT.GetExecutorCounters()); + CopyCounters(diffT->MutableAppCounters(), currentT.GetAppCounters()); + } + CopyCounters(diffT->MutableMaxExecutorCounters(), currentT.GetMaxExecutorCounters()); + CopyCounters(diffT->MutableMaxAppCounters(), currentT.GetMaxAppCounters()); + } + + for (const auto& currentR : current.Proto().GetGRpcCounters()) { + const auto grpcService = currentR.GetGRpcService(); + const auto grpcRequest = currentR.GetGRpcRequest(); + auto* diffR = diff->AddGRpcCounters(); + diffR->SetGRpcService(grpcService); + diffR->SetGRpcRequest(grpcRequest); + + auto* prevR = prev.FindGRpcCounters(grpcService, grpcRequest); + if (prevR) { + CalculateCountersDiff(diffR->MutableRequestCounters(), + currentR.GetRequestCounters(), *prevR->MutableRequestCounters()); + } else { + CopyCounters(diffR->MutableRequestCounters(), currentR.GetRequestCounters()); + } + } +} + +class TSysViewService : public TActorBootstrapped<TSysViewService> { +public: + using TBase = TActorBootstrapped<TSysViewService>; + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::SYSTEM_VIEW_SERVICE; + } + + explicit TSysViewService( + TExtCountersConfig&& config, + bool hasExternalCounters, + EProcessorMode processorMode) + : Config(std::move(config)) + , HasExternalCounters(hasExternalCounters) + , TotalInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 6 : 60)) + , CollectInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 3 : 30)) + , SendInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 2 : 20)) + {} + + void Bootstrap(const TActorContext &ctx) { + auto now = AppData()->TimeProvider->Now(); + + TopByDuration1Minute = MakeHolder<TServiceQueryHistory<TDurationGreater>>( + ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now); + TopByDuration1Hour = MakeHolder<TServiceQueryHistory<TDurationGreater>>( + ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now); + + TopByReadBytes1Minute = MakeHolder<TServiceQueryHistory<TReadBytesGreater>>( + ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now); + TopByReadBytes1Hour = MakeHolder<TServiceQueryHistory<TReadBytesGreater>>( + ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now); + + TopByCpuTime1Minute = MakeHolder<TServiceQueryHistory<TCpuTimeGreater>>( + ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now); + TopByCpuTime1Hour = MakeHolder<TServiceQueryHistory<TCpuTimeGreater>>( + ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now); + + TopByRequestUnits1Minute = MakeHolder<TServiceQueryHistory<TRequestUnitsGreater>>( + ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now); + TopByRequestUnits1Hour = MakeHolder<TServiceQueryHistory<TRequestUnitsGreater>>( + ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now); + + ctx.Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe()); + + ScanLimiter = MakeIntrusive<TScanLimiter>(ConcurrentScansLimit); + if (AppData()->FeatureFlags.GetEnablePersistentQueryStats()) { - IntervalEnd = GetNextIntervalEnd(); - Schedule(IntervalEnd, new TEvPrivate::TEvProcessInterval(IntervalEnd)); - } - - if (AppData()->FeatureFlags.GetEnableDbCounters()) { - auto intervalSize = ProcessCountersInterval.MicroSeconds(); - auto deadline = (Now().MicroSeconds() / intervalSize + 1) * intervalSize; - deadline += RandomNumber<ui64>(intervalSize / 5); - Schedule(TInstant::MicroSeconds(deadline), new TEvPrivate::TEvProcessCounters()); - - auto callback = MakeIntrusive<TServiceDbWatcherCallback>(ctx.ActorSystem()); - DbWatcherActorId = ctx.Register(CreateDbWatcherActor(callback)); - } - - if (HasExternalCounters) { - ctx.Register(CreateExtCountersUpdater(std::move(Config))); - } - - Become(&TSysViewService::StateWork); - } - - STFUNC(StateWork) { - Y_UNUSED(ctx); - switch(ev->GetTypeRewrite()) { - hFunc(TEvTenantPool::TEvTenantPoolStatus, Handle); - hFunc(TEvSysView::TEvCollectQueryStats, Handle); - hFunc(TEvSysView::TEvGetQueryStats, Handle); - hFunc(TEvSysView::TEvGetScanLimiter, Handle); - hFunc(TEvPrivate::TEvProcessInterval, Handle); - hFunc(TEvPrivate::TEvSendSummary, Handle); - hFunc(TEvPrivate::TEvProcessCounters, Handle); - hFunc(TEvPrivate::TEvRemoveDatabase, Handle); - hFunc(TEvSysView::TEvRegisterDbCounters, Handle); - hFunc(TEvSysView::TEvSendDbCountersResponse, Handle); - hFunc(TEvSysView::TEvGetIntervalMetricsRequest, Handle); - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - cFunc(TEvents::TEvPoison::EventType, PassAway); - default: - SVLOG_CRIT("NSysView::TSysViewService: unexpected event# " - << ev->GetTypeRewrite()); - } - } - -private: - struct TEvPrivate { - enum EEv { - EvProcessInterval = EventSpaceBegin(TEvents::ES_PRIVATE), - EvSendSummary, - EvProcessCounters, - EvRemoveDatabase, - EvEnd - }; - - struct TEvProcessInterval : public TEventLocal<TEvProcessInterval, EvProcessInterval> { - TInstant IntervalEnd; - - explicit TEvProcessInterval(TInstant intervalEnd) - : IntervalEnd(intervalEnd) - {} - }; - - struct TEvSendSummary : public TEventLocal<TEvSendSummary, EvSendSummary> { - TInstant IntervalEnd; - TString Database; - - TEvSendSummary(TInstant intervalEnd, const TString& database) - : IntervalEnd(intervalEnd) - , Database(database) - {} - }; - - struct TEvProcessCounters : public TEventLocal<TEvProcessCounters, EvProcessCounters> { - }; - - struct TEvRemoveDatabase : public TEventLocal<TEvRemoveDatabase, EvRemoveDatabase> { - TString Database; - TPathId PathId; - - TEvRemoveDatabase(const TString& database, TPathId pathId) - : Database(database) - , PathId(pathId) - {} - }; - }; - - class TServiceDbWatcherCallback : public TDbWatcherCallback { - TActorSystem* ActorSystem = {}; - - public: - explicit TServiceDbWatcherCallback(TActorSystem* actorSystem) - : ActorSystem(actorSystem) - {} - - void OnDatabaseRemoved(const TString& database, TPathId pathId) override { - auto evRemove = MakeHolder<TEvPrivate::TEvRemoveDatabase>(database, pathId); - auto service = MakeSysViewServiceID(ActorSystem->NodeId); - ActorSystem->Send(service, evRemove.Release()); - } - }; - - TInstant GetNextIntervalEnd() { - auto intervalSize = TotalInterval.MicroSeconds(); - auto rounded = (Now().MicroSeconds() / intervalSize + 1) * intervalSize; - return TInstant::MicroSeconds(rounded); - } - - void SendSummary(TInstant intervalEnd, const TString& database) { - auto processorId = GetProcessorId(database); - if (!processorId) { - return; - } - - auto summary = MakeHolder<TEvSysView::TEvIntervalQuerySummary>(); - auto& record = summary->Record; - record.SetDatabase(database); - record.SetIntervalEndUs(intervalEnd.MicroSeconds()); - record.SetNodeId(SelfId().NodeId()); - - auto& log = QueryLogs[database]; - log.Metrics.Report.FillSummary(*record.MutableMetrics()); - log.TopByDuration.Report.FillSummary(*record.MutableTopByDuration()); - log.TopByReadBytes.Report.FillSummary(*record.MutableTopByReadBytes()); - log.TopByCpuTime.Report.FillSummary(*record.MutableTopByCpuTime()); - log.TopByRequestUnits.Report.FillSummary(*record.MutableTopByRequestUnits()); - - SVLOG_D("Send interval summary: " - << "service id# " << SelfId() - << ", processor id# " << processorId - << ", database# " << database - << ", interval end# " << intervalEnd - << ", query count# " << record.GetMetrics().HashesSize()); - - Send(MakePipePeNodeCacheID(false), - new TEvPipeCache::TEvForward(summary.Release(), processorId, true), - IEventHandle::FlagTrackDelivery); - } - - void Rotate() { - auto summaryEnd = IntervalEnd; - auto delta = Now() - summaryEnd; - - IntervalEnd = GetNextIntervalEnd(); - Schedule(IntervalEnd, new TEvPrivate::TEvProcessInterval(IntervalEnd)); - - Attempts.clear(); - - SVLOG_D("Rotate logs: service id# " << SelfId() - << ", query logs count# " << QueryLogs.size() - << ", processor ids count# " << ProcessorIds.size() - << ", processor id to database count# " << ProcessorIdToDatabase.size()); - - if (QueryLogs.empty()) { - return; - } - - if (delta > CollectInterval) { - // too late to send summaries, skip - QueryLogs.clear(); - return; - } - - for (auto it = QueryLogs.begin(); it != QueryLogs.end(); ) { - it->second.Rotate(); - if (it->second.Metrics.Report.Empty()) { - it = QueryLogs.erase(it); - } else { - ++it; - } - } - - if (delta >= SendInterval) { - // send immediately - for (const auto& [database, _] : QueryLogs) { - SendSummary(summaryEnd, database); - } - return; - } - - // randomize send time - auto windowUs = (SendInterval - delta).MicroSeconds() / 2; - auto now = Now(); - for (const auto& [database, _] : QueryLogs) { - TInstant deadline = now + TDuration::MicroSeconds(windowUs + RandomNumber<ui64>(windowUs)); - Schedule(deadline, new TEvPrivate::TEvSendSummary(summaryEnd, database)); - Attempts[database] = std::make_pair(summaryEnd, 1); - } - } - - ui64 GetProcessorId(const TString& database) { - auto it = ProcessorIds.find(database); - if (it == ProcessorIds.end() || !it->second) { - return 0; - } - auto processorId = *it->second; - if (!processorId) { - RequestProcessorId(database); - return 0; - } - return processorId; - } - - void RequestProcessorId(const TString& database) { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - - auto request = MakeHolder<TNavigate>(); - request->ResultSet.push_back({}); - - auto& entry = request->ResultSet.back(); - entry.Path = SplitPath(database); - entry.Operation = TNavigate::EOp::OpPath; - entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; - entry.RedirectRequired = false; - - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); - } - - void SendCounters(const TString& database) { - auto processorId = GetProcessorId(database); - if (!processorId) { - return; - } - - auto& dbCounters = DatabaseCounters[database]; - - auto sendEv = MakeHolder<TEvSysView::TEvSendDbCountersRequest>(); - auto& record = sendEv->Record; - - if (dbCounters.IsConfirmed) { - for (auto& [service, state] : dbCounters.States) { - state.Counters->ToProto(state.Current); - } - ++dbCounters.Generation; - dbCounters.IsConfirmed = false; - dbCounters.IsRetrying = false; - } else { - dbCounters.IsRetrying = true; - } - - record.SetGeneration(dbCounters.Generation); - record.SetNodeId(SelfId().NodeId()); - - for (auto& [service, state] : dbCounters.States) { - auto* serviceCounters = record.AddServiceCounters(); - serviceCounters->SetService(service); - auto* diff = serviceCounters->MutableCounters(); - - CalculateCountersDiff(diff, state.Current, state.Confirmed); - } - - SVLOG_D("Send counters: " - << "service id# " << SelfId() - << ", processor id# " << processorId - << ", database# " << database - << ", generation# " << record.GetGeneration() - << ", node id# " << record.GetNodeId() - << ", is retrying# " << dbCounters.IsRetrying); - - Send(MakePipePeNodeCacheID(false), - new TEvPipeCache::TEvForward(sendEv.Release(), processorId, true), - IEventHandle::FlagTrackDelivery); - } - - void RequestDatabaseName(TPathId pathId) { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - auto request = MakeHolder<TNavigate>(); - request->ResultSet.push_back({}); - - auto& entry = request->ResultSet.back(); - entry.TableId.PathId = pathId; - entry.Operation = TNavigate::EOp::OpPath; - entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; - entry.RedirectRequired = false; - - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); - } - - void RegisterDbCounters(const TString& database, NKikimrSysView::EDbCountersService service, - TIntrusivePtr<IDbCounters> counters) - { - auto [it, inserted] = DatabaseCounters.try_emplace(database, TDbCounters()); - if (inserted) { - if (ProcessorIds.find(database) == ProcessorIds.end()) { - RequestProcessorId(database); - } - - if (DbWatcherActorId) { - auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database); - Send(DbWatcherActorId, evWatch.Release()); - } - } - it->second.States[service].Counters = counters; - } - - void Handle(TEvPrivate::TEvSendSummary::TPtr& ev) { - auto prevIntervalEnd = IntervalEnd - TotalInterval; - auto intervalEnd = ev->Get()->IntervalEnd; - if (intervalEnd != prevIntervalEnd) { - return; - } - if (Now() > intervalEnd + CollectInterval) { - return; - } - SendSummary(intervalEnd, ev->Get()->Database); - } - - void Handle(TEvPrivate::TEvProcessInterval::TPtr& ev) { - SVLOG_D("Handle TEvPrivate::TEvProcessInterval: " - << "service id# " << SelfId() - << ", interval end# " << IntervalEnd - << ", event interval end# " << ev->Get()->IntervalEnd); - - if (IntervalEnd == ev->Get()->IntervalEnd) { - Rotate(); - } - } - - void Handle(TEvSysView::TEvGetIntervalMetricsRequest::TPtr& ev) { - auto response = MakeHolder<TEvSysView::TEvGetIntervalMetricsResponse>(); - + IntervalEnd = GetNextIntervalEnd(); + Schedule(IntervalEnd, new TEvPrivate::TEvProcessInterval(IntervalEnd)); + } + + if (AppData()->FeatureFlags.GetEnableDbCounters()) { + auto intervalSize = ProcessCountersInterval.MicroSeconds(); + auto deadline = (Now().MicroSeconds() / intervalSize + 1) * intervalSize; + deadline += RandomNumber<ui64>(intervalSize / 5); + Schedule(TInstant::MicroSeconds(deadline), new TEvPrivate::TEvProcessCounters()); + + auto callback = MakeIntrusive<TServiceDbWatcherCallback>(ctx.ActorSystem()); + DbWatcherActorId = ctx.Register(CreateDbWatcherActor(callback)); + } + + if (HasExternalCounters) { + ctx.Register(CreateExtCountersUpdater(std::move(Config))); + } + + Become(&TSysViewService::StateWork); + } + + STFUNC(StateWork) { + Y_UNUSED(ctx); + switch(ev->GetTypeRewrite()) { + hFunc(TEvTenantPool::TEvTenantPoolStatus, Handle); + hFunc(TEvSysView::TEvCollectQueryStats, Handle); + hFunc(TEvSysView::TEvGetQueryStats, Handle); + hFunc(TEvSysView::TEvGetScanLimiter, Handle); + hFunc(TEvPrivate::TEvProcessInterval, Handle); + hFunc(TEvPrivate::TEvSendSummary, Handle); + hFunc(TEvPrivate::TEvProcessCounters, Handle); + hFunc(TEvPrivate::TEvRemoveDatabase, Handle); + hFunc(TEvSysView::TEvRegisterDbCounters, Handle); + hFunc(TEvSysView::TEvSendDbCountersResponse, Handle); + hFunc(TEvSysView::TEvGetIntervalMetricsRequest, Handle); + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + cFunc(TEvents::TEvPoison::EventType, PassAway); + default: + SVLOG_CRIT("NSysView::TSysViewService: unexpected event# " + << ev->GetTypeRewrite()); + } + } + +private: + struct TEvPrivate { + enum EEv { + EvProcessInterval = EventSpaceBegin(TEvents::ES_PRIVATE), + EvSendSummary, + EvProcessCounters, + EvRemoveDatabase, + EvEnd + }; + + struct TEvProcessInterval : public TEventLocal<TEvProcessInterval, EvProcessInterval> { + TInstant IntervalEnd; + + explicit TEvProcessInterval(TInstant intervalEnd) + : IntervalEnd(intervalEnd) + {} + }; + + struct TEvSendSummary : public TEventLocal<TEvSendSummary, EvSendSummary> { + TInstant IntervalEnd; + TString Database; + + TEvSendSummary(TInstant intervalEnd, const TString& database) + : IntervalEnd(intervalEnd) + , Database(database) + {} + }; + + struct TEvProcessCounters : public TEventLocal<TEvProcessCounters, EvProcessCounters> { + }; + + struct TEvRemoveDatabase : public TEventLocal<TEvRemoveDatabase, EvRemoveDatabase> { + TString Database; + TPathId PathId; + + TEvRemoveDatabase(const TString& database, TPathId pathId) + : Database(database) + , PathId(pathId) + {} + }; + }; + + class TServiceDbWatcherCallback : public TDbWatcherCallback { + TActorSystem* ActorSystem = {}; + + public: + explicit TServiceDbWatcherCallback(TActorSystem* actorSystem) + : ActorSystem(actorSystem) + {} + + void OnDatabaseRemoved(const TString& database, TPathId pathId) override { + auto evRemove = MakeHolder<TEvPrivate::TEvRemoveDatabase>(database, pathId); + auto service = MakeSysViewServiceID(ActorSystem->NodeId); + ActorSystem->Send(service, evRemove.Release()); + } + }; + + TInstant GetNextIntervalEnd() { + auto intervalSize = TotalInterval.MicroSeconds(); + auto rounded = (Now().MicroSeconds() / intervalSize + 1) * intervalSize; + return TInstant::MicroSeconds(rounded); + } + + void SendSummary(TInstant intervalEnd, const TString& database) { + auto processorId = GetProcessorId(database); + if (!processorId) { + return; + } + + auto summary = MakeHolder<TEvSysView::TEvIntervalQuerySummary>(); + auto& record = summary->Record; + record.SetDatabase(database); + record.SetIntervalEndUs(intervalEnd.MicroSeconds()); + record.SetNodeId(SelfId().NodeId()); + + auto& log = QueryLogs[database]; + log.Metrics.Report.FillSummary(*record.MutableMetrics()); + log.TopByDuration.Report.FillSummary(*record.MutableTopByDuration()); + log.TopByReadBytes.Report.FillSummary(*record.MutableTopByReadBytes()); + log.TopByCpuTime.Report.FillSummary(*record.MutableTopByCpuTime()); + log.TopByRequestUnits.Report.FillSummary(*record.MutableTopByRequestUnits()); + + SVLOG_D("Send interval summary: " + << "service id# " << SelfId() + << ", processor id# " << processorId + << ", database# " << database + << ", interval end# " << intervalEnd + << ", query count# " << record.GetMetrics().HashesSize()); + + Send(MakePipePeNodeCacheID(false), + new TEvPipeCache::TEvForward(summary.Release(), processorId, true), + IEventHandle::FlagTrackDelivery); + } + + void Rotate() { + auto summaryEnd = IntervalEnd; + auto delta = Now() - summaryEnd; + + IntervalEnd = GetNextIntervalEnd(); + Schedule(IntervalEnd, new TEvPrivate::TEvProcessInterval(IntervalEnd)); + + Attempts.clear(); + + SVLOG_D("Rotate logs: service id# " << SelfId() + << ", query logs count# " << QueryLogs.size() + << ", processor ids count# " << ProcessorIds.size() + << ", processor id to database count# " << ProcessorIdToDatabase.size()); + + if (QueryLogs.empty()) { + return; + } + + if (delta > CollectInterval) { + // too late to send summaries, skip + QueryLogs.clear(); + return; + } + + for (auto it = QueryLogs.begin(); it != QueryLogs.end(); ) { + it->second.Rotate(); + if (it->second.Metrics.Report.Empty()) { + it = QueryLogs.erase(it); + } else { + ++it; + } + } + + if (delta >= SendInterval) { + // send immediately + for (const auto& [database, _] : QueryLogs) { + SendSummary(summaryEnd, database); + } + return; + } + + // randomize send time + auto windowUs = (SendInterval - delta).MicroSeconds() / 2; + auto now = Now(); + for (const auto& [database, _] : QueryLogs) { + TInstant deadline = now + TDuration::MicroSeconds(windowUs + RandomNumber<ui64>(windowUs)); + Schedule(deadline, new TEvPrivate::TEvSendSummary(summaryEnd, database)); + Attempts[database] = std::make_pair(summaryEnd, 1); + } + } + + ui64 GetProcessorId(const TString& database) { + auto it = ProcessorIds.find(database); + if (it == ProcessorIds.end() || !it->second) { + return 0; + } + auto processorId = *it->second; + if (!processorId) { + RequestProcessorId(database); + return 0; + } + return processorId; + } + + void RequestProcessorId(const TString& database) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + + auto request = MakeHolder<TNavigate>(); + request->ResultSet.push_back({}); + + auto& entry = request->ResultSet.back(); + entry.Path = SplitPath(database); + entry.Operation = TNavigate::EOp::OpPath; + entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; + entry.RedirectRequired = false; + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); + } + + void SendCounters(const TString& database) { + auto processorId = GetProcessorId(database); + if (!processorId) { + return; + } + + auto& dbCounters = DatabaseCounters[database]; + + auto sendEv = MakeHolder<TEvSysView::TEvSendDbCountersRequest>(); + auto& record = sendEv->Record; + + if (dbCounters.IsConfirmed) { + for (auto& [service, state] : dbCounters.States) { + state.Counters->ToProto(state.Current); + } + ++dbCounters.Generation; + dbCounters.IsConfirmed = false; + dbCounters.IsRetrying = false; + } else { + dbCounters.IsRetrying = true; + } + + record.SetGeneration(dbCounters.Generation); + record.SetNodeId(SelfId().NodeId()); + + for (auto& [service, state] : dbCounters.States) { + auto* serviceCounters = record.AddServiceCounters(); + serviceCounters->SetService(service); + auto* diff = serviceCounters->MutableCounters(); + + CalculateCountersDiff(diff, state.Current, state.Confirmed); + } + + SVLOG_D("Send counters: " + << "service id# " << SelfId() + << ", processor id# " << processorId + << ", database# " << database + << ", generation# " << record.GetGeneration() + << ", node id# " << record.GetNodeId() + << ", is retrying# " << dbCounters.IsRetrying); + + Send(MakePipePeNodeCacheID(false), + new TEvPipeCache::TEvForward(sendEv.Release(), processorId, true), + IEventHandle::FlagTrackDelivery); + } + + void RequestDatabaseName(TPathId pathId) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + auto request = MakeHolder<TNavigate>(); + request->ResultSet.push_back({}); + + auto& entry = request->ResultSet.back(); + entry.TableId.PathId = pathId; + entry.Operation = TNavigate::EOp::OpPath; + entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; + entry.RedirectRequired = false; + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); + } + + void RegisterDbCounters(const TString& database, NKikimrSysView::EDbCountersService service, + TIntrusivePtr<IDbCounters> counters) + { + auto [it, inserted] = DatabaseCounters.try_emplace(database, TDbCounters()); + if (inserted) { + if (ProcessorIds.find(database) == ProcessorIds.end()) { + RequestProcessorId(database); + } + + if (DbWatcherActorId) { + auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database); + Send(DbWatcherActorId, evWatch.Release()); + } + } + it->second.States[service].Counters = counters; + } + + void Handle(TEvPrivate::TEvSendSummary::TPtr& ev) { + auto prevIntervalEnd = IntervalEnd - TotalInterval; + auto intervalEnd = ev->Get()->IntervalEnd; + if (intervalEnd != prevIntervalEnd) { + return; + } + if (Now() > intervalEnd + CollectInterval) { + return; + } + SendSummary(intervalEnd, ev->Get()->Database); + } + + void Handle(TEvPrivate::TEvProcessInterval::TPtr& ev) { + SVLOG_D("Handle TEvPrivate::TEvProcessInterval: " + << "service id# " << SelfId() + << ", interval end# " << IntervalEnd + << ", event interval end# " << ev->Get()->IntervalEnd); + + if (IntervalEnd == ev->Get()->IntervalEnd) { + Rotate(); + } + } + + void Handle(TEvSysView::TEvGetIntervalMetricsRequest::TPtr& ev) { + auto response = MakeHolder<TEvSysView::TEvGetIntervalMetricsResponse>(); + if (!AppData()->FeatureFlags.GetEnablePersistentQueryStats()) { - Send(ev->Sender, std::move(response), 0, ev->Cookie); - return; - } - - const auto& record = ev->Get()->Record; - response->Record.SetIntervalEndUs(record.GetIntervalEndUs()); - const auto& database = record.GetDatabase(); - - auto prevIntervalEnd = IntervalEnd - TotalInterval; - if (record.GetIntervalEndUs() != prevIntervalEnd.MicroSeconds()) { - SVLOG_W("Handle TEvSysView::TEvGetIntervalMetricsRequest, time mismatch: " - << "service id# " << SelfId() - << ", database# " << database - << ", prev interval end# " << prevIntervalEnd - << ", event interval end# " << record.GetIntervalEndUs()); - - Send(ev->Sender, std::move(response), 0, ev->Cookie); - return; - } - - auto it = QueryLogs.find(database); - if (it == QueryLogs.end()) { - SVLOG_W("Handle TEvSysView::TEvGetIntervalMetricsRequest, no database: " - << "service id# " << SelfId() - << ", database# " << database - << ", prev interval end# " << prevIntervalEnd); - - Send(ev->Sender, std::move(response), 0, ev->Cookie); - return; - } - - auto& log = it->second; - log.Metrics.Report.FillMetrics(record, response->Record); - log.TopByDuration.Report.FillStats( - record.GetTopByDuration(), *response->Record.MutableTopByDuration()); - log.TopByReadBytes.Report.FillStats( - record.GetTopByReadBytes(), *response->Record.MutableTopByReadBytes()); - log.TopByCpuTime.Report.FillStats( - record.GetTopByCpuTime(), *response->Record.MutableTopByCpuTime()); - log.TopByRequestUnits.Report.FillStats( - record.GetTopByRequestUnits(), *response->Record.MutableTopByRequestUnits()); - - SVLOG_D("Handle TEvSysView::TEvGetIntervalMetricsRequest: " - << "service id# " << SelfId() - << ", database# " << database - << ", prev interval end# " << prevIntervalEnd - << ", metrics count# " << response->Record.MetricsSize() - << ", texts count# " << response->Record.QueryTextsSize()); - - Send(ev->Sender, std::move(response), 0, ev->Cookie); - } - - void Handle(TEvPrivate::TEvProcessCounters::TPtr&) { - SVLOG_D("Handle TEvPrivate::TEvProcessCounters: " - << "service id# " << SelfId()); - - for (auto& [database, dbCounters] : DatabaseCounters) { - SendCounters(database); - } - - Schedule(ProcessCountersInterval, new TEvPrivate::TEvProcessCounters()); - } - - void Handle(TEvPrivate::TEvRemoveDatabase::TPtr& ev) { - auto database = ev->Get()->Database; - auto pathId = ev->Get()->PathId; - - SVLOG_D("Handle TEvPrivate::TEvRemoveDatabase: " - << "database# " << database - << ", pathId# " << pathId); - - QueryLogs.erase(database); - if (auto it = ProcessorIds.find(database); it != ProcessorIds.end()) { - if (it->second) { - ProcessorIdToDatabase.erase(*it->second); - } - } - ProcessorIds.erase(database); - Attempts.erase(database); - DatabaseCounters.erase(database); - UnresolvedTabletCounters.erase(pathId); - } - - void Handle(TEvSysView::TEvSendDbCountersResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - const auto& database = record.GetDatabase(); - const auto generation = record.GetGeneration(); - - auto it = DatabaseCounters.find(database); - if (it == DatabaseCounters.end()) { - SVLOG_W("Handle TEvSysView::TEvSendDbCountersResponse: " - << "service id# " << SelfId() - << ", unknown database# " << database); - return; - } - - auto& dbCounters = it->second; - if (generation != dbCounters.Generation) { - SVLOG_W("Handle TEvSysView::TEvSendDbCountersResponse, wrong generation: " - << "service id# " << SelfId() - << ", database# " << database - << ", generation# " << generation - << ", service generation# " << dbCounters.Generation); - return; - } - - dbCounters.IsConfirmed = true; - for (auto& [_, state] : dbCounters.States) { - state.Confirmed.Swap(state.Current); - } - - if (dbCounters.IsRetrying) { - SendCounters(database); - } - - SVLOG_D("Handle TEvSysView::TEvSendDbCountersResponse: " - << "service id# " << SelfId() - << ", database# " << database - << ", generation# " << generation); - } - - void Handle(TEvSysView::TEvRegisterDbCounters::TPtr& ev) { - const auto service = ev->Get()->Service; - - if (service == NKikimrSysView::TABLETS) { // register by path id - auto pathId = ev->Get()->PathId; - UnresolvedTabletCounters[pathId] = ev->Get()->Counters; - RequestDatabaseName(pathId); - - SVLOG_D("Handle TEvSysView::TEvRegisterDbCounters: " - << "service id# " << SelfId() - << ", path id# " << pathId - << ", service# " << (int)service); - - } else { // register by database name - const auto& database = ev->Get()->Database; - RegisterDbCounters(database, service, ev->Get()->Counters); - - SVLOG_D("Handle TEvSysView::TEvRegisterDbCounters: " - << "service id# " << SelfId() - << ", database# " << database - << ", service# " << (int)service); - } - } - - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - ui64 processorId = ev->Get()->TabletId; - TString database; - auto it = ProcessorIdToDatabase.find(processorId); - if (it != ProcessorIdToDatabase.end()) { - database = it->second; - RequestProcessorId(database); - } - - SVLOG_W("Summary delivery problem: service id# " << SelfId() - << ", processor id# " << processorId - << ", database# " << database); - - if (!database) { - return; - } - - auto attemptIt = Attempts.find(database); - if (attemptIt == Attempts.end()) { - return; - } - - auto& attempt = attemptIt->second; - if (++attempt.second > SummaryRetryAttempts) { - return; - } - - auto summaryEnd = attempt.first; - auto deadline = Now() + SummaryRetryInterval; - Schedule(deadline, new TEvPrivate::TEvSendSummary(summaryEnd, database)); - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - - THolder<TNavigate> request(ev->Get()->Request.Release()); - Y_VERIFY(request->ResultSet.size() == 1); - auto& entry = request->ResultSet.back(); - - if (entry.RequestType == TNavigate::TEntry::ERequestType::ByTableId) { - auto pathId = entry.TableId.PathId; - - if (entry.Status != TNavigate::EStatus::Ok) { - SVLOG_W("Navigate by path id failed: service id# " << SelfId() - << ", path id# " << pathId - << ", status# " << entry.Status); - return; - } - - auto database = CanonizePath(entry.Path); - auto it = UnresolvedTabletCounters.find(pathId); - if (it == UnresolvedTabletCounters.end()) { - return; - } - - RegisterDbCounters(database, NKikimrSysView::TABLETS, it->second); - UnresolvedTabletCounters.erase(it); - - SVLOG_I("Navigate by path id succeeded: service id# " << SelfId() - << ", path id# " << pathId - << ", database# " << database); - - } else { - auto database = CanonizePath(entry.Path); - - if (entry.Status != TNavigate::EStatus::Ok) { - SVLOG_W("Navigate by database failed: service id# " << SelfId() - << ", database# " << database - << ", status# " << entry.Status); - ProcessorIds.erase(database); - return; - } - - if (entry.DomainInfo->Params.HasSysViewProcessor()) { - auto processorId = entry.DomainInfo->Params.GetSysViewProcessor(); - ProcessorIds[database] = processorId; - ProcessorIdToDatabase[processorId] = database; - - SVLOG_I("Navigate by database succeeded: service id# " << SelfId() - << ", database# " << database - << ", processor id# " << processorId); - } else { - ProcessorIds[database] = 0; - - SVLOG_I("Navigate by database succeeded: service id# " << SelfId() - << ", database# " << database - << ", no sysview processor"); - } - } - } - - void Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr &ev) { - auto& record = ev->Get()->Record; - SVLOG_D("Handle TEvTenantPool::TEvTenantPoolStatus: service id# " << SelfId()); - - Tenants.clear(); - for (auto& slot : record.GetSlots()) { - Tenants.insert(slot.GetAssignedTenant()); - } - HasTenants = true; - - TopByDuration1Minute->Clear(); - TopByDuration1Hour->Clear(); - TopByReadBytes1Minute->Clear(); - TopByReadBytes1Hour->Clear(); - TopByCpuTime1Minute->Clear(); - TopByCpuTime1Hour->Clear(); - TopByRequestUnits1Minute->Clear(); - TopByRequestUnits1Hour->Clear(); - } - - void Handle(TEvSysView::TEvCollectQueryStats::TPtr& ev) { - const auto& database = ev->Get()->Database; - - auto stats = std::make_shared<NKikimrSysView::TQueryStats>(); - stats->Swap(&ev->Get()->QueryStats); - - SVLOG_T("Collect query stats: service id# " << SelfId() - << ", database# " << database - << ", query hash# " << stats->GetQueryTextHash() - << ", cpu time# " << stats->GetTotalCpuTimeUs()); - + Send(ev->Sender, std::move(response), 0, ev->Cookie); + return; + } + + const auto& record = ev->Get()->Record; + response->Record.SetIntervalEndUs(record.GetIntervalEndUs()); + const auto& database = record.GetDatabase(); + + auto prevIntervalEnd = IntervalEnd - TotalInterval; + if (record.GetIntervalEndUs() != prevIntervalEnd.MicroSeconds()) { + SVLOG_W("Handle TEvSysView::TEvGetIntervalMetricsRequest, time mismatch: " + << "service id# " << SelfId() + << ", database# " << database + << ", prev interval end# " << prevIntervalEnd + << ", event interval end# " << record.GetIntervalEndUs()); + + Send(ev->Sender, std::move(response), 0, ev->Cookie); + return; + } + + auto it = QueryLogs.find(database); + if (it == QueryLogs.end()) { + SVLOG_W("Handle TEvSysView::TEvGetIntervalMetricsRequest, no database: " + << "service id# " << SelfId() + << ", database# " << database + << ", prev interval end# " << prevIntervalEnd); + + Send(ev->Sender, std::move(response), 0, ev->Cookie); + return; + } + + auto& log = it->second; + log.Metrics.Report.FillMetrics(record, response->Record); + log.TopByDuration.Report.FillStats( + record.GetTopByDuration(), *response->Record.MutableTopByDuration()); + log.TopByReadBytes.Report.FillStats( + record.GetTopByReadBytes(), *response->Record.MutableTopByReadBytes()); + log.TopByCpuTime.Report.FillStats( + record.GetTopByCpuTime(), *response->Record.MutableTopByCpuTime()); + log.TopByRequestUnits.Report.FillStats( + record.GetTopByRequestUnits(), *response->Record.MutableTopByRequestUnits()); + + SVLOG_D("Handle TEvSysView::TEvGetIntervalMetricsRequest: " + << "service id# " << SelfId() + << ", database# " << database + << ", prev interval end# " << prevIntervalEnd + << ", metrics count# " << response->Record.MetricsSize() + << ", texts count# " << response->Record.QueryTextsSize()); + + Send(ev->Sender, std::move(response), 0, ev->Cookie); + } + + void Handle(TEvPrivate::TEvProcessCounters::TPtr&) { + SVLOG_D("Handle TEvPrivate::TEvProcessCounters: " + << "service id# " << SelfId()); + + for (auto& [database, dbCounters] : DatabaseCounters) { + SendCounters(database); + } + + Schedule(ProcessCountersInterval, new TEvPrivate::TEvProcessCounters()); + } + + void Handle(TEvPrivate::TEvRemoveDatabase::TPtr& ev) { + auto database = ev->Get()->Database; + auto pathId = ev->Get()->PathId; + + SVLOG_D("Handle TEvPrivate::TEvRemoveDatabase: " + << "database# " << database + << ", pathId# " << pathId); + + QueryLogs.erase(database); + if (auto it = ProcessorIds.find(database); it != ProcessorIds.end()) { + if (it->second) { + ProcessorIdToDatabase.erase(*it->second); + } + } + ProcessorIds.erase(database); + Attempts.erase(database); + DatabaseCounters.erase(database); + UnresolvedTabletCounters.erase(pathId); + } + + void Handle(TEvSysView::TEvSendDbCountersResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + const auto& database = record.GetDatabase(); + const auto generation = record.GetGeneration(); + + auto it = DatabaseCounters.find(database); + if (it == DatabaseCounters.end()) { + SVLOG_W("Handle TEvSysView::TEvSendDbCountersResponse: " + << "service id# " << SelfId() + << ", unknown database# " << database); + return; + } + + auto& dbCounters = it->second; + if (generation != dbCounters.Generation) { + SVLOG_W("Handle TEvSysView::TEvSendDbCountersResponse, wrong generation: " + << "service id# " << SelfId() + << ", database# " << database + << ", generation# " << generation + << ", service generation# " << dbCounters.Generation); + return; + } + + dbCounters.IsConfirmed = true; + for (auto& [_, state] : dbCounters.States) { + state.Confirmed.Swap(state.Current); + } + + if (dbCounters.IsRetrying) { + SendCounters(database); + } + + SVLOG_D("Handle TEvSysView::TEvSendDbCountersResponse: " + << "service id# " << SelfId() + << ", database# " << database + << ", generation# " << generation); + } + + void Handle(TEvSysView::TEvRegisterDbCounters::TPtr& ev) { + const auto service = ev->Get()->Service; + + if (service == NKikimrSysView::TABLETS) { // register by path id + auto pathId = ev->Get()->PathId; + UnresolvedTabletCounters[pathId] = ev->Get()->Counters; + RequestDatabaseName(pathId); + + SVLOG_D("Handle TEvSysView::TEvRegisterDbCounters: " + << "service id# " << SelfId() + << ", path id# " << pathId + << ", service# " << (int)service); + + } else { // register by database name + const auto& database = ev->Get()->Database; + RegisterDbCounters(database, service, ev->Get()->Counters); + + SVLOG_D("Handle TEvSysView::TEvRegisterDbCounters: " + << "service id# " << SelfId() + << ", database# " << database + << ", service# " << (int)service); + } + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { + ui64 processorId = ev->Get()->TabletId; + TString database; + auto it = ProcessorIdToDatabase.find(processorId); + if (it != ProcessorIdToDatabase.end()) { + database = it->second; + RequestProcessorId(database); + } + + SVLOG_W("Summary delivery problem: service id# " << SelfId() + << ", processor id# " << processorId + << ", database# " << database); + + if (!database) { + return; + } + + auto attemptIt = Attempts.find(database); + if (attemptIt == Attempts.end()) { + return; + } + + auto& attempt = attemptIt->second; + if (++attempt.second > SummaryRetryAttempts) { + return; + } + + auto summaryEnd = attempt.first; + auto deadline = Now() + SummaryRetryInterval; + Schedule(deadline, new TEvPrivate::TEvSendSummary(summaryEnd, database)); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + + THolder<TNavigate> request(ev->Get()->Request.Release()); + Y_VERIFY(request->ResultSet.size() == 1); + auto& entry = request->ResultSet.back(); + + if (entry.RequestType == TNavigate::TEntry::ERequestType::ByTableId) { + auto pathId = entry.TableId.PathId; + + if (entry.Status != TNavigate::EStatus::Ok) { + SVLOG_W("Navigate by path id failed: service id# " << SelfId() + << ", path id# " << pathId + << ", status# " << entry.Status); + return; + } + + auto database = CanonizePath(entry.Path); + auto it = UnresolvedTabletCounters.find(pathId); + if (it == UnresolvedTabletCounters.end()) { + return; + } + + RegisterDbCounters(database, NKikimrSysView::TABLETS, it->second); + UnresolvedTabletCounters.erase(it); + + SVLOG_I("Navigate by path id succeeded: service id# " << SelfId() + << ", path id# " << pathId + << ", database# " << database); + + } else { + auto database = CanonizePath(entry.Path); + + if (entry.Status != TNavigate::EStatus::Ok) { + SVLOG_W("Navigate by database failed: service id# " << SelfId() + << ", database# " << database + << ", status# " << entry.Status); + ProcessorIds.erase(database); + return; + } + + if (entry.DomainInfo->Params.HasSysViewProcessor()) { + auto processorId = entry.DomainInfo->Params.GetSysViewProcessor(); + ProcessorIds[database] = processorId; + ProcessorIdToDatabase[processorId] = database; + + SVLOG_I("Navigate by database succeeded: service id# " << SelfId() + << ", database# " << database + << ", processor id# " << processorId); + } else { + ProcessorIds[database] = 0; + + SVLOG_I("Navigate by database succeeded: service id# " << SelfId() + << ", database# " << database + << ", no sysview processor"); + } + } + } + + void Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr &ev) { + auto& record = ev->Get()->Record; + SVLOG_D("Handle TEvTenantPool::TEvTenantPoolStatus: service id# " << SelfId()); + + Tenants.clear(); + for (auto& slot : record.GetSlots()) { + Tenants.insert(slot.GetAssignedTenant()); + } + HasTenants = true; + + TopByDuration1Minute->Clear(); + TopByDuration1Hour->Clear(); + TopByReadBytes1Minute->Clear(); + TopByReadBytes1Hour->Clear(); + TopByCpuTime1Minute->Clear(); + TopByCpuTime1Hour->Clear(); + TopByRequestUnits1Minute->Clear(); + TopByRequestUnits1Hour->Clear(); + } + + void Handle(TEvSysView::TEvCollectQueryStats::TPtr& ev) { + const auto& database = ev->Get()->Database; + + auto stats = std::make_shared<NKikimrSysView::TQueryStats>(); + stats->Swap(&ev->Get()->QueryStats); + + SVLOG_T("Collect query stats: service id# " << SelfId() + << ", database# " << database + << ", query hash# " << stats->GetQueryTextHash() + << ", cpu time# " << stats->GetTotalCpuTimeUs()); + if (AppData()->FeatureFlags.GetEnablePersistentQueryStats() && !database.empty()) { - auto queryEnd = TInstant::MilliSeconds(stats->GetEndTimeMs()); - if (queryEnd < IntervalEnd - TotalInterval) { - return; - } - if (queryEnd >= IntervalEnd) { - Rotate(); - } - - QueryLogs[database].Add(stats); - - if (ProcessorIds.find(database) == ProcessorIds.end()) { - ProcessorIds[database]; - RequestProcessorId(database); - - if (DbWatcherActorId) { - auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database); - Send(DbWatcherActorId, evWatch.Release()); - } - } - } - - // gather old style stats while migrating - // TODO: remove later - TopByDuration1Minute->Add(stats); - TopByDuration1Hour->Add(stats); - TopByReadBytes1Minute->Add(stats); - TopByReadBytes1Hour->Add(stats); - TopByCpuTime1Minute->Add(stats); - TopByCpuTime1Hour->Add(stats); - TopByRequestUnits1Minute->Add(stats); - TopByRequestUnits1Hour->Add(stats); - } - - void Handle(TEvSysView::TEvGetQueryStats::TPtr& ev) { - auto& record = ev->Get()->Record; - auto result = MakeHolder<TEvSysView::TEvGetQueryStatsResult>(); - - ui64 startBucket = 0; - if (record.HasStartBucket()) { - startBucket = record.GetStartBucket(); - } - - if (!HasTenants || - !record.HasTenantName() || - !Tenants.contains(record.GetTenantName())) - { - Send(ev->Sender, std::move(result), 0, ev->Cookie); - return; - } - - switch (record.GetStatsType()) { - case NKikimrSysView::TOP_DURATION_ONE_MINUTE: - TopByDuration1Minute->ToProto(startBucket, result->Record); - break; - case NKikimrSysView::TOP_DURATION_ONE_HOUR: - TopByDuration1Hour->ToProto(startBucket, result->Record); - break; - case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE: - TopByReadBytes1Minute->ToProto(startBucket, result->Record); - break; - case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR: - TopByReadBytes1Hour->ToProto(startBucket, result->Record); - break; - case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE: - TopByCpuTime1Minute->ToProto(startBucket, result->Record); - break; - case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR: - TopByCpuTime1Hour->ToProto(startBucket, result->Record); - break; - case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE: - TopByRequestUnits1Minute->ToProto(startBucket, result->Record); - break; - case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR: - TopByRequestUnits1Hour->ToProto(startBucket, result->Record); - break; - default: - SVLOG_CRIT("NSysView::TSysViewService: unexpected query stats type# " - << (size_t)record.GetStatsType()); - // send empty result - break; - } - - Send(ev->Sender, std::move(result), 0, ev->Cookie); - } - - void Handle(TEvSysView::TEvGetScanLimiter::TPtr& ev) { - auto result = MakeHolder<TEvSysView::TEvGetScanLimiterResult>(); - result->ScanLimiter = ScanLimiter; - Send(ev->Sender, std::move(result)); - } - - void PassAway() override { - Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); - TBase::PassAway(); - } - -private: - TExtCountersConfig Config; - const bool HasExternalCounters; - const TDuration TotalInterval; - const TDuration CollectInterval; - const TDuration SendInterval; - - template <typename TInterval> - struct TDbWindow { - TInterval Collect; - TInterval Report; - - void Clear() { - Collect.Clear(); - Report.Clear(); - } - - void Rotate() { - Report.Clear(); - Report.Swap(Collect); - } - }; - - struct TDbQueryLog { - TDbWindow<TQueryInterval> Metrics; - TDbWindow<TQueryStatsDedupBucket<TDurationGreater>> TopByDuration; - TDbWindow<TQueryStatsDedupBucket<TReadBytesGreater>> TopByReadBytes; - TDbWindow<TQueryStatsDedupBucket<TCpuTimeGreater>> TopByCpuTime; - TDbWindow<TQueryStatsDedupBucket<TRequestUnitsGreater>> TopByRequestUnits; - - void Clear() { - Metrics.Clear(); - TopByDuration.Clear(); - TopByReadBytes.Clear(); - TopByCpuTime.Clear(); - TopByRequestUnits.Clear(); - } - - void Rotate() { - Metrics.Rotate(); - TopByDuration.Rotate(); - TopByReadBytes.Rotate(); - TopByCpuTime.Rotate(); - TopByRequestUnits.Rotate(); - } - - void Add(TQueryStatsPtr stats) { - Metrics.Collect.Add(stats); - TopByDuration.Collect.Add(stats); - TopByReadBytes.Collect.Add(stats); - TopByCpuTime.Collect.Add(stats); - TopByRequestUnits.Collect.Add(stats); - } - }; - std::unordered_map<TString, TDbQueryLog> QueryLogs; - - TInstant IntervalEnd; - - std::unordered_map<TString, TMaybe<ui64>> ProcessorIds; - std::unordered_map<ui64, TString> ProcessorIdToDatabase; - - std::unordered_map<TString, std::pair<TInstant, size_t>> Attempts; - - THolder<TServiceQueryHistory<TDurationGreater>> TopByDuration1Minute; - THolder<TServiceQueryHistory<TDurationGreater>> TopByDuration1Hour; - - THolder<TServiceQueryHistory<TReadBytesGreater>> TopByReadBytes1Minute; - THolder<TServiceQueryHistory<TReadBytesGreater>> TopByReadBytes1Hour; - - THolder<TServiceQueryHistory<TCpuTimeGreater>> TopByCpuTime1Minute; - THolder<TServiceQueryHistory<TCpuTimeGreater>> TopByCpuTime1Hour; - - THolder<TServiceQueryHistory<TRequestUnitsGreater>> TopByRequestUnits1Minute; - THolder<TServiceQueryHistory<TRequestUnitsGreater>> TopByRequestUnits1Hour; - - bool HasTenants = false; - THashSet<TString> Tenants; - - struct TDbCountersState { - TIntrusivePtr<IDbCounters> Counters; - NKikimr::NSysView::TDbServiceCounters Current; - NKikimr::NSysView::TDbServiceCounters Confirmed; - }; - - struct TDbCounters { - std::unordered_map<NKikimrSysView::EDbCountersService, TDbCountersState> States; - ui64 Generation; - bool IsConfirmed = true; - bool IsRetrying = false; - - TDbCounters() - : Generation(RandomNumber<ui64>()) - {} - }; - - std::unordered_map<TString, TDbCounters> DatabaseCounters; - THashMap<TPathId, TIntrusivePtr<IDbCounters>> UnresolvedTabletCounters; - TActorId DbWatcherActorId; - - static constexpr i64 ConcurrentScansLimit = 5; - TIntrusivePtr<TScanLimiter> ScanLimiter; - - static constexpr TDuration SummaryRetryInterval = TDuration::Seconds(2); - static constexpr size_t SummaryRetryAttempts = 5; - - static constexpr TDuration ProcessCountersInterval = TDuration::Seconds(5); -}; - -THolder<IActor> CreateSysViewService( - TExtCountersConfig&& config, bool hasExternalCounters) -{ - return MakeHolder<TSysViewService>( - std::move(config), hasExternalCounters, EProcessorMode::MINUTE); -} - -THolder<IActor> CreateSysViewServiceForTests() { - return MakeHolder<TSysViewService>( - TExtCountersConfig(), true, EProcessorMode::FAST); -} - -} // NSysView -} // NKikimr + auto queryEnd = TInstant::MilliSeconds(stats->GetEndTimeMs()); + if (queryEnd < IntervalEnd - TotalInterval) { + return; + } + if (queryEnd >= IntervalEnd) { + Rotate(); + } + + QueryLogs[database].Add(stats); + + if (ProcessorIds.find(database) == ProcessorIds.end()) { + ProcessorIds[database]; + RequestProcessorId(database); + + if (DbWatcherActorId) { + auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database); + Send(DbWatcherActorId, evWatch.Release()); + } + } + } + + // gather old style stats while migrating + // TODO: remove later + TopByDuration1Minute->Add(stats); + TopByDuration1Hour->Add(stats); + TopByReadBytes1Minute->Add(stats); + TopByReadBytes1Hour->Add(stats); + TopByCpuTime1Minute->Add(stats); + TopByCpuTime1Hour->Add(stats); + TopByRequestUnits1Minute->Add(stats); + TopByRequestUnits1Hour->Add(stats); + } + + void Handle(TEvSysView::TEvGetQueryStats::TPtr& ev) { + auto& record = ev->Get()->Record; + auto result = MakeHolder<TEvSysView::TEvGetQueryStatsResult>(); + + ui64 startBucket = 0; + if (record.HasStartBucket()) { + startBucket = record.GetStartBucket(); + } + + if (!HasTenants || + !record.HasTenantName() || + !Tenants.contains(record.GetTenantName())) + { + Send(ev->Sender, std::move(result), 0, ev->Cookie); + return; + } + + switch (record.GetStatsType()) { + case NKikimrSysView::TOP_DURATION_ONE_MINUTE: + TopByDuration1Minute->ToProto(startBucket, result->Record); + break; + case NKikimrSysView::TOP_DURATION_ONE_HOUR: + TopByDuration1Hour->ToProto(startBucket, result->Record); + break; + case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE: + TopByReadBytes1Minute->ToProto(startBucket, result->Record); + break; + case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR: + TopByReadBytes1Hour->ToProto(startBucket, result->Record); + break; + case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE: + TopByCpuTime1Minute->ToProto(startBucket, result->Record); + break; + case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR: + TopByCpuTime1Hour->ToProto(startBucket, result->Record); + break; + case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE: + TopByRequestUnits1Minute->ToProto(startBucket, result->Record); + break; + case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR: + TopByRequestUnits1Hour->ToProto(startBucket, result->Record); + break; + default: + SVLOG_CRIT("NSysView::TSysViewService: unexpected query stats type# " + << (size_t)record.GetStatsType()); + // send empty result + break; + } + + Send(ev->Sender, std::move(result), 0, ev->Cookie); + } + + void Handle(TEvSysView::TEvGetScanLimiter::TPtr& ev) { + auto result = MakeHolder<TEvSysView::TEvGetScanLimiterResult>(); + result->ScanLimiter = ScanLimiter; + Send(ev->Sender, std::move(result)); + } + + void PassAway() override { + Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + TBase::PassAway(); + } + +private: + TExtCountersConfig Config; + const bool HasExternalCounters; + const TDuration TotalInterval; + const TDuration CollectInterval; + const TDuration SendInterval; + + template <typename TInterval> + struct TDbWindow { + TInterval Collect; + TInterval Report; + + void Clear() { + Collect.Clear(); + Report.Clear(); + } + + void Rotate() { + Report.Clear(); + Report.Swap(Collect); + } + }; + + struct TDbQueryLog { + TDbWindow<TQueryInterval> Metrics; + TDbWindow<TQueryStatsDedupBucket<TDurationGreater>> TopByDuration; + TDbWindow<TQueryStatsDedupBucket<TReadBytesGreater>> TopByReadBytes; + TDbWindow<TQueryStatsDedupBucket<TCpuTimeGreater>> TopByCpuTime; + TDbWindow<TQueryStatsDedupBucket<TRequestUnitsGreater>> TopByRequestUnits; + + void Clear() { + Metrics.Clear(); + TopByDuration.Clear(); + TopByReadBytes.Clear(); + TopByCpuTime.Clear(); + TopByRequestUnits.Clear(); + } + + void Rotate() { + Metrics.Rotate(); + TopByDuration.Rotate(); + TopByReadBytes.Rotate(); + TopByCpuTime.Rotate(); + TopByRequestUnits.Rotate(); + } + + void Add(TQueryStatsPtr stats) { + Metrics.Collect.Add(stats); + TopByDuration.Collect.Add(stats); + TopByReadBytes.Collect.Add(stats); + TopByCpuTime.Collect.Add(stats); + TopByRequestUnits.Collect.Add(stats); + } + }; + std::unordered_map<TString, TDbQueryLog> QueryLogs; + + TInstant IntervalEnd; + + std::unordered_map<TString, TMaybe<ui64>> ProcessorIds; + std::unordered_map<ui64, TString> ProcessorIdToDatabase; + + std::unordered_map<TString, std::pair<TInstant, size_t>> Attempts; + + THolder<TServiceQueryHistory<TDurationGreater>> TopByDuration1Minute; + THolder<TServiceQueryHistory<TDurationGreater>> TopByDuration1Hour; + + THolder<TServiceQueryHistory<TReadBytesGreater>> TopByReadBytes1Minute; + THolder<TServiceQueryHistory<TReadBytesGreater>> TopByReadBytes1Hour; + + THolder<TServiceQueryHistory<TCpuTimeGreater>> TopByCpuTime1Minute; + THolder<TServiceQueryHistory<TCpuTimeGreater>> TopByCpuTime1Hour; + + THolder<TServiceQueryHistory<TRequestUnitsGreater>> TopByRequestUnits1Minute; + THolder<TServiceQueryHistory<TRequestUnitsGreater>> TopByRequestUnits1Hour; + + bool HasTenants = false; + THashSet<TString> Tenants; + + struct TDbCountersState { + TIntrusivePtr<IDbCounters> Counters; + NKikimr::NSysView::TDbServiceCounters Current; + NKikimr::NSysView::TDbServiceCounters Confirmed; + }; + + struct TDbCounters { + std::unordered_map<NKikimrSysView::EDbCountersService, TDbCountersState> States; + ui64 Generation; + bool IsConfirmed = true; + bool IsRetrying = false; + + TDbCounters() + : Generation(RandomNumber<ui64>()) + {} + }; + + std::unordered_map<TString, TDbCounters> DatabaseCounters; + THashMap<TPathId, TIntrusivePtr<IDbCounters>> UnresolvedTabletCounters; + TActorId DbWatcherActorId; + + static constexpr i64 ConcurrentScansLimit = 5; + TIntrusivePtr<TScanLimiter> ScanLimiter; + + static constexpr TDuration SummaryRetryInterval = TDuration::Seconds(2); + static constexpr size_t SummaryRetryAttempts = 5; + + static constexpr TDuration ProcessCountersInterval = TDuration::Seconds(5); +}; + +THolder<IActor> CreateSysViewService( + TExtCountersConfig&& config, bool hasExternalCounters) +{ + return MakeHolder<TSysViewService>( + std::move(config), hasExternalCounters, EProcessorMode::MINUTE); +} + +THolder<IActor> CreateSysViewServiceForTests() { + return MakeHolder<TSysViewService>( + TExtCountersConfig(), true, EProcessorMode::FAST); +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/service/sysview_service.h b/ydb/core/sys_view/service/sysview_service.h index fd923c2ffa..430dfb0261 100644 --- a/ydb/core/sys_view/service/sysview_service.h +++ b/ydb/core/sys_view/service/sysview_service.h @@ -1,28 +1,28 @@ -#pragma once - -#include "ext_counters.h" - +#pragma once + +#include "ext_counters.h" + #include <ydb/core/base/defs.h> #include <ydb/core/protos/kqp.pb.h> #include <ydb/core/protos/kqp_stats.pb.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + inline TActorId MakeSysViewServiceID(ui32 node) { - const char x[12] = "SysViewSvc!"; + const char x[12] = "SysViewSvc!"; return TActorId(node, TStringBuf(x, 12)); -} - -void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery* queryStats, - TDuration queryDuration, const TString& queryText, - const TString& userSID, ui64 parametersSize, const TString& database, - const NKikimrKqp::EQueryType type, ui64 requestUnits); - -THolder<IActor> CreateSysViewService( - TExtCountersConfig&& config, bool hasExternalCounters); - -THolder<IActor> CreateSysViewServiceForTests(); - -} // NSysView -} // NKikimr +} + +void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery* queryStats, + TDuration queryDuration, const TString& queryText, + const TString& userSID, ui64 parametersSize, const TString& database, + const NKikimrKqp::EQueryType type, ui64 requestUnits); + +THolder<IActor> CreateSysViewService( + TExtCountersConfig&& config, bool hasExternalCounters); + +THolder<IActor> CreateSysViewServiceForTests(); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/service/ut/ya.make b/ydb/core/sys_view/service/ut/ya.make index a7b7eac2e0..60f2fd4b7a 100644 --- a/ydb/core/sys_view/service/ut/ya.make +++ b/ydb/core/sys_view/service/ut/ya.make @@ -1,20 +1,20 @@ UNITTEST_FOR(ydb/core/sys_view/service) - -OWNER( - monster - g:kikimr -) - -FORK_SUBTESTS() -SIZE(MEDIUM) -TIMEOUT(600) - -PEERDIR( + +OWNER( + monster + g:kikimr +) + +FORK_SUBTESTS() +SIZE(MEDIUM) +TIMEOUT(600) + +PEERDIR( library/cpp/testing/unittest -) - -SRCS( - query_history_ut.cpp -) - -END() +) + +SRCS( + query_history_ut.cpp +) + +END() diff --git a/ydb/core/sys_view/service/ya.make b/ydb/core/sys_view/service/ya.make index f8f680ec0a..133557dbe1 100644 --- a/ydb/core/sys_view/service/ya.make +++ b/ydb/core/sys_view/service/ya.make @@ -1,30 +1,30 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - db_counters.h - db_counters.cpp - ext_counters.h - ext_counters.cpp - query_history.h - query_interval.h - query_interval.cpp - sysview_service.h - sysview_service.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + db_counters.h + db_counters.cpp + ext_counters.h + ext_counters.cpp + query_history.h + query_interval.h + query_interval.cpp + sysview_service.h + sysview_service.cpp +) + +PEERDIR( library/cpp/actors/core ydb/core/base ydb/core/protos ydb/library/aclib/protos -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/core/sys_view/storage/groups.cpp b/ydb/core/sys_view/storage/groups.cpp index 45e80fc597..dabd46f9dc 100644 --- a/ydb/core/sys_view/storage/groups.cpp +++ b/ydb/core/sys_view/storage/groups.cpp @@ -1,22 +1,22 @@ #include "groups.h" #include "base.h" - + namespace NKikimr::NSysView { - + template<> void SetField<0>(NKikimrSysView::TGroupKey& key, ui32 value) { key.SetGroupId(value); } class TGroupsScan : public TStorageScanBase<TGroupsScan, TEvSysView::TEvGetGroupsResponse> { -public: +public: using TStorageScanBase::TStorageScanBase; - + static constexpr const char *GetName() { return "TGroupsScan"; } - + TEvSysView::TEvGetGroupsRequest *CreateQuery() { - auto request = MakeHolder<TEvSysView::TEvGetGroupsRequest>(); - ConvertKeyRange<NKikimrSysView::TEvGetGroupsRequest, ui32>(request->Record, TableRange); + auto request = MakeHolder<TEvSysView::TEvGetGroupsRequest>(); + ConvertKeyRange<NKikimrSysView::TEvGetGroupsRequest, ui32>(request->Record, TableRange); return request.Release(); - } - + } + static const TFieldMap& GetFieldMap() { using T = Schema::Groups; using E = NKikimrSysView::TGroupEntry; @@ -36,15 +36,15 @@ public: {T::PutTabletLogLatency::ColumnId, {E::kInfoFieldNumber, V::kPutTabletLogLatencyFieldNumber}}, {T::PutUserDataLatency::ColumnId, {E::kInfoFieldNumber, V::kPutUserDataLatencyFieldNumber}}, {T::GetFastLatency::ColumnId, {E::kInfoFieldNumber, V::kGetFastLatencyFieldNumber}}, - }; + }; return fieldMap; - } - -}; + } + +}; THolder<IActor> CreateGroupsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ return MakeHolder<TGroupsScan>(ownerId, scanId, tableId, tableRange, columns); -} - +} + } // NKikimr::NSysView diff --git a/ydb/core/sys_view/storage/groups.h b/ydb/core/sys_view/storage/groups.h index af0f838028..ab861d94a8 100644 --- a/ydb/core/sys_view/storage/groups.h +++ b/ydb/core/sys_view/storage/groups.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + THolder<IActor> CreateGroupsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/storage/pdisks.cpp b/ydb/core/sys_view/storage/pdisks.cpp index 39fa7e4ac7..47f84ae9aa 100644 --- a/ydb/core/sys_view/storage/pdisks.cpp +++ b/ydb/core/sys_view/storage/pdisks.cpp @@ -1,23 +1,23 @@ -#include "pdisks.h" +#include "pdisks.h" #include "base.h" - + namespace NKikimr::NSysView { - + template<> void SetField<0>(NKikimrSysView::TPDiskKey& key, ui32 value) { key.SetNodeId(value); } template<> void SetField<1>(NKikimrSysView::TPDiskKey& key, ui32 value) { key.SetPDiskId(value); } class TPDisksScan : public TStorageScanBase<TPDisksScan, TEvSysView::TEvGetPDisksResponse> { -public: +public: using TStorageScanBase::TStorageScanBase; - + static constexpr const char *GetName() { return "TPDisksScan"; } - + TEvSysView::TEvGetPDisksRequest *CreateQuery() { - auto request = MakeHolder<TEvSysView::TEvGetPDisksRequest>(); - ConvertKeyRange<NKikimrSysView::TEvGetPDisksRequest, ui32, ui32>(request->Record, TableRange); + auto request = MakeHolder<TEvSysView::TEvGetPDisksRequest>(); + ConvertKeyRange<NKikimrSysView::TEvGetPDisksRequest, ui32, ui32>(request->Record, TableRange); return request.Release(); - } - + } + static const TFieldMap& GetFieldMap() { using T = Schema::PDisks; using E = NKikimrSysView::TPDiskEntry; @@ -39,15 +39,15 @@ public: {T::StatusChangeTimestamp::ColumnId, {E::kInfoFieldNumber, V::kStatusChangeTimestampFieldNumber}}, {T::ExpectedSlotCount::ColumnId, {E::kInfoFieldNumber, V::kExpectedSlotCountFieldNumber}}, {T::NumActiveSlots::ColumnId, {E::kInfoFieldNumber, V::kNumActiveSlotsFieldNumber}}, - }; + }; return fieldMap; - } -}; - + } +}; + THolder<IActor> CreatePDisksScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ return MakeHolder<TPDisksScan>(ownerId, scanId, tableId, tableRange, columns); -} - +} + } // NKikimr::NSysView diff --git a/ydb/core/sys_view/storage/pdisks.h b/ydb/core/sys_view/storage/pdisks.h index 7457f09c96..4172cb333f 100644 --- a/ydb/core/sys_view/storage/pdisks.h +++ b/ydb/core/sys_view/storage/pdisks.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + THolder<IActor> CreatePDisksScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/storage/storage_pools.cpp b/ydb/core/sys_view/storage/storage_pools.cpp index 71e5a61baf..66896a6999 100644 --- a/ydb/core/sys_view/storage/storage_pools.cpp +++ b/ydb/core/sys_view/storage/storage_pools.cpp @@ -1,23 +1,23 @@ #include "storage_pools.h" #include "base.h" - + namespace NKikimr::NSysView { - + template<> void SetField<0>(NKikimrSysView::TStoragePoolKey& key, ui64 value) { key.SetBoxId(value); } template<> void SetField<1>(NKikimrSysView::TStoragePoolKey& key, ui64 value) { key.SetStoragePoolId(value); } class TStoragePoolsScan : public TStorageScanBase<TStoragePoolsScan, TEvSysView::TEvGetStoragePoolsResponse> { -public: +public: using TStorageScanBase::TStorageScanBase; - + static constexpr const char *GetName() { return "TStoragePoolsScan"; } - + TEvSysView::TEvGetStoragePoolsRequest *CreateQuery() { - auto request = MakeHolder<TEvSysView::TEvGetStoragePoolsRequest>(); - ConvertKeyRange<NKikimrSysView::TEvGetStoragePoolsRequest, ui64, ui64>(request->Record, TableRange); + auto request = MakeHolder<TEvSysView::TEvGetStoragePoolsRequest>(); + ConvertKeyRange<NKikimrSysView::TEvGetStoragePoolsRequest, ui64, ui64>(request->Record, TableRange); return request.Release(); - } - + } + static const TFieldMap& GetFieldMap() { using T = Schema::StoragePools; using E = NKikimrSysView::TStoragePoolEntry; @@ -35,15 +35,15 @@ public: {T::EncryptionMode::ColumnId, {E::kInfoFieldNumber, V::kEncryptionModeFieldNumber}}, {T::SchemeshardId::ColumnId, {E::kInfoFieldNumber, V::kSchemeshardIdFieldNumber}}, {T::PathId::ColumnId, {E::kInfoFieldNumber, V::kPathIdFieldNumber}}, - }; + }; return fieldMap; - } -}; - + } +}; + THolder<IActor> CreateStoragePoolsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ return MakeHolder<TStoragePoolsScan>(ownerId, scanId, tableId, tableRange, columns); -} - +} + } // NKikimr::NSysView diff --git a/ydb/core/sys_view/storage/storage_pools.h b/ydb/core/sys_view/storage/storage_pools.h index 3cc4a0b530..7ced359d00 100644 --- a/ydb/core/sys_view/storage/storage_pools.h +++ b/ydb/core/sys_view/storage/storage_pools.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + THolder<IActor> CreateStoragePoolsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/storage/vslots.cpp b/ydb/core/sys_view/storage/vslots.cpp index f907a25635..987eb2750d 100644 --- a/ydb/core/sys_view/storage/vslots.cpp +++ b/ydb/core/sys_view/storage/vslots.cpp @@ -1,24 +1,24 @@ -#include "vslots.h" +#include "vslots.h" #include "base.h" - + namespace NKikimr::NSysView { - + template<> void SetField<0>(NKikimrSysView::TVSlotKey& key, ui32 value) { key.SetNodeId(value); } template<> void SetField<1>(NKikimrSysView::TVSlotKey& key, ui32 value) { key.SetPDiskId(value); } template<> void SetField<2>(NKikimrSysView::TVSlotKey& key, ui32 value) { key.SetVSlotId(value); } class TVSlotsScan : public TStorageScanBase<TVSlotsScan, TEvSysView::TEvGetVSlotsResponse> { -public: +public: using TStorageScanBase::TStorageScanBase; - + static constexpr const char *GetName() { return "TVSlotsScan"; } - + TEvSysView::TEvGetVSlotsRequest *CreateQuery() { - auto request = MakeHolder<TEvSysView::TEvGetVSlotsRequest>(); - ConvertKeyRange<NKikimrSysView::TEvGetVSlotsRequest, ui32, ui32, ui32>(request->Record, TableRange); + auto request = MakeHolder<TEvSysView::TEvGetVSlotsRequest>(); + ConvertKeyRange<NKikimrSysView::TEvGetVSlotsRequest, ui32, ui32, ui32>(request->Record, TableRange); return request.Release(); - } - + } + static const TFieldMap& GetFieldMap() { using T = Schema::VSlots; using E = NKikimrSysView::TVSlotEntry; @@ -37,15 +37,15 @@ public: {T::AvailableSize::ColumnId, {E::kInfoFieldNumber, V::kAvailableSizeFieldNumber}}, {T::Status::ColumnId, {E::kInfoFieldNumber, V::kStatusV2FieldNumber}}, {T::Kind::ColumnId, {E::kInfoFieldNumber, V::kKindFieldNumber}}, - }; + }; return fieldMap; - } -}; - + } +}; + THolder<IActor> CreateVSlotsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ return MakeHolder<TVSlotsScan>(ownerId, scanId, tableId, tableRange, columns); -} - +} + } // NKikimr::NSysView diff --git a/ydb/core/sys_view/storage/vslots.h b/ydb/core/sys_view/storage/vslots.h index 3b6026ea46..34f38f9e0f 100644 --- a/ydb/core/sys_view/storage/vslots.h +++ b/ydb/core/sys_view/storage/vslots.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + THolder<IActor> CreateVSlotsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/storage/ya.make b/ydb/core/sys_view/storage/ya.make index 7136a617f7..324b36b798 100644 --- a/ydb/core/sys_view/storage/ya.make +++ b/ydb/core/sys_view/storage/ya.make @@ -1,30 +1,30 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - groups.h - groups.cpp - pdisks.h - pdisks.cpp - storage_pools.h - storage_pools.cpp +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + groups.h + groups.cpp + pdisks.h + pdisks.cpp + storage_pools.h + storage_pools.cpp storage_stats.h storage_stats.cpp - vslots.h - vslots.cpp -) - -PEERDIR( + vslots.h + vslots.cpp +) + +PEERDIR( library/cpp/actors/core ydb/core/base ydb/core/kqp/runtime ydb/core/sys_view/common -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/sys_view/tablets/tablets.cpp b/ydb/core/sys_view/tablets/tablets.cpp index 73da45318d..88627748d8 100644 --- a/ydb/core/sys_view/tablets/tablets.cpp +++ b/ydb/core/sys_view/tablets/tablets.cpp @@ -1,77 +1,77 @@ -#include "tablets.h" - +#include "tablets.h" + #include <ydb/core/sys_view/common/events.h> #include <ydb/core/sys_view/common/schema.h> #include <ydb/core/sys_view/common/scan_actor_base_impl.h> #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/mind/hive/hive.h> #include <ydb/core/mind/hive/tablet_info.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> -#include <library/cpp/actors/core/hfunc.h> - -namespace NKikimr::NSysView { - -using namespace NActors; - -class TTabletsScan : public TScanActorBase<TTabletsScan> { -public: - using TBase = TScanActorBase<TTabletsScan>; - +#include <library/cpp/actors/core/hfunc.h> + +namespace NKikimr::NSysView { + +using namespace NActors; + +class TTabletsScan : public TScanActorBase<TTabletsScan> { +public: + using TBase = TScanActorBase<TTabletsScan>; + static constexpr auto ActorActivityType() { - return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; - } - + return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; + } + TTabletsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) - : TBase(ownerId, scanId, tableId, tableRange, columns) - { - } - - STFUNC(StateScan) { - switch (ev->GetTypeRewrite()) { - hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); - hFunc(TEvSysView::TEvGetTabletIdsResponse, Handle); - hFunc(TEvSysView::TEvGetTabletsResponse, Handle); - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); - hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution); - cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); - cFunc(TEvents::TEvPoison::EventType, PassAway); - default: - LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, - "NSysView::TTabletsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - -private: - void ProceedToScan() override { - Become(&TThis::StateScan); - if (AckReceived) { - RequestTabletIds(); - } - } - - void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { - if (TabletIds.empty()) { - RequestTabletIds(); - } else { - RequestBatch(); - } - } - + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) + : TBase(ownerId, scanId, tableId, tableRange, columns) + { + } + + STFUNC(StateScan) { + switch (ev->GetTypeRewrite()) { + hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); + hFunc(TEvSysView::TEvGetTabletIdsResponse, Handle); + hFunc(TEvSysView::TEvGetTabletsResponse, Handle); + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution); + cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); + cFunc(TEvents::TEvPoison::EventType, PassAway); + default: + LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS, + "NSysView::TTabletsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + +private: + void ProceedToScan() override { + Become(&TThis::StateScan); + if (AckReceived) { + RequestTabletIds(); + } + } + + void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) { + if (TabletIds.empty()) { + RequestTabletIds(); + } else { + RequestBatch(); + } + } + bool CalculateRangeFrom() { /* * Please note that TabletId and FollowerId do not have NULLs in columns */ - const auto& cellsFrom = TableRange.From.GetCells(); - + const auto& cellsFrom = TableRange.From.GetCells(); + // Empty means that we read from +inf, it is impossible if (cellsFrom.empty()) { YQL_ENSURE(false, "Range starts from +inf, can't read anything."); return false; } - + if (cellsFrom[0].IsNull()) { return true; } @@ -85,11 +85,11 @@ private: if (cellsFrom.size() == 2) { if (!cellsFrom[1].IsNull()) { FromFollowerId = cellsFrom[1].AsValue<ui32>(); - } - + } + if (TableRange.FromInclusive) { return true; - } + } // The range start from NULL exclusive. So, the next value after NULL will be used. if (!FromFollowerId.has_value()) { @@ -103,8 +103,8 @@ private: } FromFollowerId.reset(); - } - + } + if (FromTabletId < Max<ui64>()) { ++FromTabletId; return true; @@ -114,12 +114,12 @@ private: } bool CalculateRangeTo() { - const auto& cellsTo = TableRange.To.GetCells(); - + const auto& cellsTo = TableRange.To.GetCells(); + if (cellsTo.empty()) { return true; } - + YQL_ENSURE(!cellsTo[0].IsNull(), "Read to -inf range"); ToTabletId = cellsTo[0].AsValue<ui64>(); @@ -140,11 +140,11 @@ private: if (cellsTo.size() == 2) { if (!cellsTo[1].IsNull()) { ToFollowerId = cellsTo[1].AsValue<ui32>(); - } - + } + if (TableRange.ToInclusive) { return true; - } + } // The range ends at NULL exclusive. So, the value before NULL will be used. if (!ToFollowerId.has_value()) { @@ -158,8 +158,8 @@ private: } ToFollowerId.reset(); - } - + } + return decreaseTabletId(); } @@ -171,196 +171,196 @@ private: return; } - if (ToTabletId < FromTabletId) { - ReplyEmptyAndDie(); - return; - } - - auto& record = request->Record; - record.SetFrom(FromTabletId); - record.SetTo(ToTabletId); - - auto pipeCache = MakePipePeNodeCacheID(false); - Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), HiveId, true), - IEventHandle::FlagTrackDelivery); - } - - void RequestBatch() { - if (BatchRequested) { - return; - } - BatchRequested = true; - - auto request = MakeHolder<TEvSysView::TEvGetTabletsRequest>(); - auto& record = request->Record; - - auto it = FromIterator; - for (size_t count = 0; count < BatchSize && it != TabletIds.end(); ++count, ++it) { - record.AddTabletIds(*it); - } - FromIterator = it; - - record.SetBatchSizeLimit(BatchSize); - - auto pipeCache = MakePipePeNodeCacheID(false); - Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), HiveId, true), - IEventHandle::FlagTrackDelivery); - } - - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { - ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in TTabletsScan"); - } - - void Handle(TEvSysView::TEvGetTabletIdsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - if (record.TabletIdsSize() == 0) { - ReplyEmptyAndDie(); - return; - } - - TabletIds.reserve(record.TabletIdsSize()); - for (const auto& id : record.GetTabletIds()) { - TabletIds.push_back(id); - } - std::sort(TabletIds.begin(), TabletIds.end()); - - FromIterator = TabletIds.begin(); - RequestBatch(); - } - - void Handle(TEvSysView::TEvGetTabletsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - - using TEntry = NKikimrSysView::TTabletEntry; - using TExtractor = std::function<TCell(const TEntry&)>; - using TSchema = Schema::Tablets; - - struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { - TExtractorsMap() { - insert({TSchema::TabletId::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetTabletId()); - }}); + if (ToTabletId < FromTabletId) { + ReplyEmptyAndDie(); + return; + } + + auto& record = request->Record; + record.SetFrom(FromTabletId); + record.SetTo(ToTabletId); + + auto pipeCache = MakePipePeNodeCacheID(false); + Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), HiveId, true), + IEventHandle::FlagTrackDelivery); + } + + void RequestBatch() { + if (BatchRequested) { + return; + } + BatchRequested = true; + + auto request = MakeHolder<TEvSysView::TEvGetTabletsRequest>(); + auto& record = request->Record; + + auto it = FromIterator; + for (size_t count = 0; count < BatchSize && it != TabletIds.end(); ++count, ++it) { + record.AddTabletIds(*it); + } + FromIterator = it; + + record.SetBatchSizeLimit(BatchSize); + + auto pipeCache = MakePipePeNodeCacheID(false); + Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), HiveId, true), + IEventHandle::FlagTrackDelivery); + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { + ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in TTabletsScan"); + } + + void Handle(TEvSysView::TEvGetTabletIdsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + if (record.TabletIdsSize() == 0) { + ReplyEmptyAndDie(); + return; + } + + TabletIds.reserve(record.TabletIdsSize()); + for (const auto& id : record.GetTabletIds()) { + TabletIds.push_back(id); + } + std::sort(TabletIds.begin(), TabletIds.end()); + + FromIterator = TabletIds.begin(); + RequestBatch(); + } + + void Handle(TEvSysView::TEvGetTabletsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + + using TEntry = NKikimrSysView::TTabletEntry; + using TExtractor = std::function<TCell(const TEntry&)>; + using TSchema = Schema::Tablets; + + struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { + TExtractorsMap() { + insert({TSchema::TabletId::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetTabletId()); + }}); insert({TSchema::FollowerId::ColumnId, [] (const TEntry& entry) { return TCell::Make<ui32>(entry.GetFollowerId()); - }}); - insert({TSchema::TypeCol::ColumnId, [] (const TEntry& entry) { - const auto& type = entry.GetType(); - return TCell(type.data(), type.size()); - }}); - insert({TSchema::State::ColumnId, [] (const TEntry& entry) { - if (!entry.HasState()) { - return TCell(); - } - const auto& state = entry.GetState(); - return TCell(state.data(), state.size()); - }}); - insert({TSchema::VolatileState::ColumnId, [] (const TEntry& entry) { - const auto& state = entry.GetVolatileState(); - return TCell(state.data(), state.size()); - }}); - insert({TSchema::BootState::ColumnId, [] (const TEntry& entry) { - const auto& state = entry.GetBootState(); - return TCell(state.data(), state.size()); - }}); - insert({TSchema::Generation::ColumnId, [] (const TEntry& entry) { - if (!entry.HasGeneration()) { - return TCell(); - } + }}); + insert({TSchema::TypeCol::ColumnId, [] (const TEntry& entry) { + const auto& type = entry.GetType(); + return TCell(type.data(), type.size()); + }}); + insert({TSchema::State::ColumnId, [] (const TEntry& entry) { + if (!entry.HasState()) { + return TCell(); + } + const auto& state = entry.GetState(); + return TCell(state.data(), state.size()); + }}); + insert({TSchema::VolatileState::ColumnId, [] (const TEntry& entry) { + const auto& state = entry.GetVolatileState(); + return TCell(state.data(), state.size()); + }}); + insert({TSchema::BootState::ColumnId, [] (const TEntry& entry) { + const auto& state = entry.GetBootState(); + return TCell(state.data(), state.size()); + }}); + insert({TSchema::Generation::ColumnId, [] (const TEntry& entry) { + if (!entry.HasGeneration()) { + return TCell(); + } return TCell::Make<ui32>(entry.GetGeneration()); - }}); - insert({TSchema::NodeId::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui32>(entry.GetNodeId()); - }}); - insert({TSchema::CPU::ColumnId, [] (const TEntry& entry) { - return TCell::Make<double>(entry.GetCPU() / 1000000.); - }}); - insert({TSchema::Memory::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetMemory()); - }}); - insert({TSchema::Network::ColumnId, [] (const TEntry& entry) { - return TCell::Make<ui64>(entry.GetNetwork()); - }}); - } - }; - static TExtractorsMap extractors; - - size_t index = 0; + }}); + insert({TSchema::NodeId::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui32>(entry.GetNodeId()); + }}); + insert({TSchema::CPU::ColumnId, [] (const TEntry& entry) { + return TCell::Make<double>(entry.GetCPU() / 1000000.); + }}); + insert({TSchema::Memory::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetMemory()); + }}); + insert({TSchema::Network::ColumnId, [] (const TEntry& entry) { + return TCell::Make<ui64>(entry.GetNetwork()); + }}); + } + }; + static TExtractorsMap extractors; + + size_t index = 0; ui32 fromFollowerId = FromFollowerId.value_or(Min<ui32>()); if (record.EntriesSize() > 0 - && record.GetEntries(0).GetTabletId() == FromTabletId) - { - for (; index < record.EntriesSize(); ++index) { - const auto& entry = record.GetEntries(index); + && record.GetEntries(0).GetTabletId() == FromTabletId) + { + for (; index < record.EntriesSize(); ++index) { + const auto& entry = record.GetEntries(index); if (entry.GetTabletId() != FromTabletId || entry.GetFollowerId() >= fromFollowerId) { - break; - } - } - } - - auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); - TVector<TCell> cells; + break; + } + } + } + + auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId); + TVector<TCell> cells; ui32 toFollowerId = ToFollowerId.value_or(Max<ui32>()); - - for (; index < record.EntriesSize(); ++index) { - const auto& entry = record.GetEntries(index); - + + for (; index < record.EntriesSize(); ++index) { + const auto& entry = record.GetEntries(index); + if (entry.GetTabletId() == ToTabletId && entry.GetFollowerId() > toFollowerId) { - break; - } - - for (auto& column : Columns) { - auto extractor = extractors.find(column.Tag); - if (extractor == extractors.end()) { - cells.push_back(TCell()); - } else { - cells.push_back(extractor->second(entry)); - } - } - TArrayRef<const TCell> ref(cells); + break; + } + + for (auto& column : Columns) { + auto extractor = extractors.find(column.Tag); + if (extractor == extractors.end()) { + cells.push_back(TCell()); + } else { + cells.push_back(extractor->second(entry)); + } + } + TArrayRef<const TCell> ref(cells); batch->Rows.emplace_back(TOwnedCellVec::Make(ref)); - cells.clear(); - } - - if (record.HasNextTabletId()) { - FromIterator = std::lower_bound(TabletIds.begin(), TabletIds.end(), - record.GetNextTabletId()); - } - - if (FromIterator == TabletIds.end()) { - batch->Finished = true; - } - - BatchRequested = false; - - SendBatch(std::move(batch)); - } - - void PassAway() override { - Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); - TBase::PassAway(); - } - -private: - static constexpr size_t BatchSize = 10000; - - ui64 FromTabletId = 0; + cells.clear(); + } + + if (record.HasNextTabletId()) { + FromIterator = std::lower_bound(TabletIds.begin(), TabletIds.end(), + record.GetNextTabletId()); + } + + if (FromIterator == TabletIds.end()) { + batch->Finished = true; + } + + BatchRequested = false; + + SendBatch(std::move(batch)); + } + + void PassAway() override { + Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + TBase::PassAway(); + } + +private: + static constexpr size_t BatchSize = 10000; + + ui64 FromTabletId = 0; std::optional<ui32> FromFollowerId; - - ui64 ToTabletId = Max<ui64>(); + + ui64 ToTabletId = Max<ui64>(); std::optional<ui32> ToFollowerId; - - TVector<ui64> TabletIds; - TVector<ui64>::const_iterator FromIterator; - - bool BatchRequested = false; -}; - + + TVector<ui64> TabletIds; + TVector<ui64>::const_iterator FromIterator; + + bool BatchRequested = false; +}; + THolder<IActor> CreateTabletsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) -{ + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns) +{ return MakeHolder<TTabletsScan>(ownerId, scanId, tableId, tableRange, columns); -} - -} // NKikimr::NSysView +} + +} // NKikimr::NSysView diff --git a/ydb/core/sys_view/tablets/tablets.h b/ydb/core/sys_view/tablets/tablets.h index 321bd9b081..ca54b08d3e 100644 --- a/ydb/core/sys_view/tablets/tablets.h +++ b/ydb/core/sys_view/tablets/tablets.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/runtime/kqp_compute.h> - -namespace NKikimr { -namespace NSysView { - + +namespace NKikimr { +namespace NSysView { + THolder<IActor> CreateTabletsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); - -} // NSysView -} // NKikimr + const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/tablets/ya.make b/ydb/core/sys_view/tablets/ya.make index 95680b0ace..9da54e0712 100644 --- a/ydb/core/sys_view/tablets/ya.make +++ b/ydb/core/sys_view/tablets/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - tablets.h - tablets.cpp -) - -PEERDIR( - library/cpp/actors/core +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + tablets.h + tablets.cpp +) + +PEERDIR( + library/cpp/actors/core ydb/core/base ydb/core/kqp/runtime ydb/core/sys_view/common -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/sys_view/ut_common.cpp b/ydb/core/sys_view/ut_common.cpp index a0ef0dffa4..6165258568 100644 --- a/ydb/core/sys_view/ut_common.cpp +++ b/ydb/core/sys_view/ut_common.cpp @@ -1,92 +1,92 @@ -#include "ut_common.h" - -namespace NKikimr { -namespace NSysView { - -NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings(const TString &name, const TStoragePools &pools) { - NKikimrSubDomains::TSubDomainSettings subdomain; - subdomain.SetName(name); - for (auto& pool: pools) { - *subdomain.AddStoragePools() = pool; - } - return subdomain; -} - -NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(const TString &name, const TStoragePools &pools) { - NKikimrSubDomains::TSubDomainSettings subdomain; - subdomain.SetName(name); - subdomain.SetCoordinators(2); - subdomain.SetMediators(2); - subdomain.SetPlanResolution(50); - subdomain.SetTimeCastBucketsPerMediator(2); - for (auto& pool: pools) { - *subdomain.AddStoragePools() = pool; - } - return subdomain; -} - -TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, bool enableSVP) { - auto mbusPort = PortManager.GetPort(); - auto grpcPort = PortManager.GetPort(); - - TVector<NKikimrKqp::TKqpSetting> kqpSettings; - NKikimrKqp::TKqpSetting setting; - setting.SetName("_KqpAllowNewEngine"); - setting.SetValue("true"); - kqpSettings.push_back(setting); - - Settings = new Tests::TServerSettings(mbusPort); - Settings->SetDomainName("Root"); - Settings->SetNodeCount(staticNodes); - Settings->SetDynamicNodeCount(dynamicNodes); - Settings->SetKqpSettings(kqpSettings); - - if (enableSVP) { - Settings->SetEnablePersistentQueryStats(true); - Settings->SetEnableDbCounters(true); - } - - for (ui32 i : xrange(storagePools)) { - TString poolName = Sprintf("test%d", i); - Settings->AddStoragePool(poolName, TString("/Root:") + poolName, 2); - } - - Server = new Tests::TServer(*Settings); - Server->EnableGRpc(grpcPort); - - Client = MakeHolder<Tests::TClient>(*Settings); - - Tenants = MakeHolder<Tests::TTenants>(Server); - - Client->InitRootScheme("Root"); - - Endpoint = "localhost:" + ToString(grpcPort); - DriverConfig = NYdb::TDriverConfig().SetEndpoint(Endpoint); - Driver = MakeHolder<NYdb::TDriver>(DriverConfig); - - Server->GetRuntime()->SetLogPriority(NKikimrServices::SYSTEM_VIEWS, NActors::NLog::PRI_DEBUG); -} - -TTestEnv::~TTestEnv() { - Driver->Stop(true); -} - -TStoragePools TTestEnv::GetPools() const { - TStoragePools pools; - for (const auto& [kind, pool] : Settings->StoragePoolTypes) { - pools.emplace_back(pool.GetName(), kind); - } - return pools; -} - -TStoragePools TTestEnv::CreatePoolsForTenant(const TString& tenant) { - TStoragePools result; - for (auto& poolType: Settings->StoragePoolTypes) { - auto& poolKind = poolType.first; - result.emplace_back(Client->CreateStoragePool(poolKind, tenant), poolKind); - } - return result; -} - -} // NSysView -} // NKikimr +#include "ut_common.h" + +namespace NKikimr { +namespace NSysView { + +NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings(const TString &name, const TStoragePools &pools) { + NKikimrSubDomains::TSubDomainSettings subdomain; + subdomain.SetName(name); + for (auto& pool: pools) { + *subdomain.AddStoragePools() = pool; + } + return subdomain; +} + +NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(const TString &name, const TStoragePools &pools) { + NKikimrSubDomains::TSubDomainSettings subdomain; + subdomain.SetName(name); + subdomain.SetCoordinators(2); + subdomain.SetMediators(2); + subdomain.SetPlanResolution(50); + subdomain.SetTimeCastBucketsPerMediator(2); + for (auto& pool: pools) { + *subdomain.AddStoragePools() = pool; + } + return subdomain; +} + +TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, bool enableSVP) { + auto mbusPort = PortManager.GetPort(); + auto grpcPort = PortManager.GetPort(); + + TVector<NKikimrKqp::TKqpSetting> kqpSettings; + NKikimrKqp::TKqpSetting setting; + setting.SetName("_KqpAllowNewEngine"); + setting.SetValue("true"); + kqpSettings.push_back(setting); + + Settings = new Tests::TServerSettings(mbusPort); + Settings->SetDomainName("Root"); + Settings->SetNodeCount(staticNodes); + Settings->SetDynamicNodeCount(dynamicNodes); + Settings->SetKqpSettings(kqpSettings); + + if (enableSVP) { + Settings->SetEnablePersistentQueryStats(true); + Settings->SetEnableDbCounters(true); + } + + for (ui32 i : xrange(storagePools)) { + TString poolName = Sprintf("test%d", i); + Settings->AddStoragePool(poolName, TString("/Root:") + poolName, 2); + } + + Server = new Tests::TServer(*Settings); + Server->EnableGRpc(grpcPort); + + Client = MakeHolder<Tests::TClient>(*Settings); + + Tenants = MakeHolder<Tests::TTenants>(Server); + + Client->InitRootScheme("Root"); + + Endpoint = "localhost:" + ToString(grpcPort); + DriverConfig = NYdb::TDriverConfig().SetEndpoint(Endpoint); + Driver = MakeHolder<NYdb::TDriver>(DriverConfig); + + Server->GetRuntime()->SetLogPriority(NKikimrServices::SYSTEM_VIEWS, NActors::NLog::PRI_DEBUG); +} + +TTestEnv::~TTestEnv() { + Driver->Stop(true); +} + +TStoragePools TTestEnv::GetPools() const { + TStoragePools pools; + for (const auto& [kind, pool] : Settings->StoragePoolTypes) { + pools.emplace_back(pool.GetName(), kind); + } + return pools; +} + +TStoragePools TTestEnv::CreatePoolsForTenant(const TString& tenant) { + TStoragePools result; + for (auto& poolType: Settings->StoragePoolTypes) { + auto& poolKind = poolType.first; + result.emplace_back(Client->CreateStoragePool(poolKind, tenant), poolKind); + } + return result; +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/ut_common.h b/ydb/core/sys_view/ut_common.h index 175ec2a5ab..e142b96909 100644 --- a/ydb/core/sys_view/ut_common.h +++ b/ydb/core/sys_view/ut_common.h @@ -1,63 +1,63 @@ -#pragma once - +#pragma once + #include <ydb/core/testlib/test_client.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> - + #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { -namespace NSysView { - -NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings( - const TString &name, const TStoragePools &pools = {}); - -NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings( - const TString &name, const TStoragePools &pools = {}); - -class TTestEnv { -public: - TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 4, ui32 storagePools = 0, - bool enableSVP = false); - ~TTestEnv(); - - Tests::TServer& GetServer() const { - return *Server; - } - - Tests::TClient& GetClient() const { - return *Client; - } - - Tests::TTenants& GetTenants() const { - return *Tenants; - } - - NYdb::TDriver& GetDriver() const { - return *Driver; - } - - const TString& GetEndpoint() const { - return Endpoint; - } - - TStoragePools GetPools() const; - TStoragePools CreatePoolsForTenant(const TString& tenant); - - -private: - TPortManager PortManager; - - Tests::TServerSettings::TPtr Settings; - Tests::TServer::TPtr Server; - THolder<Tests::TClient> Client; - THolder<Tests::TTenants> Tenants; - - TString Endpoint; - NYdb::TDriverConfig DriverConfig; - THolder<NYdb::TDriver> Driver; -}; - -} // NSysView -} // NKikimr + +namespace NKikimr { +namespace NSysView { + +NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings( + const TString &name, const TStoragePools &pools = {}); + +NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings( + const TString &name, const TStoragePools &pools = {}); + +class TTestEnv { +public: + TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 4, ui32 storagePools = 0, + bool enableSVP = false); + ~TTestEnv(); + + Tests::TServer& GetServer() const { + return *Server; + } + + Tests::TClient& GetClient() const { + return *Client; + } + + Tests::TTenants& GetTenants() const { + return *Tenants; + } + + NYdb::TDriver& GetDriver() const { + return *Driver; + } + + const TString& GetEndpoint() const { + return Endpoint; + } + + TStoragePools GetPools() const; + TStoragePools CreatePoolsForTenant(const TString& tenant); + + +private: + TPortManager PortManager; + + Tests::TServerSettings::TPtr Settings; + Tests::TServer::TPtr Server; + THolder<Tests::TClient> Client; + THolder<Tests::TTenants> Tenants; + + TString Endpoint; + NYdb::TDriverConfig DriverConfig; + THolder<NYdb::TDriver> Driver; +}; + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/ut_counters.cpp b/ydb/core/sys_view/ut_counters.cpp index 7f7b6b76c3..234f52478f 100644 --- a/ydb/core/sys_view/ut_counters.cpp +++ b/ydb/core/sys_view/ut_counters.cpp @@ -1,181 +1,181 @@ -#include "ut_common.h" - +#include "ut_common.h" + #include <ydb/core/base/counters.h> #include <ydb/core/kqp/ut/common/kqp_ut_common.h> - -namespace NKikimr { -namespace NSysView { - -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NYdb::NScheme; - -namespace { - -void CreateDatabase(TTestEnv& env, const TString& databaseName) { - auto subdomain = GetSubDomainDeclareSettings(databaseName); - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().CreateExtSubdomain("/Root", subdomain)); - - env.GetTenants().Run("/Root/" + databaseName, 1); - - auto subdomainSettings = GetSubDomainDefaultSettings(databaseName, env.GetPools()); - subdomainSettings.SetExternalSysViewProcessor(true); - subdomainSettings.SetExternalSchemeShard(true); - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().AlterExtSubdomain("/Root", subdomainSettings)); -} - -void CreateDatabases(TTestEnv& env) { - CreateDatabase(env, "Database1"); - CreateDatabase(env, "Database2"); -} - -void CreateTables(TTestEnv& env) { - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - - NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( - CREATE TABLE `Root/Database1/Table1` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )").GetValueSync()); - - NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( - REPLACE INTO `Root/Database1/Table1` (Key, Value) VALUES - (1u, "A"), - (2u, "B"), - (3u, "C"); - )", TTxControl::BeginTx().CommitTx()).GetValueSync()); - - NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( - CREATE TABLE `Root/Database2/Table2` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )").GetValueSync()); - - NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( - REPLACE INTO `Root/Database2/Table2` (Key, Value) VALUES - (4u, "D"), - (5u, "E"); - )", TTxControl::BeginTx().CommitTx()).GetValueSync()); -} - -void CreateDatabasesAndTables(TTestEnv& env) { - CreateDatabases(env); - CreateTables(env); -} - -} // namespace - -Y_UNIT_TEST_SUITE(DbCounters) { - - Y_UNIT_TEST(TabletsSimple) { - TTestEnv env(1, 2, 0, true); - CreateDatabasesAndTables(env); - - for (size_t iter = 0; iter < 30; ++iter) { - Cerr << "iteration " << iter << Endl; - - auto checkTabletCounters = [] (NMonitoring::TDynamicCounterPtr databaseGroup, - const char* databaseName) - { - auto checkCounter = [databaseName] (NMonitoring::TDynamicCounterPtr group, - const char* sensorName, bool isDerivative) - { - auto value = group->GetCounter(sensorName, isDerivative)->Val(); - Cerr << "Database " << databaseName << ", sensor " << sensorName << ", value " << value << Endl; - return (value > 0); - }; - - bool isGood = true; - - auto tabletGroup = databaseGroup->GetSubgroup("host", "")->GetSubgroup("group", "tablets"); - auto datashardGroup = tabletGroup->GetSubgroup("type", "DataShard"); - { - auto executorGroup = datashardGroup->GetSubgroup("category", "executor"); - { - TStringStream ss; - executorGroup->OutputHtml(ss); - Cerr << ss.Str() << Endl; - } - - isGood &= checkCounter(executorGroup, "SUM(UsedTabletMemory)", false); - isGood &= checkCounter(executorGroup, "MAX(UsedTabletMemory)", false); - - isGood &= checkCounter(executorGroup, "TabletBytesWritten", true); - - auto appGroup = datashardGroup->GetSubgroup("category", "app"); - { - TStringStream ss; - appGroup->OutputHtml(ss); - Cerr << ss.Str() << Endl; - } - - isGood &= checkCounter(appGroup, "DataShard/EngineHostRowUpdateBytes", true); - isGood &= checkCounter(appGroup, "MAX(DataShard/EngineHostRowUpdateBytes)", false); - } - - auto schemeshardGroup = tabletGroup->GetSubgroup("type", "SchemeShard"); - { - auto executorGroup = schemeshardGroup->GetSubgroup("category", "executor"); - { - TStringStream ss; - executorGroup->OutputHtml(ss); - Cerr << ss.Str() << Endl; - } - - isGood &= checkCounter(executorGroup, "SUM(UsedTabletMemory)", false); - isGood &= checkCounter(executorGroup, "MAX(UsedTabletMemory)", false); - - isGood &= checkCounter(executorGroup, "TabletBytesWritten", true); - - auto appGroup = schemeshardGroup->GetSubgroup("category", "app"); - { - TStringStream ss; - appGroup->OutputHtml(ss); - Cerr << ss.Str() << Endl; - } - - isGood &= checkCounter(appGroup, "SUM(SchemeShard/Tables)", false); - isGood &= checkCounter(appGroup, "MAX(SchemeShard/Tables)", false); - - isGood &= checkCounter(appGroup, "SchemeShard/FinishedOps/CreateTable", true); - } - - return isGood; - }; - - bool checkDb1 = false, checkDb2 = false; - - for (ui32 nodeId = 0; nodeId < env.GetServer().GetRuntime()->GetNodeCount(); ++nodeId) { - auto counters = env.GetServer().GetRuntime()->GetAppData(nodeId).Counters; - auto dbGroup = GetServiceCounters(counters, "db", false); - - auto databaseGroup1 = dbGroup->FindSubgroup("database", "/Root/Database1"); - if (databaseGroup1) { - checkDb1 = checkTabletCounters(databaseGroup1, "/Root/Database1"); - } - auto databaseGroup2 = dbGroup->FindSubgroup("database", "/Root/Database2"); - if (databaseGroup2) { - checkDb2 = checkTabletCounters(databaseGroup2, "/Root/Database2"); - } - } - - if (checkDb1 && checkDb2) { - return; - } - - Sleep(TDuration::Seconds(5)); - } - - UNIT_ASSERT_C(false, "out of iterations"); - } -} - -} // NSysView -} // NKikimr + +namespace NKikimr { +namespace NSysView { + +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NYdb::NScheme; + +namespace { + +void CreateDatabase(TTestEnv& env, const TString& databaseName) { + auto subdomain = GetSubDomainDeclareSettings(databaseName); + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().CreateExtSubdomain("/Root", subdomain)); + + env.GetTenants().Run("/Root/" + databaseName, 1); + + auto subdomainSettings = GetSubDomainDefaultSettings(databaseName, env.GetPools()); + subdomainSettings.SetExternalSysViewProcessor(true); + subdomainSettings.SetExternalSchemeShard(true); + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().AlterExtSubdomain("/Root", subdomainSettings)); +} + +void CreateDatabases(TTestEnv& env) { + CreateDatabase(env, "Database1"); + CreateDatabase(env, "Database2"); +} + +void CreateTables(TTestEnv& env) { + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( + CREATE TABLE `Root/Database1/Table1` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )").GetValueSync()); + + NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( + REPLACE INTO `Root/Database1/Table1` (Key, Value) VALUES + (1u, "A"), + (2u, "B"), + (3u, "C"); + )", TTxControl::BeginTx().CommitTx()).GetValueSync()); + + NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( + CREATE TABLE `Root/Database2/Table2` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )").GetValueSync()); + + NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( + REPLACE INTO `Root/Database2/Table2` (Key, Value) VALUES + (4u, "D"), + (5u, "E"); + )", TTxControl::BeginTx().CommitTx()).GetValueSync()); +} + +void CreateDatabasesAndTables(TTestEnv& env) { + CreateDatabases(env); + CreateTables(env); +} + +} // namespace + +Y_UNIT_TEST_SUITE(DbCounters) { + + Y_UNIT_TEST(TabletsSimple) { + TTestEnv env(1, 2, 0, true); + CreateDatabasesAndTables(env); + + for (size_t iter = 0; iter < 30; ++iter) { + Cerr << "iteration " << iter << Endl; + + auto checkTabletCounters = [] (NMonitoring::TDynamicCounterPtr databaseGroup, + const char* databaseName) + { + auto checkCounter = [databaseName] (NMonitoring::TDynamicCounterPtr group, + const char* sensorName, bool isDerivative) + { + auto value = group->GetCounter(sensorName, isDerivative)->Val(); + Cerr << "Database " << databaseName << ", sensor " << sensorName << ", value " << value << Endl; + return (value > 0); + }; + + bool isGood = true; + + auto tabletGroup = databaseGroup->GetSubgroup("host", "")->GetSubgroup("group", "tablets"); + auto datashardGroup = tabletGroup->GetSubgroup("type", "DataShard"); + { + auto executorGroup = datashardGroup->GetSubgroup("category", "executor"); + { + TStringStream ss; + executorGroup->OutputHtml(ss); + Cerr << ss.Str() << Endl; + } + + isGood &= checkCounter(executorGroup, "SUM(UsedTabletMemory)", false); + isGood &= checkCounter(executorGroup, "MAX(UsedTabletMemory)", false); + + isGood &= checkCounter(executorGroup, "TabletBytesWritten", true); + + auto appGroup = datashardGroup->GetSubgroup("category", "app"); + { + TStringStream ss; + appGroup->OutputHtml(ss); + Cerr << ss.Str() << Endl; + } + + isGood &= checkCounter(appGroup, "DataShard/EngineHostRowUpdateBytes", true); + isGood &= checkCounter(appGroup, "MAX(DataShard/EngineHostRowUpdateBytes)", false); + } + + auto schemeshardGroup = tabletGroup->GetSubgroup("type", "SchemeShard"); + { + auto executorGroup = schemeshardGroup->GetSubgroup("category", "executor"); + { + TStringStream ss; + executorGroup->OutputHtml(ss); + Cerr << ss.Str() << Endl; + } + + isGood &= checkCounter(executorGroup, "SUM(UsedTabletMemory)", false); + isGood &= checkCounter(executorGroup, "MAX(UsedTabletMemory)", false); + + isGood &= checkCounter(executorGroup, "TabletBytesWritten", true); + + auto appGroup = schemeshardGroup->GetSubgroup("category", "app"); + { + TStringStream ss; + appGroup->OutputHtml(ss); + Cerr << ss.Str() << Endl; + } + + isGood &= checkCounter(appGroup, "SUM(SchemeShard/Tables)", false); + isGood &= checkCounter(appGroup, "MAX(SchemeShard/Tables)", false); + + isGood &= checkCounter(appGroup, "SchemeShard/FinishedOps/CreateTable", true); + } + + return isGood; + }; + + bool checkDb1 = false, checkDb2 = false; + + for (ui32 nodeId = 0; nodeId < env.GetServer().GetRuntime()->GetNodeCount(); ++nodeId) { + auto counters = env.GetServer().GetRuntime()->GetAppData(nodeId).Counters; + auto dbGroup = GetServiceCounters(counters, "db", false); + + auto databaseGroup1 = dbGroup->FindSubgroup("database", "/Root/Database1"); + if (databaseGroup1) { + checkDb1 = checkTabletCounters(databaseGroup1, "/Root/Database1"); + } + auto databaseGroup2 = dbGroup->FindSubgroup("database", "/Root/Database2"); + if (databaseGroup2) { + checkDb2 = checkTabletCounters(databaseGroup2, "/Root/Database2"); + } + } + + if (checkDb1 && checkDb2) { + return; + } + + Sleep(TDuration::Seconds(5)); + } + + UNIT_ASSERT_C(false, "out of iterations"); + } +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp index eafa84f4fc..5e90eb5992 100644 --- a/ydb/core/sys_view/ut_kqp.cpp +++ b/ydb/core/sys_view/ut_kqp.cpp @@ -1,142 +1,142 @@ -#include "ut_common.h" - +#include "ut_common.h" + #include <ydb/core/kqp/ut/common/kqp_ut_common.h> - + #include <ydb/core/sys_view/common/events.h> #include <ydb/core/sys_view/service/sysview_service.h> #include <ydb/core/tx/datashard/datashard.h> - + #include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h> - + #include <library/cpp/yson/node/node_io.h> - -namespace NKikimr { -namespace NSysView { - -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NYdb::NScheme; - -namespace { - -void CreateTenant(TTestEnv& env, const TString& tenantName, bool extSchemeShard = true) { - auto subdomain = GetSubDomainDeclareSettings(tenantName); - if (extSchemeShard) { - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().CreateExtSubdomain("/Root", subdomain)); - } else { - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().CreateSubdomain("/Root", subdomain)); - } - - env.GetTenants().Run("/Root/" + tenantName, 2); - - auto subdomainSettings = GetSubDomainDefaultSettings(tenantName, env.GetPools()); - subdomainSettings.SetExternalSysViewProcessor(true); - - if (extSchemeShard) { - subdomainSettings.SetExternalSchemeShard(true); - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().AlterExtSubdomain("/Root", subdomainSettings)); - } else { - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().AlterSubdomain("/Root", subdomainSettings)); - } -} - -void CreateTenants(TTestEnv& env, bool extSchemeShard = true) { - CreateTenant(env, "Tenant1", extSchemeShard); - CreateTenant(env, "Tenant2", extSchemeShard); -} - -void CreateTables(TTestEnv& env) { - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - - NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( - CREATE TABLE `Root/Table0` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )").GetValueSync()); - - NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( - REPLACE INTO `Root/Table0` (Key, Value) VALUES - (0u, "Z"); - )", TTxControl::BeginTx().CommitTx()).GetValueSync()); - - NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( - CREATE TABLE `Root/Tenant1/Table1` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )").GetValueSync()); - - NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( - REPLACE INTO `Root/Tenant1/Table1` (Key, Value) VALUES - (1u, "A"), - (2u, "B"), - (3u, "C"); - )", TTxControl::BeginTx().CommitTx()).GetValueSync()); - - NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( - CREATE TABLE `Root/Tenant2/Table2` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )").GetValueSync()); - - NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( - REPLACE INTO `Root/Tenant2/Table2` (Key, Value) VALUES - (4u, "D"), - (5u, "E"); - )", TTxControl::BeginTx().CommitTx()).GetValueSync()); -} - -void CreateTenantsAndTables(TTestEnv& env, bool extSchemeShard = true) { - CreateTenants(env, extSchemeShard); - CreateTables(env); -} - -void CreateRootTable(TTestEnv& env, ui64 partitionCount = 1) { - env.GetClient().CreateTable("/Root", Sprintf(R"( - Name: "Table0" - Columns { Name: "Key", Type: "Uint64" } - Columns { Name: "Value", Type: "String" } - KeyColumnNames: ["Key"] - UniformPartitionsCount: %lu - )", partitionCount)); -} - -class TYsonFieldChecker { - NYT::TNode Root; - NYT::TNode::TListType::const_iterator RowIterator; - -private: - const NYT::TNode& ExtractOptional(const NYT::TNode& opt) { - UNIT_ASSERT(opt.IsList()); - UNIT_ASSERT_VALUES_EQUAL(opt.AsList().size(), 1); - return opt.AsList().front(); - }; - -public: - TYsonFieldChecker(const TString& ysonString, size_t fieldCount) { + +namespace NKikimr { +namespace NSysView { + +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NYdb::NScheme; + +namespace { + +void CreateTenant(TTestEnv& env, const TString& tenantName, bool extSchemeShard = true) { + auto subdomain = GetSubDomainDeclareSettings(tenantName); + if (extSchemeShard) { + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().CreateExtSubdomain("/Root", subdomain)); + } else { + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().CreateSubdomain("/Root", subdomain)); + } + + env.GetTenants().Run("/Root/" + tenantName, 2); + + auto subdomainSettings = GetSubDomainDefaultSettings(tenantName, env.GetPools()); + subdomainSettings.SetExternalSysViewProcessor(true); + + if (extSchemeShard) { + subdomainSettings.SetExternalSchemeShard(true); + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().AlterExtSubdomain("/Root", subdomainSettings)); + } else { + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().AlterSubdomain("/Root", subdomainSettings)); + } +} + +void CreateTenants(TTestEnv& env, bool extSchemeShard = true) { + CreateTenant(env, "Tenant1", extSchemeShard); + CreateTenant(env, "Tenant2", extSchemeShard); +} + +void CreateTables(TTestEnv& env) { + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( + CREATE TABLE `Root/Table0` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )").GetValueSync()); + + NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( + REPLACE INTO `Root/Table0` (Key, Value) VALUES + (0u, "Z"); + )", TTxControl::BeginTx().CommitTx()).GetValueSync()); + + NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( + CREATE TABLE `Root/Tenant1/Table1` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )").GetValueSync()); + + NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( + REPLACE INTO `Root/Tenant1/Table1` (Key, Value) VALUES + (1u, "A"), + (2u, "B"), + (3u, "C"); + )", TTxControl::BeginTx().CommitTx()).GetValueSync()); + + NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( + CREATE TABLE `Root/Tenant2/Table2` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )").GetValueSync()); + + NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"( + REPLACE INTO `Root/Tenant2/Table2` (Key, Value) VALUES + (4u, "D"), + (5u, "E"); + )", TTxControl::BeginTx().CommitTx()).GetValueSync()); +} + +void CreateTenantsAndTables(TTestEnv& env, bool extSchemeShard = true) { + CreateTenants(env, extSchemeShard); + CreateTables(env); +} + +void CreateRootTable(TTestEnv& env, ui64 partitionCount = 1) { + env.GetClient().CreateTable("/Root", Sprintf(R"( + Name: "Table0" + Columns { Name: "Key", Type: "Uint64" } + Columns { Name: "Value", Type: "String" } + KeyColumnNames: ["Key"] + UniformPartitionsCount: %lu + )", partitionCount)); +} + +class TYsonFieldChecker { + NYT::TNode Root; + NYT::TNode::TListType::const_iterator RowIterator; + +private: + const NYT::TNode& ExtractOptional(const NYT::TNode& opt) { + UNIT_ASSERT(opt.IsList()); + UNIT_ASSERT_VALUES_EQUAL(opt.AsList().size(), 1); + return opt.AsList().front(); + }; + +public: + TYsonFieldChecker(const TString& ysonString, size_t fieldCount) { Root = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node); - UNIT_ASSERT(Root.IsList()); - UNIT_ASSERT_VALUES_EQUAL(Root.AsList().size(), 1); - - const auto& rowNode = Root.AsList().front(); - UNIT_ASSERT(rowNode.IsList()); - - const auto& row = rowNode.AsList(); - UNIT_ASSERT_VALUES_EQUAL(row.size(), fieldCount); - - RowIterator = row.begin(); - } - + UNIT_ASSERT(Root.IsList()); + UNIT_ASSERT_VALUES_EQUAL(Root.AsList().size(), 1); + + const auto& rowNode = Root.AsList().front(); + UNIT_ASSERT(rowNode.IsList()); + + const auto& row = rowNode.AsList(); + UNIT_ASSERT_VALUES_EQUAL(row.size(), fieldCount); + + RowIterator = row.begin(); + } + bool SkipNull() { if (RowIterator->IsNull()) { ++RowIterator; @@ -147,492 +147,492 @@ public: } void Null() { - const auto& value = *RowIterator++; + const auto& value = *RowIterator++; UNIT_ASSERT(value.IsNull()); - } - - void Bool(bool expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsBool()); - UNIT_ASSERT_VALUES_EQUAL(value.AsBool(), expected); - } - + } + + void Bool(bool expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsBool()); + UNIT_ASSERT_VALUES_EQUAL(value.AsBool(), expected); + } + void Uint64(ui64 expected, bool orNull = false) { if (!orNull || !SkipNull()) { const auto& value = ExtractOptional(*RowIterator++); UNIT_ASSERT(value.IsUint64()); UNIT_ASSERT_VALUES_EQUAL(value.AsUint64(), expected); } - } - - void Uint64Greater(ui64 expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsUint64()); - UNIT_ASSERT_GT(value.AsUint64(), expected); - } - - void Uint64GreaterOrEquals(ui64 expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsUint64()); - UNIT_ASSERT_GE(value.AsUint64(), expected); - } - + } + + void Uint64Greater(ui64 expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsUint64()); + UNIT_ASSERT_GT(value.AsUint64(), expected); + } + + void Uint64GreaterOrEquals(ui64 expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsUint64()); + UNIT_ASSERT_GE(value.AsUint64(), expected); + } + void Uint64LessOrEquals(ui64 expected) { const auto& value = ExtractOptional(*RowIterator++); UNIT_ASSERT(value.IsUint64()); UNIT_ASSERT_LE(value.AsUint64(), expected); } - void Int64(i64 expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsInt64()); - UNIT_ASSERT_VALUES_EQUAL(value.AsInt64(), expected); - } - - void Int64Greater(i64 expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsInt64()); - UNIT_ASSERT_GT(value.AsInt64(), expected); - } - - void Int64GreaterOrEquals(i64 expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsInt64()); - UNIT_ASSERT_GE(value.AsInt64(), expected); - } - - void Double(double expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsDouble()); - UNIT_ASSERT_VALUES_EQUAL(value.AsDouble(), expected); - } - - void DoubleGreaterOrEquals(double expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsDouble()); - UNIT_ASSERT_GE(value.AsDouble(), expected); - } - - void String(const TString& expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsString()); - UNIT_ASSERT_STRINGS_EQUAL(value.AsString(), expected); - } - - void StringContains(const TString& substr) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsString()); - UNIT_ASSERT(value.AsString().Contains(substr)); - } -}; - -} // namespace - -Y_UNIT_TEST_SUITE(SystemView) { - - Y_UNIT_TEST(PartitionStatsOneSchemeShard) { - TTestEnv env; - CreateTenantsAndTables(env, false); - TTableClient client(env.GetDriver()); - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT PathId, PartIdx, Path FROM `Root/.sys/partition_stats`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - NKqp::CompareYson(R"([ + void Int64(i64 expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsInt64()); + UNIT_ASSERT_VALUES_EQUAL(value.AsInt64(), expected); + } + + void Int64Greater(i64 expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsInt64()); + UNIT_ASSERT_GT(value.AsInt64(), expected); + } + + void Int64GreaterOrEquals(i64 expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsInt64()); + UNIT_ASSERT_GE(value.AsInt64(), expected); + } + + void Double(double expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsDouble()); + UNIT_ASSERT_VALUES_EQUAL(value.AsDouble(), expected); + } + + void DoubleGreaterOrEquals(double expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsDouble()); + UNIT_ASSERT_GE(value.AsDouble(), expected); + } + + void String(const TString& expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsString()); + UNIT_ASSERT_STRINGS_EQUAL(value.AsString(), expected); + } + + void StringContains(const TString& substr) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsString()); + UNIT_ASSERT(value.AsString().Contains(substr)); + } +}; + +} // namespace + +Y_UNIT_TEST_SUITE(SystemView) { + + Y_UNIT_TEST(PartitionStatsOneSchemeShard) { + TTestEnv env; + CreateTenantsAndTables(env, false); + TTableClient client(env.GetDriver()); + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT PathId, PartIdx, Path FROM `Root/.sys/partition_stats`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + NKqp::CompareYson(R"([ [[4u];[0u];["/Root/Table0"]] - ])", NKqp::StreamResultToYson(it)); - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT PathId, PartIdx, Path FROM `Root/Tenant1/.sys/partition_stats`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - NKqp::CompareYson(R"([ + ])", NKqp::StreamResultToYson(it)); + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT PathId, PartIdx, Path FROM `Root/Tenant1/.sys/partition_stats`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + NKqp::CompareYson(R"([ [[5u];[0u];["/Root/Tenant1/Table1"]] - ])", NKqp::StreamResultToYson(it)); - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT PathId, PartIdx, Path FROM `Root/Tenant2/.sys/partition_stats`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - NKqp::CompareYson(R"([ + ])", NKqp::StreamResultToYson(it)); + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT PathId, PartIdx, Path FROM `Root/Tenant2/.sys/partition_stats`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + NKqp::CompareYson(R"([ [[6u];[0u];["/Root/Tenant2/Table2"]] - ])", NKqp::StreamResultToYson(it)); - } - } - - Y_UNIT_TEST(PartitionStatsOneSchemeShardDataQueryNewEngine) { - TTestEnv env; - CreateTenantsAndTables(env, false); - + ])", NKqp::StreamResultToYson(it)); + } + } + + Y_UNIT_TEST(PartitionStatsOneSchemeShardDataQueryNewEngine) { + TTestEnv env; + CreateTenantsAndTables(env, false); + env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_DEBUG); - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - { - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - SELECT PathId, PartIdx, Path FROM `Root/.sys/partition_stats`; - )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - NKqp::CompareYson(R"([ - [[4u];[0u];["/Root/Table0"]] - ])", FormatResultSetYson(result.GetResultSet(0))); - } - { - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - SELECT PathId, PartIdx, Path FROM `Root/Tenant1/.sys/partition_stats`; - )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - - UNIT_ASSERT(result.IsSuccess()); - NKqp::CompareYson(R"([ - [[5u];[0u];["/Root/Tenant1/Table1"]] - ])", FormatResultSetYson(result.GetResultSet(0))); - } - { - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - SELECT PathId, PartIdx, Path FROM `Root/Tenant2/.sys/partition_stats`; - )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - - UNIT_ASSERT(result.IsSuccess()); - NKqp::CompareYson(R"([ - [[6u];[0u];["/Root/Tenant2/Table2"]] - ])", FormatResultSetYson(result.GetResultSet(0))); - } - } - - Y_UNIT_TEST(Nodes) { - return; // table is currenty switched off - - TTestEnv env; + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + { + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + SELECT PathId, PartIdx, Path FROM `Root/.sys/partition_stats`; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + NKqp::CompareYson(R"([ + [[4u];[0u];["/Root/Table0"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + { + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + SELECT PathId, PartIdx, Path FROM `Root/Tenant1/.sys/partition_stats`; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + + UNIT_ASSERT(result.IsSuccess()); + NKqp::CompareYson(R"([ + [[5u];[0u];["/Root/Tenant1/Table1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + { + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + SELECT PathId, PartIdx, Path FROM `Root/Tenant2/.sys/partition_stats`; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + + UNIT_ASSERT(result.IsSuccess()); + NKqp::CompareYson(R"([ + [[6u];[0u];["/Root/Tenant2/Table2"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(Nodes) { + return; // table is currenty switched off + + TTestEnv env; CreateTenantsAndTables(env, false); - TTableClient client(env.GetDriver()); - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT Host, NodeId - FROM `Root/Tenant1/.sys/nodes`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0); - auto expected = Sprintf(R"([ - [["::1"];[%du]]; - [["::1"];[%du]]; - ])", offset + 3, offset + 4); - - NKqp::CompareYson(expected, NKqp::StreamResultToYson(it)); - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT Host, NodeId - FROM `Root/Tenant2/.sys/nodes`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0); - auto expected = Sprintf(R"([ - [["::1"];[%du]]; - [["::1"];[%du]]; - ])", offset + 1, offset + 2); - - NKqp::CompareYson(expected, NKqp::StreamResultToYson(it)); - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT Host, NodeId - FROM `Root/.sys/nodes`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0); - auto expected = Sprintf(R"([ - [["::1"];[%du]]; - [["::1"];[%du]]; - [["::1"];[%du]]; - [["::1"];[%du]]; - [["::1"];[%du]]; - ])", offset, offset + 1, offset + 2, offset + 3, offset + 4); - - NKqp::CompareYson(expected, NKqp::StreamResultToYson(it)); - } - } - - Y_UNIT_TEST(QueryStats) { - TTestEnv env; - CreateTenants(env); - - auto* runtime = env.GetServer().GetRuntime(); - runtime ->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_DEBUG); - - auto oneMinute = TDuration::Minutes(1); - auto oneMinuteUs = oneMinute.MicroSeconds(); - - auto instant = TAppData::TimeProvider->Now() + oneMinute; - auto instantRounded = instant.MicroSeconds() / oneMinuteUs * oneMinuteUs; - - std::vector<ui64> buckets; - for (size_t i : xrange(4)) { - Y_UNUSED(i); - buckets.push_back(instantRounded); - instantRounded += oneMinuteUs; - } - - auto& tenant1Nodes = env.GetTenants().List("/Root/Tenant1"); - auto& tenant2Nodes = env.GetTenants().List("/Root/Tenant2"); - - UNIT_ASSERT_EQUAL(tenant1Nodes.size(), 2); - UNIT_ASSERT_EQUAL(tenant2Nodes.size(), 2); - - auto tenant1Node0 = tenant1Nodes[0]; - auto tenant1Node1 = tenant1Nodes[1]; - auto tenant2Node = tenant2Nodes.front(); - ui32 staticNode = 0; - - auto makeQueryEvent = [&runtime] (ui32 nodeIdx, ui64 endTimeUs, const TString& queryText, ui64 readBytes) { - auto stats = MakeHolder<NSysView::TEvSysView::TEvCollectQueryStats>(); - stats->QueryStats.MutableStats()->SetReadBytes(readBytes); - stats->QueryStats.SetQueryText(queryText); - stats->QueryStats.SetQueryTextHash(MurmurHash<ui64>(queryText.data(), queryText.size())); - stats->QueryStats.SetDurationMs(1); - stats->QueryStats.SetEndTimeMs(endTimeUs / 1000); - - auto serviceId = MakeSysViewServiceID(runtime->GetNodeId(nodeIdx)); + TTableClient client(env.GetDriver()); + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT Host, NodeId + FROM `Root/Tenant1/.sys/nodes`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0); + auto expected = Sprintf(R"([ + [["::1"];[%du]]; + [["::1"];[%du]]; + ])", offset + 3, offset + 4); + + NKqp::CompareYson(expected, NKqp::StreamResultToYson(it)); + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT Host, NodeId + FROM `Root/Tenant2/.sys/nodes`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0); + auto expected = Sprintf(R"([ + [["::1"];[%du]]; + [["::1"];[%du]]; + ])", offset + 1, offset + 2); + + NKqp::CompareYson(expected, NKqp::StreamResultToYson(it)); + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT Host, NodeId + FROM `Root/.sys/nodes`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0); + auto expected = Sprintf(R"([ + [["::1"];[%du]]; + [["::1"];[%du]]; + [["::1"];[%du]]; + [["::1"];[%du]]; + [["::1"];[%du]]; + ])", offset, offset + 1, offset + 2, offset + 3, offset + 4); + + NKqp::CompareYson(expected, NKqp::StreamResultToYson(it)); + } + } + + Y_UNIT_TEST(QueryStats) { + TTestEnv env; + CreateTenants(env); + + auto* runtime = env.GetServer().GetRuntime(); + runtime ->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_DEBUG); + + auto oneMinute = TDuration::Minutes(1); + auto oneMinuteUs = oneMinute.MicroSeconds(); + + auto instant = TAppData::TimeProvider->Now() + oneMinute; + auto instantRounded = instant.MicroSeconds() / oneMinuteUs * oneMinuteUs; + + std::vector<ui64> buckets; + for (size_t i : xrange(4)) { + Y_UNUSED(i); + buckets.push_back(instantRounded); + instantRounded += oneMinuteUs; + } + + auto& tenant1Nodes = env.GetTenants().List("/Root/Tenant1"); + auto& tenant2Nodes = env.GetTenants().List("/Root/Tenant2"); + + UNIT_ASSERT_EQUAL(tenant1Nodes.size(), 2); + UNIT_ASSERT_EQUAL(tenant2Nodes.size(), 2); + + auto tenant1Node0 = tenant1Nodes[0]; + auto tenant1Node1 = tenant1Nodes[1]; + auto tenant2Node = tenant2Nodes.front(); + ui32 staticNode = 0; + + auto makeQueryEvent = [&runtime] (ui32 nodeIdx, ui64 endTimeUs, const TString& queryText, ui64 readBytes) { + auto stats = MakeHolder<NSysView::TEvSysView::TEvCollectQueryStats>(); + stats->QueryStats.MutableStats()->SetReadBytes(readBytes); + stats->QueryStats.SetQueryText(queryText); + stats->QueryStats.SetQueryTextHash(MurmurHash<ui64>(queryText.data(), queryText.size())); + stats->QueryStats.SetDurationMs(1); + stats->QueryStats.SetEndTimeMs(endTimeUs / 1000); + + auto serviceId = MakeSysViewServiceID(runtime->GetNodeId(nodeIdx)); runtime->Send(new IEventHandle(serviceId, TActorId(), stats.Release()), nodeIdx); - }; - - makeQueryEvent(tenant1Node0, buckets[0], "a", 100); - makeQueryEvent(tenant1Node1, buckets[1], "b", 200); - makeQueryEvent(tenant1Node0, buckets[1], "c", 300); - makeQueryEvent(tenant1Node0, buckets[1], "d", 400); - makeQueryEvent(tenant1Node1, buckets[2], "e", 500); - - makeQueryEvent(tenant2Node, buckets[0], "f", 600); - - makeQueryEvent(staticNode, buckets[0], "g", 700); - - TTableClient client(env.GetDriver()); - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT IntervalEnd, QueryText, Rank, ReadBytes - FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - TStringBuilder result; - result << "["; - result << "[[" << buckets[1] << "u];[\"g\"];[1u];[700u]];"; - result << "]"; - - NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT IntervalEnd, QueryText, Rank, ReadBytes - FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute`; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - TStringBuilder result; - result << "["; - result << "[[" << buckets[1] << "u];[\"a\"];[1u];[100u]];"; - result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];"; - result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; - result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];"; - result << "[[" << buckets[3] << "u];[\"e\"];[1u];[500u]];"; - result << "]"; - - NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT IntervalEnd, QueryText, Rank, ReadBytes - FROM `Root/Tenant2/.sys/top_queries_by_read_bytes_one_minute`; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - TStringBuilder result; - result << "["; - result << "[[" << buckets[1] << "u];[\"f\"];[1u];[600u]];"; - result << "]"; - - NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); - } - { - TStringBuilder query; - query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes "; - query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` "; - query << "WHERE IntervalEnd >= CAST(" << buckets[1] << "ul as Timestamp) "; - query << "AND IntervalEnd < CAST(" << buckets[3] << "ul as Timestamp);"; - - auto it = client.StreamExecuteScanQuery(query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - TStringBuilder result; - result << "["; - result << "[[" << buckets[1] << "u];[\"a\"];[1u];[100u]];"; - result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];"; - result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; - result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];"; - result << "]"; - - NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); - } - { - TStringBuilder query; - query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes "; - query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` "; - query << "WHERE IntervalEnd > CAST(" << buckets[1] << "ul as Timestamp) "; - query << "AND IntervalEnd <= CAST(" << buckets[3] << "ul as Timestamp);"; - - auto it = client.StreamExecuteScanQuery(query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - TStringBuilder result; - result << "["; - result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];"; - result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; - result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];"; - result << "[[" << buckets[3] << "u];[\"e\"];[1u];[500u]];"; - result << "]"; - - NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); - } - { - TStringBuilder query; - query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes "; - query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` "; - query << "WHERE IntervalEnd = CAST(" << buckets[2] << "ul as Timestamp) "; - query << "AND Rank >= 1u AND Rank < 3u"; - - auto it = client.StreamExecuteScanQuery(query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - TStringBuilder result; - result << "["; - result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];"; - result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; - result << "]"; - - NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); - } - { - TStringBuilder query; - query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes "; - query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` "; - query << "WHERE IntervalEnd = CAST(" << buckets[2] << "ul as Timestamp) "; - query << "AND Rank > 1u AND Rank <= 3u"; - - auto it = client.StreamExecuteScanQuery(query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - TStringBuilder result; - result << "["; - result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; - result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];"; - result << "]"; - - NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); - } - } - - Y_UNIT_TEST(QueryStatsFields) { - TTestEnv env; - CreateRootTable(env, 3); - - auto nowUs = TInstant::Now().MicroSeconds(); - - TString queryText("SELECT * FROM `Root/Table0`"); - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - NKqp::AssertSuccessResult(session.ExecuteDataQuery( - queryText, TTxControl::BeginTx().CommitTx() - ).GetValueSync()); - - auto it = client.StreamExecuteScanQuery(R"( - SELECT - CPUTime, - CompileCPUTime, - CompileDuration, - ComputeNodesCount, - DeleteBytes, - DeleteRows, - Duration, - EndTime, - FromQueryCache, - IntervalEnd, - MaxComputeCPUTime, - MaxShardCPUTime, - MinComputeCPUTime, - MinShardCPUTime, - ParametersSize, - Partitions, - ProcessCPUTime, - QueryText, - Rank, - ReadBytes, - ReadRows, - RequestUnits, - ShardCount, - SumComputeCPUTime, - SumShardCPUTime, - Type, - UpdateBytes, - UpdateRows, - UserSID - FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto ysonString = NKqp::StreamResultToYson(it); - - TYsonFieldChecker check(ysonString, 29); - - check.Uint64GreaterOrEquals(0); // CPUTime - check.Uint64GreaterOrEquals(0); // CompileCPUTime - check.Int64GreaterOrEquals(0); // CompileDuration - check.Uint64(1); // ComputeNodesCount - check.Uint64(0); // DeleteBytes - check.Uint64(0); // DeleteRows - check.Int64Greater(0); // Duration - check.Uint64Greater(nowUs); // EndTime - check.Bool(false); // FromQueryCache - check.Uint64Greater(nowUs); // IntervalEnd - check.Uint64GreaterOrEquals(0); // MaxComputeCPUTime - check.Uint64GreaterOrEquals(0); // MaxShardCPUTime - check.Uint64GreaterOrEquals(0); // MinComputeCPUTime - check.Uint64GreaterOrEquals(0); // MinShardCPUTime - check.Uint64(0); // ParametersSize - check.Uint64(3); // Partitions - check.Uint64GreaterOrEquals(0); // ProcessCPUTime - check.String(queryText); // QueryText - check.Uint64(1); // Rank - check.Uint64(0); // ReadBytes - check.Uint64(0); // ReadRows - check.Uint64Greater(0); // RequestUnits - check.Uint64(3); // ShardCount - check.Uint64GreaterOrEquals(0); // SumComputeCPUTime - check.Uint64GreaterOrEquals(0); // SumShardCPUTime - check.String("data"); // Type - check.Uint64(0); // UpdateBytes - check.Uint64(0); // UpdateRows + }; + + makeQueryEvent(tenant1Node0, buckets[0], "a", 100); + makeQueryEvent(tenant1Node1, buckets[1], "b", 200); + makeQueryEvent(tenant1Node0, buckets[1], "c", 300); + makeQueryEvent(tenant1Node0, buckets[1], "d", 400); + makeQueryEvent(tenant1Node1, buckets[2], "e", 500); + + makeQueryEvent(tenant2Node, buckets[0], "f", 600); + + makeQueryEvent(staticNode, buckets[0], "g", 700); + + TTableClient client(env.GetDriver()); + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT IntervalEnd, QueryText, Rank, ReadBytes + FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + TStringBuilder result; + result << "["; + result << "[[" << buckets[1] << "u];[\"g\"];[1u];[700u]];"; + result << "]"; + + NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT IntervalEnd, QueryText, Rank, ReadBytes + FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute`; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + TStringBuilder result; + result << "["; + result << "[[" << buckets[1] << "u];[\"a\"];[1u];[100u]];"; + result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];"; + result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; + result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];"; + result << "[[" << buckets[3] << "u];[\"e\"];[1u];[500u]];"; + result << "]"; + + NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT IntervalEnd, QueryText, Rank, ReadBytes + FROM `Root/Tenant2/.sys/top_queries_by_read_bytes_one_minute`; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + TStringBuilder result; + result << "["; + result << "[[" << buckets[1] << "u];[\"f\"];[1u];[600u]];"; + result << "]"; + + NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); + } + { + TStringBuilder query; + query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes "; + query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` "; + query << "WHERE IntervalEnd >= CAST(" << buckets[1] << "ul as Timestamp) "; + query << "AND IntervalEnd < CAST(" << buckets[3] << "ul as Timestamp);"; + + auto it = client.StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + TStringBuilder result; + result << "["; + result << "[[" << buckets[1] << "u];[\"a\"];[1u];[100u]];"; + result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];"; + result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; + result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];"; + result << "]"; + + NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); + } + { + TStringBuilder query; + query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes "; + query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` "; + query << "WHERE IntervalEnd > CAST(" << buckets[1] << "ul as Timestamp) "; + query << "AND IntervalEnd <= CAST(" << buckets[3] << "ul as Timestamp);"; + + auto it = client.StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + TStringBuilder result; + result << "["; + result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];"; + result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; + result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];"; + result << "[[" << buckets[3] << "u];[\"e\"];[1u];[500u]];"; + result << "]"; + + NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); + } + { + TStringBuilder query; + query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes "; + query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` "; + query << "WHERE IntervalEnd = CAST(" << buckets[2] << "ul as Timestamp) "; + query << "AND Rank >= 1u AND Rank < 3u"; + + auto it = client.StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + TStringBuilder result; + result << "["; + result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];"; + result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; + result << "]"; + + NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); + } + { + TStringBuilder query; + query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes "; + query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` "; + query << "WHERE IntervalEnd = CAST(" << buckets[2] << "ul as Timestamp) "; + query << "AND Rank > 1u AND Rank <= 3u"; + + auto it = client.StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + TStringBuilder result; + result << "["; + result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];"; + result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];"; + result << "]"; + + NKqp::CompareYson(result, NKqp::StreamResultToYson(it)); + } + } + + Y_UNIT_TEST(QueryStatsFields) { + TTestEnv env; + CreateRootTable(env, 3); + + auto nowUs = TInstant::Now().MicroSeconds(); + + TString queryText("SELECT * FROM `Root/Table0`"); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + NKqp::AssertSuccessResult(session.ExecuteDataQuery( + queryText, TTxControl::BeginTx().CommitTx() + ).GetValueSync()); + + auto it = client.StreamExecuteScanQuery(R"( + SELECT + CPUTime, + CompileCPUTime, + CompileDuration, + ComputeNodesCount, + DeleteBytes, + DeleteRows, + Duration, + EndTime, + FromQueryCache, + IntervalEnd, + MaxComputeCPUTime, + MaxShardCPUTime, + MinComputeCPUTime, + MinShardCPUTime, + ParametersSize, + Partitions, + ProcessCPUTime, + QueryText, + Rank, + ReadBytes, + ReadRows, + RequestUnits, + ShardCount, + SumComputeCPUTime, + SumShardCPUTime, + Type, + UpdateBytes, + UpdateRows, + UserSID + FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto ysonString = NKqp::StreamResultToYson(it); + + TYsonFieldChecker check(ysonString, 29); + + check.Uint64GreaterOrEquals(0); // CPUTime + check.Uint64GreaterOrEquals(0); // CompileCPUTime + check.Int64GreaterOrEquals(0); // CompileDuration + check.Uint64(1); // ComputeNodesCount + check.Uint64(0); // DeleteBytes + check.Uint64(0); // DeleteRows + check.Int64Greater(0); // Duration + check.Uint64Greater(nowUs); // EndTime + check.Bool(false); // FromQueryCache + check.Uint64Greater(nowUs); // IntervalEnd + check.Uint64GreaterOrEquals(0); // MaxComputeCPUTime + check.Uint64GreaterOrEquals(0); // MaxShardCPUTime + check.Uint64GreaterOrEquals(0); // MinComputeCPUTime + check.Uint64GreaterOrEquals(0); // MinShardCPUTime + check.Uint64(0); // ParametersSize + check.Uint64(3); // Partitions + check.Uint64GreaterOrEquals(0); // ProcessCPUTime + check.String(queryText); // QueryText + check.Uint64(1); // Rank + check.Uint64(0); // ReadBytes + check.Uint64(0); // ReadRows + check.Uint64Greater(0); // RequestUnits + check.Uint64(3); // ShardCount + check.Uint64GreaterOrEquals(0); // SumComputeCPUTime + check.Uint64GreaterOrEquals(0); // SumShardCPUTime + check.String("data"); // Type + check.Uint64(0); // UpdateBytes + check.Uint64(0); // UpdateRows check.Null(); // UserSID - } - + } + Y_UNIT_TEST(PartitionStatsTtlFields) { TTestEnv env; env.GetClient().CreateTable("/Root", R"( @@ -687,844 +687,844 @@ Y_UNIT_TEST_SUITE(SystemView) { check.Uint64(1u); // LastTtlRowsErased } - Y_UNIT_TEST(PartitionStatsFields) { + Y_UNIT_TEST(PartitionStatsFields) { NDataShard::gDbStatsReportInterval = TDuration::Seconds(0); - - auto nowUs = TInstant::Now().MicroSeconds(); - - TTestEnv env; - CreateRootTable(env); - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - NKqp::AssertSuccessResult(session.ExecuteDataQuery( - "REPLACE INTO `Root/Table0` (Key, Value) VALUES (0u, \"A\");", - TTxControl::BeginTx().CommitTx() - ).GetValueSync()); - - // wait for stats - for (size_t iter = 0; iter < 30; ++iter) { - auto it = client.StreamExecuteScanQuery(R"( - SELECT AccessTime FROM `/Root/.sys/partition_stats`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto ysonString = NKqp::StreamResultToYson(it); - if (ysonString != "[[#]]") { - break; - } - - Sleep(TDuration::Seconds(1)); - } - - auto it = client.StreamExecuteScanQuery(R"( - SELECT - AccessTime, - CPUCores, - CoordinatedTxCompleted, - DataSize, - ImmediateTxCompleted, - IndexSize, - InFlightTxCount, - NodeId, - OwnerId, - PartIdx, - Path, - PathId, - RangeReadRows, - RangeReads, - RowCount, - RowDeletes, - RowReads, - RowUpdates, - StartTime, - TabletId, - TxRejectedByOutOfStorage, - TxRejectedByOverload, - UpdateTime - FROM `/Root/.sys/partition_stats`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto ysonString = NKqp::StreamResultToYson(it); - - TYsonFieldChecker check(ysonString, 23); - - check.Uint64GreaterOrEquals(nowUs); // AccessTime - check.DoubleGreaterOrEquals(0.0); // CPUCores - check.Uint64(1u); // CoordinatedTxCompleted - check.Uint64(608u); // DataSize - check.Uint64(1u); // ImmediateTxCompleted - check.Uint64(0u); // IndexSize - check.Uint64(0u); // InFlightTxCount - check.Uint64Greater(0u); // NodeId - check.Uint64(72057594046644480ull); // OwnerId - check.Uint64(0u); // PartIdx - check.String("/Root/Table0"); // Path - check.Uint64(2u); // PathId - check.Uint64(0u); // RangeReadRows - check.Uint64(0u); // RangeReads - check.Uint64(1u); // RowCount - check.Uint64(0u); // RowDeletes - check.Uint64(0u); // RowReads - check.Uint64(1u); // RowUpdates - check.Uint64GreaterOrEquals(nowUs); // StartTime - check.Uint64Greater(0u); // TabletId - check.Uint64(0u); // TxRejectedByOutOfStorage - check.Uint64(0u); // TxRejectedByOverload - check.Uint64GreaterOrEquals(nowUs); // UpdateTime - } - - Y_UNIT_TEST(QueryStatsAllTables) { - auto check = [&] (const TString& queryText) { - TTestEnv env; - CreateRootTable(env); - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - NKqp::AssertSuccessResult(session.ExecuteDataQuery( - "SELECT * FROM `Root/Table0`", TTxControl::BeginTx().CommitTx() - ).GetValueSync()); - - auto it = client.StreamExecuteScanQuery(queryText).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - NKqp::CompareYson(R"([ - [[0u]] - ])", NKqp::StreamResultToYson(it)); - }; - - check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_read_bytes_one_minute`"); - check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_read_bytes_one_hour`"); - check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_duration_one_minute`"); - check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_duration_one_hour`"); - check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_cpu_time_one_minute`"); - check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_cpu_time_one_hour`"); - check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_request_units_one_minute`"); - check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_request_units_one_hour`"); - } - - Y_UNIT_TEST(QueryStatsRetries) { - TTestEnv env; - CreateRootTable(env); - - TString queryText("SELECT * FROM `Root/Table0`"); - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - NKqp::AssertSuccessResult(session.ExecuteDataQuery( - queryText, TTxControl::BeginTx().CommitTx() - ).GetValueSync()); - - auto serviceToKill = MakeSysViewServiceID(env.GetServer().GetRuntime()->GetNodeId(2)); + + auto nowUs = TInstant::Now().MicroSeconds(); + + TTestEnv env; + CreateRootTable(env); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + NKqp::AssertSuccessResult(session.ExecuteDataQuery( + "REPLACE INTO `Root/Table0` (Key, Value) VALUES (0u, \"A\");", + TTxControl::BeginTx().CommitTx() + ).GetValueSync()); + + // wait for stats + for (size_t iter = 0; iter < 30; ++iter) { + auto it = client.StreamExecuteScanQuery(R"( + SELECT AccessTime FROM `/Root/.sys/partition_stats`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto ysonString = NKqp::StreamResultToYson(it); + if (ysonString != "[[#]]") { + break; + } + + Sleep(TDuration::Seconds(1)); + } + + auto it = client.StreamExecuteScanQuery(R"( + SELECT + AccessTime, + CPUCores, + CoordinatedTxCompleted, + DataSize, + ImmediateTxCompleted, + IndexSize, + InFlightTxCount, + NodeId, + OwnerId, + PartIdx, + Path, + PathId, + RangeReadRows, + RangeReads, + RowCount, + RowDeletes, + RowReads, + RowUpdates, + StartTime, + TabletId, + TxRejectedByOutOfStorage, + TxRejectedByOverload, + UpdateTime + FROM `/Root/.sys/partition_stats`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto ysonString = NKqp::StreamResultToYson(it); + + TYsonFieldChecker check(ysonString, 23); + + check.Uint64GreaterOrEquals(nowUs); // AccessTime + check.DoubleGreaterOrEquals(0.0); // CPUCores + check.Uint64(1u); // CoordinatedTxCompleted + check.Uint64(608u); // DataSize + check.Uint64(1u); // ImmediateTxCompleted + check.Uint64(0u); // IndexSize + check.Uint64(0u); // InFlightTxCount + check.Uint64Greater(0u); // NodeId + check.Uint64(72057594046644480ull); // OwnerId + check.Uint64(0u); // PartIdx + check.String("/Root/Table0"); // Path + check.Uint64(2u); // PathId + check.Uint64(0u); // RangeReadRows + check.Uint64(0u); // RangeReads + check.Uint64(1u); // RowCount + check.Uint64(0u); // RowDeletes + check.Uint64(0u); // RowReads + check.Uint64(1u); // RowUpdates + check.Uint64GreaterOrEquals(nowUs); // StartTime + check.Uint64Greater(0u); // TabletId + check.Uint64(0u); // TxRejectedByOutOfStorage + check.Uint64(0u); // TxRejectedByOverload + check.Uint64GreaterOrEquals(nowUs); // UpdateTime + } + + Y_UNIT_TEST(QueryStatsAllTables) { + auto check = [&] (const TString& queryText) { + TTestEnv env; + CreateRootTable(env); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + NKqp::AssertSuccessResult(session.ExecuteDataQuery( + "SELECT * FROM `Root/Table0`", TTxControl::BeginTx().CommitTx() + ).GetValueSync()); + + auto it = client.StreamExecuteScanQuery(queryText).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NKqp::CompareYson(R"([ + [[0u]] + ])", NKqp::StreamResultToYson(it)); + }; + + check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_read_bytes_one_minute`"); + check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_read_bytes_one_hour`"); + check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_duration_one_minute`"); + check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_duration_one_hour`"); + check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_cpu_time_one_minute`"); + check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_cpu_time_one_hour`"); + check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_request_units_one_minute`"); + check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_request_units_one_hour`"); + } + + Y_UNIT_TEST(QueryStatsRetries) { + TTestEnv env; + CreateRootTable(env); + + TString queryText("SELECT * FROM `Root/Table0`"); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + NKqp::AssertSuccessResult(session.ExecuteDataQuery( + queryText, TTxControl::BeginTx().CommitTx() + ).GetValueSync()); + + auto serviceToKill = MakeSysViewServiceID(env.GetServer().GetRuntime()->GetNodeId(2)); env.GetServer().GetRuntime()->Send(new IEventHandle(serviceToKill, TActorId(), new TEvents::TEvPoison())); - - auto it = client.StreamExecuteScanQuery(R"( - SELECT - ReadBytes - FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - NKqp::CompareYson(R"([ - [[0u]] - ])", NKqp::StreamResultToYson(it)); - } - - Y_UNIT_TEST(ConcurrentScans) { - TTestEnv env; - CreateRootTable(env); - TTableClient client(env.GetDriver()); - - TVector<TAsyncScanQueryPartIterator> futures; - for (size_t i = 0; i < 20; ++i) { - auto future0 = client.StreamExecuteScanQuery(R"( - SELECT - ReadBytes - FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; - )"); - futures.push_back(future0); - } - - for (auto& future0 : futures) { - auto it = future0.GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - auto streamPart = it.ReadNext().GetValueSync(); - if (streamPart.IsSuccess()) { - UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SUCCESS); - Cerr << "SUCCESS" << Endl; - } else { - UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::GENERIC_ERROR); - Cerr << "FAIL " << streamPart.GetIssues().ToString() << Endl; - } - } - } - - Y_UNIT_TEST(PDisksFields) { - TTestEnv env(1, 0); - - TTableClient client(env.GetDriver()); - size_t rowCount = 0; - TString ysonString; - - for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { - auto it = client.StreamExecuteScanQuery(R"( - SELECT - AvailableSize, - BoxId, - Guid, - Kind, - NodeId, - PDiskId, - Path, - ReadCentric, - SharedWithOS, - Status, + + auto it = client.StreamExecuteScanQuery(R"( + SELECT + ReadBytes + FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NKqp::CompareYson(R"([ + [[0u]] + ])", NKqp::StreamResultToYson(it)); + } + + Y_UNIT_TEST(ConcurrentScans) { + TTestEnv env; + CreateRootTable(env); + TTableClient client(env.GetDriver()); + + TVector<TAsyncScanQueryPartIterator> futures; + for (size_t i = 0; i < 20; ++i) { + auto future0 = client.StreamExecuteScanQuery(R"( + SELECT + ReadBytes + FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; + )"); + futures.push_back(future0); + } + + for (auto& future0 : futures) { + auto it = future0.GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + auto streamPart = it.ReadNext().GetValueSync(); + if (streamPart.IsSuccess()) { + UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SUCCESS); + Cerr << "SUCCESS" << Endl; + } else { + UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::GENERIC_ERROR); + Cerr << "FAIL " << streamPart.GetIssues().ToString() << Endl; + } + } + } + + Y_UNIT_TEST(PDisksFields) { + TTestEnv env(1, 0); + + TTableClient client(env.GetDriver()); + size_t rowCount = 0; + TString ysonString; + + for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { + auto it = client.StreamExecuteScanQuery(R"( + SELECT + AvailableSize, + BoxId, + Guid, + Kind, + NodeId, + PDiskId, + Path, + ReadCentric, + SharedWithOS, + Status, StatusChangeTimestamp, - TotalSize, + TotalSize, Type, ExpectedSlotCount, NumActiveSlots - FROM `/Root/.sys/ds_pdisks`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - ysonString = NKqp::StreamResultToYson(it); - + FROM `/Root/.sys/ds_pdisks`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + ysonString = NKqp::StreamResultToYson(it); + auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node); - UNIT_ASSERT(node.IsList()); - rowCount = node.AsList().size(); - - if (!rowCount) { - Sleep(TDuration::Seconds(1)); - } - } - + UNIT_ASSERT(node.IsList()); + rowCount = node.AsList().size(); + + if (!rowCount) { + Sleep(TDuration::Seconds(1)); + } + } + TYsonFieldChecker check(ysonString, 15); - - check.Uint64(0u); // AvailableSize - check.Uint64(999u); // BoxId - check.Uint64(123u); // Guid - check.Uint64(0u); // Kind - check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId - check.Uint64(1u); // PDiskId - check.StringContains("pdisk_1.dat"); // Path - check.Bool(false); // ReadCentric - check.Bool(false); // SharedWithOS - check.String("ACTIVE"); // Status + + check.Uint64(0u); // AvailableSize + check.Uint64(999u); // BoxId + check.Uint64(123u); // Guid + check.Uint64(0u); // Kind + check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId + check.Uint64(1u); // PDiskId + check.StringContains("pdisk_1.dat"); // Path + check.Bool(false); // ReadCentric + check.Bool(false); // SharedWithOS + check.String("ACTIVE"); // Status check.Null(); // StatusChangeTimestamp - check.Uint64(0u); // TotalSize - check.String("ROT"); // Type + check.Uint64(0u); // TotalSize + check.String("ROT"); // Type check.Uint64(16); // ExpectedSlotCount check.Uint64(2); // NumActiveSlots - } - - Y_UNIT_TEST(VSlotsFields) { - TTestEnv env(1, 0); - - TTableClient client(env.GetDriver()); - size_t rowCount = 0; - TString ysonString; - - for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { - auto it = client.StreamExecuteScanQuery(R"( - SELECT - AllocatedSize, - AvailableSize, - FailDomain, - FailRealm, - GroupGeneration, - GroupId, - Kind, - NodeId, - PDiskId, - Status, - VDisk, - VSlotId + } + + Y_UNIT_TEST(VSlotsFields) { + TTestEnv env(1, 0); + + TTableClient client(env.GetDriver()); + size_t rowCount = 0; + TString ysonString; + + for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { + auto it = client.StreamExecuteScanQuery(R"( + SELECT + AllocatedSize, + AvailableSize, + FailDomain, + FailRealm, + GroupGeneration, + GroupId, + Kind, + NodeId, + PDiskId, + Status, + VDisk, + VSlotId FROM `/Root/.sys/ds_vslots` WHERE GroupId >= 0x80000000; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - ysonString = NKqp::StreamResultToYson(it); - + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + ysonString = NKqp::StreamResultToYson(it); + auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node); - UNIT_ASSERT(node.IsList()); - rowCount = node.AsList().size(); - - if (!rowCount) { - Sleep(TDuration::Seconds(1)); - } - } - + UNIT_ASSERT(node.IsList()); + rowCount = node.AsList().size(); + + if (!rowCount) { + Sleep(TDuration::Seconds(1)); + } + } + TYsonFieldChecker check(ysonString, 12); - + check.Uint64(0u, true); // AllocatedSize check.Uint64(0u, true); // AvailableSize - check.Uint64(0u); // FailDomain - check.Uint64(0u); // FailRealm - check.Uint64(1u); // GroupGeneration - check.Uint64(2181038080u); // GroupId - check.String("Default"); // Kind - check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId - check.Uint64(1u); // PDiskId + check.Uint64(0u); // FailDomain + check.Uint64(0u); // FailRealm + check.Uint64(1u); // GroupGeneration + check.Uint64(2181038080u); // GroupId + check.String("Default"); // Kind + check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId + check.Uint64(1u); // PDiskId check.String("INIT_PENDING"); // Status - check.Uint64(0u); // VDisk - check.Uint64(1000u); // VSlotId - } - - Y_UNIT_TEST(GroupsFields) { - TTestEnv env; - - TTableClient client(env.GetDriver()); - size_t rowCount = 0; - TString ysonString; - - for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { - auto it = client.StreamExecuteScanQuery(R"( - SELECT - AllocatedSize, - AvailableSize, - BoxId, - EncryptionMode, - ErasureSpecies, - Generation, - GetFastLatency, - GroupId, - LifeCyclePhase, - PutTabletLogLatency, - PutUserDataLatency, + check.Uint64(0u); // VDisk + check.Uint64(1000u); // VSlotId + } + + Y_UNIT_TEST(GroupsFields) { + TTestEnv env; + + TTableClient client(env.GetDriver()); + size_t rowCount = 0; + TString ysonString; + + for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { + auto it = client.StreamExecuteScanQuery(R"( + SELECT + AllocatedSize, + AvailableSize, + BoxId, + EncryptionMode, + ErasureSpecies, + Generation, + GetFastLatency, + GroupId, + LifeCyclePhase, + PutTabletLogLatency, + PutUserDataLatency, StoragePoolId FROM `/Root/.sys/ds_groups` WHERE GroupId >= 0x80000000; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - ysonString = NKqp::StreamResultToYson(it); - + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + ysonString = NKqp::StreamResultToYson(it); + auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node); - UNIT_ASSERT(node.IsList()); - rowCount = node.AsList().size(); - - if (!rowCount) { - Sleep(TDuration::Seconds(1)); - } - } - + UNIT_ASSERT(node.IsList()); + rowCount = node.AsList().size(); + + if (!rowCount) { + Sleep(TDuration::Seconds(1)); + } + } + TYsonFieldChecker check(ysonString, 12); - - check.Uint64(0u); // AllocatedSize - check.Uint64(0u); // AvailableSize - check.Uint64(999u); // BoxId - check.Uint64(0u); // EncryptionMode - check.String("none"); // ErasureSpecies - check.Uint64(1u); // Generation + + check.Uint64(0u); // AllocatedSize + check.Uint64(0u); // AvailableSize + check.Uint64(999u); // BoxId + check.Uint64(0u); // EncryptionMode + check.String("none"); // ErasureSpecies + check.Uint64(1u); // Generation check.Null(); // GetFastLatency - check.Uint64(2181038080u); // GroupId - check.Uint64(0u); // LifeCyclePhase + check.Uint64(2181038080u); // GroupId + check.Uint64(0u); // LifeCyclePhase check.Null(); // PutTabletLogLatency check.Null(); // PutUserDataLatency - check.Uint64(2u); // StoragePoolId - } - - Y_UNIT_TEST(StoragePoolsFields) { - TTestEnv env(1, 0); - - TTableClient client(env.GetDriver()); - size_t rowCount = 0; - TString ysonString; - - for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { - auto it = client.StreamExecuteScanQuery(R"( - SELECT - BoxId, - EncryptionMode, - ErasureSpecies, - Generation, - Kind, - Name, - NumGroups, - PathId, - SchemeshardId, - StoragePoolId, - VDiskKind - FROM `/Root/.sys/ds_storage_pools`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - ysonString = NKqp::StreamResultToYson(it); - + check.Uint64(2u); // StoragePoolId + } + + Y_UNIT_TEST(StoragePoolsFields) { + TTestEnv env(1, 0); + + TTableClient client(env.GetDriver()); + size_t rowCount = 0; + TString ysonString; + + for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { + auto it = client.StreamExecuteScanQuery(R"( + SELECT + BoxId, + EncryptionMode, + ErasureSpecies, + Generation, + Kind, + Name, + NumGroups, + PathId, + SchemeshardId, + StoragePoolId, + VDiskKind + FROM `/Root/.sys/ds_storage_pools`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + ysonString = NKqp::StreamResultToYson(it); + auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node); - UNIT_ASSERT(node.IsList()); - rowCount = node.AsList().size(); - - if (!rowCount) { - Sleep(TDuration::Seconds(1)); - } - } - - TYsonFieldChecker check(ysonString, 11); - - check.Uint64(999u); // BoxId - check.Uint64(0u); // EncryptionMode - check.String("none"); // ErasureSpecies - check.Uint64(1u); // Generation - check.String("test"); // Kind - check.String("/Root:test"); // Name - check.Uint64(1u); // NumGroups + UNIT_ASSERT(node.IsList()); + rowCount = node.AsList().size(); + + if (!rowCount) { + Sleep(TDuration::Seconds(1)); + } + } + + TYsonFieldChecker check(ysonString, 11); + + check.Uint64(999u); // BoxId + check.Uint64(0u); // EncryptionMode + check.String("none"); // ErasureSpecies + check.Uint64(1u); // Generation + check.String("test"); // Kind + check.String("/Root:test"); // Name + check.Uint64(1u); // NumGroups check.Null(); // PathId check.Null(); // SchemeshardId - check.Uint64(2u); // StoragePoolId - check.String("Default"); // VDiskKind - } - - Y_UNIT_TEST(StoragePoolsRanges) { - TTestEnv env(1, 0, 3); - - TTableClient client(env.GetDriver()); - size_t rowCount = 0; - for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { - auto it = client.StreamExecuteScanQuery(R"( - SELECT BoxId FROM `/Root/.sys/ds_storage_pools`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto ysonString = NKqp::StreamResultToYson(it); + check.Uint64(2u); // StoragePoolId + check.String("Default"); // VDiskKind + } + + Y_UNIT_TEST(StoragePoolsRanges) { + TTestEnv env(1, 0, 3); + + TTableClient client(env.GetDriver()); + size_t rowCount = 0; + for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { + auto it = client.StreamExecuteScanQuery(R"( + SELECT BoxId FROM `/Root/.sys/ds_storage_pools`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto ysonString = NKqp::StreamResultToYson(it); auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node); - UNIT_ASSERT(node.IsList()); - rowCount = node.AsList().size(); - - if (!rowCount) { - Sleep(TDuration::Seconds(1)); - } - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT BoxId, StoragePoolId - FROM `/Root/.sys/ds_storage_pools` - WHERE BoxId = 999u AND StoragePoolId > 3u; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - NKqp::CompareYson(R"([ - [[999u];[4u]]; - [[999u];[5u]]; - ])", NKqp::StreamResultToYson(it)); - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT BoxId, StoragePoolId - FROM `/Root/.sys/ds_storage_pools` - WHERE BoxId = 999u AND StoragePoolId >= 3u; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - NKqp::CompareYson(R"([ - [[999u];[3u]]; - [[999u];[4u]]; - [[999u];[5u]]; - ])", NKqp::StreamResultToYson(it)); - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT BoxId, StoragePoolId - FROM `/Root/.sys/ds_storage_pools` - WHERE BoxId = 999u AND StoragePoolId < 4u; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - NKqp::CompareYson(R"([ - [[999u];[2u]]; - [[999u];[3u]]; - ])", NKqp::StreamResultToYson(it)); - } - { - auto it = client.StreamExecuteScanQuery(R"( - SELECT BoxId, StoragePoolId - FROM `/Root/.sys/ds_storage_pools` - WHERE BoxId = 999u AND StoragePoolId <= 4u; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - NKqp::CompareYson(R"([ - [[999u];[2u]]; - [[999u];[3u]]; - [[999u];[4u]]; - ])", NKqp::StreamResultToYson(it)); - } - } - - Y_UNIT_TEST(OldEngineSystemView) { - TTestEnv env; - CreateRootTable(env); - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteDataQuery( - "SELECT * from `/Root/.sys/partition_stats`", TTxControl::BeginTx().CommitTx() - ).GetValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - result.GetIssues().PrintTo(Cerr); - } - - Y_UNIT_TEST(Describe) { - TTestEnv env; - CreateRootTable(env); - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - { - auto settings = TDescribeTableSettings() - .WithKeyShardBoundary(true) - .WithTableStatistics(true) - .WithPartitionStatistics(true); - - auto result = session.DescribeTable("/Root/.sys/partition_stats", settings).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - const auto& table = result.GetTableDescription(); - const auto& columns = table.GetTableColumns(); - const auto& keyColumns = table.GetPrimaryKeyColumns(); - + UNIT_ASSERT(node.IsList()); + rowCount = node.AsList().size(); + + if (!rowCount) { + Sleep(TDuration::Seconds(1)); + } + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT BoxId, StoragePoolId + FROM `/Root/.sys/ds_storage_pools` + WHERE BoxId = 999u AND StoragePoolId > 3u; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NKqp::CompareYson(R"([ + [[999u];[4u]]; + [[999u];[5u]]; + ])", NKqp::StreamResultToYson(it)); + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT BoxId, StoragePoolId + FROM `/Root/.sys/ds_storage_pools` + WHERE BoxId = 999u AND StoragePoolId >= 3u; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NKqp::CompareYson(R"([ + [[999u];[3u]]; + [[999u];[4u]]; + [[999u];[5u]]; + ])", NKqp::StreamResultToYson(it)); + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT BoxId, StoragePoolId + FROM `/Root/.sys/ds_storage_pools` + WHERE BoxId = 999u AND StoragePoolId < 4u; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NKqp::CompareYson(R"([ + [[999u];[2u]]; + [[999u];[3u]]; + ])", NKqp::StreamResultToYson(it)); + } + { + auto it = client.StreamExecuteScanQuery(R"( + SELECT BoxId, StoragePoolId + FROM `/Root/.sys/ds_storage_pools` + WHERE BoxId = 999u AND StoragePoolId <= 4u; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NKqp::CompareYson(R"([ + [[999u];[2u]]; + [[999u];[3u]]; + [[999u];[4u]]; + ])", NKqp::StreamResultToYson(it)); + } + } + + Y_UNIT_TEST(OldEngineSystemView) { + TTestEnv env; + CreateRootTable(env); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteDataQuery( + "SELECT * from `/Root/.sys/partition_stats`", TTxControl::BeginTx().CommitTx() + ).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + result.GetIssues().PrintTo(Cerr); + } + + Y_UNIT_TEST(Describe) { + TTestEnv env; + CreateRootTable(env); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + { + auto settings = TDescribeTableSettings() + .WithKeyShardBoundary(true) + .WithTableStatistics(true) + .WithPartitionStatistics(true); + + auto result = session.DescribeTable("/Root/.sys/partition_stats", settings).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + const auto& table = result.GetTableDescription(); + const auto& columns = table.GetTableColumns(); + const auto& keyColumns = table.GetPrimaryKeyColumns(); + UNIT_ASSERT_VALUES_EQUAL(columns.size(), 26); - UNIT_ASSERT_STRINGS_EQUAL(columns[0].Name, "OwnerId"); - UNIT_ASSERT_STRINGS_EQUAL(FormatType(columns[0].Type), "Uint64?"); - - UNIT_ASSERT_VALUES_EQUAL(keyColumns.size(), 3); - UNIT_ASSERT_STRINGS_EQUAL(keyColumns[0], "OwnerId"); - - UNIT_ASSERT_VALUES_EQUAL(table.GetPartitionStats().size(), 0); - UNIT_ASSERT_VALUES_EQUAL(table.GetPartitionsCount(), 0); - } - - TSchemeClient schemeClient(env.GetDriver()); - - { - auto result = schemeClient.DescribePath("/Root/.sys/partition_stats").GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - auto entry = result.GetEntry(); - UNIT_ASSERT_VALUES_EQUAL(entry.Name, "partition_stats"); - UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Table); - } - { - auto result = schemeClient.ListDirectory("/Root/.sys/partition_stats").GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - auto entry = result.GetEntry(); - UNIT_ASSERT_VALUES_EQUAL(entry.Name, "partition_stats"); - UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Table); - } - } - - Y_UNIT_TEST(SystemViewFailOps) { - TTestEnv env; - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - auto desc = TTableBuilder() - .AddNullableColumn("Column1", EPrimitiveType::Uint64) - .SetPrimaryKeyColumn("Column1") - .Build(); - - auto result = session.CreateTable("/Root/.sys/partition_stats", std::move(desc)).GetValueSync(); + UNIT_ASSERT_STRINGS_EQUAL(columns[0].Name, "OwnerId"); + UNIT_ASSERT_STRINGS_EQUAL(FormatType(columns[0].Type), "Uint64?"); + + UNIT_ASSERT_VALUES_EQUAL(keyColumns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(keyColumns[0], "OwnerId"); + + UNIT_ASSERT_VALUES_EQUAL(table.GetPartitionStats().size(), 0); + UNIT_ASSERT_VALUES_EQUAL(table.GetPartitionsCount(), 0); + } + + TSchemeClient schemeClient(env.GetDriver()); + + { + auto result = schemeClient.DescribePath("/Root/.sys/partition_stats").GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto entry = result.GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(entry.Name, "partition_stats"); + UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Table); + } + { + auto result = schemeClient.ListDirectory("/Root/.sys/partition_stats").GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto entry = result.GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(entry.Name, "partition_stats"); + UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Table); + } + } + + Y_UNIT_TEST(SystemViewFailOps) { + TTestEnv env; + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + auto desc = TTableBuilder() + .AddNullableColumn("Column1", EPrimitiveType::Uint64) + .SetPrimaryKeyColumn("Column1") + .Build(); + + auto result = session.CreateTable("/Root/.sys/partition_stats", std::move(desc)).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = session.CopyTable("/Root/.sys/partition_stats", "/Root/Table0").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + result.GetIssues().PrintTo(Cerr); + } + { + auto settings = TAlterTableSettings() + .AppendDropColumns("OwnerId"); + + auto result = session.AlterTable("/Root/.sys/partition_stats", settings).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = session.CopyTable("/Root/.sys/partition_stats", "/Root/Table0").GetValueSync(); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = session.DropTable("/Root/.sys/partition_stats").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - { - auto settings = TAlterTableSettings() - .AppendDropColumns("OwnerId"); - - auto result = session.AlterTable("/Root/.sys/partition_stats", settings).GetValueSync(); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = session.ExecuteSchemeQuery(R"( + DROP TABLE `/Root/.sys/partition_stats`; + )").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = session.DropTable("/Root/.sys/partition_stats").GetValueSync(); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = session.ReadTable("/Root/.sys/partition_stats").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + TReadTableResultPart streamPart = result.ReadNext().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); + streamPart.GetIssues().PrintTo(Cerr); + } + { + TValueBuilder rows; + rows.BeginList().EndList(); + auto result = client.BulkUpsert("/Root/.sys/partition_stats", rows.Build()).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = session.ExecuteSchemeQuery(R"( - DROP TABLE `/Root/.sys/partition_stats`; - )").GetValueSync(); + result.GetIssues().PrintTo(Cerr); + } + + TSchemeClient schemeClient(env.GetDriver()); + + { + auto result = schemeClient.MakeDirectory("/Root/.sys").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = session.ReadTable("/Root/.sys/partition_stats").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - TReadTableResultPart streamPart = result.ReadNext().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); - streamPart.GetIssues().PrintTo(Cerr); - } - { - TValueBuilder rows; - rows.BeginList().EndList(); - auto result = client.BulkUpsert("/Root/.sys/partition_stats", rows.Build()).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - - TSchemeClient schemeClient(env.GetDriver()); - - { - auto result = schemeClient.MakeDirectory("/Root/.sys").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = schemeClient.MakeDirectory("/Root/.sys/partition_stats").GetValueSync(); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = schemeClient.MakeDirectory("/Root/.sys/partition_stats").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = schemeClient.RemoveDirectory("/Root/.sys").GetValueSync(); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = schemeClient.RemoveDirectory("/Root/.sys").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = schemeClient.RemoveDirectory("/Root/.sys/partition_stats").GetValueSync(); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = schemeClient.RemoveDirectory("/Root/.sys/partition_stats").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - { - TModifyPermissionsSettings settings; - auto result = schemeClient.ModifyPermissions("/Root/.sys/partition_stats", settings).GetValueSync(); + result.GetIssues().PrintTo(Cerr); + } + { + TModifyPermissionsSettings settings; + auto result = schemeClient.ModifyPermissions("/Root/.sys/partition_stats", settings).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - } - - Y_UNIT_TEST(DescribeSystemFolder) { - TTestEnv env; + result.GetIssues().PrintTo(Cerr); + } + } + + Y_UNIT_TEST(DescribeSystemFolder) { + TTestEnv env; CreateTenantsAndTables(env, false); - - TSchemeClient schemeClient(env.GetDriver()); - { - auto result = schemeClient.ListDirectory("/Root").GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - auto entry = result.GetEntry(); - UNIT_ASSERT_STRINGS_EQUAL(entry.Name, "Root"); - UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory); - - auto children = result.GetChildren(); - UNIT_ASSERT_VALUES_EQUAL(children.size(), 4); - UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, "Table0"); - UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, "Tenant1"); - UNIT_ASSERT_STRINGS_EQUAL(children[2].Name, "Tenant2"); - UNIT_ASSERT_STRINGS_EQUAL(children[3].Name, ".sys"); - } - { - auto result = schemeClient.ListDirectory("/Root/Tenant1").GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - auto entry = result.GetEntry(); + + TSchemeClient schemeClient(env.GetDriver()); + { + auto result = schemeClient.ListDirectory("/Root").GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto entry = result.GetEntry(); + UNIT_ASSERT_STRINGS_EQUAL(entry.Name, "Root"); + UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory); + + auto children = result.GetChildren(); + UNIT_ASSERT_VALUES_EQUAL(children.size(), 4); + UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, "Table0"); + UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, "Tenant1"); + UNIT_ASSERT_STRINGS_EQUAL(children[2].Name, "Tenant2"); + UNIT_ASSERT_STRINGS_EQUAL(children[3].Name, ".sys"); + } + { + auto result = schemeClient.ListDirectory("/Root/Tenant1").GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto entry = result.GetEntry(); UNIT_ASSERT_VALUES_EQUAL(entry.Name, "Tenant1"); - UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::SubDomain); - - auto children = result.GetChildren(); - UNIT_ASSERT_VALUES_EQUAL(children.size(), 2); - UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, "Table1"); - UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, ".sys"); - } - { - auto result = schemeClient.ListDirectory("/Root/.sys").GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - auto entry = result.GetEntry(); - UNIT_ASSERT_VALUES_EQUAL(entry.Name, ".sys"); - UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory); - - auto children = result.GetChildren(); + UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::SubDomain); + + auto children = result.GetChildren(); + UNIT_ASSERT_VALUES_EQUAL(children.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, "Table1"); + UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, ".sys"); + } + { + auto result = schemeClient.ListDirectory("/Root/.sys").GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto entry = result.GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(entry.Name, ".sys"); + UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory); + + auto children = result.GetChildren(); UNIT_ASSERT_VALUES_EQUAL(children.size(), 16); - - THashSet<TString> names; - for (const auto& child : children) { - names.insert(child.Name); - UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table); - } - UNIT_ASSERT(names.contains("partition_stats")); - } - { - auto result = schemeClient.ListDirectory("/Root/Tenant1/.sys").GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - auto entry = result.GetEntry(); - UNIT_ASSERT_VALUES_EQUAL(entry.Name, ".sys"); - UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory); - - auto children = result.GetChildren(); - UNIT_ASSERT_VALUES_EQUAL(children.size(), 10); - - THashSet<TString> names; - for (const auto& child : children) { - names.insert(child.Name); - UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table); - } - UNIT_ASSERT(names.contains("partition_stats")); - } - { - auto result = schemeClient.ListDirectory("/Root/Tenant1/Table1/.sys").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - result.GetIssues().PrintTo(Cerr); - } - } - - Y_UNIT_TEST(DescribeAccessDenied) { - TTestEnv env; + + THashSet<TString> names; + for (const auto& child : children) { + names.insert(child.Name); + UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table); + } + UNIT_ASSERT(names.contains("partition_stats")); + } + { + auto result = schemeClient.ListDirectory("/Root/Tenant1/.sys").GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto entry = result.GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(entry.Name, ".sys"); + UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory); + + auto children = result.GetChildren(); + UNIT_ASSERT_VALUES_EQUAL(children.size(), 10); + + THashSet<TString> names; + for (const auto& child : children) { + names.insert(child.Name); + UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table); + } + UNIT_ASSERT(names.contains("partition_stats")); + } + { + auto result = schemeClient.ListDirectory("/Root/Tenant1/Table1/.sys").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + result.GetIssues().PrintTo(Cerr); + } + } + + Y_UNIT_TEST(DescribeAccessDenied) { + TTestEnv env; CreateTenantsAndTables(env, false); - - auto driverConfig = TDriverConfig() - .SetEndpoint(env.GetEndpoint()) - .SetAuthToken("user0@builtin"); - auto driver = TDriver(driverConfig); - - TSchemeClient schemeClient(driver); - { - auto result = schemeClient.ListDirectory("/Root").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = schemeClient.ListDirectory("/Root/Tenant1").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = schemeClient.ListDirectory("/Root/.sys").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = schemeClient.ListDirectory("/Root/Tenant1/.sys").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = schemeClient.DescribePath("/Root/.sys/partition_stats").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); - result.GetIssues().PrintTo(Cerr); - } - { - auto result = schemeClient.DescribePath("/Root/Tenant1/.sys/partition_stats").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); - result.GetIssues().PrintTo(Cerr); - } - } - - Y_UNIT_TEST(TabletsFields) { - TTestEnv env(1, 0); - CreateRootTable(env); - - TTableClient client(env.GetDriver()); - auto it = client.StreamExecuteScanQuery(R"( - SELECT - BootState, - CPU, - Generation, - Memory, - Network, - NodeId, + + auto driverConfig = TDriverConfig() + .SetEndpoint(env.GetEndpoint()) + .SetAuthToken("user0@builtin"); + auto driver = TDriver(driverConfig); + + TSchemeClient schemeClient(driver); + { + auto result = schemeClient.ListDirectory("/Root").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = schemeClient.ListDirectory("/Root/Tenant1").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = schemeClient.ListDirectory("/Root/.sys").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = schemeClient.ListDirectory("/Root/Tenant1/.sys").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = schemeClient.DescribePath("/Root/.sys/partition_stats").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); + result.GetIssues().PrintTo(Cerr); + } + { + auto result = schemeClient.DescribePath("/Root/Tenant1/.sys/partition_stats").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); + result.GetIssues().PrintTo(Cerr); + } + } + + Y_UNIT_TEST(TabletsFields) { + TTestEnv env(1, 0); + CreateRootTable(env); + + TTableClient client(env.GetDriver()); + auto it = client.StreamExecuteScanQuery(R"( + SELECT + BootState, + CPU, + Generation, + Memory, + Network, + NodeId, FollowerId, - State, - TabletId, - Type, - VolatileState - FROM `/Root/.sys/hive_tablets`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto ysonString = NKqp::StreamResultToYson(it); - - TYsonFieldChecker check(ysonString, 11); - + State, + TabletId, + Type, + VolatileState + FROM `/Root/.sys/hive_tablets`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto ysonString = NKqp::StreamResultToYson(it); + + TYsonFieldChecker check(ysonString, 11); + check.String("Running"); // BootState - check.DoubleGreaterOrEquals(0.0); // CPU - check.Uint64(1u); // Generation - check.Uint64GreaterOrEquals(0u); // Memory - check.Uint64(0u); // Network - check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId + check.DoubleGreaterOrEquals(0.0); // CPU + check.Uint64(1u); // Generation + check.Uint64GreaterOrEquals(0u); // Memory + check.Uint64(0u); // Network + check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId check.Uint64(0u); // FollowerId - check.String("ReadyToWork"); // State - check.Uint64(72075186224037888ul); // TabletId - check.String("DataShard"); // Type - check.String("Running"); // VolatileState - } - - Y_UNIT_TEST(TabletsShards) { - TTestEnv env(1, 0); - CreateRootTable(env, 3); - - TTableClient client(env.GetDriver()); - auto it = client.StreamExecuteScanQuery(R"( + check.String("ReadyToWork"); // State + check.Uint64(72075186224037888ul); // TabletId + check.String("DataShard"); // Type + check.String("Running"); // VolatileState + } + + Y_UNIT_TEST(TabletsShards) { + TTestEnv env(1, 0); + CreateRootTable(env, 3); + + TTableClient client(env.GetDriver()); + auto it = client.StreamExecuteScanQuery(R"( SELECT FollowerId, TabletId, Type - FROM `/Root/.sys/hive_tablets`; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - NKqp::CompareYson(R"([ - [[0u];[72075186224037888u];["DataShard"]]; - [[0u];[72075186224037889u];["DataShard"]]; - [[0u];[72075186224037890u];["DataShard"]]; - ])", NKqp::StreamResultToYson(it)); - } - + FROM `/Root/.sys/hive_tablets`; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + NKqp::CompareYson(R"([ + [[0u];[72075186224037888u];["DataShard"]]; + [[0u];[72075186224037889u];["DataShard"]]; + [[0u];[72075186224037890u];["DataShard"]]; + ])", NKqp::StreamResultToYson(it)); + } + Y_UNIT_TEST(TabletsFollowers) { - TTestEnv env(1, 0); - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - - auto desc = TTableBuilder() - .AddNullableColumn("Column1", EPrimitiveType::Uint64) - .SetPrimaryKeyColumn("Column1") - .Build(); - - auto settings = TCreateTableSettings() - .ReplicationPolicy(TReplicationPolicy().ReplicasCount(3)); - - auto result = session.CreateTable("/Root/Table0", - std::move(desc), settings).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - auto it = client.StreamExecuteScanQuery(R"( + TTestEnv env(1, 0); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + auto desc = TTableBuilder() + .AddNullableColumn("Column1", EPrimitiveType::Uint64) + .SetPrimaryKeyColumn("Column1") + .Build(); + + auto settings = TCreateTableSettings() + .ReplicationPolicy(TReplicationPolicy().ReplicasCount(3)); + + auto result = session.CreateTable("/Root/Table0", + std::move(desc), settings).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto it = client.StreamExecuteScanQuery(R"( SELECT FollowerId, TabletId, Type - FROM `/Root/.sys/hive_tablets`; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - NKqp::CompareYson(R"([ - [[0u];[72075186224037888u];["DataShard"]]; - [[1u];[72075186224037888u];["DataShard"]]; - [[2u];[72075186224037888u];["DataShard"]]; - [[3u];[72075186224037888u];["DataShard"]]; - ])", NKqp::StreamResultToYson(it)); - } - - Y_UNIT_TEST(TabletsRanges) { - TTestEnv env(1, 0); - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - - auto desc = TTableBuilder() - .AddNullableColumn("Column1", EPrimitiveType::Uint64) - .SetPrimaryKeyColumn("Column1") - .Build(); - - auto settings = TCreateTableSettings() - .ReplicationPolicy(TReplicationPolicy().ReplicasCount(3)) - .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(3)); - - auto result = session.CreateTable("/Root/Table0", - std::move(desc), settings).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - + FROM `/Root/.sys/hive_tablets`; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + NKqp::CompareYson(R"([ + [[0u];[72075186224037888u];["DataShard"]]; + [[1u];[72075186224037888u];["DataShard"]]; + [[2u];[72075186224037888u];["DataShard"]]; + [[3u];[72075186224037888u];["DataShard"]]; + ])", NKqp::StreamResultToYson(it)); + } + + Y_UNIT_TEST(TabletsRanges) { + TTestEnv env(1, 0); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + auto desc = TTableBuilder() + .AddNullableColumn("Column1", EPrimitiveType::Uint64) + .SetPrimaryKeyColumn("Column1") + .Build(); + + auto settings = TCreateTableSettings() + .ReplicationPolicy(TReplicationPolicy().ReplicasCount(3)) + .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(3)); + + auto result = session.CreateTable("/Root/Table0", + std::move(desc), settings).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + std::vector<std::pair<TString, TString>> testData = { { "TabletId = 72075186224037888ul AND FollowerId > 1u", @@ -1574,27 +1574,27 @@ Y_UNIT_TEST_SUITE(SystemView) { for (auto& data: testData) { TString query = R"( SELECT FollowerId, TabletId - FROM `/Root/.sys/hive_tablets` + FROM `/Root/.sys/hive_tablets` WHERE <PREDICATE>; )"; SubstGlobal(query, "<PREDICATE>", data.first); auto it = client.StreamExecuteScanQuery(enablePredicateExtractor + query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); auto streamed = NKqp::StreamResultToYson(it); it = client.StreamExecuteScanQuery(query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); auto expected = NKqp::StreamResultToYson(it); // Compare two ways of execution NKqp::CompareYson(expected, streamed); // And check with expected result from test description NKqp::CompareYson(data.second, streamed); - } - } - + } + } + Y_UNIT_TEST(TabletsRangesPredicateExtractDisabled) { TTestEnv env(1, 0); @@ -1638,110 +1638,110 @@ Y_UNIT_TEST_SUITE(SystemView) { NKqp::CompareYson(expected, NKqp::StreamResultToYson(it)); } - void TestQueryType( - std::function<void(const TTestEnv&, const TString&)> execQuery, - const TString& type) - { - TTestEnv env(1, 0); - CreateRootTable(env); - - TString query("SELECT * FROM `Root/Table0`"); - execQuery(env, query); - - TTableClient client(env.GetDriver()); - auto it = client.StreamExecuteScanQuery(R"( - SELECT QueryText, Type - FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - NKqp::CompareYson( - Sprintf("[[[\"%s\"];[\"%s\"]]]", query.c_str(), type.c_str()), - NKqp::StreamResultToYson(it)); - } - - Y_UNIT_TEST(CollectPreparedQueries) { - TestQueryType([](const TTestEnv& env, const TString& query) { - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - auto prepareResult = session.PrepareDataQuery(query).GetValueSync(); - UNIT_ASSERT_C(prepareResult.IsSuccess(), prepareResult.GetIssues().ToString()); - auto prepared = prepareResult.GetQuery(); - auto result = prepared.Execute(TTxControl::BeginTx().CommitTx()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - }, "data"); - } - - Y_UNIT_TEST(CollectScanQueries) { - TestQueryType([](const TTestEnv& env, const TString& query) { - TTableClient client(env.GetDriver()); - auto it = client.StreamExecuteScanQuery(query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - NKqp::StreamResultToYson(it); - }, "scan"); - } - - Y_UNIT_TEST(CollectScriptingQueries) { - TestQueryType([](const TTestEnv& env, const TString& query) { - auto scriptingClient = NYdb::NScripting::TScriptingClient(env.GetDriver()); - auto result = scriptingClient.ExecuteYqlScript(query).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - }, "script"); - } - - // TODO: make a test when tenant support is provided - void QueryMetricsSimple() { - TTestEnv env(1, 2); - CreateTenant(env, "Tenant1", true); - { - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - - NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( - CREATE TABLE `Root/Tenant1/Table1` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )").GetValueSync()); - } - - auto driverConfig = TDriverConfig() - .SetEndpoint(env.GetEndpoint()) - .SetDatabase("/Root/Tenant1"); - auto driver = TDriver(driverConfig); - - TTableClient client(driver); - auto session = client.CreateSession().GetValueSync().GetSession(); - NKqp::AssertSuccessResult(session.ExecuteDataQuery( - "SELECT * FROM `Root/Tenant1/Table1`", TTxControl::BeginTx().CommitTx() - ).GetValueSync()); - - size_t rowCount = 0; - TString ysonString; - - for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { - auto it = client.StreamExecuteScanQuery(R"( - SELECT SumReadBytes FROM `Root/Tenant1/.sys/query_metrics`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - ysonString = NKqp::StreamResultToYson(it); - + void TestQueryType( + std::function<void(const TTestEnv&, const TString&)> execQuery, + const TString& type) + { + TTestEnv env(1, 0); + CreateRootTable(env); + + TString query("SELECT * FROM `Root/Table0`"); + execQuery(env, query); + + TTableClient client(env.GetDriver()); + auto it = client.StreamExecuteScanQuery(R"( + SELECT QueryText, Type + FROM `Root/.sys/top_queries_by_read_bytes_one_minute`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NKqp::CompareYson( + Sprintf("[[[\"%s\"];[\"%s\"]]]", query.c_str(), type.c_str()), + NKqp::StreamResultToYson(it)); + } + + Y_UNIT_TEST(CollectPreparedQueries) { + TestQueryType([](const TTestEnv& env, const TString& query) { + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + auto prepareResult = session.PrepareDataQuery(query).GetValueSync(); + UNIT_ASSERT_C(prepareResult.IsSuccess(), prepareResult.GetIssues().ToString()); + auto prepared = prepareResult.GetQuery(); + auto result = prepared.Execute(TTxControl::BeginTx().CommitTx()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + }, "data"); + } + + Y_UNIT_TEST(CollectScanQueries) { + TestQueryType([](const TTestEnv& env, const TString& query) { + TTableClient client(env.GetDriver()); + auto it = client.StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NKqp::StreamResultToYson(it); + }, "scan"); + } + + Y_UNIT_TEST(CollectScriptingQueries) { + TestQueryType([](const TTestEnv& env, const TString& query) { + auto scriptingClient = NYdb::NScripting::TScriptingClient(env.GetDriver()); + auto result = scriptingClient.ExecuteYqlScript(query).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + }, "script"); + } + + // TODO: make a test when tenant support is provided + void QueryMetricsSimple() { + TTestEnv env(1, 2); + CreateTenant(env, "Tenant1", true); + { + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"( + CREATE TABLE `Root/Tenant1/Table1` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )").GetValueSync()); + } + + auto driverConfig = TDriverConfig() + .SetEndpoint(env.GetEndpoint()) + .SetDatabase("/Root/Tenant1"); + auto driver = TDriver(driverConfig); + + TTableClient client(driver); + auto session = client.CreateSession().GetValueSync().GetSession(); + NKqp::AssertSuccessResult(session.ExecuteDataQuery( + "SELECT * FROM `Root/Tenant1/Table1`", TTxControl::BeginTx().CommitTx() + ).GetValueSync()); + + size_t rowCount = 0; + TString ysonString; + + for (size_t iter = 0; iter < 30 && !rowCount; ++iter) { + auto it = client.StreamExecuteScanQuery(R"( + SELECT SumReadBytes FROM `Root/Tenant1/.sys/query_metrics`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + ysonString = NKqp::StreamResultToYson(it); + auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node); - UNIT_ASSERT(node.IsList()); - rowCount = node.AsList().size(); - - if (!rowCount) { - Sleep(TDuration::Seconds(1)); - } - } - - NKqp::CompareYson(R"([ - [[0u]]; - ])", ysonString); - } -} - -} // NSysView -} // NKikimr + UNIT_ASSERT(node.IsList()); + rowCount = node.AsList().size(); + + if (!rowCount) { + Sleep(TDuration::Seconds(1)); + } + } + + NKqp::CompareYson(R"([ + [[0u]]; + ])", ysonString); + } +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/ut_kqp/ya.make b/ydb/core/sys_view/ut_kqp/ya.make index 9c19748dea..3b75f4e6fe 100644 --- a/ydb/core/sys_view/ut_kqp/ya.make +++ b/ydb/core/sys_view/ut_kqp/ya.make @@ -1,35 +1,35 @@ UNITTEST_FOR(ydb/core/sys_view) - -OWNER( - monster - g:kikimr -) - -FORK_SUBTESTS() - -IF (WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( + +OWNER( + monster + g:kikimr +) + +FORK_SUBTESTS() + +IF (WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( library/cpp/testing/unittest library/cpp/yson/node ydb/core/kqp/ut/common ydb/core/testlib ydb/public/sdk/cpp/client/draft -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - ut_kqp.cpp - ut_common.cpp - ut_counters.cpp -) - -END() +SRCS( + ut_kqp.cpp + ut_common.cpp + ut_counters.cpp +) + +END() diff --git a/ydb/core/sys_view/ya.make b/ydb/core/sys_view/ya.make index 46b32d9335..edaa0470bf 100644 --- a/ydb/core/sys_view/ya.make +++ b/ydb/core/sys_view/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - -OWNER( - monster - g:kikimr -) - -SRCS( - scan.h - scan.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:kikimr +) + +SRCS( + scan.h + scan.cpp +) + +PEERDIR( ydb/core/base ydb/core/kqp/runtime ydb/core/sys_view/common @@ -20,8 +20,8 @@ PEERDIR( ydb/core/sys_view/service ydb/core/sys_view/storage ydb/core/sys_view/tablets -) - +) + YQL_LAST_ABI_VERSION() END() diff --git a/ydb/core/tablet/node_whiteboard.cpp b/ydb/core/tablet/node_whiteboard.cpp index 8b23eb7f71..783758e99d 100644 --- a/ydb/core/tablet/node_whiteboard.cpp +++ b/ydb/core/tablet/node_whiteboard.cpp @@ -798,7 +798,7 @@ protected: if (ProcessStats.CGroupMemLim != 0) { systemStatsUpdate->Record.SetMemoryLimit(ProcessStats.CGroupMemLim); } - systemStatsUpdate->Record.SetMemoryUsedInAlloc(TAllocState::GetAllocatedMemoryEstimate()); + systemStatsUpdate->Record.SetMemoryUsedInAlloc(TAllocState::GetAllocatedMemoryEstimate()); ctx.Send(ctx.SelfID, systemStatsUpdate.Release()); ctx.Schedule(TDuration::Seconds(30), new TEvPrivate::TEvUpdateRuntimeStats()); } diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp index af724e0080..1885b26e09 100644 --- a/ydb/core/tablet/tablet_counters_aggregator.cpp +++ b/ydb/core/tablet/tablet_counters_aggregator.cpp @@ -1,5 +1,5 @@ #include "tablet_counters_aggregator.h" -#include "tablet_counters_app.h" +#include "tablet_counters_app.h" #include <library/cpp/actors/core/log.h> #include <ydb/core/mon/mon.h> #include <library/cpp/actors/core/mon.h> @@ -64,15 +64,15 @@ using TCountersVector = TVector<NMonitoring::TDynamicCounters::TCounterPtr>; struct THistogramCounter { TVector<TTabletPercentileCounter::TRangeDef> Ranges; TVector<NMonitoring::TDynamicCounters::TCounterPtr> Values; - NMonitoring::THistogramPtr Histogram; + NMonitoring::THistogramPtr Histogram; - THistogramCounter( - const TVector<TTabletPercentileCounter::TRangeDef>& ranges, - TVector<NMonitoring::TDynamicCounters::TCounterPtr>&& values, - NMonitoring::THistogramPtr histogram) + THistogramCounter( + const TVector<TTabletPercentileCounter::TRangeDef>& ranges, + TVector<NMonitoring::TDynamicCounters::TCounterPtr>&& values, + NMonitoring::THistogramPtr histogram) : Ranges(ranges) , Values(values) - , Histogram(histogram) + , Histogram(histogram) { Y_VERIFY(!Ranges.empty() && Ranges.size() == Values.size()); } @@ -81,15 +81,15 @@ struct THistogramCounter { for (const NMonitoring::TDynamicCounters::TCounterPtr& cnt : Values) { *cnt = 0; } - - Histogram->Reset(); + + Histogram->Reset(); } void IncrementFor(ui64 value) { const size_t i = Max<ssize_t>(0, std::upper_bound(Ranges.begin(), Ranges.end(), value) - Ranges.begin() - 1); Values[i]->Inc(); - - Histogram->Collect(value); + + Histogram->Collect(value); } }; @@ -119,33 +119,33 @@ public: HistSimpleCounters.emplace_back(std::move(percentileAggregate)); } - ui64 GetSum(ui32 counterIndex) const { - Y_VERIFY(counterIndex < SumSimpleCounters.size(), - "inconsistent sum simple counters, %u >= %lu", counterIndex, SumSimpleCounters.size()); - return *SumSimpleCounters[counterIndex]; - } - - void SetSum(ui32 counterIndex, ui64 value) { - Y_VERIFY(counterIndex < SumSimpleCounters.size(), - "inconsistent sum simple counters, %u >= %lu", counterIndex, SumSimpleCounters.size()); - *SumSimpleCounters[counterIndex] = value; - } - - ui64 GetMax(ui32 counterIndex) const { - Y_VERIFY(counterIndex < MaxSimpleCounters.size(), - "inconsistent max simple counters, %u >= %lu", counterIndex, MaxSimpleCounters.size()); - return *MaxSimpleCounters[counterIndex]; - } - - void SetMax(ui32 counterIndex, ui64 value) { - Y_VERIFY(counterIndex < MaxSimpleCounters.size(), - "inconsistent max simple counters, %u >= %lu", counterIndex, MaxSimpleCounters.size()); - *MaxSimpleCounters[counterIndex] = value; - } - + ui64 GetSum(ui32 counterIndex) const { + Y_VERIFY(counterIndex < SumSimpleCounters.size(), + "inconsistent sum simple counters, %u >= %lu", counterIndex, SumSimpleCounters.size()); + return *SumSimpleCounters[counterIndex]; + } + + void SetSum(ui32 counterIndex, ui64 value) { + Y_VERIFY(counterIndex < SumSimpleCounters.size(), + "inconsistent sum simple counters, %u >= %lu", counterIndex, SumSimpleCounters.size()); + *SumSimpleCounters[counterIndex] = value; + } + + ui64 GetMax(ui32 counterIndex) const { + Y_VERIFY(counterIndex < MaxSimpleCounters.size(), + "inconsistent max simple counters, %u >= %lu", counterIndex, MaxSimpleCounters.size()); + return *MaxSimpleCounters[counterIndex]; + } + + void SetMax(ui32 counterIndex, ui64 value) { + Y_VERIFY(counterIndex < MaxSimpleCounters.size(), + "inconsistent max simple counters, %u >= %lu", counterIndex, MaxSimpleCounters.size()); + *MaxSimpleCounters[counterIndex] = value; + } + void SetValue(ui64 tabletID, ui32 counterIndex, ui64 value, TTabletTypes::EType tabletType) { - Y_VERIFY(counterIndex < CountersByTabletID.size(), - "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType)); + Y_VERIFY(counterIndex < CountersByTabletID.size(), + "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType)); auto it = CountersByTabletID[counterIndex].find(tabletID); if (it != CountersByTabletID[counterIndex].end()) { if (it->second != value) { @@ -234,18 +234,18 @@ public: HistCumulativeCounters.emplace_back(std::move(percentileAggregate)); } - ui64 GetMax(ui32 counterIndex) const { - Y_VERIFY(counterIndex < MaxCumulativeCounters.size(), - "inconsistent max cumulative counters, %u >= %lu", counterIndex, MaxCumulativeCounters.size()); - return *MaxCumulativeCounters[counterIndex]; - } - - void SetMax(ui32 counterIndex, ui64 value) { - Y_VERIFY(counterIndex < MaxCumulativeCounters.size(), - "inconsistent max cumulative counters, %u >= %lu", counterIndex, MaxCumulativeCounters.size()); - *MaxCumulativeCounters[counterIndex] = value; - } - + ui64 GetMax(ui32 counterIndex) const { + Y_VERIFY(counterIndex < MaxCumulativeCounters.size(), + "inconsistent max cumulative counters, %u >= %lu", counterIndex, MaxCumulativeCounters.size()); + return *MaxCumulativeCounters[counterIndex]; + } + + void SetMax(ui32 counterIndex, ui64 value) { + Y_VERIFY(counterIndex < MaxCumulativeCounters.size(), + "inconsistent max cumulative counters, %u >= %lu", counterIndex, MaxCumulativeCounters.size()); + *MaxCumulativeCounters[counterIndex] = value; + } + void SetValue(ui64 tabletID, ui32 counterIndex, ui64 value, TTabletTypes::EType tabletType) { Y_VERIFY(counterIndex < CountersByTabletID.size(), "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType)); auto it = CountersByTabletID[counterIndex].find(tabletID); @@ -474,31 +474,31 @@ public: : Counters(GetServiceCounters(counters, isFollower ? "followers" : "tablets")) , AllTypes(Counters.Get(), "type", "all", true) , IsFollower(isFollower) - , DbWatcherActorId(dbWatcherActorId) + , DbWatcherActorId(dbWatcherActorId) { if (!IsFollower) { - YdbCounters = MakeIntrusive<TYdbTabletCounters>(GetServiceCounters(counters, "ydb")); - } + YdbCounters = MakeIntrusive<TYdbTabletCounters>(GetServiceCounters(counters, "ydb")); + } } - void Apply(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId, - const TTabletCountersBase* executorCounters, const TTabletCountersBase* appCounters, - const TActorContext& ctx) - { - AllTypes.Apply(tabletID, executorCounters, nullptr, tabletType); + void Apply(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId, + const TTabletCountersBase* executorCounters, const TTabletCountersBase* appCounters, + const TActorContext& ctx) + { + AllTypes.Apply(tabletID, executorCounters, nullptr, tabletType); // - auto* typeCounters = GetOrAddCountersByTabletType(tabletType, CountersByTabletType, Counters); - if (typeCounters) { - typeCounters->Apply(tabletID, executorCounters, appCounters, tabletType); + auto* typeCounters = GetOrAddCountersByTabletType(tabletType, CountersByTabletType, Counters); + if (typeCounters) { + typeCounters->Apply(tabletID, executorCounters, appCounters, tabletType); } // if (!IsFollower && AppData(ctx)->FeatureFlags.GetEnableDbCounters() && tenantPathId) { - auto dbCounters = GetDbCounters(tenantPathId, ctx); - if (dbCounters) { - auto* limitedAppCounters = GetOrAddLimitedAppCounters(tabletType); - dbCounters->Apply(tabletID, executorCounters, appCounters, tabletType, limitedAppCounters); - } - } + auto dbCounters = GetDbCounters(tenantPathId, ctx); + if (dbCounters) { + auto* limitedAppCounters = GetOrAddLimitedAppCounters(tabletType); + dbCounters->Apply(tabletID, executorCounters, appCounters, tabletType, limitedAppCounters); + } + } // auto& quietStats = QuietTabletCounters[tabletID]; @@ -553,17 +553,17 @@ public: } } - void ForgetTablet(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId) { + void ForgetTablet(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId) { AllTypes.Forget(tabletID); // and now erase from every other path auto iterTabletType = CountersByTabletType.find(tabletType); if (iterTabletType != CountersByTabletType.end()) { iterTabletType->second->Forget(tabletID); } - // from db counters - if (auto itPath = CountersByPathId.find(tenantPathId); itPath != CountersByPathId.end()) { - itPath->second->Forget(tabletID, tabletType); - } + // from db counters + if (auto itPath = CountersByPathId.find(tenantPathId); itPath != CountersByPathId.end()) { + itPath->second->Forget(tabletID, tabletType); + } //and from all labeledCounters that could have this tablet auto iterTabletTypeAndGroup = LabeledCountersByTabletTypeAndGroup.lower_bound(std::make_pair(tabletType, TString())); for (; iterTabletTypeAndGroup != LabeledCountersByTabletTypeAndGroup.end() && iterTabletTypeAndGroup->first.first == tabletType; ) { @@ -687,17 +687,17 @@ public: for (auto& c : CountersByTabletType) { c.second->RecalcAll(); } - - if (YdbCounters) { - YdbCounters->Initialize(Counters, CountersByTabletType); - YdbCounters->Transform(); - } + + if (YdbCounters) { + YdbCounters->Initialize(Counters, CountersByTabletType); + YdbCounters->Transform(); + } + } + + void RemoveTabletsByPathId(TPathId pathId) { + CountersByPathId.erase(pathId); } - void RemoveTabletsByPathId(TPathId pathId) { - CountersByPathId.erase(pathId); - } - private: // subgroups class TTabletCountersForTabletType { @@ -711,12 +711,12 @@ private: , TabletAppCounters(TabletAppCountersSection, doAggregateSimpleCountrers) {} - void Apply(ui64 tabletID, - const TTabletCountersBase* executorCounters, - const TTabletCountersBase* appCounters, - TTabletTypes::EType tabletType, - const TTabletCountersBase* limitedAppCounters = {}) - { + void Apply(ui64 tabletID, + const TTabletCountersBase* executorCounters, + const TTabletCountersBase* appCounters, + TTabletTypes::EType tabletType, + const TTabletCountersBase* limitedAppCounters = {}) + { Y_VERIFY(executorCounters); if (executorCounters) { @@ -728,7 +728,7 @@ private: if (appCounters) { if (!TabletAppCounters.IsInitialized) { - TabletAppCounters.Initialize(limitedAppCounters ? limitedAppCounters : appCounters); + TabletAppCounters.Initialize(limitedAppCounters ? limitedAppCounters : appCounters); } TabletAppCounters.Apply(tabletID, appCounters, tabletType); } @@ -752,48 +752,48 @@ private: } } - // db counters - - bool IsInitialized() const { - return TabletExecutorCounters.IsInitialized; - } - - void Initialize(const TTabletCountersBase* executorCounters, const TTabletCountersBase* appCounters) { - Y_VERIFY(executorCounters); - - if (!TabletExecutorCounters.IsInitialized) { - TabletExecutorCounters.Initialize(executorCounters); - } - - if (appCounters && !TabletAppCounters.IsInitialized) { - TabletAppCounters.Initialize(appCounters); - } - } - - void ToProto(NKikimrSysView::TDbTabletCounters& tabletCounters) { - if (TabletExecutorCounters.IsInitialized) { - TabletExecutorCounters.RecalcAll(); - TabletExecutorCounters.ToProto(*tabletCounters.MutableExecutorCounters(), - *tabletCounters.MutableMaxExecutorCounters()); - } - if (TabletAppCounters.IsInitialized) { - TabletAppCounters.RecalcAll(); - TabletAppCounters.ToProto(*tabletCounters.MutableAppCounters(), - *tabletCounters.MutableMaxAppCounters()); - } - } - - void FromProto(NKikimrSysView::TDbTabletCounters& tabletCounters) { - if (TabletExecutorCounters.IsInitialized) { - TabletExecutorCounters.FromProto(*tabletCounters.MutableExecutorCounters(), - *tabletCounters.MutableMaxExecutorCounters()); - } - if (TabletAppCounters.IsInitialized) { - TabletAppCounters.FromProto(*tabletCounters.MutableAppCounters(), - *tabletCounters.MutableMaxAppCounters()); - } - } - + // db counters + + bool IsInitialized() const { + return TabletExecutorCounters.IsInitialized; + } + + void Initialize(const TTabletCountersBase* executorCounters, const TTabletCountersBase* appCounters) { + Y_VERIFY(executorCounters); + + if (!TabletExecutorCounters.IsInitialized) { + TabletExecutorCounters.Initialize(executorCounters); + } + + if (appCounters && !TabletAppCounters.IsInitialized) { + TabletAppCounters.Initialize(appCounters); + } + } + + void ToProto(NKikimrSysView::TDbTabletCounters& tabletCounters) { + if (TabletExecutorCounters.IsInitialized) { + TabletExecutorCounters.RecalcAll(); + TabletExecutorCounters.ToProto(*tabletCounters.MutableExecutorCounters(), + *tabletCounters.MutableMaxExecutorCounters()); + } + if (TabletAppCounters.IsInitialized) { + TabletAppCounters.RecalcAll(); + TabletAppCounters.ToProto(*tabletCounters.MutableAppCounters(), + *tabletCounters.MutableMaxAppCounters()); + } + } + + void FromProto(NKikimrSysView::TDbTabletCounters& tabletCounters) { + if (TabletExecutorCounters.IsInitialized) { + TabletExecutorCounters.FromProto(*tabletCounters.MutableExecutorCounters(), + *tabletCounters.MutableMaxExecutorCounters()); + } + if (TabletAppCounters.IsInitialized) { + TabletAppCounters.FromProto(*tabletCounters.MutableAppCounters(), + *tabletCounters.MutableMaxAppCounters()); + } + } + private: // class TSolomonCounters { @@ -817,14 +817,14 @@ private: THashMap<TString, THolder<THistogramCounter>> histogramAggregates; // percentile counters - FullSizePercentile = counters->Percentile().Size(); - for (ui32 i = 0; i < FullSizePercentile; ++i) { + FullSizePercentile = counters->Percentile().Size(); + for (ui32 i = 0; i < FullSizePercentile; ++i) { if (!counters->PercentileCounterName(i)) { - DeprecatedPercentile.insert(i); + DeprecatedPercentile.insert(i); continue; } - // old style + // old style PercentileCounters.push_back(TVector<NMonitoring::TDynamicCounters::TCounterPtr>()); auto counterRBeginIter = PercentileCounters.rbegin(); @@ -833,44 +833,44 @@ private: TStringBuf counterName(percentileCounterName); TStringBuf simpleCounterName = GetHistogramAggregateSimpleName(counterName); bool histogramAggregate = !simpleCounterName.empty(); - - bool isDerivative = !histogramAggregate && !percentileCounter.GetIntegral(); - - auto rangeCount = percentileCounter.GetRangeCount(); - for (ui32 r = 0; r < rangeCount; ++r) { + + bool isDerivative = !histogramAggregate && !percentileCounter.GetIntegral(); + + auto rangeCount = percentileCounter.GetRangeCount(); + for (ui32 r = 0; r < rangeCount; ++r) { const char* rangeName = percentileCounter.GetRangeName(r); auto subgroup = CounterGroup->GetSubgroup("range", rangeName); - auto counter = subgroup->GetCounter(percentileCounterName, isDerivative); + auto counter = subgroup->GetCounter(percentileCounterName, isDerivative); counterRBeginIter->push_back(counter); } - - // new style - NMonitoring::TBucketBounds bucketBounds; - for (ui32 r = 1; r < rangeCount; ++r) { // values in proto are lower bounds, thus shift - bucketBounds.push_back(percentileCounter.GetRangeBound(r)); - } - auto histogram = CounterGroup->GetHistogram( - percentileCounterName, NMonitoring::ExplicitHistogram(bucketBounds), isDerivative); - Histograms.push_back(histogram); - + + // new style + NMonitoring::TBucketBounds bucketBounds; + for (ui32 r = 1; r < rangeCount; ++r) { // values in proto are lower bounds, thus shift + bucketBounds.push_back(percentileCounter.GetRangeBound(r)); + } + auto histogram = CounterGroup->GetHistogram( + percentileCounterName, NMonitoring::ExplicitHistogram(bucketBounds), isDerivative); + Histograms.push_back(histogram); + if (histogramAggregate) { - histogramAggregates.emplace(simpleCounterName, new THistogramCounter( - percentileCounter.GetRanges(), std::move(*counterRBeginIter), histogram)); + histogramAggregates.emplace(simpleCounterName, new THistogramCounter( + percentileCounter.GetRanges(), std::move(*counterRBeginIter), histogram)); } } // simple counters - FullSizeSimple = counters->Simple().Size(); - for (ui32 i = 0; i < FullSizeSimple; ++i) { + FullSizeSimple = counters->Simple().Size(); + for (ui32 i = 0; i < FullSizeSimple; ++i) { const char* name = counters->SimpleCounterName(i); if (!name) { - DeprecatedSimple.insert(i); + DeprecatedSimple.insert(i); continue; } if (DoAggregateSimpleCounters) { - auto itHistogramAggregate = histogramAggregates.find(name); - if (itHistogramAggregate != histogramAggregates.end()) { - AggregatedSimpleCounters.AddSimpleCounter(name, std::move(itHistogramAggregate->second)); + auto itHistogramAggregate = histogramAggregates.find(name); + if (itHistogramAggregate != histogramAggregates.end()) { + AggregatedSimpleCounters.AddSimpleCounter(name, std::move(itHistogramAggregate->second)); } else { AggregatedSimpleCounters.AddSimpleCounter(name); } @@ -881,17 +881,17 @@ private: } // cumulative counters - FullSizeCumulative = counters->Cumulative().Size(); - for (ui32 i = 0; i < FullSizeCumulative; ++i) { + FullSizeCumulative = counters->Cumulative().Size(); + for (ui32 i = 0; i < FullSizeCumulative; ++i) { const char* name = counters->CumulativeCounterName(i); if (!name) { - DeprecatedCumulative.insert(i); + DeprecatedCumulative.insert(i); continue; } if (DoAggregateCumulativeCounters) { - auto itHistogramAggregate = histogramAggregates.find(name); - if (itHistogramAggregate != histogramAggregates.end()) { - AggregatedCumulativeCounters.AddCumulativeCounter(name, std::move(itHistogramAggregate->second)); + auto itHistogramAggregate = histogramAggregates.find(name); + if (itHistogramAggregate != histogramAggregates.end()) { + AggregatedCumulativeCounters.AddCumulativeCounter(name, std::move(itHistogramAggregate->second)); } else { AggregatedCumulativeCounters.AddCumulativeCounter(name); } @@ -920,7 +920,7 @@ private: // simple counters ui32 nextSimpleOffset = 0; - for (ui32 i = 0; i < FullSizeSimple; ++i) { + for (ui32 i = 0; i < FullSizeSimple; ++i) { if (!counters->SimpleCounterName(i)) { continue; } @@ -936,7 +936,7 @@ private: // cumulative counters ui32 nextCumulativeOffset = 0; - for (ui32 i = 0; i < FullSizeCumulative; ++i) { + for (ui32 i = 0; i < FullSizeCumulative; ++i) { if (!counters->CumulativeCounterName(i)) { continue; } @@ -954,7 +954,7 @@ private: // percentile counters ui32 nextPercentileOffset = 0; - for (ui32 i = 0; i < FullSizePercentile; ++i) { + for (ui32 i = 0; i < FullSizePercentile; ++i) { if (!counters->PercentileCounterName(i)) { continue; } @@ -968,34 +968,34 @@ private: } auto&& percentileCounter = counters->Percentile()[i]; - auto rangeCount = percentileCounter.GetRangeCount(); - Y_VERIFY(rangeCount <= pcx.size(), - "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType)); - - for (ui32 r = 0; r < rangeCount; ++r) { + auto rangeCount = percentileCounter.GetRangeCount(); + Y_VERIFY(rangeCount <= pcx.size(), + "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType)); + + for (ui32 r = 0; r < rangeCount; ++r) { if (percentileCounter.GetIntegral()) { *pcx[r] = percentileCounter.GetRangeValue(r); } else { *pcx[r] += percentileCounter.GetRangeValue(r); } } - - if (rangeCount < 2) { - continue; - } - - auto& histogram = Histograms[offset]; - if (percentileCounter.GetIntegral()) { - histogram->Reset(); - } - for (ui32 r = 0; r < rangeCount - 1; ++r) { - histogram->Collect( - (NMonitoring::TBucketBound)percentileCounter.GetRangeBound(r + 1), - percentileCounter.GetRangeValue(r)); - } - histogram->Collect( - Max<NMonitoring::TBucketBound>(), - percentileCounter.GetRangeValue(rangeCount - 1)); + + if (rangeCount < 2) { + continue; + } + + auto& histogram = Histograms[offset]; + if (percentileCounter.GetIntegral()) { + histogram->Reset(); + } + for (ui32 r = 0; r < rangeCount - 1; ++r) { + histogram->Collect( + (NMonitoring::TBucketBound)percentileCounter.GetRangeBound(r + 1), + percentileCounter.GetRangeValue(r)); + } + histogram->Collect( + Max<NMonitoring::TBucketBound>(), + percentileCounter.GetRangeValue(rangeCount - 1)); } } @@ -1017,120 +1017,120 @@ private: } void RecalcAll() { - if (DoAggregateSimpleCounters) { + if (DoAggregateSimpleCounters) { AggregatedSimpleCounters.RecalcAll(); } - if (DoAggregateCumulativeCounters) { + if (DoAggregateCumulativeCounters) { AggregatedCumulativeCounters.RecalcAll(); } } - template <bool IsSaving> - void Convert(NKikimrSysView::TDbCounters& sumCounters, - NKikimrSysView::TDbCounters& maxCounters) - { - if (!DoAggregateSimpleCounters || !DoAggregateCumulativeCounters) { - return; - } - // simple counters - auto* simpleSum = sumCounters.MutableSimple(); - auto* simpleMax = maxCounters.MutableSimple(); - simpleSum->Resize(FullSizeSimple, 0); - simpleMax->Resize(FullSizeSimple, 0); - ui32 nextSimpleOffset = 0; - for (ui32 i = 0; i < FullSizeSimple; ++i) { - if (DeprecatedSimple.find(i) != DeprecatedSimple.end()) { - if constexpr (IsSaving) { - (*simpleSum)[i] = 0; - (*simpleMax)[i] = 0; - } - continue; - } - const ui32 offset = nextSimpleOffset++; - if constexpr (IsSaving) { - (*simpleSum)[i] = AggregatedSimpleCounters.GetSum(offset); - (*simpleMax)[i] = AggregatedSimpleCounters.GetMax(offset); - } else { - AggregatedSimpleCounters.SetSum(offset, (*simpleSum)[i]); - AggregatedSimpleCounters.SetMax(offset, (*simpleMax)[i]); - } - } - // cumulative counters - auto* cumulativeSum = sumCounters.MutableCumulative(); - auto* cumulativeMax = maxCounters.MutableCumulative(); - cumulativeSum->Resize(FullSizeCumulative, 0); - cumulativeMax->Resize(FullSizeCumulative, 0); - ui32 nextCumulativeOffset = 0; - for (ui32 i = 0; i < FullSizeCumulative; ++i) { - if (DeprecatedCumulative.find(i) != DeprecatedCumulative.end()) { - if constexpr (IsSaving) { - (*cumulativeSum)[i] = 0; - (*cumulativeMax)[i] = 0; - } - continue; - } - const ui32 offset = nextCumulativeOffset++; - Y_VERIFY(offset < CumulativeCounters.size(), - "inconsistent cumulative counters %u >= %lu", offset, CumulativeCounters.size()); - if constexpr (IsSaving) { - (*cumulativeSum)[i] = *CumulativeCounters[offset]; - (*cumulativeMax)[i] = AggregatedCumulativeCounters.GetMax(offset); - } else { - *CumulativeCounters[offset] = (*cumulativeSum)[i]; - AggregatedCumulativeCounters.SetMax(offset, (*cumulativeMax)[i]); - } - } - // percentile counters - auto* histogramSum = sumCounters.MutableHistogram(); - if (sumCounters.HistogramSize() < FullSizePercentile) { - auto missing = FullSizePercentile - sumCounters.HistogramSize(); - for (; missing > 0; --missing) { - sumCounters.AddHistogram(); - } - } - ui32 nextPercentileOffset = 0; - for (ui32 i = 0; i < FullSizePercentile; ++i) { - if (DeprecatedPercentile.find(i) != DeprecatedPercentile.end()) { - continue; - } - auto* buckets = (*histogramSum)[i].MutableBuckets(); - const ui32 offset = nextPercentileOffset++; - auto& histogram = Histograms[offset]; - auto snapshot = histogram->Snapshot(); - auto count = snapshot->Count(); - buckets->Resize(count, 0); - if constexpr (!IsSaving) { - histogram->Reset(); - } - for (ui32 r = 0; r < count; ++r) { - if constexpr (IsSaving) { - (*buckets)[r] = snapshot->Value(r); - } else { - histogram->Collect(snapshot->UpperBound(r), (*buckets)[r]); - } - } - } - } - - void ToProto(NKikimrSysView::TDbCounters& sumCounters, - NKikimrSysView::TDbCounters& maxCounters) - { - Convert<true>(sumCounters, maxCounters); - } - - void FromProto(NKikimrSysView::TDbCounters& sumCounters, - NKikimrSysView::TDbCounters& maxCounters) - { - Convert<false>(sumCounters, maxCounters); - } - + template <bool IsSaving> + void Convert(NKikimrSysView::TDbCounters& sumCounters, + NKikimrSysView::TDbCounters& maxCounters) + { + if (!DoAggregateSimpleCounters || !DoAggregateCumulativeCounters) { + return; + } + // simple counters + auto* simpleSum = sumCounters.MutableSimple(); + auto* simpleMax = maxCounters.MutableSimple(); + simpleSum->Resize(FullSizeSimple, 0); + simpleMax->Resize(FullSizeSimple, 0); + ui32 nextSimpleOffset = 0; + for (ui32 i = 0; i < FullSizeSimple; ++i) { + if (DeprecatedSimple.find(i) != DeprecatedSimple.end()) { + if constexpr (IsSaving) { + (*simpleSum)[i] = 0; + (*simpleMax)[i] = 0; + } + continue; + } + const ui32 offset = nextSimpleOffset++; + if constexpr (IsSaving) { + (*simpleSum)[i] = AggregatedSimpleCounters.GetSum(offset); + (*simpleMax)[i] = AggregatedSimpleCounters.GetMax(offset); + } else { + AggregatedSimpleCounters.SetSum(offset, (*simpleSum)[i]); + AggregatedSimpleCounters.SetMax(offset, (*simpleMax)[i]); + } + } + // cumulative counters + auto* cumulativeSum = sumCounters.MutableCumulative(); + auto* cumulativeMax = maxCounters.MutableCumulative(); + cumulativeSum->Resize(FullSizeCumulative, 0); + cumulativeMax->Resize(FullSizeCumulative, 0); + ui32 nextCumulativeOffset = 0; + for (ui32 i = 0; i < FullSizeCumulative; ++i) { + if (DeprecatedCumulative.find(i) != DeprecatedCumulative.end()) { + if constexpr (IsSaving) { + (*cumulativeSum)[i] = 0; + (*cumulativeMax)[i] = 0; + } + continue; + } + const ui32 offset = nextCumulativeOffset++; + Y_VERIFY(offset < CumulativeCounters.size(), + "inconsistent cumulative counters %u >= %lu", offset, CumulativeCounters.size()); + if constexpr (IsSaving) { + (*cumulativeSum)[i] = *CumulativeCounters[offset]; + (*cumulativeMax)[i] = AggregatedCumulativeCounters.GetMax(offset); + } else { + *CumulativeCounters[offset] = (*cumulativeSum)[i]; + AggregatedCumulativeCounters.SetMax(offset, (*cumulativeMax)[i]); + } + } + // percentile counters + auto* histogramSum = sumCounters.MutableHistogram(); + if (sumCounters.HistogramSize() < FullSizePercentile) { + auto missing = FullSizePercentile - sumCounters.HistogramSize(); + for (; missing > 0; --missing) { + sumCounters.AddHistogram(); + } + } + ui32 nextPercentileOffset = 0; + for (ui32 i = 0; i < FullSizePercentile; ++i) { + if (DeprecatedPercentile.find(i) != DeprecatedPercentile.end()) { + continue; + } + auto* buckets = (*histogramSum)[i].MutableBuckets(); + const ui32 offset = nextPercentileOffset++; + auto& histogram = Histograms[offset]; + auto snapshot = histogram->Snapshot(); + auto count = snapshot->Count(); + buckets->Resize(count, 0); + if constexpr (!IsSaving) { + histogram->Reset(); + } + for (ui32 r = 0; r < count; ++r) { + if constexpr (IsSaving) { + (*buckets)[r] = snapshot->Value(r); + } else { + histogram->Collect(snapshot->UpperBound(r), (*buckets)[r]); + } + } + } + } + + void ToProto(NKikimrSysView::TDbCounters& sumCounters, + NKikimrSysView::TDbCounters& maxCounters) + { + Convert<true>(sumCounters, maxCounters); + } + + void FromProto(NKikimrSysView::TDbCounters& sumCounters, + NKikimrSysView::TDbCounters& maxCounters) + { + Convert<false>(sumCounters, maxCounters); + } + private: - ui32 FullSizeSimple = 0; - THashSet<ui32> DeprecatedSimple; - ui32 FullSizeCumulative = 0; - THashSet<ui32> DeprecatedCumulative; - ui32 FullSizePercentile = 0; - THashSet<ui32> DeprecatedPercentile; + ui32 FullSizeSimple = 0; + THashSet<ui32> DeprecatedSimple; + ui32 FullSizeCumulative = 0; + THashSet<ui32> DeprecatedCumulative; + ui32 FullSizePercentile = 0; + THashSet<ui32> DeprecatedPercentile; // bool DoAggregateSimpleCounters; TCountersVector SimpleCounters; @@ -1141,8 +1141,8 @@ private: TAggregatedCumulativeCounters AggregatedCumulativeCounters; THashMap<ui64, TInstant> LastAggregateUpdateTime; - TVector<TCountersVector> PercentileCounters; // old style - TVector<NMonitoring::THistogramPtr> Histograms; // new style + TVector<TCountersVector> PercentileCounters; // old style + TVector<NMonitoring::THistogramPtr> Histograms; // new style NMonitoring::TDynamicCounterPtr CounterGroup; }; @@ -1157,59 +1157,59 @@ private: TSolomonCounters TabletAppCounters; }; - typedef TMap<TTabletTypes::EType, TAutoPtr<TTabletCountersForTabletType> > TCountersByTabletType; - - static TTabletCountersForTabletType* FindCountersByTabletType( - TTabletTypes::EType tabletType, - TCountersByTabletType& countersByTabletType) - { - auto iterTabletType = countersByTabletType.find(tabletType); - if (iterTabletType != countersByTabletType.end()) { - return iterTabletType->second.Get(); - } - return {}; - } - - static TTabletCountersForTabletType* GetOrAddCountersByTabletType( - TTabletTypes::EType tabletType, - TCountersByTabletType& countersByTabletType, - NMonitoring::TDynamicCounterPtr counters) - { - auto* typeCounters = FindCountersByTabletType(tabletType, countersByTabletType); - if (!typeCounters) { - TString tabletTypeStr = TTabletTypes::TypeToStr(tabletType); - typeCounters = new TTabletCountersForTabletType( - counters.Get(), "type", tabletTypeStr.data(), true); - countersByTabletType.emplace(tabletType, typeCounters); - } - return typeCounters; - } - - const TTabletCountersBase* GetOrAddLimitedAppCounters(TTabletTypes::EType tabletType) { - if (auto it = LimitedAppCounters.find(tabletType); it != LimitedAppCounters.end()) { - return it->second.Get(); - } - auto appCounters = CreateAppCountersByTabletType(tabletType); - return LimitedAppCounters.emplace(tabletType, std::move(appCounters)).first->second.Get(); - } - - class TYdbTabletCounters : public TThrRefBase { - using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; - using THistogramPtr = NMonitoring::THistogramPtr; - - private: - TCounterPtr WriteRowCount; - TCounterPtr WriteBytes; - TCounterPtr ReadRowCount; - TCounterPtr ReadBytes; - TCounterPtr EraseRowCount; - TCounterPtr EraseBytes; - TCounterPtr BulkUpsertRowCount; - TCounterPtr BulkUpsertBytes; - TCounterPtr ScanRowCount; - TCounterPtr ScanBytes; - TCounterPtr DatashardRowCount; - TCounterPtr DatashardSizeBytes; + typedef TMap<TTabletTypes::EType, TAutoPtr<TTabletCountersForTabletType> > TCountersByTabletType; + + static TTabletCountersForTabletType* FindCountersByTabletType( + TTabletTypes::EType tabletType, + TCountersByTabletType& countersByTabletType) + { + auto iterTabletType = countersByTabletType.find(tabletType); + if (iterTabletType != countersByTabletType.end()) { + return iterTabletType->second.Get(); + } + return {}; + } + + static TTabletCountersForTabletType* GetOrAddCountersByTabletType( + TTabletTypes::EType tabletType, + TCountersByTabletType& countersByTabletType, + NMonitoring::TDynamicCounterPtr counters) + { + auto* typeCounters = FindCountersByTabletType(tabletType, countersByTabletType); + if (!typeCounters) { + TString tabletTypeStr = TTabletTypes::TypeToStr(tabletType); + typeCounters = new TTabletCountersForTabletType( + counters.Get(), "type", tabletTypeStr.data(), true); + countersByTabletType.emplace(tabletType, typeCounters); + } + return typeCounters; + } + + const TTabletCountersBase* GetOrAddLimitedAppCounters(TTabletTypes::EType tabletType) { + if (auto it = LimitedAppCounters.find(tabletType); it != LimitedAppCounters.end()) { + return it->second.Get(); + } + auto appCounters = CreateAppCountersByTabletType(tabletType); + return LimitedAppCounters.emplace(tabletType, std::move(appCounters)).first->second.Get(); + } + + class TYdbTabletCounters : public TThrRefBase { + using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; + using THistogramPtr = NMonitoring::THistogramPtr; + + private: + TCounterPtr WriteRowCount; + TCounterPtr WriteBytes; + TCounterPtr ReadRowCount; + TCounterPtr ReadBytes; + TCounterPtr EraseRowCount; + TCounterPtr EraseBytes; + TCounterPtr BulkUpsertRowCount; + TCounterPtr BulkUpsertBytes; + TCounterPtr ScanRowCount; + TCounterPtr ScanBytes; + TCounterPtr DatashardRowCount; + TCounterPtr DatashardSizeBytes; TCounterPtr ResourcesStorageUsedBytes; TCounterPtr ResourcesStorageLimitBytes; TCounterPtr ResourcesStreamUsedShards; @@ -1219,24 +1219,24 @@ private: TCounterPtr ResourcesStreamReservedStorage; TCounterPtr ResourcesStreamReservedStorageLimit; - THistogramPtr ShardCpuUtilization; - - TCounterPtr RowUpdates; - TCounterPtr RowUpdateBytes; - TCounterPtr RowReads; - TCounterPtr RangeReadRows; - TCounterPtr RowReadBytes; - TCounterPtr RangeReadBytes; - TCounterPtr RowErases; - TCounterPtr RowEraseBytes; - TCounterPtr UploadRows; - TCounterPtr UploadRowsBytes; - TCounterPtr ScannedRows; - TCounterPtr ScannedBytes; - TCounterPtr DbUniqueRowsTotal; - TCounterPtr DbUniqueDataBytes; - THistogramPtr ConsumedCpuHistogram; - + THistogramPtr ShardCpuUtilization; + + TCounterPtr RowUpdates; + TCounterPtr RowUpdateBytes; + TCounterPtr RowReads; + TCounterPtr RangeReadRows; + TCounterPtr RowReadBytes; + TCounterPtr RangeReadBytes; + TCounterPtr RowErases; + TCounterPtr RowEraseBytes; + TCounterPtr UploadRows; + TCounterPtr UploadRowsBytes; + TCounterPtr ScannedRows; + TCounterPtr ScannedBytes; + TCounterPtr DbUniqueRowsTotal; + TCounterPtr DbUniqueDataBytes; + THistogramPtr ConsumedCpuHistogram; + TCounterPtr DiskSpaceTablesTotalBytes; TCounterPtr DiskSpaceSoftQuotaBytes; @@ -1247,39 +1247,39 @@ private: TCounterPtr StreamReservedStorageLimit; - public: - explicit TYdbTabletCounters(const NMonitoring::TDynamicCounterPtr& ydbGroup) { - WriteRowCount = ydbGroup->GetNamedCounter("name", - "table.datashard.write.rows", true); - WriteBytes = ydbGroup->GetNamedCounter("name", - "table.datashard.write.bytes", true); - ReadRowCount = ydbGroup->GetNamedCounter("name", - "table.datashard.read.rows", true); - ReadBytes = ydbGroup->GetNamedCounter("name", - "table.datashard.read.bytes", true); - EraseRowCount = ydbGroup->GetNamedCounter("name", - "table.datashard.erase.rows", true); - EraseBytes = ydbGroup->GetNamedCounter("name", - "table.datashard.erase.bytes", true); - BulkUpsertRowCount = ydbGroup->GetNamedCounter("name", - "table.datashard.bulk_upsert.rows", true); - BulkUpsertBytes = ydbGroup->GetNamedCounter("name", - "table.datashard.bulk_upsert.bytes", true); - ScanRowCount = ydbGroup->GetNamedCounter("name", - "table.datashard.scan.rows", true); - ScanBytes = ydbGroup->GetNamedCounter("name", - "table.datashard.scan.bytes", true); - - DatashardRowCount = ydbGroup->GetNamedCounter("name", - "table.datashard.row_count", false); - DatashardSizeBytes = ydbGroup->GetNamedCounter("name", - "table.datashard.size_bytes", false); + public: + explicit TYdbTabletCounters(const NMonitoring::TDynamicCounterPtr& ydbGroup) { + WriteRowCount = ydbGroup->GetNamedCounter("name", + "table.datashard.write.rows", true); + WriteBytes = ydbGroup->GetNamedCounter("name", + "table.datashard.write.bytes", true); + ReadRowCount = ydbGroup->GetNamedCounter("name", + "table.datashard.read.rows", true); + ReadBytes = ydbGroup->GetNamedCounter("name", + "table.datashard.read.bytes", true); + EraseRowCount = ydbGroup->GetNamedCounter("name", + "table.datashard.erase.rows", true); + EraseBytes = ydbGroup->GetNamedCounter("name", + "table.datashard.erase.bytes", true); + BulkUpsertRowCount = ydbGroup->GetNamedCounter("name", + "table.datashard.bulk_upsert.rows", true); + BulkUpsertBytes = ydbGroup->GetNamedCounter("name", + "table.datashard.bulk_upsert.bytes", true); + ScanRowCount = ydbGroup->GetNamedCounter("name", + "table.datashard.scan.rows", true); + ScanBytes = ydbGroup->GetNamedCounter("name", + "table.datashard.scan.bytes", true); + + DatashardRowCount = ydbGroup->GetNamedCounter("name", + "table.datashard.row_count", false); + DatashardSizeBytes = ydbGroup->GetNamedCounter("name", + "table.datashard.size_bytes", false); ResourcesStorageUsedBytes = ydbGroup->GetNamedCounter("name", "resources.storage.used_bytes", false); ResourcesStorageLimitBytes = ydbGroup->GetNamedCounter("name", "resources.storage.limit_bytes", false); - + ResourcesStreamUsedShards = ydbGroup->GetNamedCounter("name", "resources.stream.used_shards", false); ResourcesStreamLimitShards = ydbGroup->GetNamedCounter("name", @@ -1297,40 +1297,40 @@ private: ResourcesStreamReservedStorageLimit = ydbGroup->GetNamedCounter("name", "resources.stream.storage.limit_bytes", false); - ShardCpuUtilization = ydbGroup->GetNamedHistogram("name", - "table.datashard.used_core_percents", NMonitoring::LinearHistogram(12, 0, 10), false); - }; - - void Initialize( - NMonitoring::TDynamicCounterPtr counters, - TCountersByTabletType& countersByTabletType) - { - auto datashard = FindCountersByTabletType( - TTabletTypes::FLAT_DATASHARD, countersByTabletType); - - if (datashard && !RowUpdates) { - auto datashardGroup = counters->GetSubgroup("type", "DataShard"); - auto appGroup = datashardGroup->GetSubgroup("category", "app"); - - RowUpdates = appGroup->GetCounter("DataShard/EngineHostRowUpdates"); - RowUpdateBytes = appGroup->GetCounter("DataShard/EngineHostRowUpdateBytes"); - RowReads = appGroup->GetCounter("DataShard/EngineHostRowReads"); - RangeReadRows = appGroup->GetCounter("DataShard/EngineHostRangeReadRows"); - RowReadBytes = appGroup->GetCounter("DataShard/EngineHostRowReadBytes"); - RangeReadBytes = appGroup->GetCounter("DataShard/EngineHostRangeReadBytes"); - RowErases = appGroup->GetCounter("DataShard/EngineHostRowErases"); - RowEraseBytes = appGroup->GetCounter("DataShard/EngineHostRowEraseBytes"); - UploadRows = appGroup->GetCounter("DataShard/UploadRows"); - UploadRowsBytes = appGroup->GetCounter("DataShard/UploadRowsBytes"); - ScannedRows = appGroup->GetCounter("DataShard/ScannedRows"); - ScannedBytes = appGroup->GetCounter("DataShard/ScannedBytes"); - - auto execGroup = datashardGroup->GetSubgroup("category", "executor"); - - DbUniqueRowsTotal = execGroup->GetCounter("SUM(DbUniqueRowsTotal)"); - DbUniqueDataBytes = execGroup->GetCounter("SUM(DbUniqueDataBytes)"); - ConsumedCpuHistogram = execGroup->FindHistogram("HIST(ConsumedCPU)"); - } + ShardCpuUtilization = ydbGroup->GetNamedHistogram("name", + "table.datashard.used_core_percents", NMonitoring::LinearHistogram(12, 0, 10), false); + }; + + void Initialize( + NMonitoring::TDynamicCounterPtr counters, + TCountersByTabletType& countersByTabletType) + { + auto datashard = FindCountersByTabletType( + TTabletTypes::FLAT_DATASHARD, countersByTabletType); + + if (datashard && !RowUpdates) { + auto datashardGroup = counters->GetSubgroup("type", "DataShard"); + auto appGroup = datashardGroup->GetSubgroup("category", "app"); + + RowUpdates = appGroup->GetCounter("DataShard/EngineHostRowUpdates"); + RowUpdateBytes = appGroup->GetCounter("DataShard/EngineHostRowUpdateBytes"); + RowReads = appGroup->GetCounter("DataShard/EngineHostRowReads"); + RangeReadRows = appGroup->GetCounter("DataShard/EngineHostRangeReadRows"); + RowReadBytes = appGroup->GetCounter("DataShard/EngineHostRowReadBytes"); + RangeReadBytes = appGroup->GetCounter("DataShard/EngineHostRangeReadBytes"); + RowErases = appGroup->GetCounter("DataShard/EngineHostRowErases"); + RowEraseBytes = appGroup->GetCounter("DataShard/EngineHostRowEraseBytes"); + UploadRows = appGroup->GetCounter("DataShard/UploadRows"); + UploadRowsBytes = appGroup->GetCounter("DataShard/UploadRowsBytes"); + ScannedRows = appGroup->GetCounter("DataShard/ScannedRows"); + ScannedBytes = appGroup->GetCounter("DataShard/ScannedBytes"); + + auto execGroup = datashardGroup->GetSubgroup("category", "executor"); + + DbUniqueRowsTotal = execGroup->GetCounter("SUM(DbUniqueRowsTotal)"); + DbUniqueDataBytes = execGroup->GetCounter("SUM(DbUniqueDataBytes)"); + ConsumedCpuHistogram = execGroup->FindHistogram("HIST(ConsumedCPU)"); + } auto schemeshard = FindCountersByTabletType( TTabletTypes::FLAT_SCHEMESHARD, countersByTabletType); @@ -1349,31 +1349,31 @@ private: StreamReservedStorageLimit = appGroup->GetCounter("SUM(SchemeShard/StreamReservedStorageQuota)"); } - } - - void Transform() { - if (RowUpdates) { - WriteRowCount->Set(RowUpdates->Val()); - WriteBytes->Set(RowUpdateBytes->Val()); - ReadRowCount->Set(RowReads->Val() + RangeReadRows->Val()); - ReadBytes->Set(RowReadBytes->Val() + RangeReadBytes->Val()); - EraseRowCount->Set(RowErases->Val()); - EraseBytes->Set(RowEraseBytes->Val()); - BulkUpsertRowCount->Set(UploadRows->Val()); - BulkUpsertBytes->Set(UploadRowsBytes->Val()); - ScanRowCount->Set(ScannedRows->Val()); - ScanBytes->Set(ScannedBytes->Val()); - DatashardRowCount->Set(DbUniqueRowsTotal->Val()); - DatashardSizeBytes->Set(DbUniqueDataBytes->Val()); - - if (ConsumedCpuHistogram) { - TransferBuckets(ShardCpuUtilization, ConsumedCpuHistogram); - } - } + } + + void Transform() { + if (RowUpdates) { + WriteRowCount->Set(RowUpdates->Val()); + WriteBytes->Set(RowUpdateBytes->Val()); + ReadRowCount->Set(RowReads->Val() + RangeReadRows->Val()); + ReadBytes->Set(RowReadBytes->Val() + RangeReadBytes->Val()); + EraseRowCount->Set(RowErases->Val()); + EraseBytes->Set(RowEraseBytes->Val()); + BulkUpsertRowCount->Set(UploadRows->Val()); + BulkUpsertBytes->Set(UploadRowsBytes->Val()); + ScanRowCount->Set(ScannedRows->Val()); + ScanBytes->Set(ScannedBytes->Val()); + DatashardRowCount->Set(DbUniqueRowsTotal->Val()); + DatashardSizeBytes->Set(DbUniqueDataBytes->Val()); + + if (ConsumedCpuHistogram) { + TransferBuckets(ShardCpuUtilization, ConsumedCpuHistogram); + } + } if (DiskSpaceTablesTotalBytes) { - ResourcesStorageUsedBytes->Set(DiskSpaceTablesTotalBytes->Val()); - ResourcesStorageLimitBytes->Set(DiskSpaceSoftQuotaBytes->Val()); + ResourcesStorageUsedBytes->Set(DiskSpaceTablesTotalBytes->Val()); + ResourcesStorageLimitBytes->Set(DiskSpaceSoftQuotaBytes->Val()); auto quota = StreamShardsQuota->Val(); ResourcesStreamUsedShards->Set(StreamShardsCount->Val()); @@ -1388,171 +1388,171 @@ private: ResourcesStreamReservedStorage->Set(StreamReservedStorage->Val()); ResourcesStreamReservedStorageLimit->Set(StreamReservedStorageLimit->Val()); } - } - - void TransferBuckets(THistogramPtr dst, THistogramPtr src) { - auto srcSnapshot = src->Snapshot(); - auto srcCount = srcSnapshot->Count(); - auto dstSnapshot = dst->Snapshot(); - auto dstCount = dstSnapshot->Count(); - - dst->Reset(); - for (ui32 b = 0; b < std::min(srcCount, dstCount); ++b) { - dst->Collect(dstSnapshot->UpperBound(b), srcSnapshot->Value(b)); - } - } - }; - - using TYdbTabletCountersPtr = TIntrusivePtr<TYdbTabletCounters>; - -public: - class TTabletCountersForDb : public NSysView::IDbCounters { - public: - TTabletCountersForDb() - : SolomonCounters(new NMonitoring::TDynamicCounters) - , AllTypes(SolomonCounters.Get(), "type", "all", true) - {} - - TTabletCountersForDb(NMonitoring::TDynamicCounterPtr externalGroup, - NMonitoring::TDynamicCounterPtr internalGroup, - THolder<TTabletCountersBase> executorCounters) - : SolomonCounters(internalGroup->GetSubgroup("group", "tablets")) - , ExecutorCounters(std::move(executorCounters)) - , AllTypes(SolomonCounters.Get(), "type", "all", true) - { - YdbCounters = MakeIntrusive<TYdbTabletCounters>(externalGroup); - } - - void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override { - auto* proto = counters.FindOrAddTabletCounters(TTabletTypes::Unknown); - AllTypes.ToProto(*proto); - - for (auto& [type, tabletCounters] : CountersByTabletType) { - auto* proto = counters.FindOrAddTabletCounters(type); - tabletCounters->ToProto(*proto); - } - } - - void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override { - for (auto& proto : *counters.Proto().MutableTabletCounters()) { - auto type = proto.GetType(); - TTabletCountersForTabletType* tabletCounters = {}; - if (type == TTabletTypes::Unknown) { - tabletCounters = &AllTypes; - } else { - tabletCounters = GetOrAddCountersByTabletType(type, CountersByTabletType, SolomonCounters); - } - if (tabletCounters) { - if (!tabletCounters->IsInitialized()) { - Y_VERIFY(ExecutorCounters.Get()); - auto appCounters = CreateAppCountersByTabletType(type); - tabletCounters->Initialize(ExecutorCounters.Get(), appCounters.Get()); - } - tabletCounters->FromProto(proto); - } - } - if (YdbCounters) { - YdbCounters->Initialize(SolomonCounters, CountersByTabletType); - YdbCounters->Transform(); - } - } - - void Apply(ui64 tabletId, const TTabletCountersBase* executorCounters, - const TTabletCountersBase* appCounters, TTabletTypes::EType type, - const TTabletCountersBase* limitedAppCounters) - { - AllTypes.Apply(tabletId, executorCounters, nullptr, type); - auto* tabletCounters = GetOrAddCountersByTabletType(type, CountersByTabletType, SolomonCounters); - if (tabletCounters) { - tabletCounters->Apply(tabletId, executorCounters, appCounters, type, limitedAppCounters); - } - } - - void Forget(ui64 tabletId, TTabletTypes::EType type) { - if (auto it = CountersByTabletType.find(type); it != CountersByTabletType.end()) { - it->second->Forget(tabletId); - } - } - - private: - NMonitoring::TDynamicCounterPtr SolomonCounters; - THolder<TTabletCountersBase> ExecutorCounters; - - TTabletCountersForTabletType AllTypes; - TCountersByTabletType CountersByTabletType; - - TYdbTabletCountersPtr YdbCounters; - }; - - class TTabletsDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback { - TActorSystem* ActorSystem = {}; - - public: - explicit TTabletsDbWatcherCallback(TActorSystem* actorSystem) - : ActorSystem(actorSystem) - {} - - void OnDatabaseRemoved(const TString&, TPathId pathId) override { - auto evRemove = MakeHolder<TEvTabletCounters::TEvRemoveDatabase>(pathId); - auto aggregator = MakeTabletCountersAggregatorID(ActorSystem->NodeId, false); - ActorSystem->Send(aggregator, evRemove.Release()); - } - }; - -private: - TIntrusivePtr<TTabletCountersForDb> GetDbCounters(TPathId pathId, const TActorContext& ctx) { - auto it = CountersByPathId.find(pathId); - if (it != CountersByPathId.end()) { - return it->second; - } - - auto dbCounters = MakeIntrusive<TTabletMon::TTabletCountersForDb>(); - CountersByPathId[pathId] = dbCounters; - - auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>( - NKikimrSysView::TABLETS, pathId, dbCounters); - ctx.Send(NSysView::MakeSysViewServiceID(ctx.SelfID.NodeId()), evRegister.Release()); - - if (DbWatcherActorId) { - auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(pathId); - ctx.Send(DbWatcherActorId, evWatch.Release()); - } - - return dbCounters; - } - -private: + } + + void TransferBuckets(THistogramPtr dst, THistogramPtr src) { + auto srcSnapshot = src->Snapshot(); + auto srcCount = srcSnapshot->Count(); + auto dstSnapshot = dst->Snapshot(); + auto dstCount = dstSnapshot->Count(); + + dst->Reset(); + for (ui32 b = 0; b < std::min(srcCount, dstCount); ++b) { + dst->Collect(dstSnapshot->UpperBound(b), srcSnapshot->Value(b)); + } + } + }; + + using TYdbTabletCountersPtr = TIntrusivePtr<TYdbTabletCounters>; + +public: + class TTabletCountersForDb : public NSysView::IDbCounters { + public: + TTabletCountersForDb() + : SolomonCounters(new NMonitoring::TDynamicCounters) + , AllTypes(SolomonCounters.Get(), "type", "all", true) + {} + + TTabletCountersForDb(NMonitoring::TDynamicCounterPtr externalGroup, + NMonitoring::TDynamicCounterPtr internalGroup, + THolder<TTabletCountersBase> executorCounters) + : SolomonCounters(internalGroup->GetSubgroup("group", "tablets")) + , ExecutorCounters(std::move(executorCounters)) + , AllTypes(SolomonCounters.Get(), "type", "all", true) + { + YdbCounters = MakeIntrusive<TYdbTabletCounters>(externalGroup); + } + + void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override { + auto* proto = counters.FindOrAddTabletCounters(TTabletTypes::Unknown); + AllTypes.ToProto(*proto); + + for (auto& [type, tabletCounters] : CountersByTabletType) { + auto* proto = counters.FindOrAddTabletCounters(type); + tabletCounters->ToProto(*proto); + } + } + + void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override { + for (auto& proto : *counters.Proto().MutableTabletCounters()) { + auto type = proto.GetType(); + TTabletCountersForTabletType* tabletCounters = {}; + if (type == TTabletTypes::Unknown) { + tabletCounters = &AllTypes; + } else { + tabletCounters = GetOrAddCountersByTabletType(type, CountersByTabletType, SolomonCounters); + } + if (tabletCounters) { + if (!tabletCounters->IsInitialized()) { + Y_VERIFY(ExecutorCounters.Get()); + auto appCounters = CreateAppCountersByTabletType(type); + tabletCounters->Initialize(ExecutorCounters.Get(), appCounters.Get()); + } + tabletCounters->FromProto(proto); + } + } + if (YdbCounters) { + YdbCounters->Initialize(SolomonCounters, CountersByTabletType); + YdbCounters->Transform(); + } + } + + void Apply(ui64 tabletId, const TTabletCountersBase* executorCounters, + const TTabletCountersBase* appCounters, TTabletTypes::EType type, + const TTabletCountersBase* limitedAppCounters) + { + AllTypes.Apply(tabletId, executorCounters, nullptr, type); + auto* tabletCounters = GetOrAddCountersByTabletType(type, CountersByTabletType, SolomonCounters); + if (tabletCounters) { + tabletCounters->Apply(tabletId, executorCounters, appCounters, type, limitedAppCounters); + } + } + + void Forget(ui64 tabletId, TTabletTypes::EType type) { + if (auto it = CountersByTabletType.find(type); it != CountersByTabletType.end()) { + it->second->Forget(tabletId); + } + } + + private: + NMonitoring::TDynamicCounterPtr SolomonCounters; + THolder<TTabletCountersBase> ExecutorCounters; + + TTabletCountersForTabletType AllTypes; + TCountersByTabletType CountersByTabletType; + + TYdbTabletCountersPtr YdbCounters; + }; + + class TTabletsDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback { + TActorSystem* ActorSystem = {}; + + public: + explicit TTabletsDbWatcherCallback(TActorSystem* actorSystem) + : ActorSystem(actorSystem) + {} + + void OnDatabaseRemoved(const TString&, TPathId pathId) override { + auto evRemove = MakeHolder<TEvTabletCounters::TEvRemoveDatabase>(pathId); + auto aggregator = MakeTabletCountersAggregatorID(ActorSystem->NodeId, false); + ActorSystem->Send(aggregator, evRemove.Release()); + } + }; + +private: + TIntrusivePtr<TTabletCountersForDb> GetDbCounters(TPathId pathId, const TActorContext& ctx) { + auto it = CountersByPathId.find(pathId); + if (it != CountersByPathId.end()) { + return it->second; + } + + auto dbCounters = MakeIntrusive<TTabletMon::TTabletCountersForDb>(); + CountersByPathId[pathId] = dbCounters; + + auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>( + NKikimrSysView::TABLETS, pathId, dbCounters); + ctx.Send(NSysView::MakeSysViewServiceID(ctx.SelfID.NodeId()), evRegister.Release()); + + if (DbWatcherActorId) { + auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(pathId); + ctx.Send(DbWatcherActorId, evWatch.Release()); + } + + return dbCounters; + } + +private: // NMonitoring::TDynamicCounterPtr Counters; TTabletCountersForTabletType AllTypes; bool IsFollower = false; - typedef THashMap<TPathId, TIntrusivePtr<TTabletCountersForDb>> TCountersByPathId; - typedef TMap<TTabletTypes::EType, THolder<TTabletCountersBase>> TAppCountersByTabletType; + typedef THashMap<TPathId, TIntrusivePtr<TTabletCountersForDb>> TCountersByPathId; + typedef TMap<TTabletTypes::EType, THolder<TTabletCountersBase>> TAppCountersByTabletType; typedef TMap<std::pair<TTabletTypes::EType, TString>, TAutoPtr<TAggregatedLabeledCounters> > TLabeledCountersByTabletTypeAndGroup; TCountersByTabletType CountersByTabletType; - TCountersByPathId CountersByPathId; - TActorId DbWatcherActorId; - TAppCountersByTabletType LimitedAppCounters; // without txs - - TYdbTabletCountersPtr YdbCounters; - + TCountersByPathId CountersByPathId; + TActorId DbWatcherActorId; + TAppCountersByTabletType LimitedAppCounters; // without txs + + TYdbTabletCountersPtr YdbCounters; + TLabeledCountersByTabletTypeAndGroup LabeledCountersByTabletTypeAndGroup; THashMap<ui64, std::pair<TAutoPtr<TTabletCountersBase>, TAutoPtr<TTabletCountersBase>>> QuietTabletCounters; }; - -TIntrusivePtr<NSysView::IDbCounters> CreateTabletDbCounters( - NMonitoring::TDynamicCounterPtr externalGroup, - NMonitoring::TDynamicCounterPtr internalGroup, - THolder<TTabletCountersBase> executorCounters) -{ - return MakeIntrusive<TTabletMon::TTabletCountersForDb>( - externalGroup, internalGroup, std::move(executorCounters)); -} - + +TIntrusivePtr<NSysView::IDbCounters> CreateTabletDbCounters( + NMonitoring::TDynamicCounterPtr externalGroup, + NMonitoring::TDynamicCounterPtr internalGroup, + THolder<TTabletCountersBase> executorCounters) +{ + return MakeIntrusive<TTabletMon::TTabletCountersForDb>( + externalGroup, internalGroup, std::move(executorCounters)); +} + //////////////////////////////////////////// /// The TTabletCountersAggregatorActor class //////////////////////////////////////////// @@ -1582,11 +1582,11 @@ private: void HandleWork(TEvTabletCounters::TEvTabletLabeledCountersResponse::TPtr &ev, const TActorContext &ctx);//from cluster aggregator void HandleWork(NMon::TEvHttpInfo::TPtr& ev, const TActorContext &ctx); void HandleWakeup(const TActorContext &ctx); - void HandleWork(TEvTabletCounters::TEvRemoveDatabase::TPtr& ev); + void HandleWork(TEvTabletCounters::TEvRemoveDatabase::TPtr& ev); // TAutoPtr<TTabletMon> TabletMon; - TActorId DbWatcherActorId; + TActorId DbWatcherActorId; THashMap<TActorId, std::pair<TActorId, TAutoPtr<NMon::TEvHttpInfo>>> HttpRequestHandlers; THashSet<ui32> TabletTypeOfReceivedLabeledCounters; bool Follower; @@ -1610,12 +1610,12 @@ TTabletCountersAggregatorActor::Bootstrap(const TActorContext &ctx) { TAppData* appData = AppData(ctx); Y_VERIFY(!TabletMon); - + if (AppData(ctx)->FeatureFlags.GetEnableDbCounters() && !Follower) { - auto callback = MakeIntrusive<TTabletMon::TTabletsDbWatcherCallback>(ctx.ActorSystem()); - DbWatcherActorId = ctx.Register(NSysView::CreateDbWatcherActor(callback)); - } - + auto callback = MakeIntrusive<TTabletMon::TTabletsDbWatcherCallback>(ctx.ActorSystem()); + DbWatcherActorId = ctx.Register(NSysView::CreateDbWatcherActor(callback)); + } + TabletMon = new TTabletMon(appData->Counters, Follower, DbWatcherActorId); auto mon = appData->Mon; if (mon) { @@ -1634,7 +1634,7 @@ void TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvTabletAddCounters::TPtr &ev, const TActorContext &ctx) { Y_UNUSED(ctx); TEvTabletCounters::TEvTabletAddCounters* msg = ev->Get(); - TabletMon->Apply(msg->TabletID, msg->TabletType, msg->TenantPathId, msg->ExecutorCounters.Get(), msg->AppCounters.Get(), ctx); + TabletMon->Apply(msg->TabletID, msg->TabletType, msg->TenantPathId, msg->ExecutorCounters.Get(), msg->AppCounters.Get(), ctx); } //////////////////////////////////////////// @@ -1652,7 +1652,7 @@ void TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvTabletCountersForgetTablet::TPtr &ev, const TActorContext &ctx) { Y_UNUSED(ctx); TEvTabletCounters::TEvTabletCountersForgetTablet* msg = ev->Get(); - TabletMon->ForgetTablet(msg->TabletID, msg->TabletType, msg->TenantPathId); + TabletMon->ForgetTablet(msg->TabletID, msg->TabletType, msg->TenantPathId); } //////////////////////////////////////////// @@ -1786,12 +1786,12 @@ TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvTabletLabeledCo //////////////////////////////////////////// -void TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvRemoveDatabase::TPtr& ev) { - - TabletMon->RemoveTabletsByPathId(ev->Get()->PathId); -} - -//////////////////////////////////////////// +void TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvRemoveDatabase::TPtr& ev) { + + TabletMon->RemoveTabletsByPathId(ev->Get()->PathId); +} + +//////////////////////////////////////////// void TTabletCountersAggregatorActor::HandleWork(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx) { @@ -1839,7 +1839,7 @@ STFUNC(TTabletCountersAggregatorActor::StateWork) { HFunc(TEvTabletCounters::TEvTabletAddLabeledCounters, HandleWork); HFunc(TEvTabletCounters::TEvTabletLabeledCountersRequest, HandleWork); HFunc(TEvTabletCounters::TEvTabletLabeledCountersResponse, HandleWork); //from cluster aggregator, for http requests - hFunc(TEvTabletCounters::TEvRemoveDatabase, HandleWork); + hFunc(TEvTabletCounters::TEvRemoveDatabase, HandleWork); HFunc(NMon::TEvHttpInfo, HandleWork); CFunc(TEvents::TSystem::Wakeup, HandleWakeup); @@ -1858,7 +1858,7 @@ CreateTabletCountersAggregator(bool follower) { void TabletCountersForgetTablet(ui64 tabletId, TTabletTypes::EType tabletType, TPathId tenantPathId, bool follower, TActorIdentity identity) { const TActorId countersAggregator = MakeTabletCountersAggregatorID(identity.NodeId(), follower); - identity.Send(countersAggregator, new TEvTabletCounters::TEvTabletCountersForgetTablet(tabletId, tabletType, tenantPathId)); + identity.Send(countersAggregator, new TEvTabletCounters::TEvTabletCountersForgetTablet(tabletId, tabletType, tenantPathId)); } /////////////////////////////////////////// diff --git a/ydb/core/tablet/tablet_counters_aggregator.h b/ydb/core/tablet/tablet_counters_aggregator.h index cfc645305c..db350d41cb 100644 --- a/ydb/core/tablet/tablet_counters_aggregator.h +++ b/ydb/core/tablet/tablet_counters_aggregator.h @@ -26,14 +26,14 @@ struct TEvTabletCounters { // enum EEv { EvTabletAddCounters = EventSpaceBegin(TKikimrEvents::ES_TABLET_COUNTERS_AGGREGATOR), - EvDeprecated1, + EvDeprecated1, EvTabletCountersForgetTablet, EvTabletCountersRequest, EvTabletCountersResponse, EvTabletAddLabeledCounters, EvTabletLabeledCountersRequest, EvTabletLabeledCountersResponse, - EvRemoveDatabase, + EvRemoveDatabase, EvEnd }; @@ -46,16 +46,16 @@ struct TEvTabletCounters { // const ui64 TabletID; const TTabletTypes::EType TabletType; - const TPathId TenantPathId; + const TPathId TenantPathId; TAutoPtr<TTabletCountersBase> ExecutorCounters; TAutoPtr<TTabletCountersBase> AppCounters; TIntrusivePtr<TInFlightCookie> InFlightCounter; // Used to detect when previous event has been consumed by the aggregator - TEvTabletAddCounters(TIntrusivePtr<TInFlightCookie> inFlightCounter, ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId, - TAutoPtr<TTabletCountersBase> executorCounters, TAutoPtr<TTabletCountersBase> appCounters) + TEvTabletAddCounters(TIntrusivePtr<TInFlightCookie> inFlightCounter, ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId, + TAutoPtr<TTabletCountersBase> executorCounters, TAutoPtr<TTabletCountersBase> appCounters) : TabletID(tabletID) , TabletType(tabletType) - , TenantPathId(tenantPathId) + , TenantPathId(tenantPathId) , ExecutorCounters(executorCounters) , AppCounters(appCounters) , InFlightCounter(inFlightCounter) @@ -81,12 +81,12 @@ struct TEvTabletCounters { // const ui64 TabletID; const TTabletTypes::EType TabletType; - const TPathId TenantPathId; + const TPathId TenantPathId; - TEvTabletCountersForgetTablet(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId) + TEvTabletCountersForgetTablet(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId) : TabletID(tabletID) , TabletType(tabletType) - , TenantPathId(tenantPathId) + , TenantPathId(tenantPathId) {} }; @@ -104,29 +104,29 @@ struct TEvTabletCounters { struct TEvTabletLabeledCountersResponse : public TEventPB<TEvTabletLabeledCountersResponse, NKikimrTabletCountersAggregator::TEvTabletLabeledCountersResponse, EvTabletLabeledCountersResponse> { }; - struct TEvRemoveDatabase : public TEventLocal<TEvRemoveDatabase, EvRemoveDatabase> { - const TPathId PathId; + struct TEvRemoveDatabase : public TEventLocal<TEvRemoveDatabase, EvRemoveDatabase> { + const TPathId PathId; + + explicit TEvRemoveDatabase(TPathId pathId) + : PathId(pathId) + {} + }; - explicit TEvRemoveDatabase(TPathId pathId) - : PathId(pathId) - {} - }; - }; //////////////////////////////////////////// void TabletCountersForgetTablet(ui64 tabletId, TTabletTypes::EType tabletType, TPathId tenantPathId, bool follower, TActorIdentity identity); - + TStringBuf GetHistogramAggregateSimpleName(TStringBuf name); bool IsHistogramAggregateSimpleName(TStringBuf name); //////////////////////////////////////////// -TIntrusivePtr<NSysView::IDbCounters> CreateTabletDbCounters( - NMonitoring::TDynamicCounterPtr externalGroup, - NMonitoring::TDynamicCounterPtr internalGroup, - THolder<TTabletCountersBase> executorCounters); - -//////////////////////////////////////////// +TIntrusivePtr<NSysView::IDbCounters> CreateTabletDbCounters( + NMonitoring::TDynamicCounterPtr externalGroup, + NMonitoring::TDynamicCounterPtr internalGroup, + THolder<TTabletCountersBase> executorCounters); + +//////////////////////////////////////////// IActor* CreateTabletCountersAggregator(bool follower); diff --git a/ydb/core/tablet/tablet_counters_app.cpp b/ydb/core/tablet/tablet_counters_app.cpp index c7f09ffcfe..bfed7ffd13 100644 --- a/ydb/core/tablet/tablet_counters_app.cpp +++ b/ydb/core/tablet/tablet_counters_app.cpp @@ -1,43 +1,43 @@ -#include "tablet_counters_app.h" - -#include "tablet_counters_protobuf.h" - +#include "tablet_counters_app.h" + +#include "tablet_counters_protobuf.h" + #include <ydb/core/protos/counters_schemeshard.pb.h> #include <ydb/core/protos/counters_datashard.pb.h> #include <ydb/core/protos/counters_hive.pb.h> #include <ydb/core/protos/counters_kesus.pb.h> - -namespace NKikimr { - -THolder<TTabletCountersBase> CreateAppCountersByTabletType(TTabletTypes::EType type) { - switch (type) { - case TTabletTypes::SchemeShard: + +namespace NKikimr { + +THolder<TTabletCountersBase> CreateAppCountersByTabletType(TTabletTypes::EType type) { + switch (type) { + case TTabletTypes::SchemeShard: return MakeHolder<TAppProtobufTabletCounters< NSchemeShard::ESimpleCounters_descriptor, NSchemeShard::ECumulativeCounters_descriptor, NSchemeShard::EPercentileCounters_descriptor >>(); - case TTabletTypes::DataShard: + case TTabletTypes::DataShard: return MakeHolder<TAppProtobufTabletCounters< NDataShard::ESimpleCounters_descriptor, NDataShard::ECumulativeCounters_descriptor, NDataShard::EPercentileCounters_descriptor >>(); - case TTabletTypes::Hive: + case TTabletTypes::Hive: return MakeHolder<TAppProtobufTabletCounters< - NHive::ESimpleCounters_descriptor, - NHive::ECumulativeCounters_descriptor, - NHive::EPercentileCounters_descriptor + NHive::ESimpleCounters_descriptor, + NHive::ECumulativeCounters_descriptor, + NHive::EPercentileCounters_descriptor >>(); - case TTabletTypes::Kesus: + case TTabletTypes::Kesus: return MakeHolder<TAppProtobufTabletCounters< - NKesus::ESimpleCounters_descriptor, - NKesus::ECumulativeCounters_descriptor, - NKesus::EPercentileCounters_descriptor + NKesus::ESimpleCounters_descriptor, + NKesus::ECumulativeCounters_descriptor, + NKesus::EPercentileCounters_descriptor >>(); - default: - return {}; - } -} - -} + default: + return {}; + } +} + +} diff --git a/ydb/core/tablet/tablet_counters_app.h b/ydb/core/tablet/tablet_counters_app.h index 9b4efe098e..a8b473126e 100644 --- a/ydb/core/tablet/tablet_counters_app.h +++ b/ydb/core/tablet/tablet_counters_app.h @@ -1,11 +1,11 @@ -#pragma once - -#include "tablet_counters.h" - +#pragma once + +#include "tablet_counters.h" + #include <ydb/core/base/tablet_types.h> - -namespace NKikimr { - -THolder<TTabletCountersBase> CreateAppCountersByTabletType(TTabletTypes::EType type); - -} + +namespace NKikimr { + +THolder<TTabletCountersBase> CreateAppCountersByTabletType(TTabletTypes::EType type); + +} diff --git a/ydb/core/tablet/tablet_resolver.cpp b/ydb/core/tablet/tablet_resolver.cpp index 67cce60510..c0a39a2216 100644 --- a/ydb/core/tablet/tablet_resolver.cpp +++ b/ydb/core/tablet/tablet_resolver.cpp @@ -88,7 +88,7 @@ class TTabletResolver : public TActorBootstrapped<TTabletResolver> { } } - struct TQueueEntry { + struct TQueueEntry { TInstant AddInstant; TEvTabletResolver::TEvForward::TPtr Ev; diff --git a/ydb/core/tablet/ya.make b/ydb/core/tablet/ya.make index 461d50af09..25e611b408 100644 --- a/ydb/core/tablet/ya.make +++ b/ydb/core/tablet/ya.make @@ -20,8 +20,8 @@ SRCS( tablet_counters.h tablet_counters_aggregator.cpp tablet_counters_aggregator.h - tablet_counters_app.cpp - tablet_counters_app.h + tablet_counters_app.cpp + tablet_counters_app.h tablet_counters_protobuf.h tablet_exception.h tablet_impl.h diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 0ffcd492ec..59afc46215 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -135,7 +135,7 @@ void TExecutor::Broken() { BootLogic->Cancel(); if (Owner) { - TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(), + TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(), Owner->Info()->TenantPathId, Stats->IsFollower, SelfId()); Owner->Detach(OwnerCtx()); } @@ -332,7 +332,7 @@ void TExecutor::ActivateFollower(const TActorContext &ctx) { CounterCacheMemTable = new NMonitoring::TCounterForPtr; ResourceMetrics = MakeHolder<NMetrics::TResourceMetrics>(Owner->TabletID(), FollowerId, Launcher); - + PendingBlobQueue.Config.TabletID = Owner->TabletID(); PendingBlobQueue.Config.Generation = Generation(); PendingBlobQueue.Config.Follower = true; @@ -617,9 +617,9 @@ TExecutorCaches TExecutor::CleanupState() { void TExecutor::Boot(TEvTablet::TEvBoot::TPtr &ev, const TActorContext &ctx) { if (Stats->IsFollower) { - TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(), + TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(), Owner->Info()->TenantPathId, Stats->IsFollower, SelfId()); - } + } RegisterTabletFlatProbes(); @@ -696,7 +696,7 @@ void TExecutor::Restored(TEvTablet::TEvRestored::TPtr &ev, const TActorContext & } void TExecutor::DetachTablet(const TActorContext &) { - TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(), + TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(), Owner->Info()->TenantPathId, Stats->IsFollower, SelfId()); return PassAway(); } @@ -3342,11 +3342,11 @@ void TExecutor::UpdateCounters(const TActorContext &ctx) { // tablet id + tablet type ui64 tabletId = Owner->TabletID(); auto tabletType = Owner->TabletType(); - auto tenantPathId = Owner->Info()->TenantPathId; + auto tenantPathId = Owner->Info()->TenantPathId; TActorId countersAggregator = MakeTabletCountersAggregatorID(SelfId().NodeId(), Stats->IsFollower); Send(countersAggregator, new TEvTabletCounters::TEvTabletAddCounters( - CounterEventsInFlight, tabletId, tabletType, tenantPathId, executorCounters, externalTabletCounters)); + CounterEventsInFlight, tabletId, tabletType, tenantPathId, executorCounters, externalTabletCounters)); if (ResourceMetrics) { ResourceMetrics->TryUpdate(ctx); diff --git a/ydb/core/tablet_flat/flat_executor.h b/ydb/core/tablet_flat/flat_executor.h index 7aeabff1b5..df08b3a338 100644 --- a/ydb/core/tablet_flat/flat_executor.h +++ b/ydb/core/tablet_flat/flat_executor.h @@ -355,7 +355,7 @@ class TExecutor TAutoPtr<NUtil::ILogger> Logger; ui32 FollowerId = 0; - + using TActivationQueue = TOneOneQueueInplace<TSeat *, 64>; THolder<TActivationQueue, TActivationQueue::TPtrCleanDestructor> ActivationQueue; THolder<TActivationQueue, TActivationQueue::TPtrCleanDestructor> PendingQueue; diff --git a/ydb/core/tablet_flat/flat_range_cache.h b/ydb/core/tablet_flat/flat_range_cache.h index 851ce9b8df..2005831095 100644 --- a/ydb/core/tablet_flat/flat_range_cache.h +++ b/ydb/core/tablet_flat/flat_range_cache.h @@ -6,8 +6,8 @@ #include <ydb/core/base/row_version.h> #include <ydb/core/scheme/scheme_tablecell.h> -#include <library/cpp/actors/core/memory_track.h> - +#include <library/cpp/actors/core/memory_track.h> + #include <util/generic/deque.h> #include <util/generic/intrlist.h> @@ -16,8 +16,8 @@ namespace NKikimr { namespace NTable { -static constexpr char MemoryLabelKeyRangeCache[] = "Tablet/TKeyRangeCache"; - +static constexpr char MemoryLabelKeyRangeCache[] = "Tablet/TKeyRangeCache"; + struct TKeyRangeEntry { TArrayRef<TCell> FromKey; TArrayRef<TCell> ToKey; @@ -190,10 +190,10 @@ private: return Used() == 0; } - size_t AllocSize() const noexcept { - return Next_ - (char*)this + Left_; - } - + size_t AllocSize() const noexcept { + return Next_ - (char*)this + Left_; + } + private: char* Next_; size_t Left_; @@ -288,12 +288,12 @@ private: void* ptr = ::operator new(allocSize); TChunk* chunk = new(ptr) TChunk(allocSize - sizeof(TChunk)); TotalAllocated_ += allocSize; - NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Add(allocSize); + NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Add(allocSize); return chunk; } void DeallocateChunk(TChunk* chunk) { - NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Sub(chunk->AllocSize()); + NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Sub(chunk->AllocSize()); chunk->~TChunk(); ::operator delete((void*)chunk); } @@ -335,17 +335,17 @@ private: { } T* allocate(size_t n) { - auto size = sizeof(T) * n; - *UsedMemory += size; - NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Add(size); + auto size = sizeof(T) * n; + *UsedMemory += size; + NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Add(size); return TBase::allocate(n); } void deallocate(T* ptr, size_t n) noexcept { - auto size = sizeof(T) * n; - Y_VERIFY_DEBUG(*UsedMemory >= size); - *UsedMemory -= size; - NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Sub(size); + auto size = sizeof(T) * n; + Y_VERIFY_DEBUG(*UsedMemory >= size); + *UsedMemory -= size; + NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Sub(size); return TBase::deallocate(ptr, n); } diff --git a/ydb/core/tablet_flat/tablet_flat_executed.h b/ydb/core/tablet_flat/tablet_flat_executed.h index 8126ff6ef7..117d7532bb 100644 --- a/ydb/core/tablet_flat/tablet_flat_executed.h +++ b/ydb/core/tablet_flat/tablet_flat_executed.h @@ -21,7 +21,7 @@ protected: TTabletExecutedFlat(TTabletStorageInfo *info, const TActorId &tablet, IMiniKQLFactory *factory); IExecutor* Executor() const { return Executor0; } - const TInstant StartTime() const { return StartTime0; } + const TInstant StartTime() const { return StartTime0; } void Execute(TAutoPtr<ITransaction> transaction, const TActorContext &ctx); void Execute(TAutoPtr<ITransaction> transaction); diff --git a/ydb/core/testlib/basics/helpers.h b/ydb/core/testlib/basics/helpers.h index 964f8aad55..7a70063f7a 100644 --- a/ydb/core/testlib/basics/helpers.h +++ b/ydb/core/testlib/basics/helpers.h @@ -51,7 +51,7 @@ namespace NFake { void SetupNodeTabletMonitor(TTestActorRuntime& runtime, ui32 nodeIndex); void SetupSchemeCache(TTestActorRuntime& runtime, ui32 nodeIndex, const TString& root); void SetupQuoterService(TTestActorRuntime& runtime, ui32 nodeIndex); - void SetupSysViewService(TTestActorRuntime& runtime, ui32 nodeIndex); + void SetupSysViewService(TTestActorRuntime& runtime, ui32 nodeIndex); // StateStorage, NodeWarden, TabletResolver, ResourceBroker, SharedPageCache void SetupBasicServices(TTestActorRuntime &runtime, TAppPrepare &app, bool mockDisk = false, diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp index 4a7644d902..a853040b93 100644 --- a/ydb/core/testlib/tablet_helpers.cpp +++ b/ydb/core/testlib/tablet_helpers.cpp @@ -1183,8 +1183,8 @@ namespace NKikimr { bootstrapperActorId = Boot(ctx, type, [=](const TActorId& tablet, TTabletStorageInfo* info) { return new TFakeHive(tablet, info, state, &TFakeHive::DefaultGetTabletCreationFunc); }, DataGroupErasure); - } else if (type == defaultTabletTypes.SysViewProcessor) { - bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure); + } else if (type == defaultTabletTypes.SysViewProcessor) { + bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure); } else if (type == defaultTabletTypes.SequenceShard) { bootstrapperActorId = Boot(ctx, type, &NSequenceShard::CreateSequenceShard, DataGroupErasure); } else if (type == defaultTabletTypes.ReplicationController) { diff --git a/ydb/core/testlib/tenant_runtime.cpp b/ydb/core/testlib/tenant_runtime.cpp index f047a9bca9..974cc166d0 100644 --- a/ydb/core/testlib/tenant_runtime.cpp +++ b/ydb/core/testlib/tenant_runtime.cpp @@ -431,8 +431,8 @@ class TFakeHive : public TActor<TFakeHive>, public TTabletExecutedFlat { bootstrapperActorId = Boot(ctx, type, &CreateFlatTxSchemeShard, DataGroupErasure); } else if (type == defaultTabletTypes.Hive) { bootstrapperActorId = Boot(ctx, type, &CreateDefaultHive, DataGroupErasure); - } else if (type == defaultTabletTypes.SysViewProcessor) { - bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure); + } else if (type == defaultTabletTypes.SysViewProcessor) { + bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure); } else if (type == defaultTabletTypes.SequenceShard) { bootstrapperActorId = Boot(ctx, type, &NSequenceShard::CreateSequenceShard, DataGroupErasure); } else if (type == defaultTabletTypes.ReplicationController) { diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index f2f0ed782f..d4907c26f1 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -537,10 +537,10 @@ namespace Tests { TLocalConfig::TTabletClassInfo(new TTabletSetupInfo( &CreateDefaultHive, TMailboxType::Revolving, appData.UserPoolId, TMailboxType::Revolving, appData.SystemPoolId)); - localConfig.TabletClassInfo[appData.DefaultTabletTypes.SysViewProcessor] = - TLocalConfig::TTabletClassInfo(new TTabletSetupInfo( - &NSysView::CreateSysViewProcessorForTests, TMailboxType::Revolving, appData.UserPoolId, - TMailboxType::Revolving, appData.SystemPoolId)); + localConfig.TabletClassInfo[appData.DefaultTabletTypes.SysViewProcessor] = + TLocalConfig::TTabletClassInfo(new TTabletSetupInfo( + &NSysView::CreateSysViewProcessorForTests, TMailboxType::Revolving, appData.UserPoolId, + TMailboxType::Revolving, appData.SystemPoolId)); localConfig.TabletClassInfo[appData.DefaultTabletTypes.SequenceShard] = TLocalConfig::TTabletClassInfo(new TTabletSetupInfo( &NSequenceShard::CreateSequenceShard, TMailboxType::Revolving, appData.UserPoolId, @@ -568,13 +568,13 @@ namespace Tests { Runtime->RegisterService(NConsole::MakeConfigsDispatcherID(Runtime->GetNodeId(nodeIdx)), aid); } Runtime->Register(CreateLabelsMaintainer({}), nodeIdx, appData.SystemPoolId, TMailboxType::Revolving, 0); - - auto sysViewService = NSysView::CreateSysViewServiceForTests(); + + auto sysViewService = NSysView::CreateSysViewServiceForTests(); TActorId sysViewServiceId = Runtime->Register(sysViewService.Release(), nodeIdx); - Runtime->RegisterService(NSysView::MakeSysViewServiceID(Runtime->GetNodeId(nodeIdx)), sysViewServiceId, nodeIdx); - - auto tenantPublisher = CreateTenantNodeEnumerationPublisher(); - Runtime->Register(tenantPublisher, nodeIdx); + Runtime->RegisterService(NSysView::MakeSysViewServiceID(Runtime->GetNodeId(nodeIdx)), sysViewServiceId, nodeIdx); + + auto tenantPublisher = CreateTenantNodeEnumerationPublisher(); + Runtime->Register(tenantPublisher, nodeIdx); } void TServer::SetupConfigurators(ui32 nodeIdx) { diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 0734a1b4df..2064752ab4 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -153,7 +153,7 @@ namespace Tests { TServerSettings& SetDomainPlanResolution(ui64 resolution) { DomainPlanResolution = resolution; return *this; } TServerSettings& SetFeatureFlags(const NKikimrConfig::TFeatureFlags& value) { FeatureFlags = value; return *this; } TServerSettings& SetCompactionConfig(const NKikimrConfig::TCompactionConfig& value) { CompactionConfig = value; return *this; } - TServerSettings& SetEnableDbCounters(bool value) { FeatureFlags.SetEnableDbCounters(value); return *this; } + TServerSettings& SetEnableDbCounters(bool value) { FeatureFlags.SetEnableDbCounters(value); return *this; } TServerSettings& SetEnableYq(bool value) { EnableYq = value; return *this; } TServerSettings& SetKeepSnapshotTimeout(TDuration value) { KeepSnapshotTimeout = value; return *this; } TServerSettings& SetChangesQueueItemsLimit(ui64 value) { ChangesQueueItemsLimit = value; return *this; } diff --git a/ydb/core/tx/coordinator/coordinator__plan_step.cpp b/ydb/core/tx/coordinator/coordinator__plan_step.cpp index 313393d4da..c1fac2ffa5 100644 --- a/ydb/core/tx/coordinator/coordinator__plan_step.cpp +++ b/ydb/core/tx/coordinator/coordinator__plan_step.cpp @@ -72,9 +72,9 @@ struct TTxCoordinator::TTxPlanStep : public TTransactionBase<TTxCoordinator> { Y_VERIFY(txId); Self->MonCounters.StepConsideredTx->Inc(); - auto durationMs = (ExecStartMoment - proposal->AcceptMoment).MilliSeconds(); - Self->MonCounters.LegacyTxFromReceiveToPlan.Add(durationMs); - Self->MonCounters.TxFromReceiveToPlan->Collect(durationMs); + auto durationMs = (ExecStartMoment - proposal->AcceptMoment).MilliSeconds(); + Self->MonCounters.LegacyTxFromReceiveToPlan.Add(durationMs); + Self->MonCounters.TxFromReceiveToPlan->Collect(durationMs); if (proposal->MaxStep < PlanOnStep) { Self->MonCounters.StepOutdatedTx->Inc(); @@ -213,9 +213,9 @@ struct TTxCoordinator::TTxPlanStep : public TTransactionBase<TTxCoordinator> { } void Complete(const TActorContext &ctx) override { - auto durationMs = (ctx.Now() - ExecStartMoment).MilliSeconds(); - Self->MonCounters.LegacyTxPlanLatency.Add(durationMs); - Self->MonCounters.TxPlanLatency->Collect(durationMs); + auto durationMs = (ctx.Now() - ExecStartMoment).MilliSeconds(); + Self->MonCounters.LegacyTxPlanLatency.Add(durationMs); + Self->MonCounters.TxPlanLatency->Collect(durationMs); for (auto &cx : StepsToConfirm) { const ui64 mediatorId = cx.first; diff --git a/ydb/core/tx/coordinator/coordinator_impl.cpp b/ydb/core/tx/coordinator/coordinator_impl.cpp index ed21df845d..b4eeb184e2 100644 --- a/ydb/core/tx/coordinator/coordinator_impl.cpp +++ b/ydb/core/tx/coordinator/coordinator_impl.cpp @@ -334,12 +334,12 @@ void TTxCoordinator::TryInitMonCounters(const TActorContext &ctx) { MonCounters.StepPlannedTx = MonCounters.Coordinator->GetCounter("Step/PlannedTx", true); MonCounters.StepDeclinedNoSpaceTx = MonCounters.Coordinator->GetCounter("Step/DeclinedNoSpaceTx", true); - MonCounters.LegacyTxFromReceiveToPlan.Init(MonCounters.Coordinator.Get(), "TxFromReceiveToPlan", "ms", 1, 20); - MonCounters.TxFromReceiveToPlan = MonCounters.Coordinator->GetHistogram( - "TxFromReceiveToPlanMs", NMonitoring::ExponentialHistogram(20, 2, 1)); - MonCounters.LegacyTxPlanLatency.Init(MonCounters.Coordinator.Get(), "TxPlanLatency", "ms", 1, 20); - MonCounters.TxPlanLatency = MonCounters.Coordinator->GetHistogram( - "TxPlanLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1)); + MonCounters.LegacyTxFromReceiveToPlan.Init(MonCounters.Coordinator.Get(), "TxFromReceiveToPlan", "ms", 1, 20); + MonCounters.TxFromReceiveToPlan = MonCounters.Coordinator->GetHistogram( + "TxFromReceiveToPlanMs", NMonitoring::ExponentialHistogram(20, 2, 1)); + MonCounters.LegacyTxPlanLatency.Init(MonCounters.Coordinator.Get(), "TxPlanLatency", "ms", 1, 20); + MonCounters.TxPlanLatency = MonCounters.Coordinator->GetHistogram( + "TxPlanLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1)); } void TTxCoordinator::SendMediatorStep(TMediator &mediator, const TActorContext &ctx) { diff --git a/ydb/core/tx/coordinator/coordinator_impl.h b/ydb/core/tx/coordinator/coordinator_impl.h index 3fec141a61..39018aff1e 100644 --- a/ydb/core/tx/coordinator/coordinator_impl.h +++ b/ydb/core/tx/coordinator/coordinator_impl.h @@ -64,7 +64,7 @@ struct TTransactionProposal { }; struct TCoordinatorStepConfirmations { - struct TEntry { + struct TEntry { TTxId TxId; TActorId ProxyId; TEvTxProxy::TEvProposeTransactionStatus::EStatus Status; @@ -396,10 +396,10 @@ private: NMonitoring::TDynamicCounters::TCounterPtr StepPlannedTx; NMonitoring::TDynamicCounters::TCounterPtr StepDeclinedNoSpaceTx; - NMon::THistogramCounterHelper LegacyTxFromReceiveToPlan; - NMonitoring::THistogramPtr TxFromReceiveToPlan; - NMon::THistogramCounterHelper LegacyTxPlanLatency; - NMonitoring::THistogramPtr TxPlanLatency; + NMon::THistogramCounterHelper LegacyTxFromReceiveToPlan; + NMonitoring::THistogramPtr TxFromReceiveToPlan; + NMon::THistogramCounterHelper LegacyTxPlanLatency; + NMonitoring::THistogramPtr TxPlanLatency; i64 CurrentTxInFly; }; diff --git a/ydb/core/tx/datashard/datashard.cpp b/ydb/core/tx/datashard/datashard.cpp index 789b81b796..d86568b4a1 100644 --- a/ydb/core/tx/datashard/datashard.cpp +++ b/ydb/core/tx/datashard/datashard.cpp @@ -1062,7 +1062,7 @@ TUserTable::TSpecialUpdate TDataShard::SpecialUpdates(const NTable::TDatabase& d Y_VERIFY(tableId.PathId.OwnerId == PathOwnerId, "%" PRIu64 " vs %" PRIu64, tableId.PathId.OwnerId, PathOwnerId); - auto it = TableInfos.find(tableId.PathId.LocalPathId); + auto it = TableInfos.find(tableId.PathId.LocalPathId); Y_VERIFY(it != TableInfos.end()); const TUserTable& tableInfo = *it->second; Y_VERIFY(tableInfo.LocalTid != Max<ui32>()); @@ -1092,14 +1092,14 @@ TUserTable::TSpecialUpdate TDataShard::SpecialUpdates(const NTable::TDatabase& d void TDataShard::SetTableAccessTime(const TTableId& tableId, TInstant ts) { Y_VERIFY(!TSysTables::IsSystemTable(tableId)); - auto iter = TableInfos.find(tableId.PathId.LocalPathId); + auto iter = TableInfos.find(tableId.PathId.LocalPathId); Y_VERIFY(iter != TableInfos.end()); iter->second->Stats.AccessTime = ts; } void TDataShard::SetTableUpdateTime(const TTableId& tableId, TInstant ts) { Y_VERIFY(!TSysTables::IsSystemTable(tableId)); - auto iter = TableInfos.find(tableId.PathId.LocalPathId); + auto iter = TableInfos.find(tableId.PathId.LocalPathId); Y_VERIFY(iter != TableInfos.end()); iter->second->Stats.AccessTime = ts; iter->second->Stats.UpdateTime = ts; diff --git a/ydb/core/tx/datashard/datashard__engine_host.cpp b/ydb/core/tx/datashard/datashard__engine_host.cpp index fd5b7f9320..876c337093 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.cpp +++ b/ydb/core/tx/datashard/datashard__engine_host.cpp @@ -195,7 +195,7 @@ public: : TableId(tableId) , Self(self) { - switch (TableId.PathId.LocalPathId) { + switch (TableId.PathId.LocalPathId) { case TSysTables::SysTableLocks: TSysTables::TLocksTable::GetInfo(Columns, KeyTypes, false); break; @@ -217,10 +217,10 @@ public: auto lock = Self->SysLocksTable().GetLock(row); Y_VERIFY(!lock.IsError()); - if (TableId.PathId.LocalPathId == TSysTables::SysTableLocks2) { + if (TableId.PathId.LocalPathId == TSysTables::SysTableLocks2) { return result.CreateResult(lock.MakeRow(true), holderFactory); } - Y_VERIFY(TableId.PathId.LocalPathId == TSysTables::SysTableLocks); + Y_VERIFY(TableId.PathId.LocalPathId == TSysTables::SysTableLocks); return result.CreateResult(lock.MakeRow(false), holderFactory); } @@ -262,11 +262,11 @@ public: , Locks2(TTableId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2), self) {} - const TDataShardSysTable& Get(const TTableId& tableId) const { - if (tableId.PathId.LocalPathId == TSysTables::SysTableLocks2) + const TDataShardSysTable& Get(const TTableId& tableId) const { + if (tableId.PathId.LocalPathId == TSysTables::SysTableLocks2) return Locks2; - if (tableId.PathId.LocalPathId == TSysTables::SysTableLocks) + if (tableId.PathId.LocalPathId == TSysTables::SysTableLocks) return Locks; Y_FAIL("unexpected sys table id"); @@ -455,7 +455,7 @@ public: if (TSysTables::IsSystemTable(tableId)) return DataShardSysTable(tableId).IsMyKey(row); - auto iter = Self->TableInfos.find(tableId.PathId.LocalPathId); + auto iter = Self->TableInfos.find(tableId.PathId.LocalPathId); if (iter == Self->TableInfos.end()) { // TODO: can this happen? return false; @@ -492,7 +492,7 @@ public: } private: - const TDataShardSysTable& DataShardSysTable(const TTableId& tableId) const { + const TDataShardSysTable& DataShardSysTable(const TTableId& tableId) const { return static_cast<const TDataShardSysTables *>(Self->GetDataShardSysTables())->Get(tableId); } diff --git a/ydb/core/tx/datashard/datashard__read_columns.cpp b/ydb/core/tx/datashard/datashard__read_columns.cpp index 42a30e90b7..2b56a192d1 100644 --- a/ydb/core/tx/datashard/datashard__read_columns.cpp +++ b/ydb/core/tx/datashard/datashard__read_columns.cpp @@ -182,8 +182,8 @@ private: TSmallVec<TRawTypeValue> KeyTo; bool InclusiveFrom; bool InclusiveTo; - ui64 RowsLimit = 100000; - ui64 BytesLimit = 1024*1024; + ui64 RowsLimit = 100000; + ui64 BytesLimit = 1024*1024; ui64 Restarts = 0; TRowVersion ReadVersion = TRowVersion::Max(); diff --git a/ydb/core/tx/datashard/datashard_active_transaction.cpp b/ydb/core/tx/datashard/datashard_active_transaction.cpp index 5af7903fa4..0226bd42e2 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.cpp +++ b/ydb/core/tx/datashard/datashard_active_transaction.cpp @@ -7,8 +7,8 @@ #include "datashard_failpoints.h" #include "key_conflicts.h" -#include <library/cpp/actors/core/memory_track.h> - +#include <library/cpp/actors/core/memory_track.h> + namespace NKikimr { namespace NDataShard { @@ -38,9 +38,9 @@ TValidatedDataTx::TValidatedDataTx(TDataShard *self, return; } - ComputeTxSize(); - NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Add(TxSize); - + ComputeTxSize(); + NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Add(TxSize); + Y_VERIFY(Tx.HasMiniKQL() || Tx.HasReadTableTransaction() || Tx.HasKqpTransaction(), "One of the fields should be set: MiniKQL, ReadTableTransaction, KqpTransaction"); @@ -195,10 +195,10 @@ TValidatedDataTx::TValidatedDataTx(TDataShard *self, ComputeDeadline(); } -TValidatedDataTx::~TValidatedDataTx() { - NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Sub(TxSize); -} - +TValidatedDataTx::~TValidatedDataTx() { + NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Sub(TxSize); +} + const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& TValidatedDataTx::GetKqpTasks() const { Y_VERIFY(IsKqpTx()); return Tx.GetKqpTransaction().GetTasks(); @@ -309,10 +309,10 @@ void TValidatedDataTx::ReleaseTxData() { Tx.SetLockTxId(lock); EngineBay.DestroyEngine(); IsReleased = true; - - NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Sub(TxSize); - ComputeTxSize(); - NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Add(TxSize); + + NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Sub(TxSize); + ComputeTxSize(); + NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Add(TxSize); } void TValidatedDataTx::ComputeTxSize() { @@ -335,7 +335,7 @@ TActiveTransaction::TActiveTransaction(const TBasicOpInfo &op, TValidatedDataTx::TPtr dataTx) : TActiveTransaction(op) { - TrackMemory(); + TrackMemory(); FillTxData(dataTx); } @@ -349,15 +349,15 @@ TActiveTransaction::TActiveTransaction(TDataShard *self, ui64 artifactFlags) : TActiveTransaction(op) { - TrackMemory(); + TrackMemory(); FillTxData(self, txc, ctx, target, txBody, locks, artifactFlags); } -TActiveTransaction::~TActiveTransaction() -{ - UntrackMemory(); -} - +TActiveTransaction::~TActiveTransaction() +{ + UntrackMemory(); +} + void TActiveTransaction::FillTxData(TValidatedDataTx::TPtr dataTx) { Y_VERIFY(!DataTx); @@ -378,8 +378,8 @@ void TActiveTransaction::FillTxData(TDataShard *self, const TVector<TSysTables::TLocksTable::TLock> &locks, ui64 artifactFlags) { - UntrackMemory(); - + UntrackMemory(); + Y_VERIFY(!DataTx); Y_VERIFY(TxBody.empty()); @@ -406,8 +406,8 @@ void TActiveTransaction::FillTxData(TDataShard *self, } else if (IsCommitWritesTx()) { BuildCommitWritesTx(); } - - TrackMemory(); + + TrackMemory(); } TValidatedDataTx::TPtr TActiveTransaction::BuildDataTx(TDataShard *self, @@ -546,11 +546,11 @@ void TActiveTransaction::ReleaseTxData(NTabletFlatExecutor::TTxMemoryProviderBas DataTx->ReleaseTxData(); // Immediate transactions have no body stored. - if (!IsImmediate()) { - UntrackMemory(); + if (!IsImmediate()) { + UntrackMemory(); TxBody.clear(); - TrackMemory(); - } + TrackMemory(); + } //InReadSets.clear(); OutReadSets().clear(); @@ -613,8 +613,8 @@ ERestoreDataStatus TActiveTransaction::RestoreTxData( return ERestoreDataStatus::Ok; } - UntrackMemory(); - + UntrackMemory(); + // For immediate transactions we should restore just // from the TxBody. For planned transaction we should // restore from local database. @@ -632,8 +632,8 @@ ERestoreDataStatus TActiveTransaction::RestoreTxData( Y_VERIFY(TxBody); } - TrackMemory(); - + TrackMemory(); + for (auto &lock : locks) LocksCache().Locks[lock.LockId] = lock; @@ -912,12 +912,12 @@ void TActiveTransaction::KillAsyncJobActor(const TActorContext& ctx) { SetAsyncJobActor(TActorId()); } -void TActiveTransaction::TrackMemory() const { - NActors::NMemory::TLabel<MemoryLabelActiveTransactionBody>::Add(TxBody.size()); -} - -void TActiveTransaction::UntrackMemory() const { - NActors::NMemory::TLabel<MemoryLabelActiveTransactionBody>::Sub(TxBody.size()); -} - +void TActiveTransaction::TrackMemory() const { + NActors::NMemory::TLabel<MemoryLabelActiveTransactionBody>::Add(TxBody.size()); +} + +void TActiveTransaction::UntrackMemory() const { + NActors::NMemory::TLabel<MemoryLabelActiveTransactionBody>::Sub(TxBody.size()); +} + }} diff --git a/ydb/core/tx/datashard/datashard_active_transaction.h b/ydb/core/tx/datashard/datashard_active_transaction.h index 15b732398a..43abd525e1 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.h +++ b/ydb/core/tx/datashard/datashard_active_transaction.h @@ -16,9 +16,9 @@ class TBalanceCoverageBuilder; namespace NDataShard { -static constexpr char MemoryLabelValidatedDataTx[] = "Datashard/TValidatedDataTx"; -static constexpr char MemoryLabelActiveTransactionBody[] = "Datashard/TActiveTransaction/TxBody"; - +static constexpr char MemoryLabelValidatedDataTx[] = "Datashard/TValidatedDataTx"; +static constexpr char MemoryLabelActiveTransactionBody[] = "Datashard/TActiveTransaction/TxBody"; + using NTabletFlatExecutor::TTransactionContext; using NTabletFlatExecutor::TTableSnapshotContext; @@ -122,8 +122,8 @@ public: TInstant receivedAt, const TString &txBody); - ~TValidatedDataTx(); - + ~TValidatedDataTx(); + static constexpr ui64 MaxReorderTxKeys() { return 100; } NKikimrTxDataShard::TError::EKind Code() const { return ErrCode; } @@ -321,9 +321,9 @@ public: , SchemeTxType(TSchemaOperation::ETypeUnknown) , ScanSnapshotId(0) , ScanTask(0) - { - TrackMemory(); - } + { + TrackMemory(); + } TActiveTransaction(const TBasicOpInfo &op, TValidatedDataTx::TPtr savedTx); @@ -336,8 +336,8 @@ public: const TVector<TSysTables::TLocksTable::TLock> &locks, ui64 artifactFlags); - ~TActiveTransaction(); - + ~TActiveTransaction(); + void FillTxData(TValidatedDataTx::TPtr dataTx); void FillTxData(TDataShard *self, TTransactionContext &txc, @@ -348,16 +348,16 @@ public: ui64 artifactFlags); const TString &GetTxBody() const { return TxBody; } - void SetTxBody(const TString &txBody) { - UntrackMemory(); - TxBody = txBody; - TrackMemory(); - } - void ClearTxBody() { - UntrackMemory(); - TxBody.clear(); - TrackMemory(); - } + void SetTxBody(const TString &txBody) { + UntrackMemory(); + TxBody = txBody; + TrackMemory(); + } + void ClearTxBody() { + UntrackMemory(); + TxBody.clear(); + TrackMemory(); + } ui64 GetSchemeShardId() const { return SchemeShardId; } void SetSchemeShardId(ui64 id) { SchemeShardId = id; } @@ -534,10 +534,10 @@ public: } private: - void TrackMemory() const; - void UntrackMemory() const; - -private: + void TrackMemory() const; + void UntrackMemory() const; + +private: TValidatedDataTx::TPtr DataTx; THolder<NKikimrTxDataShard::TFlatSchemeTransaction> SchemeTx; THolder<NKikimrTxDataShard::TSnapshotTransaction> SnapshotTx; diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 5ac43aaa45..11313474f8 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -1188,7 +1188,7 @@ public: ui64 GetLocalTableId(const TTableId& tableId) const { Y_VERIFY(!TSysTables::IsSystemTable(tableId)); - auto it = TableInfos.find(tableId.PathId.LocalPathId); + auto it = TableInfos.find(tableId.PathId.LocalPathId); return it == TableInfos.end() ? 0 : it->second->LocalTid; } @@ -2398,9 +2398,9 @@ protected: ev->Record.AddSysTablesPartOwners(pi); } - ev->Record.SetNodeId(ctx.ExecutorThread.ActorSystem->NodeId); - ev->Record.SetStartTime(StartTime().MilliSeconds()); - + ev->Record.SetNodeId(ctx.ExecutorThread.ActorSystem->NodeId); + ev->Record.SetStartTime(StartTime().MilliSeconds()); + NTabletPipe::SendData(ctx, DbStatsReportPipe, ev.Release()); } diff --git a/ydb/core/tx/datashard/datashard_ut_locks.cpp b/ydb/core/tx/datashard/datashard_ut_locks.cpp index d894033406..1d25c39f99 100644 --- a/ydb/core/tx/datashard/datashard_ut_locks.cpp +++ b/ydb/core/tx/datashard/datashard_ut_locks.cpp @@ -88,7 +88,7 @@ using namespace NDataShard; namespace NTest { bool TFakeDataShard::IsUserTable(const TTableId& tableId) const { - return tableId.PathId.LocalPathId >= NumSysTables(); + return tableId.PathId.LocalPathId >= NumSysTables(); } /// @@ -140,7 +140,7 @@ namespace NTest { : TableId(tableId) , Locks(&DataShard) { - ui64 tid = tableId.PathId.LocalPathId; + ui64 tid = tableId.PathId.LocalPathId; ui64 sid = tableId.PathId.OwnerId; TmpLock.PathId = tid; diff --git a/ydb/core/tx/datashard/sys_tables.h b/ydb/core/tx/datashard/sys_tables.h index 18885f54f7..e9887bd8db 100644 --- a/ydb/core/tx/datashard/sys_tables.h +++ b/ydb/core/tx/datashard/sys_tables.h @@ -31,9 +31,9 @@ struct TSysTables { SysTableMAX = 999 }; - static bool IsSystemTable(const TTableId& table) { return table.PathId.OwnerId == SysSchemeShard; } - static bool IsLocksTable(const TTableId& table) { - return IsSystemTable(table) && (table.PathId.LocalPathId == SysTableLocks || table.PathId.LocalPathId == SysTableLocks2); + static bool IsSystemTable(const TTableId& table) { return table.PathId.OwnerId == SysSchemeShard; } + static bool IsLocksTable(const TTableId& table) { + return IsSystemTable(table) && (table.PathId.LocalPathId == SysTableLocks || table.PathId.LocalPathId == SysTableLocks2); } struct TLocksTable { diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index b175c1d490..8e778a4a1d 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -1265,7 +1265,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { Kind = TNavigate::KindPath; Created = true; PathId = TPathId(TSysTables::SysSchemeShard, 0); - Path = "/sys"; + Path = "/sys"; IsVirtual = true; } @@ -1278,7 +1278,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { Kind = TNavigate::KindTable; Created = true; PathId = TPathId(TSysTables::SysSchemeShard, v2 ? TSysTables::SysTableLocks2 : TSysTables::SysTableLocks); - Path = v2 ? "/sys/locks2" : "/sys/locks"; + Path = v2 ? "/sys/locks2" : "/sys/locks"; TVector<ui32> keyColumnTypes; TSysTables::TLocksTable::GetInfo(Columns, keyColumnTypes, v2); @@ -1309,9 +1309,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { Y_VERIFY(notify.DescribeSchemeResult.HasPathDescription()); auto& pathDesc = *notify.DescribeSchemeResult.MutablePathDescription(); - Y_VERIFY(notify.DescribeSchemeResult.HasPath()); - Path = notify.DescribeSchemeResult.GetPath(); - + Y_VERIFY(notify.DescribeSchemeResult.HasPath()); + Path = notify.DescribeSchemeResult.GetPath(); + const auto& abandoned = pathDesc.GetAbandonedTenantsSchemeShards(); AbandonedSchemeShardsIds = TSet<ui64>(abandoned.begin(), abandoned.end()); @@ -1523,38 +1523,38 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { return AbandonedSchemeShardsIds; } - void FillSystemViewEntry(TNavigateContext* context, TNavigate::TEntry& entry, - NSysView::ISystemViewResolver::ETarget target) const - { - auto sysViewInfo = entry.TableId.SysViewInfo; - - if (sysViewInfo == NSysView::SysPathName) { - if (entry.Operation == TNavigate::OpTable) { + void FillSystemViewEntry(TNavigateContext* context, TNavigate::TEntry& entry, + NSysView::ISystemViewResolver::ETarget target) const + { + auto sysViewInfo = entry.TableId.SysViewInfo; + + if (sysViewInfo == NSysView::SysPathName) { + if (entry.Operation == TNavigate::OpTable) { return SetError(context, entry, TNavigate::EStatus::PathNotTable); - } - - auto listNodeEntry = MakeIntrusive<TNavigate::TListNodeEntry>(); - - auto names = Owner->SystemViewResolver->GetSystemViewNames(target); - std::sort(names.begin(), names.end()); - - listNodeEntry->Kind = TNavigate::KindPath; - listNodeEntry->Children.reserve(names.size()); - for (const auto& name : names) { + } + + auto listNodeEntry = MakeIntrusive<TNavigate::TListNodeEntry>(); + + auto names = Owner->SystemViewResolver->GetSystemViewNames(target); + std::sort(names.begin(), names.end()); + + listNodeEntry->Kind = TNavigate::KindPath; + listNodeEntry->Children.reserve(names.size()); + for (const auto& name : names) { listNodeEntry->Children.emplace_back(name, TPathId(), TNavigate::KindTable); - } - - entry.Kind = TNavigate::KindPath; - entry.ListNodeEntry = listNodeEntry; - entry.TableId = TTableId(PathId.OwnerId, InvalidLocalPathId, sysViewInfo); - - } else { - auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target); - if (!schema) { + } + + entry.Kind = TNavigate::KindPath; + entry.ListNodeEntry = listNodeEntry; + entry.TableId = TTableId(PathId.OwnerId, InvalidLocalPathId, sysViewInfo); + + } else { + auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target); + if (!schema) { return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); - } - - entry.Kind = TNavigate::KindTable; + } + + entry.Kind = TNavigate::KindTable; if (target == NSysView::ISystemViewResolver::ETarget::OlapStore || target == NSysView::ISystemViewResolver::ETarget::OlapTable) { @@ -1562,23 +1562,23 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { entry.Kind =TNavigate::KindOlapTable; } - entry.Columns = std::move(schema->Columns); - - if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) { - entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId, sysViewInfo); - } else { - entry.Path = SplitPath(Path); - entry.Path.emplace_back(NKikimr::NSysView::SysPathName); - entry.Path.emplace_back(sysViewInfo); - } - } - - entry.Status = TNavigate::EStatus::Ok; - - entry.SecurityObject = SecurityObject; - entry.DomainInfo = DomainInfo; - } - + entry.Columns = std::move(schema->Columns); + + if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) { + entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId, sysViewInfo); + } else { + entry.Path = SplitPath(Path); + entry.Path.emplace_back(NKikimr::NSysView::SysPathName); + entry.Path.emplace_back(sysViewInfo); + } + } + + entry.Status = TNavigate::EStatus::Ok; + + entry.SecurityObject = SecurityObject; + entry.DomainInfo = DomainInfo; + } + void FillEntry(TNavigateContext* context, TNavigate::TEntry& entry, const TResponseProps& props = TResponseProps()) const { SBC_LOG_D("FillEntry for TNavigate" << ": self# " << Owner->SelfId() @@ -1598,13 +1598,13 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { return SetError(context, entry, TNavigate::EStatus::LookupError); } - if (!entry.TableId.SysViewInfo.empty()) { - if (Kind == TNavigate::KindPath) { - auto split = SplitPath(Path); - if (split.size() == 1) { + if (!entry.TableId.SysViewInfo.empty()) { + if (Kind == TNavigate::KindPath) { + auto split = SplitPath(Path); + if (split.size() == 1) { return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain); - } - } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) { + } + } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) { return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain); } else if (Kind == TNavigate::KindOlapStore) { FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::OlapStore); @@ -1615,11 +1615,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { entry.OlapStoreInfo = OlapStoreInfo; entry.OlapTableInfo = OlapTableInfo; return; - } + } return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); - } - + } + const bool isTable = Kind == TNavigate::KindTable || Kind == TNavigate::KindOlapTable; if (entry.Operation == TNavigate::OpTable && !isTable) { return SetError(context, entry, TNavigate::EStatus::PathNotTable); @@ -1647,14 +1647,14 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { entry.Kind = Kind; entry.CreateStep = CreateStep; - if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) { - if (Kind == TNavigate::KindTable) { + if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) { + if (Kind == TNavigate::KindTable) { entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId, SchemaVersion); - } else { + } else { entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId); - } + } } else { - entry.Path = SplitPath(Path); + entry.Path = SplitPath(Path); // update schema version entry.TableId.SchemaVersion = SchemaVersion; @@ -1683,25 +1683,25 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry, - const TVector<NScheme::TTypeId>& keyColumnTypes, - const THashMap<ui32, TSysTables::TTableColumnInfo>& columns) const - { + const TVector<NScheme::TTypeId>& keyColumnTypes, + const THashMap<ui32, TSysTables::TTableColumnInfo>& columns) const + { TKeyDesc& keyDesc = *entry.KeyDescription; - // check key types - if (keyDesc.KeyColumnTypes.size() > keyColumnTypes.size()) { + // check key types + if (keyDesc.KeyColumnTypes.size() > keyColumnTypes.size()) { SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::TypeCheckFailed); - return false; + return false; } for (ui32 i : xrange(keyDesc.KeyColumnTypes.size())) { - if (keyDesc.KeyColumnTypes[i] != keyColumnTypes[i]) { + if (keyDesc.KeyColumnTypes[i] != keyColumnTypes[i]) { SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::TypeCheckFailed); - return false; + return false; } } - // check operations + // check operations keyDesc.ColumnInfos.clear(); keyDesc.ColumnInfos.reserve(keyDesc.Columns.size()); for (auto& columnOp : keyDesc.Columns) { @@ -1711,10 +1711,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { if (IsSysTable() && columnOp.Operation == TKeyDesc::EColumnOperation::InplaceUpdate) { SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::OperationNotSupported); - return false; + return false; } - const auto* column = columns.FindPtr(columnOp.Column); + const auto* column = columns.FindPtr(columnOp.Column); if (!column) { entry.Status = TResolve::EStatus::TypeCheckError; @@ -1744,36 +1744,36 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } } - return true; - } - - void FillSystemViewEntry(TResolveContext* context, TResolve::TEntry& entry, - NSysView::ISystemViewResolver::ETarget target) const - { - TKeyDesc& keyDesc = *entry.KeyDescription; - auto sysViewInfo = keyDesc.TableId.SysViewInfo; - - auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target); - if (!schema) { + return true; + } + + void FillSystemViewEntry(TResolveContext* context, TResolve::TEntry& entry, + NSysView::ISystemViewResolver::ETarget target) const + { + TKeyDesc& keyDesc = *entry.KeyDescription; + auto sysViewInfo = keyDesc.TableId.SysViewInfo; + + auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target); + if (!schema) { return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); - } - + } + if (!CheckColumns(context, entry, schema->KeyColumnTypes, schema->Columns)) { - return; - } - - if (keyDesc.Status != TKeyDesc::EStatus::Unknown) { - ++context->Request->ErrorCount; - return; - } - - entry.DomainInfo = DomainInfo; - keyDesc.SecurityObject = SecurityObject; - - entry.Status = TResolve::EStatus::OkData; - keyDesc.Status = TKeyDesc::EStatus::Ok; - } - + return; + } + + if (keyDesc.Status != TKeyDesc::EStatus::Unknown) { + ++context->Request->ErrorCount; + return; + } + + entry.DomainInfo = DomainInfo; + keyDesc.SecurityObject = SecurityObject; + + entry.Status = TResolve::EStatus::OkData; + keyDesc.Status = TKeyDesc::EStatus::Ok; + } + void FillEntry(TResolveContext* context, TResolve::TEntry& entry, const TResponseProps& props = TResponseProps()) const { SBC_LOG_D("FillEntry for TResolve" << ": self# " << Owner->SelfId() @@ -1781,27 +1781,27 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { << ", entry# " << entry.ToString() << ", props# " << props.ToString()); - TKeyDesc& keyDesc = *entry.KeyDescription; - + TKeyDesc& keyDesc = *entry.KeyDescription; + if (props.IsSync && props.Partial) { return SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists); } if (Status && Status == NKikimrScheme::StatusPathDoesNotExist) { return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); - } - + } + if (!Status || Status != NKikimrScheme::StatusSuccess) { return SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists); - } - + } + if (!keyDesc.TableId.SysViewInfo.empty()) { - if (Kind == TNavigate::KindPath) { - auto split = SplitPath(Path); - if (split.size() == 1) { + if (Kind == TNavigate::KindPath) { + auto split = SplitPath(Path); + if (split.size() == 1) { return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain); - } - } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) { + } + } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) { return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain); } else if (Kind == TNavigate::KindOlapStore) { FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::OlapStore); @@ -1819,23 +1819,23 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { keyDesc.Partitions.back().Range = TKeyDesc::TPartitionRangeInfo(); } return; - } + } return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); - } - - if (!Created) { + } + + if (!Created) { return SetError(context, entry, TResolve::EStatus::NotMaterialized, TKeyDesc::EStatus::NotExists); - } - + } + entry.Kind = TableKind; - entry.DomainInfo = DomainInfo; - + entry.DomainInfo = DomainInfo; + if (!CheckColumns(context, entry, KeyColumnTypes, Columns)) { - return; - } - - // fill partition info + return; + } + + // fill partition info if (IsSysTable()) { if (keyDesc.Range.Point) { ui64 shard = 0; @@ -1866,7 +1866,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { keyDesc.Status = TKeyDesc::EStatus::OperationNotSupported; } - // fill ACL info + // fill ACL info keyDesc.SecurityObject = SecurityObject; if (keyDesc.Status != TKeyDesc::EStatus::Unknown) { @@ -1893,7 +1893,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { bool Created; ui64 CreateStep; TPathId PathId; - TString Path; + TString Path; TSet<ui64> AbandonedSchemeShardsIds; TIntrusivePtr<TSecurityObject> SecurityObject; NSchemeCache::TDomainInfo::TPtr DomainInfo; @@ -2010,19 +2010,19 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { return CreateSubscriber(pathId, pathId.OwnerId, domainOwnerId); } - template <typename TContextPtr, typename TEntry, typename TPathExtractor, typename TTabletIdExtractor> - void HandleEntry(TContextPtr context, TEntry& entry, size_t index, - TPathExtractor pathExtractor, TTabletIdExtractor tabletIdExtractor) - { - auto path = pathExtractor(entry); - TCacheItem* cacheItem = Cache.FindPtr(path); - - if (!cacheItem) { - const EPathType pathType = PathType(path); - switch (pathType) { - case EPathType::RegularPath: - { - const ui64 tabletId = tabletIdExtractor(entry); + template <typename TContextPtr, typename TEntry, typename TPathExtractor, typename TTabletIdExtractor> + void HandleEntry(TContextPtr context, TEntry& entry, size_t index, + TPathExtractor pathExtractor, TTabletIdExtractor tabletIdExtractor) + { + auto path = pathExtractor(entry); + TCacheItem* cacheItem = Cache.FindPtr(path); + + if (!cacheItem) { + const EPathType pathType = PathType(path); + switch (pathType) { + case EPathType::RegularPath: + { + const ui64 tabletId = tabletIdExtractor(entry); const ui32 domainId = AppData()->DomainsInfo->GetDomainUidByTabletId(tabletId); if (tabletId == ui64(NSchemeShard::InvalidTabletId) || domainId == TDomainsInfo::BadDomainId) { return SetRootUnknown(context.Get(), entry); @@ -2053,33 +2053,33 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } cacheItem = &Cache.Upsert(path, TCacheItem(this, CreateSubscriber(path, tabletId, domainOwnerId), false)); - break; + break; } - case EPathType::SysPath: - cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true)); - cacheItem->FillAsSysPath(); - break; - case EPathType::SysLocksV1: - case EPathType::SysLocksV2: - cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true)); - cacheItem->FillAsSysLocks(pathType == EPathType::SysLocksV2); - break; + case EPathType::SysPath: + cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true)); + cacheItem->FillAsSysPath(); + break; + case EPathType::SysLocksV1: + case EPathType::SysLocksV2: + cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true)); + cacheItem->FillAsSysLocks(pathType == EPathType::SysLocksV2); + break; } } - Cache.Promote(path); - + Cache.Promote(path); + if (cacheItem->IsFilled() && !entry.SyncVersion) { cacheItem->FillEntry(context.Get(), entry); } - if (entry.SyncVersion) { - cacheItem->AddInFlight(context, index, true); + if (entry.SyncVersion) { + cacheItem->AddInFlight(context, index, true); + } + + if (!cacheItem->IsFilled()) { + cacheItem->AddInFlight(context, index, false); } - - if (!cacheItem->IsFilled()) { - cacheItem->AddInFlight(context, index, false); - } } TCacheItem* SwapSubscriberAndUpsert(TCacheItem* byPath, const TPathId& notifyPathId, const TString& notifyPath) { @@ -2369,59 +2369,59 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { void Handle(TEvTxProxySchemeCache::TEvNavigateKeySet::TPtr& ev) { SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySet" << ": self# " << SelfId() - << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry)); + << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry)); if (MaybeRunDbResolver(ev)) { return; } TIntrusivePtr<TNavigateContext> context(new TNavigateContext(ev->Sender, ev->Get()->Request, Now())); - - for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) { - auto& entry = context->Request->ResultSet[i]; - if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) { - auto pathExtractor = [this](TNavigate::TEntry& entry) { + for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) { + auto& entry = context->Request->ResultSet[i]; + + if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) { + auto pathExtractor = [this](TNavigate::TEntry& entry) { if (AppData()->FeatureFlags.GetEnableSystemViews() - && (entry.Operation == TNavigate::OpPath || entry.Operation == TNavigate::OpTable)) - { - NSysView::ISystemViewResolver::TSystemViewPath sysViewPath; - if (SystemViewResolver->IsSystemViewPath(entry.Path, sysViewPath)) { - entry.TableId.SysViewInfo = sysViewPath.ViewName; - return CanonizePath(sysViewPath.Parent); - } - } - - TString path = CanonizePath(entry.Path); - return path ? path : TString("/"); - }; - - auto tabletIdExtractor = [this](const TNavigate::TEntry& entry) { - if (entry.Path.empty()) { + && (entry.Operation == TNavigate::OpPath || entry.Operation == TNavigate::OpTable)) + { + NSysView::ISystemViewResolver::TSystemViewPath sysViewPath; + if (SystemViewResolver->IsSystemViewPath(entry.Path, sysViewPath)) { + entry.TableId.SysViewInfo = sysViewPath.ViewName; + return CanonizePath(sysViewPath.Parent); + } + } + + TString path = CanonizePath(entry.Path); + return path ? path : TString("/"); + }; + + auto tabletIdExtractor = [this](const TNavigate::TEntry& entry) { + if (entry.Path.empty()) { return ui64(NSchemeShard::InvalidTabletId); - } - - auto it = Roots.find(entry.Path.front()); - if (it == Roots.end()) { + } + + auto it = Roots.find(entry.Path.front()); + if (it == Roots.end()) { return ui64(NSchemeShard::InvalidTabletId); - } - - return it->second; - }; - - HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor); - } else { - auto pathExtractor = [](const TNavigate::TEntry& entry) { - return entry.TableId.PathId; - }; - - auto tabletIdExtractor = [](const TNavigate::TEntry& entry) { - return entry.TableId.PathId.OwnerId; - }; - - HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor); - } - } + } + + return it->second; + }; + + HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor); + } else { + auto pathExtractor = [](const TNavigate::TEntry& entry) { + return entry.TableId.PathId; + }; + + auto tabletIdExtractor = [](const TNavigate::TEntry& entry) { + return entry.TableId.PathId.OwnerId; + }; + + HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor); + } + } MaybeComplete(context); } @@ -2436,22 +2436,22 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } TIntrusivePtr<TResolveContext> context(new TResolveContext(ev->Sender, ev->Get()->Request, Now())); - + auto pathExtractor = [](const TResolve::TEntry& entry) { const TKeyDesc* keyDesc = entry.KeyDescription.Get(); Y_VERIFY(keyDesc != nullptr); - return TPathId(keyDesc->TableId.PathId); + return TPathId(keyDesc->TableId.PathId); }; auto tabletIdExtractor = [](const TResolve::TEntry& entry) { - return entry.KeyDescription->TableId.PathId.OwnerId; + return entry.KeyDescription->TableId.PathId.OwnerId; }; - for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) { - auto& entry = context->Request->ResultSet[i]; - HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor); - } - + for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) { + auto& entry = context->Request->ResultSet[i]; + HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor); + } + MaybeComplete(context); } @@ -2538,7 +2538,7 @@ public: TSchemeCache(NSchemeCache::TSchemeCacheConfig* config) : Counters(config->Counters) , Cache(TDuration::Minutes(2), TThis::Now) - , SystemViewResolver(NSysView::CreateSystemViewResolver()) + , SystemViewResolver(NSysView::CreateSystemViewResolver()) { for (const auto& root : config->Roots) { Roots.emplace(root.Name, root.RootSchemeShard); @@ -2578,7 +2578,7 @@ private: TCounters Counters; TDoubleIndexedCache<TString, TPathId, TCacheItem, TMerger, TEvicter> Cache; - THolder<NSysView::ISystemViewResolver> SystemViewResolver; + THolder<NSysView::ISystemViewResolver> SystemViewResolver; TActorId WatchCache; diff --git a/ydb/core/tx/scheme_board/cache_ut.cpp b/ydb/core/tx/scheme_board/cache_ut.cpp index 969cf901fb..c27b3517b3 100644 --- a/ydb/core/tx/scheme_board/cache_ut.cpp +++ b/ydb/core/tx/scheme_board/cache_ut.cpp @@ -47,9 +47,9 @@ public: UNIT_TEST(RacyRecreateAndSync); UNIT_TEST(RacyCreateAndSync); UNIT_TEST(CheckAccess); - UNIT_TEST(CheckSystemViewAccess); - UNIT_TEST(SystemView); - UNIT_TEST(SysLocks); + UNIT_TEST(CheckSystemViewAccess); + UNIT_TEST(SystemView); + UNIT_TEST(SysLocks); UNIT_TEST(TableSchemaVersion); UNIT_TEST(MigrationCommon); UNIT_TEST(MigrationCommit); @@ -66,9 +66,9 @@ public: void RacyRecreateAndSync(); void RacyCreateAndSync(); void CheckAccess(); - void CheckSystemViewAccess(); - void SystemView(); - void SysLocks(); + void CheckSystemViewAccess(); + void SystemView(); + void SysLocks(); void TableSchemaVersion(); void MigrationCommon(); void MigrationCommit(); @@ -78,18 +78,18 @@ public: void WatchRoot(); protected: - TNavigate::TEntry TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus, + TNavigate::TEntry TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus, const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired); - + TNavigate::TEntry TestNavigate(const TString& path, TNavigate::EStatus expectedStatus = TNavigate::EStatus::Ok, const TString& sid = TString(), TNavigate::EOp op = TNavigate::EOp::OpPath, bool showPrivatePath = false, bool redirectRequired = true, bool syncVersion = false); - TNavigate::TEntry TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus, - const TString& expectedPath, const TString& sid = TString(), + TNavigate::TEntry TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus, + const TString& expectedPath, const TString& sid = TString(), TNavigate::EOp op = TNavigate::EOp::OpPath, bool showPrivatePath = false); - - TResolve::TEntry TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus = TResolve::EStatus::OkData, + + TResolve::TEntry TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus = TResolve::EStatus::OkData, const TString& sid = TString()); TActorId TestWatch(const TPathId& pathId, const TActorId& watcher = {}, ui64 key = 0); @@ -108,15 +108,15 @@ UNIT_TEST_SUITE_REGISTRATION(TCacheTest); void TCacheTest::Navigate() { TestNavigate("/Root", TNavigate::EStatus::Ok); - - ui64 txId = 100; - - TestMkDir(*Context, ++txId, "/Root", "DirA"); + + ui64 txId = 100; + + TestMkDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - - auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); - - TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/Root/DirA"); + + auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); + + TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/Root/DirA"); TestNavigateByTableId(TTableId(1ull << 56, 1), TNavigate::EStatus::RootUnknown, ""); } @@ -166,9 +166,9 @@ void TCacheTest::Recreate() { TestRmDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); - auto pathId = ev->Get()->PathId; - TTableId tableId(pathId.OwnerId, pathId.LocalPathId); - TestResolve(tableId, TResolve::EStatus::PathErrorNotExist); + auto pathId = ev->Get()->PathId; + TTableId tableId(pathId.OwnerId, pathId.LocalPathId); + TestResolve(tableId, TResolve::EStatus::PathErrorNotExist); TestMkDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); @@ -293,49 +293,49 @@ void TCacheTest::CheckAccess() { TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/Root/DirA", "user0@builtin"); } -void TCacheTest::CheckSystemViewAccess() { - ui64 txId = 100; - TestCreateSubDomain(*Context, ++txId, "/Root", "Name: \"SubDomainA\""); +void TCacheTest::CheckSystemViewAccess() { + ui64 txId = 100; + TestCreateSubDomain(*Context, ++txId, "/Root", "Name: \"SubDomainA\""); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - TestModifyACL(*Context, ++txId, "/Root", "SubDomainA", TString(), "user0@builtin"); - - auto entry = TestNavigate("/Root/SubDomainA/.sys/partition_stats", - TNavigate::EStatus::Ok, TString(), TNavigate::OpTable); - - auto tableId = entry.TableId; - UNIT_ASSERT_VALUES_EQUAL(tableId.SysViewInfo, "partition_stats"); - - TestNavigate("/Root/SubDomainA/.sys/partition_stats", - TNavigate::EStatus::Ok, "user0@builtin", TNavigate::OpTable); - - TestNavigate("/Root/SubDomainA/.sys/partition_stats", - TNavigate::EStatus::PathErrorUnknown, "user1@builtin", TNavigate::OpTable); - - TestResolve(tableId, TResolve::EStatus::OkData); - TestResolve(tableId, TResolve::EStatus::OkData, "user0@builtin"); - TestResolve(tableId, TResolve::EStatus::PathErrorNotExist, "user1@builtin"); -} - + TestModifyACL(*Context, ++txId, "/Root", "SubDomainA", TString(), "user0@builtin"); + + auto entry = TestNavigate("/Root/SubDomainA/.sys/partition_stats", + TNavigate::EStatus::Ok, TString(), TNavigate::OpTable); + + auto tableId = entry.TableId; + UNIT_ASSERT_VALUES_EQUAL(tableId.SysViewInfo, "partition_stats"); + + TestNavigate("/Root/SubDomainA/.sys/partition_stats", + TNavigate::EStatus::Ok, "user0@builtin", TNavigate::OpTable); + + TestNavigate("/Root/SubDomainA/.sys/partition_stats", + TNavigate::EStatus::PathErrorUnknown, "user1@builtin", TNavigate::OpTable); + + TestResolve(tableId, TResolve::EStatus::OkData); + TestResolve(tableId, TResolve::EStatus::OkData, "user0@builtin"); + TestResolve(tableId, TResolve::EStatus::PathErrorNotExist, "user1@builtin"); +} + void TCacheTest::SystemView() { - auto entry = TestNavigate("/Root/.sys/partition_stats", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable); - - auto tableId = entry.TableId; - UNIT_ASSERT_VALUES_EQUAL(tableId.SysViewInfo, "partition_stats"); - - TestNavigateByTableId(tableId, TNavigate::EStatus::Ok, "/Root/.sys/partition_stats", TString(), TNavigate::OpTable); + auto entry = TestNavigate("/Root/.sys/partition_stats", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable); + + auto tableId = entry.TableId; + UNIT_ASSERT_VALUES_EQUAL(tableId.SysViewInfo, "partition_stats"); + + TestNavigateByTableId(tableId, TNavigate::EStatus::Ok, "/Root/.sys/partition_stats", TString(), TNavigate::OpTable); +} + +void TCacheTest::SysLocks() { + { + auto entry = TestNavigate("/sys/locks", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable, true); + TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/sys/locks", TString(), TNavigate::OpTable, true); + } + { + auto entry = TestNavigate("/sys/locks2", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable, true); + TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/sys/locks2", TString(), TNavigate::OpTable, true); + } } -void TCacheTest::SysLocks() { - { - auto entry = TestNavigate("/sys/locks", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable, true); - TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/sys/locks", TString(), TNavigate::OpTable, true); - } - { - auto entry = TestNavigate("/sys/locks2", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable, true); - TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/sys/locks2", TString(), TNavigate::OpTable, true); - } -} - void TCacheTest::TableSchemaVersion() { ui64 txId = 100; TestCreateTable(*Context, ++txId, "/Root", R"( @@ -366,11 +366,11 @@ void TCacheTest::TableSchemaVersion() { } } -TNavigate::TEntry TCacheTest::TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus, +TNavigate::TEntry TCacheTest::TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus, const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired) -{ +{ auto& entry = request->ResultSet.back(); - entry.Operation = op; + entry.Operation = op; entry.ShowPrivatePath = showPrivatePath; entry.RedirectRequired = redirectRequired; @@ -390,40 +390,40 @@ TNavigate::TEntry TCacheTest::TestNavigateImpl(THolder<TNavigate> request, TNavi return result; } -TNavigate::TEntry TCacheTest::TestNavigate(const TString& path, TNavigate::EStatus expectedStatus, +TNavigate::TEntry TCacheTest::TestNavigate(const TString& path, TNavigate::EStatus expectedStatus, const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired, bool syncVersion) -{ - auto request = MakeHolder<TNavigate>(); - request->ResultSet.push_back({}); - auto& entry = request->ResultSet.back(); - - entry.Path = SplitPath(path); +{ + auto request = MakeHolder<TNavigate>(); + request->ResultSet.push_back({}); + auto& entry = request->ResultSet.back(); + + entry.Path = SplitPath(path); entry.SyncVersion = syncVersion; - + auto result = TestNavigateImpl(std::move(request), expectedStatus, sid, op, showPrivatePath, redirectRequired); - return result; -} - -TNavigate::TEntry TCacheTest::TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus, + return result; +} + +TNavigate::TEntry TCacheTest::TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus, const TString& expectedPath, const TString& sid, TNavigate::EOp op, bool showPrivatePath) -{ - auto request = MakeHolder<TNavigate>(); - request->ResultSet.push_back({}); - auto& entry = request->ResultSet.back(); - - entry.TableId = tableId; - entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; - +{ + auto request = MakeHolder<TNavigate>(); + request->ResultSet.push_back({}); + auto& entry = request->ResultSet.back(); + + entry.TableId = tableId; + entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; + auto result = TestNavigateImpl(std::move(request), expectedStatus, sid, op, showPrivatePath, true); - UNIT_ASSERT_VALUES_EQUAL(CanonizePath(result.Path), expectedPath); - return result; -} - -TResolve::TEntry TCacheTest::TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus, const TString& sid) { + UNIT_ASSERT_VALUES_EQUAL(CanonizePath(result.Path), expectedPath); + return result; +} + +TResolve::TEntry TCacheTest::TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus, const TString& sid) { auto request = MakeHolder<TResolve>(); auto keyDesc = MakeHolder<TKeyDesc>( - tableId, + tableId, TTableRange({}), TKeyDesc::ERowOperation::Unknown, TVector<NScheme::TTypeId>(), TVector<TKeyDesc::TColumnOp>() diff --git a/ydb/core/tx/scheme_board/replica.cpp b/ydb/core/tx/scheme_board/replica.cpp index a7aa4dd8f4..638656e91d 100644 --- a/ydb/core/tx/scheme_board/replica.cpp +++ b/ydb/core/tx/scheme_board/replica.cpp @@ -13,7 +13,7 @@ #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/core/log.h> -#include <library/cpp/actors/core/memory_track.h> +#include <library/cpp/actors/core/memory_track.h> #include <util/generic/hash.h> #include <util/generic/map.h> @@ -53,7 +53,7 @@ private: , WaitForAck(false) , LastVersionSent(0) , NotifiedStrongly(true) - , SyncRequestCookie(0) + , SyncRequestCookie(0) , SyncResponseCookie(0) { } @@ -145,8 +145,8 @@ private: public: class TDescription { - static constexpr char MemoryLabelDescribeResult[] = "SchemeBoard/Replica/DescribeSchemeResult"; - + static constexpr char MemoryLabelDescribeResult[] = "SchemeBoard/Replica/DescribeSchemeResult"; + void Notify() { if (!Subscribers) { return; @@ -166,59 +166,59 @@ public: MultiSend(subscribers, Owner->SelfId(), std::move(notify)); } - void CalculateResultSize() { - ResultSize = DescribeSchemeResult.ByteSizeLong(); - } - - size_t FullSize() const { - size_t size = ResultSize; - if (PreSerializedDescribeSchemeResult) { - size += PreSerializedDescribeSchemeResult->size(); - } - return size; - } - - void TrackMemory() const { - NActors::NMemory::TLabel<MemoryLabelDescribeResult>::Add(FullSize()); - } - - void UntrackMemory() const { - NActors::NMemory::TLabel<MemoryLabelDescribeResult>::Sub(FullSize()); - } - - void Move(TDescription&& other) { - UntrackMemory(); - other.UntrackMemory(); - - Owner = other.Owner; - Path = std::move(other.Path); - PathId = std::move(other.PathId); - DescribeSchemeResult = std::move(other.DescribeSchemeResult); - PreSerializedDescribeSchemeResult = std::move(other.PreSerializedDescribeSchemeResult); - ExplicitlyDeleted = other.ExplicitlyDeleted; - Subscribers = std::move(other.Subscribers); - - ResultSize = other.ResultSize; - other.ResultSize = 0; - TrackNotify = other.TrackNotify; - - TrackMemory(); - other.TrackMemory(); - } - + void CalculateResultSize() { + ResultSize = DescribeSchemeResult.ByteSizeLong(); + } + + size_t FullSize() const { + size_t size = ResultSize; + if (PreSerializedDescribeSchemeResult) { + size += PreSerializedDescribeSchemeResult->size(); + } + return size; + } + + void TrackMemory() const { + NActors::NMemory::TLabel<MemoryLabelDescribeResult>::Add(FullSize()); + } + + void UntrackMemory() const { + NActors::NMemory::TLabel<MemoryLabelDescribeResult>::Sub(FullSize()); + } + + void Move(TDescription&& other) { + UntrackMemory(); + other.UntrackMemory(); + + Owner = other.Owner; + Path = std::move(other.Path); + PathId = std::move(other.PathId); + DescribeSchemeResult = std::move(other.DescribeSchemeResult); + PreSerializedDescribeSchemeResult = std::move(other.PreSerializedDescribeSchemeResult); + ExplicitlyDeleted = other.ExplicitlyDeleted; + Subscribers = std::move(other.Subscribers); + + ResultSize = other.ResultSize; + other.ResultSize = 0; + TrackNotify = other.TrackNotify; + + TrackMemory(); + other.TrackMemory(); + } + public: explicit TDescription(TReplica* owner, const TString& path) : Owner(owner) , Path(path) { - TrackMemory(); + TrackMemory(); } explicit TDescription(TReplica* owner, const TPathId& pathId) : Owner(owner) , PathId(pathId) { - TrackMemory(); + TrackMemory(); } explicit TDescription(TReplica* owner, const TString& path, const TPathId& pathId) @@ -226,7 +226,7 @@ public: , Path(path) , PathId(pathId) { - TrackMemory(); + TrackMemory(); } explicit TDescription( @@ -237,8 +237,8 @@ public: , PathId(pathId) , DescribeSchemeResult(std::move(describeSchemeResult)) { - CalculateResultSize(); - TrackMemory(); + CalculateResultSize(); + TrackMemory(); } explicit TDescription( @@ -251,28 +251,28 @@ public: , PathId(pathId) , DescribeSchemeResult(std::move(describeSchemeResult)) { - CalculateResultSize(); - TrackMemory(); - } - - TDescription(TDescription&& other) { - TrackMemory(); - Move(std::move(other)); - } - - TDescription& operator=(TDescription&& other) { - Move(std::move(other)); - return *this; - } - - TDescription(const TDescription& other) = delete; - TDescription& operator=(const TDescription& other) = delete; - - ~TDescription() - { - UntrackMemory(); - } - + CalculateResultSize(); + TrackMemory(); + } + + TDescription(TDescription&& other) { + TrackMemory(); + Move(std::move(other)); + } + + TDescription& operator=(TDescription&& other) { + Move(std::move(other)); + return *this; + } + + TDescription(const TDescription& other) = delete; + TDescription& operator=(const TDescription& other) = delete; + + ~TDescription() + { + UntrackMemory(); + } + bool operator<(const TDescription& other) const { return GetVersion() < other.GetVersion(); } @@ -309,11 +309,11 @@ public: << ", rigth pathId# " << other.PathId << ", rigth version# " << other.GetVersion()); - UntrackMemory(); - other.UntrackMemory(); - TrackNotify = false; - other.TrackNotify = false; - + UntrackMemory(); + other.UntrackMemory(); + TrackNotify = false; + other.TrackNotify = false; + if (*this > other) { other.DescribeSchemeResult.Swap(&DescribeSchemeResult); other.PreSerializedDescribeSchemeResult.Clear(); @@ -328,14 +328,14 @@ public: Notify(); } - CalculateResultSize(); - other.CalculateResultSize(); - - TrackNotify = true; - other.TrackNotify = true; - TrackMemory(); - other.TrackMemory(); - + CalculateResultSize(); + other.CalculateResultSize(); + + TrackNotify = true; + other.TrackNotify = true; + TrackMemory(); + other.TrackMemory(); + Subscribers.insert(other.Subscribers.begin(), other.Subscribers.end()); return *this; @@ -388,11 +388,11 @@ public: void Clear() { ExplicitlyDeleted = true; - UntrackMemory(); + UntrackMemory(); TDescribeSchemeResult().Swap(&DescribeSchemeResult); PreSerializedDescribeSchemeResult.Clear(); - ResultSize = 0; - TrackMemory(); + ResultSize = 0; + TrackMemory(); Notify(); } @@ -415,13 +415,13 @@ public: if (!PreSerializedDescribeSchemeResult) { TString serialized; Y_PROTOBUF_SUPPRESS_NODISCARD DescribeSchemeResult.SerializeToString(&serialized); - if (TrackNotify) { - UntrackMemory(); - } + if (TrackNotify) { + UntrackMemory(); + } PreSerializedDescribeSchemeResult = std::move(serialized); - if (TrackNotify) { - TrackMemory(); - } + if (TrackNotify) { + TrackMemory(); + } } notify->SetDescribeSchemeResult(*PreSerializedDescribeSchemeResult); @@ -479,10 +479,10 @@ public: // subscribers THashMap<TActorId, TSubscriberInfo> Subscribers; - // memory tracking - size_t ResultSize = 0; - bool TrackNotify = true; - + // memory tracking + size_t ResultSize = 0; + bool TrackNotify = true; + }; // TDescription struct TMerger { diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h index 5c74daa5e8..26a07897a1 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.h +++ b/ydb/core/tx/scheme_cache/scheme_cache.h @@ -202,15 +202,15 @@ struct TSchemeCacheNavigate { }; struct TEntry { - enum class ERequestType : ui8 { - ByPath, - ByTableId - }; + enum class ERequestType : ui8 { + ByPath, + ByTableId + }; // in TVector<TString> Path; TTableId TableId; - ERequestType RequestType = ERequestType::ByPath; + ERequestType RequestType = ERequestType::ByPath; EOp Operation = OpUnknown; bool RedirectRequired = true; bool ShowPrivatePath = false; @@ -523,8 +523,8 @@ public: }; }; -inline TActorId MakeSchemeCacheID() { +inline TActorId MakeSchemeCacheID() { return TActorId(0, TStringBuf("SchmCcheSrv")); } - + } // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard.h b/ydb/core/tx/schemeshard/schemeshard.h index 17d3f5947e..d33173508c 100644 --- a/ydb/core/tx/schemeshard/schemeshard.h +++ b/ydb/core/tx/schemeshard/schemeshard.h @@ -235,8 +235,8 @@ struct TEvSchemeShard { TEvDescribeScheme(TTableId tableId) { - Record.SetSchemeshardId(tableId.PathId.OwnerId); - Record.SetPathId(tableId.PathId.LocalPathId); + Record.SetSchemeshardId(tableId.PathId.OwnerId); + Record.SetPathId(tableId.PathId.LocalPathId); } TEvDescribeScheme(NKikimr::TPathId pathId) @@ -425,7 +425,7 @@ struct TEvSchemeShard { ui64 effectiveACLVersion, ui64 subdomainVersion, ui64 userAttrsVersion, - ui64 tenantHive, + ui64 tenantHive, ui64 tenantSysViewProcessor, const TString& rootACL) { @@ -440,7 +440,7 @@ struct TEvSchemeShard { Record.SetUserAttributesVersion(userAttrsVersion); Record.SetTenantHive(tenantHive); - Record.SetTenantSysViewProcessor(tenantSysViewProcessor); + Record.SetTenantSysViewProcessor(tenantSysViewProcessor); Record.SetTenantRootACL(rootACL); } @@ -483,10 +483,10 @@ struct TEvSchemeShard { void SetTenantHive(ui64 hive) { Record.SetTenantHive(hive); } - - void SetTenantSysViewProcessor(ui64 svp) { - Record.SetTenantSysViewProcessor(svp); - } + + void SetTenantSysViewProcessor(ui64 svp) { + Record.SetTenantSysViewProcessor(svp); + } void SetUpdateTenantRootACL(const TString& acl) { Record.SetUpdateTenantRootACL(acl); diff --git a/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp b/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp index 20fe55ed24..855553cddc 100644 --- a/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp @@ -89,9 +89,9 @@ struct TSchemeShard::TTxDeleteTabletReply : public TSchemeShard::TRwTxBase { case ETabletType::PersQueueReadBalancer: Self->TabletCounters->Simple()[COUNTER_PQ_RB_SHARD_COUNT].Sub(1); break; - case ETabletType::SysViewProcessor: + case ETabletType::SysViewProcessor: Self->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Sub(1); - break; + break; case ETabletType::ColumnShard: Self->TabletCounters->Simple()[COUNTER_OLAP_COLUMN_SHARDS].Sub(-1); break; diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 8a44d513c6..650506549e 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -2022,7 +2022,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { Y_VERIFY(!partitions.empty()); Y_VERIFY(Self->Tables.contains(prevTableId)); TTableInfo::TPtr tableInfo = Self->Tables.at(prevTableId); - Self->SetPartitioning(prevTableId, tableInfo, std::move(partitions)); + Self->SetPartitioning(prevTableId, tableInfo, std::move(partitions)); partitions.clear(); } @@ -2055,7 +2055,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { Y_VERIFY(!partitions.empty()); Y_VERIFY(Self->Tables.contains(prevTableId)); TTableInfo::TPtr tableInfo = Self->Tables.at(prevTableId); - Self->SetPartitioning(prevTableId, tableInfo, std::move(partitions)); + Self->SetPartitioning(prevTableId, tableInfo, std::move(partitions)); } } @@ -3710,9 +3710,9 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { case ETabletType::Hive: Self->TabletCounters->Simple()[COUNTER_SUB_DOMAIN_HIVE_COUNT].Add(1); break; - case ETabletType::SysViewProcessor: - Self->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Add(1); - break; + case ETabletType::SysViewProcessor: + Self->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Add(1); + break; case ETabletType::ColumnShard: Self->TabletCounters->Simple()[COUNTER_OLAP_COLUMN_SHARDS].Add(1); break; diff --git a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp index a771d55612..74399680cd 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp @@ -328,9 +328,9 @@ struct TSchemeShard::TTxInitTenantSchemeShard : public TSchemeShard::TRwTxBase { if (processingParams.HasHive()) { RegisterShard(db, subdomain, TVector<ui64>{processingParams.GetHive()}, TTabletTypes::Hive); } - if (processingParams.HasSysViewProcessor()) { - RegisterShard(db, subdomain, TVector<ui64>{processingParams.GetSysViewProcessor()}, TTabletTypes::SysViewProcessor); - } + if (processingParams.HasSysViewProcessor()) { + RegisterShard(db, subdomain, TVector<ui64>{processingParams.GetSysViewProcessor()}, TTabletTypes::SysViewProcessor); + } subdomain->Initialize(Self->ShardInfos); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.cpp b/ydb/core/tx/schemeshard/schemeshard__operation.cpp index 9d6c966229..daf1218bcf 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation.cpp @@ -210,7 +210,7 @@ THolder<TProposeResponse> TSchemeShard::IgniteOperation(TProposeRequest& request struct TSchemeShard::TTxOperationPropose: public NTabletFlatExecutor::TTransactionBase<TSchemeShard> { using TBase = NTabletFlatExecutor::TTransactionBase<TSchemeShard>; - + TProposeRequest::TPtr Request; THolder<TProposeResponse> Response = nullptr; @@ -538,14 +538,14 @@ TOperation::TConsumeQuotaResult TOperation::ConsumeQuota(const TTxTransaction& t return result; } -TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context) { - TSplitTransactionsResult result; - +TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context) { + TSplitTransactionsResult result; + const TPath parentPath = TPath::Resolve(tx.GetWorkingDir(), context.SS); { TPath::TChecker checks = parentPath.Check(); checks - .NotUnderDomainUpgrade() + .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() .NotDeleted() @@ -554,8 +554,8 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx .IsLikeDirectory(); if (!checks) { - result.Transactions.push_back(tx); - return result; + result.Transactions.push_back(tx); + return result; } } @@ -567,8 +567,8 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable: if (tx.GetCreateTable().HasCopyFromTable()) { - result.Transactions.push_back(tx); - return result; + result.Transactions.push_back(tx); + return result; } targetName = tx.GetCreateTable().GetName(); break; @@ -604,13 +604,13 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx targetName = tx.GetCreateColumnTable().GetName(); break; default: - result.Transactions.push_back(tx); - return result; + result.Transactions.push_back(tx); + return result; } if (!targetName || targetName.StartsWith('/') || targetName.EndsWith('/')) { - result.Transactions.push_back(tx); - return result; + result.Transactions.push_back(tx); + return result; } TPath path = TPath::Resolve(JoinPath(tx.GetWorkingDir(), targetName), context.SS); @@ -640,9 +640,9 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx } if (!checks) { - result.Status = checks.GetStatus(&result.Reason); - result.Transactions.push_back(tx); - return result; + result.Status = checks.GetStatus(&result.Reason); + result.Transactions.push_back(tx); + return result; } const TString name = path.LeafName(); @@ -694,7 +694,7 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx Y_UNREACHABLE(); } - result.Transactions.push_back(create); + result.Transactions.push_back(create); if (exists) { return result; @@ -703,9 +703,9 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx while (path != parentPath) { TPath::TChecker checks = path.Check(); - checks - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard(); + checks + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard(); if (path.IsResolved()) { checks.IsResolved(); @@ -733,7 +733,7 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx checks .IsValidLeafName() .DepthLimit() - .PathsLimit(result.Transactions.size() + 1); + .PathsLimit(result.Transactions.size() + 1); } if (checks && path.Parent().IsResolved()) { @@ -741,10 +741,10 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx } if (!checks) { - result.Status = checks.GetStatus(&result.Reason); - result.Transactions.clear(); - result.Transactions.push_back(tx); - return result; + result.Status = checks.GetStatus(&result.Reason); + result.Transactions.clear(); + result.Transactions.push_back(tx); + return result; } const TString name = path.LeafName(); @@ -755,10 +755,10 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx mkdir.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpMkDir); mkdir.SetWorkingDir(path.PathString()); mkdir.MutableMkDir()->SetName(name); - result.Transactions.push_back(mkdir); + result.Transactions.push_back(mkdir); } - Reverse(result.Transactions.begin(), result.Transactions.end()); + Reverse(result.Transactions.begin(), result.Transactions.end()); return result; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.h b/ydb/core/tx/schemeshard/schemeshard__operation.h index b4fe170b5e..52f1848aa1 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation.h @@ -51,12 +51,12 @@ struct TOperation: TSimpleRefCount<TOperation> { TString Reason; }; - struct TSplitTransactionsResult { + struct TSplitTransactionsResult { NKikimrScheme::EStatus Status = NKikimrScheme::StatusSuccess; - TString Reason; - TVector<TTxTransaction> Transactions; - }; - + TString Reason; + TVector<TTxTransaction> Transactions; + }; + TOperation(TTxId txId) : TxId(txId) {} @@ -66,7 +66,7 @@ struct TOperation: TSimpleRefCount<TOperation> { static TConsumeQuotaResult ConsumeQuota(const TTxTransaction& tx, const TOperationContext& context); - static TSplitTransactionsResult SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context); + static TSplitTransactionsResult SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context); ISubOperationBase::TPtr RestorePart(TTxState::ETxType opType, TTxState::ETxState opState); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp index af683144e3..463f3bb3cf 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp @@ -254,8 +254,8 @@ public: "Malformed subdomain request: unable to change ExternalHive, only set it up"); return result; } - } - + } + if (addExternalHive) { shardsToCreate += 1; allowOverTheLimitShards += 1; @@ -418,8 +418,8 @@ public: if (addViewProcessors) { DeclareShards(txState, OperationId.GetTxId(), subDomain->PathId, 1, TTabletTypes::SysViewProcessor, channelBindings, context.SS); - } - + } + for (auto& shard: txState.Shards) { alterData->AddPrivateShard(shard.Idx); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp index 2f547ce4a4..5a88636ffa 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp @@ -436,9 +436,9 @@ void NTableState::UpdatePartitioningForCopyTable(TOperationId operationId, TTxSt TShardInfo datashardInfo = TShardInfo::DataShardInfo(operationId.GetTxId(), txState.TargetPathId); datashardInfo.BindedChannels = channelsBinding; - context.SS->SetPartitioning(txState.TargetPathId, dstTableInfo, - ApplyPartitioningCopyTable(datashardInfo, srcTableInfo, txState, context.SS)); - + context.SS->SetPartitioning(txState.TargetPathId, dstTableInfo, + ApplyPartitioningCopyTable(datashardInfo, srcTableInfo, txState, context.SS)); + ui32 newShardCout = dstTableInfo->GetPartitions().size(); dstPath->SetShardsInside(newShardCout); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_common.h index f3bb8fca02..5a3da32cad 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_common.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.h @@ -317,16 +317,16 @@ public: context.OnComplete.BindMsgToPipe(OperationId, tabletID, idx, event); break; } - case ETabletType::SysViewProcessor: { - LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Send configure request to sysview processor: " << tabletID << - " opId: " << OperationId << - " schemeshard: " << ssId); - auto event = new NSysView::TEvSysView::TEvConfigureProcessor(path.PathString()); - shard.Operation = TTxState::ConfigureParts; - context.OnComplete.BindMsgToPipe(OperationId, tabletID, idx, event); - break; - } + case ETabletType::SysViewProcessor: { + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Send configure request to sysview processor: " << tabletID << + " opId: " << OperationId << + " schemeshard: " << ssId); + auto event = new NSysView::TEvSysView::TEvConfigureProcessor(path.PathString()); + shard.Operation = TTxState::ConfigureParts; + context.OnComplete.BindMsgToPipe(OperationId, tabletID, idx, event); + break; + } case ETabletType::SchemeShard: { auto event = new TEvSchemeShard::TEvInitTenantSchemeShard(ui64(ssId), pathId.LocalPathId, path.PathString(), @@ -622,9 +622,9 @@ public: case ETabletType::Hive: context.SS->TabletCounters->Simple()[COUNTER_SUB_DOMAIN_HIVE_COUNT].Add(1); break; - case ETabletType::SysViewProcessor: - context.SS->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Add(1); - break; + case ETabletType::SysViewProcessor: + context.SS->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Add(1); + break; default: break; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp index a7ab5a79e2..5004522529 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp @@ -122,7 +122,7 @@ void ApplyPartitioning(TTxId txId, partitions[i].ShardIdx = idx; } - ss->SetPartitioning(pathId, tableInfo, std::move(partitions)); + ss->SetPartitioning(pathId, tableInfo, std::move(partitions)); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp index bbec44fc46..645d7f37e1 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp @@ -272,7 +272,7 @@ public: // Delete the whole old partitioning and persist the whole new partitionig as the indexes have changed context.SS->DeleteTablePartitioning(db, tableId, tableInfo); - context.SS->SetPartitioning(tableId, tableInfo, std::move(newPartitioning)); + context.SS->SetPartitioning(tableId, tableInfo, std::move(newPartitioning)); context.SS->PersistTablePartitioning(db, tableId, tableInfo); context.SS->PersistTablePartitionStats(db, tableId, tableInfo); context.SS->TabletCounters->Simple()[COUNTER_TABLE_SHARD_ACTIVE_COUNT].Sub(allSrcShardIdxs.size()); diff --git a/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp b/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp index a81218a216..d04be46080 100644 --- a/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp @@ -134,47 +134,47 @@ struct TSchemeShard::TTxUpdateTenant : public TSchemeShard::TRwTxBase { MakeSync(); } - auto addPrivateShard = [&] (TTabletId tabletId, TTabletTypes::EType tabletType) { + auto addPrivateShard = [&] (TTabletId tabletId, TTabletTypes::EType tabletType) { const auto shardIdx = Self->RegisterShardInfo( TShardInfo(InvalidTxId, Self->RootPathId(), tabletType) .WithTabletID(tabletId)); - Self->PersistUpdateNextShardIdx(db); + Self->PersistUpdateNextShardIdx(db); Self->PersistShardMapping(db, shardIdx, tabletId, Self->RootPathId(), InvalidTxId, tabletType); - Y_VERIFY(record.GetSubdomainVersion() >= subdomain->GetVersion()); - if (record.GetSubdomainVersion() > subdomain->GetVersion()) { - subdomain->SetVersion(record.GetSubdomainVersion()); - } - + Y_VERIFY(record.GetSubdomainVersion() >= subdomain->GetVersion()); + if (record.GetSubdomainVersion() > subdomain->GetVersion()) { + subdomain->SetVersion(record.GetSubdomainVersion()); + } + subdomain->AddPrivateShard(shardIdx); subdomain->AddInternalShard(shardIdx); - subdomain->Initialize(Self->ShardInfos); - Self->PersistSubDomain(db, Self->RootPathId(), *subdomain); + subdomain->Initialize(Self->ShardInfos); + Self->PersistSubDomain(db, Self->RootPathId(), *subdomain); path->IncShardsInside(1); - SideEffects.PublishToSchemeBoard(InvalidOperationId, Self->RootPathId()); - MakeSync(); - }; - - if (record.HasTenantHive()) { - TTabletId tenantHive = TTabletId(record.GetTenantHive()); - if (!subdomain->GetTenantHiveID()) { - addPrivateShard(tenantHive, ETabletType::Hive); + SideEffects.PublishToSchemeBoard(InvalidOperationId, Self->RootPathId()); + MakeSync(); + }; + + if (record.HasTenantHive()) { + TTabletId tenantHive = TTabletId(record.GetTenantHive()); + if (!subdomain->GetTenantHiveID()) { + addPrivateShard(tenantHive, ETabletType::Hive); + } + Y_VERIFY(tenantHive == subdomain->GetTenantHiveID()); + } + + if (record.HasTenantSysViewProcessor()) { + TTabletId tenantSVP = TTabletId(record.GetTenantSysViewProcessor()); + if (!subdomain->GetTenantSysViewProcessorID()) { + addPrivateShard(tenantSVP, ETabletType::SysViewProcessor); } - Y_VERIFY(tenantHive == subdomain->GetTenantHiveID()); + Y_VERIFY(tenantSVP == subdomain->GetTenantSysViewProcessorID()); } - if (record.HasTenantSysViewProcessor()) { - TTabletId tenantSVP = TTabletId(record.GetTenantSysViewProcessor()); - if (!subdomain->GetTenantSysViewProcessorID()) { - addPrivateShard(tenantSVP, ETabletType::SysViewProcessor); - } - Y_VERIFY(tenantSVP == subdomain->GetTenantSysViewProcessorID()); - } - if (record.HasUpdateTenantRootACL()) { // KIKIMR-10699: transfer tenants root ACL from GSS to the TSS // here TSS sees the ACL from GSS diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp index 68c53c94fc..b2e7db88b7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp @@ -22,44 +22,44 @@ static ui64 GetIops(const T& c) { } void TSchemeShard::Handle(NSysView::TEvSysView::TEvGetPartitionStats::TPtr& ev, const TActorContext& ctx) { - ctx.Send(ev->Forward(SysPartitionStatsCollector)); -} - + ctx.Send(ev->Forward(SysPartitionStatsCollector)); +} + auto TSchemeShard::BuildStatsForCollector(TPathId pathId, TShardIdx shardIdx, TTabletId datashardId, - TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TTableInfo::TPartitionStats& stats) -{ - auto ev = MakeHolder<NSysView::TEvSysView::TEvSendPartitionStats>( - GetDomainKey(pathId), pathId, std::make_pair(ui64(shardIdx.GetOwnerId()), ui64(shardIdx.GetLocalId()))); - - auto& sysStats = ev->Stats; - sysStats.SetDataSize(stats.DataSize); - sysStats.SetRowCount(stats.RowCount); - sysStats.SetIndexSize(stats.IndexSize); + TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TTableInfo::TPartitionStats& stats) +{ + auto ev = MakeHolder<NSysView::TEvSysView::TEvSendPartitionStats>( + GetDomainKey(pathId), pathId, std::make_pair(ui64(shardIdx.GetOwnerId()), ui64(shardIdx.GetLocalId()))); + + auto& sysStats = ev->Stats; + sysStats.SetDataSize(stats.DataSize); + sysStats.SetRowCount(stats.RowCount); + sysStats.SetIndexSize(stats.IndexSize); sysStats.SetCPUCores(std::min(stats.GetCurrentRawCpuUsage() / 1000000., 1.0)); sysStats.SetTabletId(ui64(datashardId)); - sysStats.SetAccessTime(stats.LastAccessTime.MilliSeconds()); - sysStats.SetUpdateTime(stats.LastUpdateTime.MilliSeconds()); - sysStats.SetInFlightTxCount(stats.InFlightTxCount); - sysStats.SetRowUpdates(stats.RowUpdates); - sysStats.SetRowDeletes(stats.RowDeletes); - sysStats.SetRowReads(stats.RowReads); - sysStats.SetRangeReads(stats.RangeReads); - sysStats.SetRangeReadRows(stats.RangeReadRows); - sysStats.SetImmediateTxCompleted(stats.ImmediateTxCompleted); - sysStats.SetPlannedTxCompleted(stats.PlannedTxCompleted); - sysStats.SetTxRejectedByOverload(stats.TxRejectedByOverload); - sysStats.SetTxRejectedBySpace(stats.TxRejectedBySpace); - - if (nodeId) { - sysStats.SetNodeId(*nodeId); - } - if (startTime) { - sysStats.SetStartTime(*startTime); - } - + sysStats.SetAccessTime(stats.LastAccessTime.MilliSeconds()); + sysStats.SetUpdateTime(stats.LastUpdateTime.MilliSeconds()); + sysStats.SetInFlightTxCount(stats.InFlightTxCount); + sysStats.SetRowUpdates(stats.RowUpdates); + sysStats.SetRowDeletes(stats.RowDeletes); + sysStats.SetRowReads(stats.RowReads); + sysStats.SetRangeReads(stats.RangeReads); + sysStats.SetRangeReadRows(stats.RangeReadRows); + sysStats.SetImmediateTxCompleted(stats.ImmediateTxCompleted); + sysStats.SetPlannedTxCompleted(stats.PlannedTxCompleted); + sysStats.SetTxRejectedByOverload(stats.TxRejectedByOverload); + sysStats.SetTxRejectedBySpace(stats.TxRejectedBySpace); + + if (nodeId) { + sysStats.SetNodeId(*nodeId); + } + if (startTime) { + sysStats.SetStartTime(*startTime); + } + return ev; -} - +} + class TTxStorePartitionStats: public NTabletFlatExecutor::TTransactionBase<TSchemeShard> { TEvDataShard::TEvPeriodicTableStats::TPtr Ev; @@ -219,17 +219,17 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte Self->PersistTablePartitionStats(db, tableId, shardIdx, table); if (AppData(ctx)->FeatureFlags.GetEnableSystemViews()) { - TMaybe<ui32> nodeId; - if (rec.HasNodeId()) { - nodeId = rec.GetNodeId(); - } - TMaybe<ui64> startTime; - if (rec.HasStartTime()) { - startTime = rec.GetStartTime(); - } + TMaybe<ui32> nodeId; + if (rec.HasNodeId()) { + nodeId = rec.GetNodeId(); + } + TMaybe<ui64> startTime; + if (rec.HasStartTime()) { + startTime = rec.GetStartTime(); + } StatsCollectorEv = Self->BuildStatsForCollector(tableId, shardIdx, datashardId, nodeId, startTime, newStats); - } - + } + const auto& shardToPartition = table->GetShard2PartitionIdx(); if (table->IsTTLEnabled() && shardToPartition.contains(shardIdx)) { const ui64 partitionIdx = shardToPartition.at(shardIdx); diff --git a/ydb/core/tx/schemeshard/schemeshard_domain_links.cpp b/ydb/core/tx/schemeshard/schemeshard_domain_links.cpp index 3011e04229..b50aaa509a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_domain_links.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_domain_links.cpp @@ -26,7 +26,7 @@ THolder<TEvSchemeShard::TEvSyncTenantSchemeShard> TParentDomainLink::MakeSyncMsg Self->ParentDomainEffectiveACLVersion, rootSubdomain->GetVersion(), rootPath->UserAttrs->AlterVersion, - ui64(rootSubdomain->GetTenantHiveID()), + ui64(rootSubdomain->GetTenantHiveID()), ui64(rootSubdomain->GetTenantSysViewProcessorID()), rootPath->ACL); } @@ -106,7 +106,7 @@ void TSubDomainsLinks::TLink::Out(IOutputStream& stream) const { << ", SubdomainVersion: " << SubdomainVersion << ", UserAttributesVersion: " << UserAttributesVersion << ", TenantHive: " << TenantHive - << ", TenantSysViewProcessor: " << TenantSysViewProcessor + << ", TenantSysViewProcessor: " << TenantSysViewProcessor << ", TenantRootACL: " << TenantRootACL << "}"; } @@ -119,8 +119,8 @@ TSubDomainsLinks::TLink::TLink(const NKikimrScheme::TEvSyncTenantSchemeShard &re , SubdomainVersion(record.GetSubdomainVersion()) , UserAttributesVersion(record.GetUserAttributesVersion()) , TenantHive(record.HasTenantHive() ? TTabletId(record.GetTenantHive()) : InvalidTabletId) - , TenantSysViewProcessor(record.HasTenantSysViewProcessor() ? - TTabletId(record.GetTenantSysViewProcessor()) : InvalidTabletId) + , TenantSysViewProcessor(record.HasTenantSysViewProcessor() ? + TTabletId(record.GetTenantSysViewProcessor()) : InvalidTabletId) , TenantRootACL(record.GetTenantRootACL()) {} diff --git a/ydb/core/tx/schemeshard/schemeshard_domain_links.h b/ydb/core/tx/schemeshard/schemeshard_domain_links.h index 0acbc4c9d4..d9cb6133c4 100644 --- a/ydb/core/tx/schemeshard/schemeshard_domain_links.h +++ b/ydb/core/tx/schemeshard/schemeshard_domain_links.h @@ -40,7 +40,7 @@ public: ui64 SubdomainVersion = 0; ui64 UserAttributesVersion = 0; TTabletId TenantHive = InvalidTabletId; - TTabletId TenantSysViewProcessor = InvalidTabletId; + TTabletId TenantSysViewProcessor = InvalidTabletId; TString TenantRootACL; TLink() = default; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index bf193ff06f..0715543a22 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -990,12 +990,12 @@ TSubDomainInfo::TPtr TSchemeShard::ResolveDomainInfo(TPathElement::TPtr pathEl) } TPathId TSchemeShard::GetDomainKey(TPathId pathId) const { - auto domainId = ResolveDomainId(pathId); - TPathElement::TPtr domainElement = PathsById.at(domainId); - Y_VERIFY(domainElement); - return domainElement->IsRoot() ? ParentDomainId : domainId; -} - + auto domainId = ResolveDomainId(pathId); + TPathElement::TPtr domainElement = PathsById.at(domainId); + Y_VERIFY(domainElement); + return domainElement->IsRoot() ? ParentDomainId : domainId; +} + const NKikimrSubDomains::TProcessingParams &TSchemeShard::SelectProcessingPrarams(TPathId id) const { TPathElement::TPtr item = PathsById.at(id); return SelectProcessingPrarams(item); @@ -3248,11 +3248,11 @@ void TSchemeShard::PersistRemoveTable(NIceDb::TNiceDb& db, TPathId pathId, const Tables.erase(pathId); DecrementPathDbRefCount(pathId, "remove table"); - + if (AppData()->FeatureFlags.GetEnableSystemViews()) { - auto ev = MakeHolder<NSysView::TEvSysView::TEvRemoveTable>(GetDomainKey(pathId), pathId); - Send(SysPartitionStatsCollector, ev.Release()); - } + auto ev = MakeHolder<NSysView::TEvSysView::TEvRemoveTable>(GetDomainKey(pathId), pathId); + Send(SysPartitionStatsCollector, ev.Release()); + } } void TSchemeShard::PersistRemoveTableIndex(NIceDb::TNiceDb &db, TPathId pathId) @@ -3699,8 +3699,8 @@ bool TSchemeShard::IsShemeShardConfigured() const { void TSchemeShard::Die(const TActorContext &ctx) { ctx.Send(SchemeBoardPopulator, new TEvents::TEvPoisonPill()); ctx.Send(TxAllocatorClient, new TEvents::TEvPoisonPill()); - ctx.Send(SysPartitionStatsCollector, new TEvents::TEvPoisonPill()); - + ctx.Send(SysPartitionStatsCollector, new TEvents::TEvPoisonPill()); + ShardDeleter.Shutdown(ctx); ParentDomainLink.Shutdown(ctx); @@ -3773,8 +3773,8 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) { TxAllocatorClient = RegisterWithSameMailbox(CreateTxAllocatorClient(CollectTxAllocators(appData))); - SysPartitionStatsCollector = Register(NSysView::CreatePartitionStatsCollector().Release()); - + SysPartitionStatsCollector = Register(NSysView::CreatePartitionStatsCollector().Release()); + SplitSettings.Register(appData->Icb); Executor()->RegisterExternalTabletCounters(TabletCountersPtr); @@ -3915,8 +3915,8 @@ void TSchemeShard::StateWork(STFUNC_SIG) { HFuncTraced(TEvPrivate::TEvServerlessStorageBilling, Handle); - HFuncTraced(NSysView::TEvSysView::TEvGetPartitionStats, Handle); - + HFuncTraced(NSysView::TEvSysView::TEvGetPartitionStats, Handle); + HFuncTraced(TEvSubDomain::TEvConfigureStatus, Handle); HFuncTraced(TEvSchemeShard::TEvInitTenantSchemeShard, Handle); HFuncTraced(TEvSchemeShard::TEvInitTenantSchemeShardResult, Handle); @@ -5766,19 +5766,19 @@ bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 key void TSchemeShard::SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, TVector<TTableShardInfo>&& newPartitioning) { if (AppData()->FeatureFlags.GetEnableSystemViews()) { - TVector<std::pair<ui64, ui64>> shardIndices; - shardIndices.reserve(newPartitioning.size()); - for (auto& info : newPartitioning) { - shardIndices.push_back( - std::make_pair(ui64(info.ShardIdx.GetOwnerId()), ui64(info.ShardIdx.GetLocalId()))); - } - - auto path = TPath::Init(pathId, this); - auto ev = MakeHolder<NSysView::TEvSysView::TEvSetPartitioning>(GetDomainKey(pathId), pathId, path.PathString()); - ev->ShardIndices.swap(shardIndices); - Send(SysPartitionStatsCollector, ev.Release()); - } - + TVector<std::pair<ui64, ui64>> shardIndices; + shardIndices.reserve(newPartitioning.size()); + for (auto& info : newPartitioning) { + shardIndices.push_back( + std::make_pair(ui64(info.ShardIdx.GetOwnerId()), ui64(info.ShardIdx.GetLocalId()))); + } + + auto path = TPath::Init(pathId, this); + auto ev = MakeHolder<NSysView::TEvSysView::TEvSetPartitioning>(GetDomainKey(pathId), pathId, path.PathString()); + ev->ShardIndices.swap(shardIndices); + Send(SysPartitionStatsCollector, ev.Release()); + } + if (!tableInfo->IsBackup) { for (const auto& p: newPartitioning) { ui64 searchHeight = 0; @@ -5793,9 +5793,9 @@ void TSchemeShard::SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, T } } - tableInfo->SetPartitioning(std::move(newPartitioning)); -} - + tableInfo->SetPartitioning(std::move(newPartitioning)); +} + void TSchemeShard::FillAsyncIndexInfo(const TPathId& tableId, NKikimrTxDataShard::TFlatSchemeTransaction& tx) { Y_VERIFY(PathsById.contains(tableId)); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index fcbcffa291..e1a1d08b9a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -210,7 +210,7 @@ public: TAutoPtr<TSelfPinger> SelfPinger; TActorId SysPartitionStatsCollector; - + TSet<TPathId> CleanDroppedPathsCandidates; TSet<TPathId> CleanDroppedSubDomainsCandidates; bool CleanDroppedPathsInFly = false; @@ -368,8 +368,8 @@ public: TSubDomainInfo::TPtr ResolveDomainInfo(TPathId pathId) const; TSubDomainInfo::TPtr ResolveDomainInfo(TPathElement::TPtr pathEl) const; - TPathId GetDomainKey(TPathId pathId) const; - + TPathId GetDomainKey(TPathId pathId) const; + const NKikimrSubDomains::TProcessingParams& SelectProcessingPrarams(TPathId id) const; const NKikimrSubDomains::TProcessingParams& SelectProcessingPrarams(TPathElement::TPtr pathEl) const; @@ -412,9 +412,9 @@ public: void DoShardsDeletion(const THashSet<TShardIdx>& shardIdx, const TActorContext& ctx); - void SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, TVector<TTableShardInfo>&& newPartitioning); + void SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, TVector<TTableShardInfo>&& newPartitioning); auto BuildStatsForCollector(TPathId tableId, TShardIdx shardIdx, TTabletId datashardId, - TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TTableInfo::TPartitionStats& stats); + TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TTableInfo::TPartitionStats& stats); bool ReadSysValue(NIceDb::TNiceDb& db, ui64 sysTag, TString& value, TString defValue = TString()); bool ReadSysValue(NIceDb::TNiceDb& db, ui64 sysTag, ui64& value, ui64 defVal = 0); @@ -818,8 +818,8 @@ public: void Handle(TEvPrivate::TEvRunConditionalErase::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev, const TActorContext& ctx); - void Handle(NSysView::TEvSysView::TEvGetPartitionStats::TPtr& ev, const TActorContext& ctx); - + void Handle(NSysView::TEvSysView::TEvGetPartitionStats::TPtr& ev, const TActorContext& ctx); + void ScheduleServerlessStorageBilling(const TActorContext& ctx); void Handle(TEvPrivate::TEvServerlessStorageBilling::TPtr& ev, const TActorContext& ctx); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 4bb201c7c2..2e5710c3d3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -1264,13 +1264,13 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> { return TTabletId(ProcessingParams.GetHive()); } - TTabletId GetTenantSysViewProcessorID() const { - if (!ProcessingParams.HasSysViewProcessor()) { - return InvalidTabletId; - } - return TTabletId(ProcessingParams.GetSysViewProcessor()); - } - + TTabletId GetTenantSysViewProcessorID() const { + if (!ProcessingParams.HasSysViewProcessor()) { + return InvalidTabletId; + } + return TTabletId(ProcessingParams.GetSysViewProcessor()); + } + ui64 GetPathsInside() const { return PathsInsideCount; } @@ -1606,13 +1606,13 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> { ProcessingParams.SetHive(ui64(hives.front())); SetSharedHive(InvalidTabletId); // set off shared hive when our own hive has found } - - ProcessingParams.ClearSysViewProcessor(); - TVector<TTabletId> sysViewProcessors = FilterTablets(ETabletType::SysViewProcessor, allShards); - Y_VERIFY_S(sysViewProcessors.size() <= 1, "size was: " << sysViewProcessors.size()); - if (sysViewProcessors.size()) { - ProcessingParams.SetSysViewProcessor(ui64(sysViewProcessors.front())); - } + + ProcessingParams.ClearSysViewProcessor(); + TVector<TTabletId> sysViewProcessors = FilterTablets(ETabletType::SysViewProcessor, allShards); + Y_VERIFY_S(sysViewProcessors.size() <= 1, "size was: " << sysViewProcessors.size()); + if (sysViewProcessors.size()) { + ProcessingParams.SetSysViewProcessor(ui64(sysViewProcessors.front())); + } } void InitializeAsGlobal(NKikimrSubDomains::TProcessingParams&& processingParams) { diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index 74d299fbde..7535ee673c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -4,7 +4,7 @@ #include <ydb/core/base/path.h> #include <ydb/core/sys_view/common/path.h> - + namespace NKikimr { namespace NSchemeShard { @@ -1617,11 +1617,11 @@ bool TPath::IsValidLeafName(TString& explain) const { } if (AppData()->FeatureFlags.GetEnableSystemViews() && leaf == NSysView::SysPathName) { - explain += TStringBuilder() - << "path part '" << NSysView::SysPathName << "' is reserved by the system"; - return false; - } - + explain += TStringBuilder() + << "path part '" << NSysView::SysPathName << "' is reserved by the system"; + return false; + } + auto brokenAt = PathPartBrokenAt(leaf, schemeLimits.ExtraPathSymbolsAllowed); if (brokenAt != leaf.end()) { explain += TStringBuilder() diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.h b/ydb/core/tx/schemeshard/schemeshard_path_element.h index e9131da453..9912880c05 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_element.h +++ b/ydb/core/tx/schemeshard/schemeshard_path_element.h @@ -2,7 +2,7 @@ #include "schemeshard_types.h" #include "schemeshard_effective_acl.h" -#include "schemeshard_user_attr_limits.h" +#include "schemeshard_user_attr_limits.h" #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/library/aclib/aclib.h> @@ -170,7 +170,7 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> { bool CheckLimits(TString& errStr) const { const ui64 bytes = Bytes(); - if (bytes > TUserAttributesLimits::MaxBytes) { + if (bytes > TUserAttributesLimits::MaxBytes) { errStr = Sprintf("UserArttibutes::CheckLimits: user attributes too big: %" PRIu64, bytes); return false; } @@ -184,13 +184,13 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> { return true; } - if (name.size() > TUserAttributesLimits::MaxNameLen) { + if (name.size() > TUserAttributesLimits::MaxNameLen) { errStr = Sprintf("UserArttibutes: name too long, name# '%s' value# '%s'" , name.c_str(), value.c_str()); return false; } - if (value.size() > TUserAttributesLimits::MaxValueLen) { + if (value.size() > TUserAttributesLimits::MaxValueLen) { errStr = Sprintf("UserArttibutes: value too long, name# '%s' value# '%s'" , name.c_str(), value.c_str()); return false; diff --git a/ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h b/ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h index b28102fff2..aeddaa22be 100644 --- a/ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h +++ b/ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h @@ -1,15 +1,15 @@ -#pragma once - -#include <util/system/types.h> - -namespace NKikimr { +#pragma once + +#include <util/system/types.h> + +namespace NKikimr { namespace NSchemeShard { - -struct TUserAttributesLimits { - static constexpr ui32 MaxNameLen = 100; - static constexpr ui32 MaxValueLen = 4 * 1024; - static constexpr ui32 MaxBytes = 10 * 1024; -}; - -} -} + +struct TUserAttributesLimits { + static constexpr ui32 MaxNameLen = 100; + static constexpr ui32 MaxValueLen = 4 * 1024; + static constexpr ui32 MaxBytes = 10 * 1024; +}; + +} +} diff --git a/ydb/core/tx/schemeshard/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base.cpp index 2df6f0b712..a76dcf9d5b 100644 --- a/ydb/core/tx/schemeshard/ut_base.cpp +++ b/ydb/core/tx/schemeshard/ut_base.cpp @@ -9303,22 +9303,22 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { env.TestWaitNotification(runtime, txId); env.TestWaitTabletDeletion(runtime, xrange(TTestTxConfig::FakeHiveTablets+2, TTestTxConfig::FakeHiveTablets+10)); } - - Y_UNIT_TEST(RejectSystemViewPath) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, 4, true, &CreateFlatTxSchemeShard, true); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", - "Name: \".sys\"" - "Columns { Name: \"key\" Type: \"Uint32\"}" - "KeyColumnNames: [\"key\"]", + + Y_UNIT_TEST(RejectSystemViewPath) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, 4, true, &CreateFlatTxSchemeShard, true); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", + "Name: \".sys\"" + "Columns { Name: \"key\" Type: \"Uint32\"}" + "KeyColumnNames: [\"key\"]", {NKikimrScheme::StatusSchemeError}); - + TestMkDir(runtime, ++txId, "/MyRoot", ".sys", {NKikimrScheme::StatusSchemeError}); TestMkDir(runtime, ++txId, "/MyRoot", ".sys/partition_stats", {NKikimrScheme::StatusSchemeError}); TestMkDir(runtime, ++txId, "/MyRoot", "DirA/.sys/partition_stats", {NKikimrScheme::StatusSchemeError}); - } + } Y_UNIT_TEST(DocumentApiVersion) { TTestBasicRuntime runtime; diff --git a/ydb/core/tx/schemeshard/ut_extsubdomain.cpp b/ydb/core/tx/schemeshard/ut_extsubdomain.cpp index c895f6bacf..0fa6a27bce 100644 --- a/ydb/core/tx/schemeshard/ut_extsubdomain.cpp +++ b/ydb/core/tx/schemeshard/ut_extsubdomain.cpp @@ -674,23 +674,23 @@ Y_UNIT_TEST_SUITE(TSchemeShardExtSubDomainTest) { UNIT_ASSERT(!CheckLocalRowExists(runtime, TTestTxConfig::SchemeShard, "SubDomains", "PathId", 2)); UNIT_ASSERT(!CheckLocalRowExists(runtime, TTestTxConfig::SchemeShard, "Paths", "Id", 2)); } - - Y_UNIT_TEST(SysViewProcessorSync) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - + + Y_UNIT_TEST(SysViewProcessorSync) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + NSchemeShard::TSchemeLimits lowLimits; lowLimits.MaxShardsInPath = 3; SetSchemeshardSchemaLimits(runtime, lowLimits); - TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", - "Name: \"USER_0\""); - + TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", + "Name: \"USER_0\""); + // check that limits have a power, try create 4 shards - TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", - "Name: \"USER_0\" " - "PlanResolution: 50 " + TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", + "Name: \"USER_0\" " + "PlanResolution: 50 " "Coordinators: 2 " "Mediators: 1 " "TimeCastBucketsPerMediator: 2 " @@ -701,68 +701,68 @@ Y_UNIT_TEST_SUITE(TSchemeShardExtSubDomainTest) { TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", "Name: \"USER_0\" " "PlanResolution: 50 " - "Coordinators: 1 " - "Mediators: 1 " - "TimeCastBucketsPerMediator: 2 " + "Coordinators: 1 " + "Mediators: 1 " + "TimeCastBucketsPerMediator: 2 " "ExternalSchemeShard: true " "StoragePools { " " Name: \"/dc-1/users/tenant-1:hdd\" " " Kind: \"hdd\" " "} "); - env.TestWaitNotification(runtime, {txId, txId - 1}); - + env.TestWaitNotification(runtime, {txId, txId - 1}); + lowLimits.MaxShardsInPath = 2; SetSchemeshardSchemaLimits(runtime, lowLimits); // one more, but for free - TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", - "Name: \"USER_0\" " - "ExternalSysViewProcessor: true "); - - env.TestWaitNotification(runtime, txId); - - ui64 tenantSchemeShard = 0; - ui64 tenantSVP = 0; - TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"), - {NLs::PathExist, - NLs::IsExternalSubDomain("USER_0"), - NLs::ExtractTenantSchemeshard(&tenantSchemeShard), - NLs::ExtractTenantSysViewProcessor(&tenantSVP)}); - - UNIT_ASSERT(tenantSchemeShard != 0 - && tenantSchemeShard != (ui64)-1 + TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", + "Name: \"USER_0\" " + "ExternalSysViewProcessor: true "); + + env.TestWaitNotification(runtime, txId); + + ui64 tenantSchemeShard = 0; + ui64 tenantSVP = 0; + TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"), + {NLs::PathExist, + NLs::IsExternalSubDomain("USER_0"), + NLs::ExtractTenantSchemeshard(&tenantSchemeShard), + NLs::ExtractTenantSysViewProcessor(&tenantSVP)}); + + UNIT_ASSERT(tenantSchemeShard != 0 + && tenantSchemeShard != (ui64)-1 && tenantSchemeShard != TTestTxConfig::SchemeShard); - - UNIT_ASSERT(tenantSVP != 0 && tenantSVP != (ui64)-1); - - ui64 tenantSVPOnTSS = 0; - TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0"), - {NLs::PathExist, - NLs::ExtractTenantSysViewProcessor(&tenantSVPOnTSS)}); - - UNIT_ASSERT_EQUAL(tenantSVP, tenantSVPOnTSS); - + + UNIT_ASSERT(tenantSVP != 0 && tenantSVP != (ui64)-1); + + ui64 tenantSVPOnTSS = 0; + TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0"), + {NLs::PathExist, + NLs::ExtractTenantSysViewProcessor(&tenantSVPOnTSS)}); + + UNIT_ASSERT_EQUAL(tenantSVP, tenantSVPOnTSS); + RebootTablet(runtime, tenantSchemeShard, runtime.AllocateEdgeActor()); - - TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/USER_0", - "Name: \"table\"" - "Columns { Name: \"RowId\" Type: \"Uint64\"}" - "Columns { Name: \"Value\" Type: \"Utf8\"}" - "KeyColumnNames: [\"RowId\"]"); - - env.TestWaitNotification(runtime, txId, tenantSchemeShard); - - TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/table"), - {NLs::PathExist}); - + + TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/USER_0", + "Name: \"table\"" + "Columns { Name: \"RowId\" Type: \"Uint64\"}" + "Columns { Name: \"Value\" Type: \"Utf8\"}" + "KeyColumnNames: [\"RowId\"]"); + + env.TestWaitNotification(runtime, txId, tenantSchemeShard); + + TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/table"), + {NLs::PathExist}); + RebootTablet(runtime, tenantSchemeShard, runtime.AllocateEdgeActor()); - - TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0"), - {NLs::PathExist, - NLs::ExtractTenantSysViewProcessor(&tenantSVPOnTSS)}); - - UNIT_ASSERT_EQUAL(tenantSVP, tenantSVPOnTSS); - } + + TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0"), + {NLs::PathExist, + NLs::ExtractTenantSysViewProcessor(&tenantSVPOnTSS)}); + + UNIT_ASSERT_EQUAL(tenantSVP, tenantSVPOnTSS); + } Y_UNIT_TEST(SchemeQuotas) { TTestBasicRuntime runtime; diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 8dd69c9c68..c5d6c5a973 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -99,15 +99,15 @@ TCheckFunc ExtractTenantSchemeshard(ui64* tenantSchemeShardId) { TCheckFunc ExtractTenantSysViewProcessor(ui64* tenantSVPId) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); - const auto& pathDescr = record.GetPathDescription(); - UNIT_ASSERT(pathDescr.HasDomainDescription()); - const auto& domainDesc = pathDescr.GetDomainDescription(); - UNIT_ASSERT(domainDesc.HasProcessingParams()); - const auto& procParams = domainDesc.GetProcessingParams(); - *tenantSVPId = procParams.GetSysViewProcessor(); - }; -} - + const auto& pathDescr = record.GetPathDescription(); + UNIT_ASSERT(pathDescr.HasDomainDescription()); + const auto& domainDesc = pathDescr.GetDomainDescription(); + UNIT_ASSERT(domainDesc.HasProcessingParams()); + const auto& procParams = domainDesc.GetProcessingParams(); + *tenantSVPId = procParams.GetSysViewProcessor(); + }; +} + void InExternalSubdomain(const NKikimrScheme::TEvDescribeSchemeResult& record) { PathRedirected(record); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index 6c8c9eedfb..33022e4257 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -60,7 +60,7 @@ namespace NLs { TCheckFunc IsExternalSubDomain(const TString& name); void InExternalSubdomain(const NKikimrScheme::TEvDescribeSchemeResult& record); TCheckFunc ExtractTenantSchemeshard(ui64* tenantSchemeShard); - TCheckFunc ExtractTenantSysViewProcessor(ui64* tenantSVP); + TCheckFunc ExtractTenantSysViewProcessor(ui64* tenantSVP); void NotFinished(const NKikimrScheme::TEvDescribeSchemeResult& record); void Finished(const NKikimrScheme::TEvDescribeSchemeResult& record); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp index 430c5fabcb..9cfd8037b7 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp @@ -506,7 +506,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe app.SetEnableProtoSourceIdInfo(opts.EnableProtoSourceIdInfo_); app.SetEnableBackgroundCompaction(opts.EnableBackgroundCompaction_); app.FeatureFlags.SetEnablePublicApiExternalBlobs(true); - + for (const auto& sid : opts.SystemBackupSIDs_) { app.AddSystemBackupSID(sid); } diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.h b/ydb/core/tx/schemeshard/ut_helpers/test_env.h index 39128d54ea..72b934c570 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.h +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.h @@ -62,8 +62,8 @@ namespace NSchemeShardUT_Private { TActorId MeteringFake; public: - TTestEnv(TTestActorRuntime& runtime, ui32 nchannels = 4, bool enablePipeRetries = true, - TSchemeShardFactory ssFactory = &CreateFlatTxSchemeShard, bool enableSystemViews = false); + TTestEnv(TTestActorRuntime& runtime, ui32 nchannels = 4, bool enablePipeRetries = true, + TSchemeShardFactory ssFactory = &CreateFlatTxSchemeShard, bool enableSystemViews = false); TTestEnv(TTestActorRuntime& runtime, const TTestEnvOptions& opts, TSchemeShardFactory ssFactory = &CreateFlatTxSchemeShard, std::shared_ptr<NKikimr::NDataShard::IExportFactory> dsExportFactory = {}); diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index 3fcaa3d02c..ceb87ce388 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -179,7 +179,7 @@ SRCS( schemeshard_types.cpp schemeshard_types.h schemeshard_ui64id.cpp - schemeshard_user_attr_limits.h + schemeshard_user_attr_limits.h schemeshard_utils.cpp schemeshard_utils.h schemeshard_export__cancel.cpp diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index 0433ff7ff9..af48b42d60 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -788,7 +788,7 @@ void TDataReq::ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus const auto* e = PerTablet.FindPtr(shard); if (!e) return ""; - return Sprintf("%" PRIu64 "/%" PRIu64, e->TableId.PathId.OwnerId, e->TableId.PathId.LocalPathId); + return Sprintf("%" PRIu64 "/%" PRIu64, e->TableId.PathId.OwnerId, e->TableId.PathId.LocalPathId); }; if (FlatMKQLRequest && FlatMKQLRequest->CollectStats) { @@ -906,12 +906,12 @@ void TDataReq::BuildTxStats(NKikimrQueryStats::TTxStats& stats) { TTableId tableId(table.GetTableInfo().GetSchemeshardId(), table.GetTableInfo().GetPathId()); auto& tableStats = byTable[tableId]; if (!tableStats.HasTableInfo()) { - tableStats.MutableTableInfo()->SetSchemeshardId(tableId.PathId.OwnerId); - tableStats.MutableTableInfo()->SetPathId(tableId.PathId.LocalPathId); + tableStats.MutableTableInfo()->SetSchemeshardId(tableId.PathId.OwnerId); + tableStats.MutableTableInfo()->SetPathId(tableId.PathId.LocalPathId); tableStats.MutableTableInfo()->SetName(table.GetTableInfo().GetName()); } - Aggregate(tableStats, table); - tableStats.SetShardCount(tableStats.GetShardCount() + 1); + Aggregate(tableStats, table); + tableStats.SetShardCount(tableStats.GetShardCount() + 1); } if (shard.second.Stats->PerShardStatsSize() == 1) { auto shardStats = stats.AddPerShardStats(); @@ -1418,14 +1418,14 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, auto &res = resp->ResultSet[0]; ReadTableRequest->TableId = res.TableId; - if (res.TableId.IsSystemView()) { - IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, - Sprintf("Table '%s' is a system view. Read table is not supported", ReadTableRequest->TablePath.data()))); - ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, NKikimrIssues::TStatusIds::SCHEME_ERROR, true, ctx); - TxProxyMon->ResolveKeySetWrongRequest->Inc(); - return Die(ctx); - } - + if (res.TableId.IsSystemView()) { + IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, + Sprintf("Table '%s' is a system view. Read table is not supported", ReadTableRequest->TablePath.data()))); + ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, NKikimrIssues::TStatusIds::SCHEME_ERROR, true, ctx); + TxProxyMon->ResolveKeySetWrongRequest->Inc(); + return Die(ctx); + } + TVector<NScheme::TTypeId> keyColumnTypes(res.Columns.size()); TVector<TKeyDesc::TColumnOp> columns(res.Columns.size()); size_t keySize = 0; @@ -1544,14 +1544,14 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev, c switch (x.Status) { case NSchemeCache::TSchemeCacheRequest::EStatus::PathErrorNotExist: gotHardResolveError = true; - ss << "table not exists: " << x.KeyDescription->TableId; + ss << "table not exists: " << x.KeyDescription->TableId; break; case NSchemeCache::TSchemeCacheRequest::EStatus::TypeCheckError: gotHardResolveError = true; - ss << "type check error: " << x.KeyDescription->TableId; + ss << "type check error: " << x.KeyDescription->TableId; break; default: - ss << "unresolved table: " << x.KeyDescription->TableId << ". Status: " << x.Status; + ss << "unresolved table: " << x.KeyDescription->TableId << ". Status: " << x.Status; break; } @@ -1599,7 +1599,7 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev, c TStringStream explanation; explanation << "Access denied for " << UserToken->GetUserSID() << " with access " << NACLib::AccessRightsToString(access) - << " to tableId# " << entry.KeyDescription->TableId; + << " to tableId# " << entry.KeyDescription->TableId; LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, explanation.Str()); IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::ACCESS_DENIED, explanation.Str())); diff --git a/ydb/core/tx/tx_proxy/describe.cpp b/ydb/core/tx/tx_proxy/describe.cpp index 4065ebb8b1..deb38ca2b7 100644 --- a/ydb/core/tx/tx_proxy/describe.cpp +++ b/ydb/core/tx/tx_proxy/describe.cpp @@ -67,115 +67,115 @@ class TDescribeReq : public TActor<TDescribeReq> { } void FillSystemViewDescr(NKikimrSchemeOp::TDirEntry* descr, ui64 schemeShardId) { - descr->SetSchemeshardId(schemeShardId); - descr->SetPathId(InvalidLocalPathId); - descr->SetParentPathId(InvalidLocalPathId); - descr->SetCreateFinished(true); - descr->SetCreateTxId(0); - descr->SetCreateStep(0); - } - - void SendSystemViewResult(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& path, - const TActorContext& ctx) - { - auto schemeShardId = entry.DomainInfo->DomainKey.OwnerId; - + descr->SetSchemeshardId(schemeShardId); + descr->SetPathId(InvalidLocalPathId); + descr->SetParentPathId(InvalidLocalPathId); + descr->SetCreateFinished(true); + descr->SetCreateTxId(0); + descr->SetCreateStep(0); + } + + void SendSystemViewResult(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& path, + const TActorContext& ctx) + { + auto schemeShardId = entry.DomainInfo->DomainKey.OwnerId; + auto result = MakeHolder<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResultBuilder>( - path, schemeShardId, TPathId()); - - auto* pathDescription = result->Record.MutablePathDescription(); - auto* self = pathDescription->MutableSelf(); - - Y_VERIFY(!entry.Path.empty()); - self->SetName(entry.Path.back()); + path, schemeShardId, TPathId()); + + auto* pathDescription = result->Record.MutablePathDescription(); + auto* self = pathDescription->MutableSelf(); + + Y_VERIFY(!entry.Path.empty()); + self->SetName(entry.Path.back()); self->SetPathType(NKikimrSchemeOp::EPathTypeTable); - FillSystemViewDescr(self, schemeShardId); - - auto* table = pathDescription->MutableTable(); - - TVector<ui32> keyColumnIds(entry.Columns.size()); - size_t keySize = 0; - - table->SetName(entry.Path.back()); - table->MutableColumns()->Reserve(entry.Columns.size()); - - for (const auto& [id, column] : entry.Columns) { - auto* col = table->AddColumns(); - col->SetName(column.Name); - col->SetType(AppData(ctx)->TypeRegistry->GetTypeName(column.PType)); - col->SetTypeId(column.PType); - col->SetId(id); - if (column.KeyOrder >= 0) { - Y_VERIFY((size_t)column.KeyOrder < keyColumnIds.size()); - keyColumnIds[column.KeyOrder] = id; - ++keySize; - } - } - - table->MutableKeyColumnNames()->Reserve(keySize); - table->MutableKeyColumnIds()->Reserve(keySize); - - for (size_t i = 0; i < keySize; ++i) { - auto columnId = keyColumnIds[i]; - auto columnIt = entry.Columns.find(columnId); - Y_VERIFY(columnIt != entry.Columns.end()); - table->AddKeyColumnIds(columnId); - table->AddKeyColumnNames(columnIt->second.Name); - } - - auto* stats = pathDescription->MutableTableStats(); - stats->SetPartCount(0); - - LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, - "Actor# " << ctx.SelfID.ToString() << - " Send sysview TEvDescribeSchemeResult to# " << Source.ToString() << - " Cookie: " << SourceCookie << - " TEvDescribeSchemeResult: " << result->ToString()); - - TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds()); - + FillSystemViewDescr(self, schemeShardId); + + auto* table = pathDescription->MutableTable(); + + TVector<ui32> keyColumnIds(entry.Columns.size()); + size_t keySize = 0; + + table->SetName(entry.Path.back()); + table->MutableColumns()->Reserve(entry.Columns.size()); + + for (const auto& [id, column] : entry.Columns) { + auto* col = table->AddColumns(); + col->SetName(column.Name); + col->SetType(AppData(ctx)->TypeRegistry->GetTypeName(column.PType)); + col->SetTypeId(column.PType); + col->SetId(id); + if (column.KeyOrder >= 0) { + Y_VERIFY((size_t)column.KeyOrder < keyColumnIds.size()); + keyColumnIds[column.KeyOrder] = id; + ++keySize; + } + } + + table->MutableKeyColumnNames()->Reserve(keySize); + table->MutableKeyColumnIds()->Reserve(keySize); + + for (size_t i = 0; i < keySize; ++i) { + auto columnId = keyColumnIds[i]; + auto columnIt = entry.Columns.find(columnId); + Y_VERIFY(columnIt != entry.Columns.end()); + table->AddKeyColumnIds(columnId); + table->AddKeyColumnNames(columnIt->second.Name); + } + + auto* stats = pathDescription->MutableTableStats(); + stats->SetPartCount(0); + + LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, + "Actor# " << ctx.SelfID.ToString() << + " Send sysview TEvDescribeSchemeResult to# " << Source.ToString() << + " Cookie: " << SourceCookie << + " TEvDescribeSchemeResult: " << result->ToString()); + + TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds()); + result->Record.SetStatus(NKikimrScheme::StatusSuccess); - ctx.Send(Source, result.Release(), 0, SourceCookie); - return Die(ctx); - } - - void SendSystemViewFolderResult(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& path, - const TActorContext& ctx) - { - auto schemeShardId = entry.DomainInfo->DomainKey.OwnerId; - + ctx.Send(Source, result.Release(), 0, SourceCookie); + return Die(ctx); + } + + void SendSystemViewFolderResult(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& path, + const TActorContext& ctx) + { + auto schemeShardId = entry.DomainInfo->DomainKey.OwnerId; + auto result = MakeHolder<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResultBuilder>( - path, schemeShardId, TPathId()); - - auto* pathDescription = result->Record.MutablePathDescription(); - auto* self = pathDescription->MutableSelf(); - - self->SetName(TString(NSysView::SysPathName)); + path, schemeShardId, TPathId()); + + auto* pathDescription = result->Record.MutablePathDescription(); + auto* self = pathDescription->MutableSelf(); + + self->SetName(TString(NSysView::SysPathName)); self->SetPathType(NKikimrSchemeOp::EPathTypeDir); - FillSystemViewDescr(self, schemeShardId); - - if (entry.ListNodeEntry) { - for (const auto& child : entry.ListNodeEntry->Children) { - auto descr = pathDescription->AddChildren(); - descr->SetName(child.Name); + FillSystemViewDescr(self, schemeShardId); + + if (entry.ListNodeEntry) { + for (const auto& child : entry.ListNodeEntry->Children) { + auto descr = pathDescription->AddChildren(); + descr->SetName(child.Name); descr->SetPathType(NKikimrSchemeOp::EPathTypeTable); - FillSystemViewDescr(descr, schemeShardId); - } - }; - - LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, - "Actor# " << ctx.SelfID.ToString() << - " Send sysview TEvDescribeSchemeResult to# " << Source.ToString() << - " Cookie: " << SourceCookie << - " TEvDescribeSchemeResult: " << result->ToString()); - - TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds()); - + FillSystemViewDescr(descr, schemeShardId); + } + }; + + LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, + "Actor# " << ctx.SelfID.ToString() << + " Send sysview TEvDescribeSchemeResult to# " << Source.ToString() << + " Cookie: " << SourceCookie << + " TEvDescribeSchemeResult: " << result->ToString()); + + TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds()); + result->Record.SetStatus(NKikimrScheme::StatusSuccess); - ctx.Send(Source, result.Release(), 0, SourceCookie); - return Die(ctx); - } - + ctx.Send(Source, result.Release(), 0, SourceCookie); + return Die(ctx); + } + void Handle(TEvTxProxyReq::TEvNavigateScheme::TPtr &ev, const TActorContext &ctx); void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr &ev, const TActorContext &ctx); void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, const TActorContext &ctx); @@ -283,9 +283,9 @@ void TDescribeReq::Handle(TEvTxProxyReq::TEvNavigateScheme::TPtr &ev, const TAct } request->ResultSet.emplace_back(entry); - + ctx.Send(Services.SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request), 0, SourceCookie); - + SchemeRequest = ev->Release(); Become(&TThis::StateWaitResolve); } @@ -350,24 +350,24 @@ void TDescribeReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr & } } - const auto& describePath = SchemeRequest->Ev->Get()->Record.GetDescribePath(); - - if (entry.TableId.IsSystemView()) { - // don't go to schemeshard - const auto& path = describePath.GetPath(); - - if (entry.TableId.SysViewInfo == NSysView::SysPathName) { - return SendSystemViewFolderResult(entry, path, ctx); - } - - return SendSystemViewResult(entry, path, ctx); - } - + const auto& describePath = SchemeRequest->Ev->Get()->Record.GetDescribePath(); + + if (entry.TableId.IsSystemView()) { + // don't go to schemeshard + const auto& path = describePath.GetPath(); + + if (entry.TableId.SysViewInfo == NSysView::SysPathName) { + return SendSystemViewFolderResult(entry, path, ctx); + } + + return SendSystemViewResult(entry, path, ctx); + } + const ui64 shardToRequest = entry.DomainInfo->ExtractSchemeShard(); - + TAutoPtr<NSchemeShard::TEvSchemeShard::TEvDescribeScheme> req( new NSchemeShard::TEvSchemeShard::TEvDescribeScheme(describePath)); - + LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, "Actor# " << ctx.SelfID.ToString() << " SEND to# " << shardToRequest << " shardToRequest " << req->ToString()); @@ -387,48 +387,48 @@ void TDescribeReq::Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult: TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds()); if (AppData()->FeatureFlags.GetEnableSystemViews()) { - const auto& pathDescription = ev->Get()->GetRecord().GetPathDescription(); - const auto& self = pathDescription.GetSelf(); - - TDomainsInfo *domainsInfo = AppData()->DomainsInfo.Get(); - Y_VERIFY(!domainsInfo->Domains.empty()); - - bool needSysFolder = false; + const auto& pathDescription = ev->Get()->GetRecord().GetPathDescription(); + const auto& self = pathDescription.GetSelf(); + + TDomainsInfo *domainsInfo = AppData()->DomainsInfo.Get(); + Y_VERIFY(!domainsInfo->Domains.empty()); + + bool needSysFolder = false; if (self.GetPathType() == NKikimrSchemeOp::EPathType::EPathTypeSubDomain || self.GetPathType() == NKikimrSchemeOp::EPathType::EPathTypeColumnStore || self.GetPathType() == NKikimrSchemeOp::EPathType::EPathTypeColumnTable) { - needSysFolder = true; + needSysFolder = true; } else if (self.GetPathId() == NSchemeShard::RootPathId) { - for (const auto& [_, domain] : domainsInfo->Domains) { - if (domain->SchemeRoot == self.GetSchemeshardId()) { - needSysFolder = true; - break; - } - } - } - - if (needSysFolder) { - bool hasSysFolder = false; - - const auto& children = pathDescription.GetChildren(); - if (!children.empty()) { - auto size = children.size(); - if (children[size - 1].GetName() == NSysView::SysPathName) { - hasSysFolder = true; - } - } - - if (!hasSysFolder) { - auto* record = ev->Get()->MutableRecord(); - auto* descr = record->MutablePathDescription()->AddChildren(); - descr->SetName(TString(NSysView::SysPathName)); + for (const auto& [_, domain] : domainsInfo->Domains) { + if (domain->SchemeRoot == self.GetSchemeshardId()) { + needSysFolder = true; + break; + } + } + } + + if (needSysFolder) { + bool hasSysFolder = false; + + const auto& children = pathDescription.GetChildren(); + if (!children.empty()) { + auto size = children.size(); + if (children[size - 1].GetName() == NSysView::SysPathName) { + hasSysFolder = true; + } + } + + if (!hasSysFolder) { + auto* record = ev->Get()->MutableRecord(); + auto* descr = record->MutablePathDescription()->AddChildren(); + descr->SetName(TString(NSysView::SysPathName)); descr->SetPathType(NKikimrSchemeOp::EPathTypeDir); - FillSystemViewDescr(descr, self.GetSchemeshardId()); - } - } - } - + FillSystemViewDescr(descr, self.GetSchemeshardId()); + } + } + } + ctx.ExecutorThread.Send(ev->Forward(Source)); return Die(ctx); } diff --git a/ydb/core/tx/tx_proxy/mon.cpp b/ydb/core/tx/tx_proxy/mon.cpp index d18b6d5edb..14f5d47064 100644 --- a/ydb/core/tx/tx_proxy/mon.cpp +++ b/ydb/core/tx/tx_proxy/mon.cpp @@ -34,29 +34,29 @@ TTxProxyMon::TTxProxyMon(const TIntrusivePtr<NMonitoring::TDynamicCounters>& cou ReportStatusNotOK = DataReqGroup->GetCounter("ReportStatus/NotOK", true); ReportStatusStreamData = DataReqGroup->GetCounter("ReportStatus/StreamData", true); - TxPrepareTimeHgram = DataReqGroup->GetHistogram("TxPrepareTimesMs", - NMonitoring::ExponentialHistogram(20, 2, 1)); - TxExecuteTimeHgram = DataReqGroup->GetHistogram("TxExecuteTimesMs", - NMonitoring::ExponentialHistogram(20, 2, 1)); - TxTotalTimeHgram = DataReqGroup->GetHistogram("TxTotalTimesMs", - NMonitoring::ExponentialHistogram(20, 2, 1)); - TxPrepareSpreadHgram = DataReqGroup->GetHistogram("TxPrepareSpreadMs", - NMonitoring::ExponentialHistogram(10, 2, 1)); - TxPrepareArriveSpreadHgram = DataReqGroup->GetHistogram("TxPrepareArriveSpreadMs", - NMonitoring::ExponentialHistogram(10, 2, 1)); - TxPrepareCompleteSpreadHgram = DataReqGroup->GetHistogram("TxPrepareCompleteSpreadMs", - NMonitoring::ExponentialHistogram(10, 2, 1)); - TxExecSpreadHgram = DataReqGroup->GetHistogram("TxExecSpreadMs", - NMonitoring::ExponentialHistogram(10, 2, 1)); - - TxPrepareSetProgramHgram = DataReqGroup->GetHistogram("TxPrepareSetProgramUs", - NMonitoring::ExponentialHistogram(10, 2, 100)); - TxPrepareResolveHgram = DataReqGroup->GetHistogram("TxPrepareResolveUs", - NMonitoring::ExponentialHistogram(10, 2, 100)); - TxPrepareBuildShardProgramsHgram = DataReqGroup->GetHistogram("TxPrepareBuildShardProgramsUs", - NMonitoring::ExponentialHistogram(10, 2, 100)); - TxPrepareSendShardProgramsHgram = DataReqGroup->GetHistogram("TxPrepareSendShardProgramsUs", - NMonitoring::ExponentialHistogram(10, 2, 100)); + TxPrepareTimeHgram = DataReqGroup->GetHistogram("TxPrepareTimesMs", + NMonitoring::ExponentialHistogram(20, 2, 1)); + TxExecuteTimeHgram = DataReqGroup->GetHistogram("TxExecuteTimesMs", + NMonitoring::ExponentialHistogram(20, 2, 1)); + TxTotalTimeHgram = DataReqGroup->GetHistogram("TxTotalTimesMs", + NMonitoring::ExponentialHistogram(20, 2, 1)); + TxPrepareSpreadHgram = DataReqGroup->GetHistogram("TxPrepareSpreadMs", + NMonitoring::ExponentialHistogram(10, 2, 1)); + TxPrepareArriveSpreadHgram = DataReqGroup->GetHistogram("TxPrepareArriveSpreadMs", + NMonitoring::ExponentialHistogram(10, 2, 1)); + TxPrepareCompleteSpreadHgram = DataReqGroup->GetHistogram("TxPrepareCompleteSpreadMs", + NMonitoring::ExponentialHistogram(10, 2, 1)); + TxExecSpreadHgram = DataReqGroup->GetHistogram("TxExecSpreadMs", + NMonitoring::ExponentialHistogram(10, 2, 1)); + + TxPrepareSetProgramHgram = DataReqGroup->GetHistogram("TxPrepareSetProgramUs", + NMonitoring::ExponentialHistogram(10, 2, 100)); + TxPrepareResolveHgram = DataReqGroup->GetHistogram("TxPrepareResolveUs", + NMonitoring::ExponentialHistogram(10, 2, 100)); + TxPrepareBuildShardProgramsHgram = DataReqGroup->GetHistogram("TxPrepareBuildShardProgramsUs", + NMonitoring::ExponentialHistogram(10, 2, 100)); + TxPrepareSendShardProgramsHgram = DataReqGroup->GetHistogram("TxPrepareSendShardProgramsUs", + NMonitoring::ExponentialHistogram(10, 2, 100)); MiniKQLResolveSentToShard = DataReqGroup->GetCounter("MiniKQLResolve/SentToShard", true); MiniKQLWrongRequest = DataReqGroup->GetCounter("MiniKQLResolve/WrongRequest", true); diff --git a/ydb/core/tx/tx_proxy/proxy.h b/ydb/core/tx/tx_proxy/proxy.h index 45a2f236e0..86d4f1ac79 100644 --- a/ydb/core/tx/tx_proxy/proxy.h +++ b/ydb/core/tx/tx_proxy/proxy.h @@ -164,8 +164,8 @@ struct TEvTxUserProxy { struct TEvInvalidateTable : public TEventPB<TEvInvalidateTable, NKikimrTxUserProxy::TEvInvalidateTable, EvInvalidateTable> { TEvInvalidateTable() = default; TEvInvalidateTable(const TTableId& tableId) { - Record.SetSchemeShardId(tableId.PathId.OwnerId); - Record.SetTableId(tableId.PathId.LocalPathId); + Record.SetSchemeShardId(tableId.PathId.OwnerId); + Record.SetTableId(tableId.PathId.LocalPathId); } }; diff --git a/ydb/core/tx/tx_proxy/proxy_ut_helpers.h b/ydb/core/tx/tx_proxy/proxy_ut_helpers.h index 9afd28473e..846b0cf568 100644 --- a/ydb/core/tx/tx_proxy/proxy_ut_helpers.h +++ b/ydb/core/tx/tx_proxy/proxy_ut_helpers.h @@ -195,7 +195,7 @@ public: GetSettings().SetNodeCount(staticNodes); GetSettings().SetDynamicNodeCount(dynamicNodes); - GetSettings().SetEnableSystemViews(false); + GetSettings().SetEnableSystemViews(false); for (ui32 poolNum = 1; poolNum <= poolsCount; ++poolNum) { GetSettings().AddStoragePoolType("storage-pool-number-" + ToString(poolNum)); diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h index fb859c90a2..0aaee7e7f5 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h @@ -475,11 +475,11 @@ private: TableKind = request.ResultSet.front().Kind; bool isOlapTable = (TableKind == NSchemeCache::TSchemeCacheNavigate::KindOlapTable); - if (request.ResultSet.front().TableId.IsSystemView()) { - return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, + if (request.ResultSet.front().TableId.IsSystemView()) { + return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Sprintf("Table '%s' is a system view. Bulk upsert is not supported.", GetTable().c_str()), ctx); - } - + } + ResolveNamesResult = ev->Get()->Request; bool makeYdbSchema = isOlapTable || (GetSourceType() != EUploadSource::ProtoValues); diff --git a/ydb/core/util/concurrent_rw_hash.cpp b/ydb/core/util/concurrent_rw_hash.cpp index f3aabfc056..8d86320f02 100644 --- a/ydb/core/util/concurrent_rw_hash.cpp +++ b/ydb/core/util/concurrent_rw_hash.cpp @@ -1 +1 @@ -#include "concurrent_rw_hash.h" +#include "concurrent_rw_hash.h" diff --git a/ydb/core/util/concurrent_rw_hash.h b/ydb/core/util/concurrent_rw_hash.h index 46ca5b55f4..4a471bc567 100644 --- a/ydb/core/util/concurrent_rw_hash.h +++ b/ydb/core/util/concurrent_rw_hash.h @@ -1,117 +1,117 @@ -#pragma once - -#include <util/generic/hash.h> -#include <util/system/rwlock.h> - -#include <array> - -template <typename K, typename V, size_t BucketCount = 64> -class TConcurrentRWHashMap { -public: - using TActualMap = THashMap<K, V>; - - struct TBucket { - friend class TConcurrentRWHashMap; - - private: - TActualMap Map; - mutable TRWMutex RWLock; - - public: - TRWMutex& GetLock() const { - return RWLock; - } - - TActualMap& GetMap() { - return Map; - } - - const TActualMap& GetMap() const { - return Map; - } - }; - - std::array<TBucket, BucketCount> Buckets; - -public: - Y_FORCE_INLINE TBucket& GetBucketForKey(const K& key) { - return Buckets[THash<K>()(key) % BucketCount]; - } - - Y_FORCE_INLINE const TBucket& GetBucketForKey(const K& key) const { - return Buckets[THash<K>()(key) % BucketCount]; - } - - void Insert(const K& key, const V& value) { - TBucket& bucket = GetBucketForKey(key); - TWriteGuard guard(bucket.RWLock); - - bucket.Map[key] = value; - } - - V& InsertIfAbsent(const K& key, const V& value) { - TBucket& bucket = GetBucketForKey(key); - TWriteGuard guard(bucket.RWLock); - - return bucket.Map.insert(std::make_pair(key, value)).first->second; - } - - template <typename Callable> - V& InsertIfAbsentWithInit(const K& key, Callable initFunc) { - TBucket& bucket = GetBucketForKey(key); - TWriteGuard guard(bucket.RWLock); - - typename TActualMap::iterator it = bucket.Map.find(key); - if (it != bucket.Map.end()) { - return it->second; - } - return bucket.Map.insert(std::make_pair(key, initFunc())).first->second; - } - - V Get(const K& key) const { - const TBucket& bucket = GetBucketForKey(key); - TReadGuard guard(bucket.RWLock); - - typename TActualMap::const_iterator it = bucket.Map.find(key); - Y_VERIFY(it != bucket.Map.end(), "not found by key"); - return it->second; - } - - bool Get(const K& key, V& result) const { - const TBucket& bucket = GetBucketForKey(key); - TReadGuard guard(bucket.RWLock); - - typename TActualMap::const_iterator it = bucket.Map.find(key); - if (it != bucket.Map.end()) { - result = it->second; - return true; - } - return false; - } - - V Remove(const K& key) { - TBucket& bucket = GetBucketForKey(key); - TWriteGuard guard(bucket.RWLock); - - typename TActualMap::iterator it = bucket.Map.find(key); - Y_VERIFY(it != bucket.Map.end(), "removing non-existent key"); - V r = it->second; - bucket.Map.erase(it); - return r; - } - - bool Has(const K& key) const { - const TBucket& bucket = GetBucketForKey(key); - TReadGuard guard(bucket.RWLock); - - typename TActualMap::const_iterator it = bucket.Map.find(key); - return (it != bucket.Map.end()); - } - - bool Erase(const K& key) { - TBucket& bucket = GetBucketForKey(key); - TWriteGuard guard(bucket.RWLock); - - return bucket.Map.erase(key); - } -}; +#pragma once + +#include <util/generic/hash.h> +#include <util/system/rwlock.h> + +#include <array> + +template <typename K, typename V, size_t BucketCount = 64> +class TConcurrentRWHashMap { +public: + using TActualMap = THashMap<K, V>; + + struct TBucket { + friend class TConcurrentRWHashMap; + + private: + TActualMap Map; + mutable TRWMutex RWLock; + + public: + TRWMutex& GetLock() const { + return RWLock; + } + + TActualMap& GetMap() { + return Map; + } + + const TActualMap& GetMap() const { + return Map; + } + }; + + std::array<TBucket, BucketCount> Buckets; + +public: + Y_FORCE_INLINE TBucket& GetBucketForKey(const K& key) { + return Buckets[THash<K>()(key) % BucketCount]; + } + + Y_FORCE_INLINE const TBucket& GetBucketForKey(const K& key) const { + return Buckets[THash<K>()(key) % BucketCount]; + } + + void Insert(const K& key, const V& value) { + TBucket& bucket = GetBucketForKey(key); + TWriteGuard guard(bucket.RWLock); + + bucket.Map[key] = value; + } + + V& InsertIfAbsent(const K& key, const V& value) { + TBucket& bucket = GetBucketForKey(key); + TWriteGuard guard(bucket.RWLock); + + return bucket.Map.insert(std::make_pair(key, value)).first->second; + } + + template <typename Callable> + V& InsertIfAbsentWithInit(const K& key, Callable initFunc) { + TBucket& bucket = GetBucketForKey(key); + TWriteGuard guard(bucket.RWLock); + + typename TActualMap::iterator it = bucket.Map.find(key); + if (it != bucket.Map.end()) { + return it->second; + } + return bucket.Map.insert(std::make_pair(key, initFunc())).first->second; + } + + V Get(const K& key) const { + const TBucket& bucket = GetBucketForKey(key); + TReadGuard guard(bucket.RWLock); + + typename TActualMap::const_iterator it = bucket.Map.find(key); + Y_VERIFY(it != bucket.Map.end(), "not found by key"); + return it->second; + } + + bool Get(const K& key, V& result) const { + const TBucket& bucket = GetBucketForKey(key); + TReadGuard guard(bucket.RWLock); + + typename TActualMap::const_iterator it = bucket.Map.find(key); + if (it != bucket.Map.end()) { + result = it->second; + return true; + } + return false; + } + + V Remove(const K& key) { + TBucket& bucket = GetBucketForKey(key); + TWriteGuard guard(bucket.RWLock); + + typename TActualMap::iterator it = bucket.Map.find(key); + Y_VERIFY(it != bucket.Map.end(), "removing non-existent key"); + V r = it->second; + bucket.Map.erase(it); + return r; + } + + bool Has(const K& key) const { + const TBucket& bucket = GetBucketForKey(key); + TReadGuard guard(bucket.RWLock); + + typename TActualMap::const_iterator it = bucket.Map.find(key); + return (it != bucket.Map.end()); + } + + bool Erase(const K& key) { + TBucket& bucket = GetBucketForKey(key); + TWriteGuard guard(bucket.RWLock); + + return bucket.Map.erase(key); + } +}; diff --git a/ydb/core/util/concurrent_rw_hash_ut.cpp b/ydb/core/util/concurrent_rw_hash_ut.cpp index e761b69ece..35d175a544 100644 --- a/ydb/core/util/concurrent_rw_hash_ut.cpp +++ b/ydb/core/util/concurrent_rw_hash_ut.cpp @@ -1,154 +1,154 @@ -#include <concurrent_rw_hash.h> - -#include <library/cpp/testing/unittest/gtest.h> - -#include <util/generic/string.h> -#include <util/generic/vector.h> - -TEST(TConcurrentRWHashTest, TEmptyGetTest) { - TConcurrentRWHashMap<TString, ui32> h; - - EXPECT_FALSE(h.Has("key")); - - ui32 res = 100; - EXPECT_FALSE(h.Get("key", res)); - EXPECT_EQ(res, 100); - - // Can't check h.Get("key") here because it has Y_VERIFY inside o_O -} - -TEST(TConcurrentRWHashTest, TInsertTest) { - TConcurrentRWHashMap<TString, ui32> h; - - h.Insert("key1", 1); - h.Insert("key2", 2); - - EXPECT_EQ(h.Get("key1"), 1); - EXPECT_EQ(h.Get("key2"), 2); - - ui32 res = 100; - EXPECT_TRUE(h.Has("key1")); - EXPECT_TRUE(h.Get("key1", res)); - EXPECT_EQ(res, 1); - - EXPECT_TRUE(h.Has("key2")); - EXPECT_TRUE(h.Get("key2", res)); - EXPECT_EQ(res, 2); - - EXPECT_FALSE(h.Has("key3")); - EXPECT_FALSE(h.Get("key3", res)); - EXPECT_EQ(res, 2); -} - -TEST(TConcurrentRWHashTest, TInsertIfAbsentTest) { - TConcurrentRWHashMap<TString, ui32> h; - - ui32 res; - EXPECT_FALSE(h.Has("key")); - EXPECT_FALSE(h.Get("key", res)); - - EXPECT_EQ(h.InsertIfAbsent("key", 1), static_cast<ui32>(1)); - EXPECT_EQ(h.Get("key"), 1); - - EXPECT_EQ(h.InsertIfAbsent("key", 2), static_cast<ui32>(1)); - EXPECT_EQ(h.Get("key"), 1); -} - -TEST(TConcurrentRWHashTest, TInsertIfAbsentTestFunc) { - TConcurrentRWHashMap<TString, ui32> h; - - bool initialized = false; - auto f = [&initialized]() { - initialized = true; - return static_cast<ui32>(1); - }; - - ui32 res = 0; - EXPECT_FALSE(h.Get("key", res)); - EXPECT_EQ(res, 0); - - EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 1); - EXPECT_EQ(h.Get("key"), 1); - EXPECT_TRUE(initialized); - - initialized = false; - EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 1); - EXPECT_EQ(h.Get("key"), 1); - EXPECT_FALSE(initialized); - - h.Insert("key", 2); - EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 2); - EXPECT_EQ(h.Get("key"), 2); - EXPECT_FALSE(initialized); -} - -TEST(TConcurrentRWHashTest, TRemoveTest) { - TConcurrentRWHashMap<TString, ui32> h; - - EXPECT_FALSE(h.Has("key1")); - EXPECT_FALSE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - - h.Insert("key1", 1); - EXPECT_TRUE(h.Has("key1")); - EXPECT_FALSE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - EXPECT_EQ(h.Get("key1"), 1); - - EXPECT_EQ(h.Remove("key1"), 1); - EXPECT_FALSE(h.Has("key1")); - EXPECT_FALSE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - - h.Insert("key1", 1); - h.Insert("key2", 2); - - EXPECT_TRUE(h.Has("key1")); - EXPECT_TRUE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - EXPECT_EQ(h.Get("key1"), 1); - EXPECT_EQ(h.Get("key2"), 2); - - EXPECT_EQ(h.Remove("key2"), 2); - EXPECT_TRUE(h.Has("key1")); - EXPECT_FALSE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - EXPECT_EQ(h.Get("key1"), 1); -} - -TEST(TConcurrentRWHashTest, TEraseTest) { - TConcurrentRWHashMap<TString, ui32> h; - - EXPECT_FALSE(h.Has("key1")); - EXPECT_FALSE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - - h.Insert("key1", 1); - EXPECT_TRUE(h.Has("key1")); - EXPECT_FALSE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - EXPECT_EQ(h.Get("key1"), 1); - - EXPECT_TRUE(h.Erase("key1")); - EXPECT_FALSE(h.Has("key1")); - EXPECT_FALSE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - - h.Insert("key1", 1); - h.Insert("key2", 2); - - EXPECT_TRUE(h.Has("key1")); - EXPECT_TRUE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - EXPECT_EQ(h.Get("key1"), 1); - EXPECT_EQ(h.Get("key2"), 2); - - EXPECT_TRUE(h.Erase("key2")); - EXPECT_TRUE(h.Has("key1")); - EXPECT_FALSE(h.Has("key2")); - EXPECT_FALSE(h.Has("key3")); - EXPECT_EQ(h.Get("key1"), 1); - - EXPECT_FALSE(h.Erase("key2")); -} - +#include <concurrent_rw_hash.h> + +#include <library/cpp/testing/unittest/gtest.h> + +#include <util/generic/string.h> +#include <util/generic/vector.h> + +TEST(TConcurrentRWHashTest, TEmptyGetTest) { + TConcurrentRWHashMap<TString, ui32> h; + + EXPECT_FALSE(h.Has("key")); + + ui32 res = 100; + EXPECT_FALSE(h.Get("key", res)); + EXPECT_EQ(res, 100); + + // Can't check h.Get("key") here because it has Y_VERIFY inside o_O +} + +TEST(TConcurrentRWHashTest, TInsertTest) { + TConcurrentRWHashMap<TString, ui32> h; + + h.Insert("key1", 1); + h.Insert("key2", 2); + + EXPECT_EQ(h.Get("key1"), 1); + EXPECT_EQ(h.Get("key2"), 2); + + ui32 res = 100; + EXPECT_TRUE(h.Has("key1")); + EXPECT_TRUE(h.Get("key1", res)); + EXPECT_EQ(res, 1); + + EXPECT_TRUE(h.Has("key2")); + EXPECT_TRUE(h.Get("key2", res)); + EXPECT_EQ(res, 2); + + EXPECT_FALSE(h.Has("key3")); + EXPECT_FALSE(h.Get("key3", res)); + EXPECT_EQ(res, 2); +} + +TEST(TConcurrentRWHashTest, TInsertIfAbsentTest) { + TConcurrentRWHashMap<TString, ui32> h; + + ui32 res; + EXPECT_FALSE(h.Has("key")); + EXPECT_FALSE(h.Get("key", res)); + + EXPECT_EQ(h.InsertIfAbsent("key", 1), static_cast<ui32>(1)); + EXPECT_EQ(h.Get("key"), 1); + + EXPECT_EQ(h.InsertIfAbsent("key", 2), static_cast<ui32>(1)); + EXPECT_EQ(h.Get("key"), 1); +} + +TEST(TConcurrentRWHashTest, TInsertIfAbsentTestFunc) { + TConcurrentRWHashMap<TString, ui32> h; + + bool initialized = false; + auto f = [&initialized]() { + initialized = true; + return static_cast<ui32>(1); + }; + + ui32 res = 0; + EXPECT_FALSE(h.Get("key", res)); + EXPECT_EQ(res, 0); + + EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 1); + EXPECT_EQ(h.Get("key"), 1); + EXPECT_TRUE(initialized); + + initialized = false; + EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 1); + EXPECT_EQ(h.Get("key"), 1); + EXPECT_FALSE(initialized); + + h.Insert("key", 2); + EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 2); + EXPECT_EQ(h.Get("key"), 2); + EXPECT_FALSE(initialized); +} + +TEST(TConcurrentRWHashTest, TRemoveTest) { + TConcurrentRWHashMap<TString, ui32> h; + + EXPECT_FALSE(h.Has("key1")); + EXPECT_FALSE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + + h.Insert("key1", 1); + EXPECT_TRUE(h.Has("key1")); + EXPECT_FALSE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + EXPECT_EQ(h.Get("key1"), 1); + + EXPECT_EQ(h.Remove("key1"), 1); + EXPECT_FALSE(h.Has("key1")); + EXPECT_FALSE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + + h.Insert("key1", 1); + h.Insert("key2", 2); + + EXPECT_TRUE(h.Has("key1")); + EXPECT_TRUE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + EXPECT_EQ(h.Get("key1"), 1); + EXPECT_EQ(h.Get("key2"), 2); + + EXPECT_EQ(h.Remove("key2"), 2); + EXPECT_TRUE(h.Has("key1")); + EXPECT_FALSE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + EXPECT_EQ(h.Get("key1"), 1); +} + +TEST(TConcurrentRWHashTest, TEraseTest) { + TConcurrentRWHashMap<TString, ui32> h; + + EXPECT_FALSE(h.Has("key1")); + EXPECT_FALSE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + + h.Insert("key1", 1); + EXPECT_TRUE(h.Has("key1")); + EXPECT_FALSE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + EXPECT_EQ(h.Get("key1"), 1); + + EXPECT_TRUE(h.Erase("key1")); + EXPECT_FALSE(h.Has("key1")); + EXPECT_FALSE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + + h.Insert("key1", 1); + h.Insert("key2", 2); + + EXPECT_TRUE(h.Has("key1")); + EXPECT_TRUE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + EXPECT_EQ(h.Get("key1"), 1); + EXPECT_EQ(h.Get("key2"), 2); + + EXPECT_TRUE(h.Erase("key2")); + EXPECT_TRUE(h.Has("key1")); + EXPECT_FALSE(h.Has("key2")); + EXPECT_FALSE(h.Has("key3")); + EXPECT_EQ(h.Get("key1"), 1); + + EXPECT_FALSE(h.Erase("key2")); +} + diff --git a/ydb/core/util/memory_tracker.cpp b/ydb/core/util/memory_tracker.cpp index fab4282420..848b36e276 100644 --- a/ydb/core/util/memory_tracker.cpp +++ b/ydb/core/util/memory_tracker.cpp @@ -1,307 +1,307 @@ -#include "memory_tracker.h" - +#include "memory_tracker.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> #include <ydb/core/mon/mon.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/memory_tracker.h> -#include <library/cpp/html/escape/escape.h> -#include <library/cpp/monlib/service/pages/templates.h> - -#include <util/generic/xrange.h> - -using namespace NMonitoring; -using namespace NActors::NMemory::NPrivate; - -namespace NKikimr { -namespace NMemory { - -class TMemoryTrackerActor : public TActorBootstrapped<TMemoryTrackerActor> { -public: - TMemoryTrackerActor(TDuration updateInterval, TDynamicCounterPtr appCounters) - : UpdateInterval(updateInterval) - , AppCounters(appCounters) - { - auto utilsCounters = GetServiceCounters(AppCounters, "utils"); - Counters = utilsCounters->GetSubgroup("component", "memory_tracker"); - } - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::MEMORY_TRACKER; - } - - void Bootstrap(const TActorContext&) { -#if defined(ENABLE_MEMORY_TRACKING) - TMemoryTracker::Instance()->Initialize(); - RegisterCounters(Counters); - - TMon* mon = AppData()->Mon; - if (mon) { - auto *actorsMonPage = mon->RegisterIndexPage("actors", "Actors"); - mon->RegisterActorPage(actorsMonPage, "memory_tracker", "Memory tracker", - false, TlsActivationContext->ExecutorThread.ActorSystem, SelfId()); - } - - Collect(); -#endif - Become(&TMemoryTrackerActor::StateWork); - } - - STRICT_STFUNC(StateWork, - cFunc(TEvents::TEvWakeup::EventType, Wakeup) - hFunc(NMon::TEvHttpInfo, HttpInfo) - cFunc(TEvents::TEvPoison::EventType, PassAway) - ) - -private: - void Wakeup() { - Collect(); - } - - void Collect() { - Schedule(UpdateInterval, new TEvents::TEvWakeup()); - - std::vector<TMetric> metrics; - TMemoryTracker::Instance()->GatherMetrics(metrics); - - auto count = metrics.size(); - if (Static.size() != count) { - return; - } - - const auto& sensors = TMemoryTracker::Instance()->GetSensors(); - - for (auto i : xrange(count)) { - auto& metric = Static[i]; - metric.Current = metrics[i]; - metric.Peak.CalculatePeak(metric.Current); - - if (sensors.find(i) != sensors.end()) { - *metric.CurrentMemCounter = metric.Current.GetMemory(); - *metric.PeakMemCounter = metric.Peak.GetMemory(); - *metric.CurrentCountCounter = metric.Current.GetCount(); - *metric.PeakCountCounter = metric.Peak.GetCount(); - } - } - - GatherDynamicMetrics(); - } - - void RegisterCounters(NMonitoring::TDynamicCounterPtr counters) { - auto componentGroup = counters->GetSubgroup("component", "memory_tracker"); - - auto count = TMemoryTracker::Instance()->GetCount(); - Static.resize(count); - - const auto& sensors = TMemoryTracker::Instance()->GetSensors(); - for (auto index : sensors) { - auto name = TMemoryTracker::Instance()->GetName(index); - auto& metric = Static[index]; - auto group = componentGroup->GetSubgroup("label", name); - metric.CurrentMemCounter = group->GetCounter("MT/Memory"); - metric.PeakMemCounter = group->GetCounter("MT/PeakMemory"); - metric.CurrentCountCounter = group->GetCounter("MT/Count"); - metric.PeakCountCounter = group->GetCounter("MT/PeakCount"); - } - } - - void UpdateDynamicCounters() { - auto utils = GetServiceCounters(AppCounters, "utils"); - - if (Dynamic.find(AnonRssSize) == Dynamic.end()) { - Dynamic.emplace(AnonRssSize, utils->GetCounter("Process/AnonRssSize")); - } - - if (Dynamic.find(LFMmapped) == Dynamic.end()) { - auto lfAllocGroup = utils->FindSubgroup("component", "lfalloc"); - if (lfAllocGroup) { - Dynamic.emplace(LFMmapped, lfAllocGroup->GetCounter("ActiveBytesMmapped")); - Dynamic.emplace(LFSmall, lfAllocGroup->GetCounter("ActiveBytesSmall")); - Dynamic.emplace(LFLarge, lfAllocGroup->GetCounter("ActiveBytesLarge")); - Dynamic.emplace(LFSystem, lfAllocGroup->GetCounter("ActiveBytesSystem")); - } - } - - if (Dynamic.find(MkqlGlobalPoolTotalBytes) == Dynamic.end()) { - auto mkqlAllocGroup = utils->FindSubgroup("subsystem", "mkqlalloc"); - if (mkqlAllocGroup) { - Dynamic.emplace(MkqlGlobalPoolTotalBytes, mkqlAllocGroup->GetCounter("GlobalPoolTotalBytes")); - - auto getNamedPagePoolCounter = [&mkqlAllocGroup](const char* name) { - return mkqlAllocGroup->GetCounter(TString(name) + "/TotalBytesAllocated"); - }; - Dynamic.emplace(MkqlKqp, getNamedPagePoolCounter("kqp")); - Dynamic.emplace(MkqlCompile, getNamedPagePoolCounter("compile")); - Dynamic.emplace(MkqlDatashard, getNamedPagePoolCounter("datashard")); - Dynamic.emplace(MkqlLocalTx, getNamedPagePoolCounter("local_tx")); - Dynamic.emplace(MkqlTxProxy, getNamedPagePoolCounter("tx_proxy")); - } - } - } - - void GatherDynamicMetrics() { - UpdateDynamicCounters(); - - for (auto& [_, metric] : Dynamic) { - if (metric.Counter) { - metric.Current.SetMemory(metric.Counter->Val()); - metric.Peak.CalculatePeak(metric.Current); - } - } - } - - static TString PrettyNumber(ssize_t val) { - if (val == 0) { - return "0"; - } - TStringStream str; - if (val < 0) { - str << "- "; - } - ui64 value = val < 0 ? -val : val; - TSmallVec<ui64> parts; - while (value > 0) { - auto head = value / 1000; - parts.push_back(value - head * 1000); - value = head; - } - str << parts.back(); - parts.pop_back(); - while (!parts.empty()) { - str << Sprintf(" %03" PRIu64, parts.back()); - parts.pop_back(); - } - return str.Str(); - } - - void HttpInfo(NMon::TEvHttpInfo::TPtr& ev) { - TStringStream str; - - HTML(str) { -#if defined(ENABLE_MEMORY_TRACKING) - TABLE_CLASS("table") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "Metric"; } - TABLEH() { str << "Memory"; } - TABLEH() { str << "Peak memory"; } - } - } - TABLEBODY() { - auto outputDynamic = [&str, &__stream] (const char* name, const TDynamicMetric& metric) { - TABLED() { str << name; } - TAG_CLASS_STYLE(TTableD, "", "text-align:right") { - str << PrettyNumber(metric.Current.GetMemory()); } - TAG_CLASS_STYLE(TTableD, "", "text-align:right") { - str << PrettyNumber(metric.Peak.GetMemory()); } - }; - - if (Dynamic.find(AnonRssSize) != Dynamic.end()) { - TABLER() { outputDynamic("AnonRssSize", Dynamic[AnonRssSize]); } - } - if (Dynamic.find(LFMmapped) != Dynamic.end()) { - TABLER() { - TABLED() { str << "LFAlloc"; } - } - TABLER() { outputDynamic("Mmapped", Dynamic[LFMmapped]); } - TABLER() { outputDynamic("Small", Dynamic[LFSmall]); } - TABLER() { outputDynamic("Large", Dynamic[LFLarge]); } - TABLER() { outputDynamic("System", Dynamic[LFSystem]); } - } - if (Dynamic.find(MkqlGlobalPoolTotalBytes) != Dynamic.end()) { - TABLER() { - TABLED() { str << "MKQLAlloc"; } - } - TABLER() { outputDynamic("GlobalPoolTotalBytes", Dynamic[MkqlGlobalPoolTotalBytes]); } - TABLER() { outputDynamic("Kqp", Dynamic[MkqlKqp]); } - TABLER() { outputDynamic("Compile", Dynamic[MkqlCompile]); } - TABLER() { outputDynamic("Datashard", Dynamic[MkqlDatashard]); } - TABLER() { outputDynamic("LocalTx", Dynamic[MkqlLocalTx]); } - TABLER() { outputDynamic("TxProxy", Dynamic[MkqlTxProxy]); } - } - } - } - TABLE_CLASS("table") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "Metric"; } - TABLEH() { str << "Count"; } - TABLEH() { str << "Peak count"; } - TABLEH() { str << "Memory"; } - TABLEH() { str << "Peak memory"; } - } - } - TABLEBODY() { - const auto& indices = TMemoryTracker::Instance()->GetMetricIndices(); - for (const auto& [name, index] : indices) { - TABLER() { - auto& metric = Static[index]; - TABLED() { str << NHtml::EscapeText(name); } - TAG_CLASS_STYLE(TTableD, "", "text-align:right") { - str << PrettyNumber(metric.Current.GetCount()); } - TAG_CLASS_STYLE(TTableD, "", "text-align:right") { - str << PrettyNumber(metric.Peak.GetCount()); } - TAG_CLASS_STYLE(TTableD, "", "text-align:right") { - str << PrettyNumber(metric.Current.GetMemory()); } - TAG_CLASS_STYLE(TTableD, "", "text-align:right") { - str << PrettyNumber(metric.Peak.GetMemory()); } - } - } - } - } -#endif - } - - Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); - } - -private: - const TDuration UpdateInterval; - - enum EDynamicMetric { - AnonRssSize, - LFMmapped, - LFSmall, - LFLarge, - LFSystem, - MkqlGlobalPoolTotalBytes, - MkqlKqp, - MkqlCompile, - MkqlDatashard, - MkqlLocalTx, - MkqlTxProxy - }; - - TDynamicCounterPtr AppCounters; - TDynamicCounterPtr Counters; - - struct TStaticMetric { - TMetric Current; - TMetric Peak; - TDynamicCounters::TCounterPtr CurrentMemCounter; - TDynamicCounters::TCounterPtr PeakMemCounter; - TDynamicCounters::TCounterPtr CurrentCountCounter; - TDynamicCounters::TCounterPtr PeakCountCounter; - }; - - struct TDynamicMetric { - TMetric Current; - TMetric Peak; - TDynamicCounters::TCounterPtr Counter; - - TDynamicMetric(TDynamicCounters::TCounterPtr counter = {}) - : Counter(counter) - {} - }; - - std::vector<TStaticMetric> Static; - std::unordered_map<EDynamicMetric, TDynamicMetric> Dynamic; -}; - -NActors::IActor* CreateMemoryTrackerActor(TDuration updateInterval, TDynamicCounterPtr counters) { - return new TMemoryTrackerActor(updateInterval, counters); -} - -} -} + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/memory_tracker.h> +#include <library/cpp/html/escape/escape.h> +#include <library/cpp/monlib/service/pages/templates.h> + +#include <util/generic/xrange.h> + +using namespace NMonitoring; +using namespace NActors::NMemory::NPrivate; + +namespace NKikimr { +namespace NMemory { + +class TMemoryTrackerActor : public TActorBootstrapped<TMemoryTrackerActor> { +public: + TMemoryTrackerActor(TDuration updateInterval, TDynamicCounterPtr appCounters) + : UpdateInterval(updateInterval) + , AppCounters(appCounters) + { + auto utilsCounters = GetServiceCounters(AppCounters, "utils"); + Counters = utilsCounters->GetSubgroup("component", "memory_tracker"); + } + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::MEMORY_TRACKER; + } + + void Bootstrap(const TActorContext&) { +#if defined(ENABLE_MEMORY_TRACKING) + TMemoryTracker::Instance()->Initialize(); + RegisterCounters(Counters); + + TMon* mon = AppData()->Mon; + if (mon) { + auto *actorsMonPage = mon->RegisterIndexPage("actors", "Actors"); + mon->RegisterActorPage(actorsMonPage, "memory_tracker", "Memory tracker", + false, TlsActivationContext->ExecutorThread.ActorSystem, SelfId()); + } + + Collect(); +#endif + Become(&TMemoryTrackerActor::StateWork); + } + + STRICT_STFUNC(StateWork, + cFunc(TEvents::TEvWakeup::EventType, Wakeup) + hFunc(NMon::TEvHttpInfo, HttpInfo) + cFunc(TEvents::TEvPoison::EventType, PassAway) + ) + +private: + void Wakeup() { + Collect(); + } + + void Collect() { + Schedule(UpdateInterval, new TEvents::TEvWakeup()); + + std::vector<TMetric> metrics; + TMemoryTracker::Instance()->GatherMetrics(metrics); + + auto count = metrics.size(); + if (Static.size() != count) { + return; + } + + const auto& sensors = TMemoryTracker::Instance()->GetSensors(); + + for (auto i : xrange(count)) { + auto& metric = Static[i]; + metric.Current = metrics[i]; + metric.Peak.CalculatePeak(metric.Current); + + if (sensors.find(i) != sensors.end()) { + *metric.CurrentMemCounter = metric.Current.GetMemory(); + *metric.PeakMemCounter = metric.Peak.GetMemory(); + *metric.CurrentCountCounter = metric.Current.GetCount(); + *metric.PeakCountCounter = metric.Peak.GetCount(); + } + } + + GatherDynamicMetrics(); + } + + void RegisterCounters(NMonitoring::TDynamicCounterPtr counters) { + auto componentGroup = counters->GetSubgroup("component", "memory_tracker"); + + auto count = TMemoryTracker::Instance()->GetCount(); + Static.resize(count); + + const auto& sensors = TMemoryTracker::Instance()->GetSensors(); + for (auto index : sensors) { + auto name = TMemoryTracker::Instance()->GetName(index); + auto& metric = Static[index]; + auto group = componentGroup->GetSubgroup("label", name); + metric.CurrentMemCounter = group->GetCounter("MT/Memory"); + metric.PeakMemCounter = group->GetCounter("MT/PeakMemory"); + metric.CurrentCountCounter = group->GetCounter("MT/Count"); + metric.PeakCountCounter = group->GetCounter("MT/PeakCount"); + } + } + + void UpdateDynamicCounters() { + auto utils = GetServiceCounters(AppCounters, "utils"); + + if (Dynamic.find(AnonRssSize) == Dynamic.end()) { + Dynamic.emplace(AnonRssSize, utils->GetCounter("Process/AnonRssSize")); + } + + if (Dynamic.find(LFMmapped) == Dynamic.end()) { + auto lfAllocGroup = utils->FindSubgroup("component", "lfalloc"); + if (lfAllocGroup) { + Dynamic.emplace(LFMmapped, lfAllocGroup->GetCounter("ActiveBytesMmapped")); + Dynamic.emplace(LFSmall, lfAllocGroup->GetCounter("ActiveBytesSmall")); + Dynamic.emplace(LFLarge, lfAllocGroup->GetCounter("ActiveBytesLarge")); + Dynamic.emplace(LFSystem, lfAllocGroup->GetCounter("ActiveBytesSystem")); + } + } + + if (Dynamic.find(MkqlGlobalPoolTotalBytes) == Dynamic.end()) { + auto mkqlAllocGroup = utils->FindSubgroup("subsystem", "mkqlalloc"); + if (mkqlAllocGroup) { + Dynamic.emplace(MkqlGlobalPoolTotalBytes, mkqlAllocGroup->GetCounter("GlobalPoolTotalBytes")); + + auto getNamedPagePoolCounter = [&mkqlAllocGroup](const char* name) { + return mkqlAllocGroup->GetCounter(TString(name) + "/TotalBytesAllocated"); + }; + Dynamic.emplace(MkqlKqp, getNamedPagePoolCounter("kqp")); + Dynamic.emplace(MkqlCompile, getNamedPagePoolCounter("compile")); + Dynamic.emplace(MkqlDatashard, getNamedPagePoolCounter("datashard")); + Dynamic.emplace(MkqlLocalTx, getNamedPagePoolCounter("local_tx")); + Dynamic.emplace(MkqlTxProxy, getNamedPagePoolCounter("tx_proxy")); + } + } + } + + void GatherDynamicMetrics() { + UpdateDynamicCounters(); + + for (auto& [_, metric] : Dynamic) { + if (metric.Counter) { + metric.Current.SetMemory(metric.Counter->Val()); + metric.Peak.CalculatePeak(metric.Current); + } + } + } + + static TString PrettyNumber(ssize_t val) { + if (val == 0) { + return "0"; + } + TStringStream str; + if (val < 0) { + str << "- "; + } + ui64 value = val < 0 ? -val : val; + TSmallVec<ui64> parts; + while (value > 0) { + auto head = value / 1000; + parts.push_back(value - head * 1000); + value = head; + } + str << parts.back(); + parts.pop_back(); + while (!parts.empty()) { + str << Sprintf(" %03" PRIu64, parts.back()); + parts.pop_back(); + } + return str.Str(); + } + + void HttpInfo(NMon::TEvHttpInfo::TPtr& ev) { + TStringStream str; + + HTML(str) { +#if defined(ENABLE_MEMORY_TRACKING) + TABLE_CLASS("table") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "Metric"; } + TABLEH() { str << "Memory"; } + TABLEH() { str << "Peak memory"; } + } + } + TABLEBODY() { + auto outputDynamic = [&str, &__stream] (const char* name, const TDynamicMetric& metric) { + TABLED() { str << name; } + TAG_CLASS_STYLE(TTableD, "", "text-align:right") { + str << PrettyNumber(metric.Current.GetMemory()); } + TAG_CLASS_STYLE(TTableD, "", "text-align:right") { + str << PrettyNumber(metric.Peak.GetMemory()); } + }; + + if (Dynamic.find(AnonRssSize) != Dynamic.end()) { + TABLER() { outputDynamic("AnonRssSize", Dynamic[AnonRssSize]); } + } + if (Dynamic.find(LFMmapped) != Dynamic.end()) { + TABLER() { + TABLED() { str << "LFAlloc"; } + } + TABLER() { outputDynamic("Mmapped", Dynamic[LFMmapped]); } + TABLER() { outputDynamic("Small", Dynamic[LFSmall]); } + TABLER() { outputDynamic("Large", Dynamic[LFLarge]); } + TABLER() { outputDynamic("System", Dynamic[LFSystem]); } + } + if (Dynamic.find(MkqlGlobalPoolTotalBytes) != Dynamic.end()) { + TABLER() { + TABLED() { str << "MKQLAlloc"; } + } + TABLER() { outputDynamic("GlobalPoolTotalBytes", Dynamic[MkqlGlobalPoolTotalBytes]); } + TABLER() { outputDynamic("Kqp", Dynamic[MkqlKqp]); } + TABLER() { outputDynamic("Compile", Dynamic[MkqlCompile]); } + TABLER() { outputDynamic("Datashard", Dynamic[MkqlDatashard]); } + TABLER() { outputDynamic("LocalTx", Dynamic[MkqlLocalTx]); } + TABLER() { outputDynamic("TxProxy", Dynamic[MkqlTxProxy]); } + } + } + } + TABLE_CLASS("table") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "Metric"; } + TABLEH() { str << "Count"; } + TABLEH() { str << "Peak count"; } + TABLEH() { str << "Memory"; } + TABLEH() { str << "Peak memory"; } + } + } + TABLEBODY() { + const auto& indices = TMemoryTracker::Instance()->GetMetricIndices(); + for (const auto& [name, index] : indices) { + TABLER() { + auto& metric = Static[index]; + TABLED() { str << NHtml::EscapeText(name); } + TAG_CLASS_STYLE(TTableD, "", "text-align:right") { + str << PrettyNumber(metric.Current.GetCount()); } + TAG_CLASS_STYLE(TTableD, "", "text-align:right") { + str << PrettyNumber(metric.Peak.GetCount()); } + TAG_CLASS_STYLE(TTableD, "", "text-align:right") { + str << PrettyNumber(metric.Current.GetMemory()); } + TAG_CLASS_STYLE(TTableD, "", "text-align:right") { + str << PrettyNumber(metric.Peak.GetMemory()); } + } + } + } + } +#endif + } + + Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); + } + +private: + const TDuration UpdateInterval; + + enum EDynamicMetric { + AnonRssSize, + LFMmapped, + LFSmall, + LFLarge, + LFSystem, + MkqlGlobalPoolTotalBytes, + MkqlKqp, + MkqlCompile, + MkqlDatashard, + MkqlLocalTx, + MkqlTxProxy + }; + + TDynamicCounterPtr AppCounters; + TDynamicCounterPtr Counters; + + struct TStaticMetric { + TMetric Current; + TMetric Peak; + TDynamicCounters::TCounterPtr CurrentMemCounter; + TDynamicCounters::TCounterPtr PeakMemCounter; + TDynamicCounters::TCounterPtr CurrentCountCounter; + TDynamicCounters::TCounterPtr PeakCountCounter; + }; + + struct TDynamicMetric { + TMetric Current; + TMetric Peak; + TDynamicCounters::TCounterPtr Counter; + + TDynamicMetric(TDynamicCounters::TCounterPtr counter = {}) + : Counter(counter) + {} + }; + + std::vector<TStaticMetric> Static; + std::unordered_map<EDynamicMetric, TDynamicMetric> Dynamic; +}; + +NActors::IActor* CreateMemoryTrackerActor(TDuration updateInterval, TDynamicCounterPtr counters) { + return new TMemoryTrackerActor(updateInterval, counters); +} + +} +} diff --git a/ydb/core/util/memory_tracker.h b/ydb/core/util/memory_tracker.h index af99e84424..1db4ee3e9e 100644 --- a/ydb/core/util/memory_tracker.h +++ b/ydb/core/util/memory_tracker.h @@ -1,17 +1,17 @@ -#pragma once - -#include <library/cpp/monlib/dynamic_counters/counters.h> - -namespace NActors { - class IActor; -} - -namespace NKikimr { -namespace NMemory { - -NActors::IActor* CreateMemoryTrackerActor( - TDuration updateInterval, - NMonitoring::TDynamicCounterPtr counters); - -} -} +#pragma once + +#include <library/cpp/monlib/dynamic_counters/counters.h> + +namespace NActors { + class IActor; +} + +namespace NKikimr { +namespace NMemory { + +NActors::IActor* CreateMemoryTrackerActor( + TDuration updateInterval, + NMonitoring::TDynamicCounterPtr counters); + +} +} diff --git a/ydb/core/util/ut/ya.make b/ydb/core/util/ut/ya.make index 295b6895f0..315713cad8 100644 --- a/ydb/core/util/ut/ya.make +++ b/ydb/core/util/ut/ya.make @@ -26,7 +26,7 @@ SRCS( btree_ut.cpp cache_ut.cpp circular_queue_ut.cpp - concurrent_rw_hash_ut.cpp + concurrent_rw_hash_ut.cpp fast_tls_ut.cpp fragmented_buffer_ut.cpp hazard_ut.cpp diff --git a/ydb/core/util/ya.make b/ydb/core/util/ya.make index 34b6df67aa..4f325d8fcb 100644 --- a/ydb/core/util/ya.make +++ b/ydb/core/util/ya.make @@ -11,8 +11,8 @@ SRCS( cache.h cache_cache.h circular_queue.h - concurrent_rw_hash.cpp - concurrent_rw_hash.h + concurrent_rw_hash.cpp + concurrent_rw_hash.h console.cpp console.h counted_leaky_bucket.h @@ -33,8 +33,8 @@ SRCS( intrusive_heap.h intrusive_stack.h log_priority_mute_checker.h - memory_tracker.cpp - memory_tracker.h + memory_tracker.cpp + memory_tracker.h operation_queue.h page_map.cpp pb.h @@ -65,7 +65,7 @@ PEERDIR( library/cpp/actors/interconnect/mock library/cpp/actors/util library/cpp/containers/stack_vector - library/cpp/html/escape + library/cpp/html/escape library/cpp/ipmath library/cpp/lwtrace library/cpp/monlib/dynamic_counters diff --git a/ydb/library/yql/core/common_opt/yql_co_flow1.cpp b/ydb/library/yql/core/common_opt/yql_co_flow1.cpp index 37a89608f3..728838dc8e 100644 --- a/ydb/library/yql/core/common_opt/yql_co_flow1.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_flow1.cpp @@ -1291,8 +1291,8 @@ TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ct auto hoppingSetting = GetSetting(settings.Ref(), "hopping"); if (hoppingSetting) { return node.Ptr(); - } - + } + if (GetSetting(settings.Ref(), "session")) { // TODO: support return node.Ptr(); diff --git a/ydb/library/yql/core/common_opt/yql_co_simple2.cpp b/ydb/library/yql/core/common_opt/yql_co_simple2.cpp index 0666f8c545..6d59989b6a 100644 --- a/ydb/library/yql/core/common_opt/yql_co_simple2.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_simple2.cpp @@ -2,7 +2,7 @@ #include <ydb/library/yql/core/yql_opt_utils.h> #include <ydb/library/yql/core/yql_expr_csee.h> - + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/yql_panic.h> @@ -198,8 +198,8 @@ TExprNode::TPtr DeduplicateAggregateSameTraits(const TExprNode::TPtr& node, TExp .Atom(1, "_yql_time", TNodeFlags::Default) .Seal() .Seal(); - } - + } + return structObj.Seal(); }) .Seal() diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h index af5680a3f5..01cc777118 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h @@ -180,19 +180,19 @@ public: } TNodeBuilder<TParent, TCoLambda>& Args(const std::vector<TCoArgument>& list) - { - Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid()); - + { + Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid()); + TExprNode::TListType argNodes; - for (auto arg : list) { - argNodes.push_back(arg.Ptr()); - } - - auto argsNode = this->Ctx.NewArguments(this->Pos, std::move(argNodes)); + for (auto arg : list) { + argNodes.push_back(arg.Ptr()); + } + + auto argsNode = this->Ctx.NewArguments(this->Pos, std::move(argNodes)); this->ArgsHolder = TCoArguments(argsNode); - return *this; - } - + return *this; + } + TNodeBuilder<TParent, TCoLambda>& Args(const TExprNode::TListType& list) { Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid()); diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json index b4b7a97918..edfcaa1530 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json @@ -549,18 +549,18 @@ }, { "Name": "TCoHoppingTraits", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "HoppingTraits"}, - "Children": [ + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "HoppingTraits"}, + "Children": [ {"Index": 0, "Name": "ItemType", "Type": "TExprBase"}, {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"}, {"Index": 2, "Name": "Hop", "Type": "TExprBase"}, {"Index": 3, "Name": "Interval", "Type": "TExprBase"}, {"Index": 4, "Name": "Delay", "Type": "TExprBase"}, {"Index": 5, "Name": "DataWatermarks", "Type": "TExprBase"} - ] - }, - { + ] + }, + { "Name": "TCoSessionWindowTraits", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "SessionWindowTraits"}, @@ -583,8 +583,8 @@ {"Index": 3, "Name": "SaveHandler", "Type": "TCoLambda"}, {"Index": 4, "Name": "LoadHandler", "Type": "TCoLambda"}, {"Index": 5, "Name": "MergeHandler", "Type": "TCoLambda"}, - {"Index": 6, "Name": "FinishHandler", "Type": "TCoLambda"}, - {"Index": 7, "Name": "DefVal", "Type": "TExprBase"} + {"Index": 6, "Name": "FinishHandler", "Type": "TCoLambda"}, + {"Index": 7, "Name": "DefVal", "Type": "TExprBase"} ] }, { @@ -1579,28 +1579,28 @@ {"Index": 2, "Name": "MatchLambda", "Type": "TCoLambda"}, {"Index": 3, "Name": "NotMatchLambda", "Type": "TCoLambda"} ] - }, - { + }, + { "Name": "TCoIterator", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "Iterator"}, - "Children": [ - {"Index": 0, "Name": "List", "Type": "TExprBase"} - ] - }, - { + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "Iterator"}, + "Children": [ + {"Index": 0, "Name": "List", "Type": "TExprBase"} + ] + }, + { "Name": "TCoForwardList", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "ForwardList"}, - "Children": [ - {"Index": 0, "Name": "Stream", "Type": "TExprBase"} - ] - }, - { + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "ForwardList"}, + "Children": [ + {"Index": 0, "Name": "Stream", "Type": "TExprBase"} + ] + }, + { "Name": "TCoHoppingCore", "Base": "TCoInputBase", - "Match": {"Type": "Callable", "Name": "HoppingCore"}, - "Children": [ + "Match": {"Type": "Callable", "Name": "HoppingCore"}, + "Children": [ {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"}, {"Index": 2, "Name": "Hop", "Type": "TExprBase"}, {"Index": 3, "Name": "Interval", "Type": "TExprBase"}, @@ -1611,7 +1611,7 @@ {"Index": 8, "Name": "LoadHandler", "Type": "TExprBase"}, {"Index": 9, "Name": "MergeHandler", "Type": "TCoLambda"}, {"Index": 10, "Name": "FinishHandler", "Type": "TCoLambda"} - ] + ] }, { "Name": "TCoMultiHoppingCore", @@ -1673,8 +1673,8 @@ "Children": [ {"Index": 1, "Name": "Type", "Type": "TCoAtom"} ] - }, - { + }, + { "Name": "TCoCondense", "Base": "TCoInputBase", "Match": {"Type": "Callable", "Name": "Condense"}, diff --git a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj index 9f2327dbf2..4b5733693f 100644 --- a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj +++ b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj @@ -203,9 +203,9 @@ public: {%- for child in node.aux.allChildren %} TBuilder<TParent, TDerived>& {{child.Name}}(const TStringBuf& argName) { {{child.aux.holderName}} = {{child.Type}}(this->GetArgFunc(argName)); return *static_cast<TBuilder<TParent, TDerived>*>(this); } - {% if child.Optional %} - TBuilder<TParent, TDerived>& {{child.Name}}(const TMaybe<{{child.Type}}>& node) { {{child.aux.holderName}} = node; return *static_cast<TBuilder<TParent, TDerived>*>(this); } - {% endif %} + {% if child.Optional %} + TBuilder<TParent, TDerived>& {{child.Name}}(const TMaybe<{{child.Type}}>& node) { {{child.aux.holderName}} = node; return *static_cast<TBuilder<TParent, TDerived>*>(this); } + {% endif %} TBuilder<TParent, TDerived>& {{child.Name}}(const {{child.Type}}& node) { {{child.aux.holderName}} = node; return *static_cast<TBuilder<TParent, TDerived>*>(this); } TBuilder<TParent, TDerived>& {{child.Name}}(const TExprNode::TPtr& node) { {{child.aux.holderName}} = {{child.Type}}(node); return *static_cast<TBuilder<TParent, TDerived>*>(this); } diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp index 4b5a7558ee..b74abc1c34 100644 --- a/ydb/library/yql/core/facade/yql_facade.cpp +++ b/ydb/library/yql/core/facade/yql_facade.cpp @@ -1085,40 +1085,40 @@ IGraphTransformer::TStatistics TProgram::GetRawDiagnostics() { return Transformer_ ? Transformer_->GetStatistics() : IGraphTransformer::TStatistics::NotPresent(); } -TMaybe<TString> TProgram::GetTasksInfo() { - if (!TypeCtx_) { - return Nothing(); - } - - bool hasTasks = false; - - TStringStream out; +TMaybe<TString> TProgram::GetTasksInfo() { + if (!TypeCtx_) { + return Nothing(); + } + + bool hasTasks = false; + + TStringStream out; NYson::TYsonWriter writer(&out, ResultFormat_); - - writer.OnBeginMap(); - writer.OnKeyedItem("Write"); - writer.OnBeginList(); - writer.OnListItem(); - writer.OnBeginMap(); - writer.OnKeyedItem("Tasks"); - writer.OnBeginList(); - - for (auto& datasink : TypeCtx_->DataSinks) { + + writer.OnBeginMap(); + writer.OnKeyedItem("Write"); + writer.OnBeginList(); + writer.OnListItem(); + writer.OnBeginMap(); + writer.OnKeyedItem("Tasks"); + writer.OnBeginList(); + + for (auto& datasink : TypeCtx_->DataSinks) { hasTasks = hasTasks || datasink->GetTasksInfo(writer); - } - - writer.OnEndList(); - writer.OnEndMap(); - writer.OnEndList(); - writer.OnEndMap(); - - if (hasTasks) { - return out.Str(); - } else { - return Nothing(); - } -} - + } + + writer.OnEndList(); + writer.OnEndMap(); + writer.OnEndList(); + writer.OnEndMap(); + + if (hasTasks) { + return out.Str(); + } else { + return Nothing(); + } +} + TMaybe<TString> TProgram::GetStatistics(bool totalOnly) { if (!TypeCtx_) { return Nothing(); @@ -1311,9 +1311,9 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us } if (providerNames.contains(RtmrProviderName)) { - resultProviderDataSources.push_back(TString(RtmrProviderName)); - } - + resultProviderDataSources.push_back(TString(RtmrProviderName)); + } + if (providerNames.contains(PqProviderName)) { resultProviderDataSources.push_back(TString(PqProviderName)); } @@ -1384,13 +1384,13 @@ void TProgram::Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanP InstantTransform(*compositeTransformer, ExprRoot_, *ExprCtx_); } -bool TProgram::HasActiveProcesses() { - for (const auto& dp : DataProviders_) { - if (dp.HasActiveProcesses && dp.HasActiveProcesses()) { - return true; - } - } - return false; -} - +bool TProgram::HasActiveProcesses() { + for (const auto& dp : DataProviders_) { + if (dp.HasActiveProcesses && dp.HasActiveProcesses()) { + return true; + } + } + return false; +} + } // namespace NYql diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h index 7e3308e1c2..1f8f47681d 100644 --- a/ydb/library/yql/core/facade/yql_facade.h +++ b/ydb/library/yql/core/facade/yql_facade.h @@ -156,8 +156,8 @@ public: TFutureStatus ContinueAsync(); - bool HasActiveProcesses(); - + bool HasActiveProcesses(); + void Abort(); inline TIssues Issues() { @@ -215,8 +215,8 @@ public: TMaybe<TString> GetDiagnostics(); IGraphTransformer::TStatistics GetRawDiagnostics(); - TMaybe<TString> GetTasksInfo(); - + TMaybe<TString> GetTasksInfo(); + TMaybe<TString> GetStatistics(bool totalOnly = false); TMaybe<TString> GetDiscoveredData(); diff --git a/ydb/library/yql/core/services/yql_plan.cpp b/ydb/library/yql/core/services/yql_plan.cpp index c3f3534511..4d7cfffaf5 100644 --- a/ydb/library/yql/core/services/yql_plan.cpp +++ b/ydb/library/yql/core/services/yql_plan.cpp @@ -374,9 +374,9 @@ public: info.IsVisible = (*provider)->GetPlanFormatter().GetDependencies(*node, dependencies, true); } else { info.IsVisible = false; - for (auto& child : node->Children()) { - dependencies.push_back(child.Get()); - } + for (auto& child : node->Children()) { + dependencies.push_back(child.Get()); + } } for (const auto& child : dependencies) { diff --git a/ydb/library/yql/core/services/yql_plan.h b/ydb/library/yql/core/services/yql_plan.h index f13a84d33e..608c54fb8e 100644 --- a/ydb/library/yql/core/services/yql_plan.h +++ b/ydb/library/yql/core/services/yql_plan.h @@ -3,7 +3,7 @@ #include <ydb/library/yql/core/type_ann/type_ann_core.h> #include <library/cpp/yson/writer.h> - + namespace NYql { class IPlanBuilder { diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index 1388aa77cc..5846e6cb10 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -2043,7 +2043,7 @@ namespace NTypeAnnImpl { } template <bool Equal, bool Order> - IGraphTransformer::TStatus AggrCompareWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus AggrCompareWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -2067,18 +2067,18 @@ namespace NTypeAnnImpl { if (!Order) { input->SetUnorderedChildren(); } - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus AggrMinMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus AggrMinMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureComparableType(input->Pos(), *input->Head().GetTypeAnn(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + return IGraphTransformer::TStatus::Error; + } + if (!IsSameAnnotation(*input->Head().GetTypeAnn(), *input->Tail().GetTypeAnn())) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: " << *input->Head().GetTypeAnn() << ", right:" << *input->Tail().GetTypeAnn())); @@ -12994,8 +12994,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["CommonJoinCore"] = &CommonJoinCoreWrapper; Functions["CombineCore"] = &CombineCoreWrapper; Functions["GroupingCore"] = &GroupingCoreWrapper; - Functions["HoppingTraits"] = &HoppingTraitsWrapper; - Functions["HoppingCore"] = &HoppingCoreWrapper; + Functions["HoppingTraits"] = &HoppingTraitsWrapper; + Functions["HoppingCore"] = &HoppingCoreWrapper; Functions["MultiHoppingCore"] = &MultiHoppingCoreWrapper; Functions["EquiJoin"] = &EquiJoinWrapper; Functions["OptionalReduce"] = &OptionalReduceWrapper; diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp index a1790c8957..b145cde389 100644 --- a/ydb/library/yql/core/type_ann/type_ann_list.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp @@ -3456,7 +3456,7 @@ namespace { IGraphTransformer::TStatus SqueezeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); - if (!EnsureArgsCount(*input, 5, ctx.Expr)) { + if (!EnsureArgsCount(*input, 5, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -3468,16 +3468,16 @@ namespace { return IGraphTransformer::TStatus::Error; } - auto& lambda = input->ChildRef(2); - - auto status = ConvertToLambda(lambda, ctx.Expr, 2); + auto& lambda = input->ChildRef(2); + + auto status = ConvertToLambda(lambda, ctx.Expr, 2); if (status.Level != IGraphTransformer::TStatus::Ok) { return status; } auto stateType = input->Child(1)->GetTypeAnn(); auto itemType = input->Head().GetTypeAnn()->Cast<TStreamExprType>()->GetItemType(); - if (!UpdateLambdaAllArgumentsTypes(lambda, { itemType, stateType }, ctx.Expr)) { + if (!UpdateLambdaAllArgumentsTypes(lambda, { itemType, stateType }, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } if (!lambda->GetTypeAnn()) { @@ -3490,48 +3490,48 @@ namespace { return IGraphTransformer::TStatus::Error; } - auto& saveLambda = input->ChildRef(3); - auto& loadLambda = input->ChildRef(4); - - if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { + auto& saveLambda = input->ChildRef(3); + auto& loadLambda = input->ChildRef(4); + + if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << - "Save and load lambdas must be specified at the same time")); - return IGraphTransformer::TStatus::Error; - } - - if (!saveLambda->IsCallable("Void")) { - auto status = ConvertToLambda(saveLambda, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1)); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + "Save and load lambdas must be specified at the same time")); + return IGraphTransformer::TStatus::Error; + } + + if (!saveLambda->IsCallable("Void")) { + auto status = ConvertToLambda(saveLambda, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1)); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!saveLambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + auto savedStateType = saveLambda->GetTypeAnn(); - if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!loadLambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " << *loadLambda->GetTypeAnn() << " != " << *stateType)); - return IGraphTransformer::TStatus::Error; - } - } - + return IGraphTransformer::TStatus::Error; + } + } + auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Deprecated Squeeze, use Condense instead."); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_YQL_DEPRECATED_FUNCTION_OR_SIGNATURE, issue); if (!ctx.Expr.AddWarning(issue)) { @@ -3544,7 +3544,7 @@ namespace { IGraphTransformer::TStatus Squeeze1Wrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); - if (!EnsureArgsCount(*input, 5, ctx.Expr)) { + if (!EnsureArgsCount(*input, 5, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -3552,11 +3552,11 @@ namespace { return IGraphTransformer::TStatus::Error; } - auto& initLambda = input->ChildRef(1); - auto& updateLambda = input->ChildRef(2); - - auto status = ConvertToLambda(initLambda, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2)); + auto& initLambda = input->ChildRef(1); + auto& updateLambda = input->ChildRef(2); + + auto status = ConvertToLambda(initLambda, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2)); if (status.Level != IGraphTransformer::TStatus::Ok) { return status; } @@ -3585,48 +3585,48 @@ namespace { return IGraphTransformer::TStatus::Error; } - auto& saveLambda = input->ChildRef(3); - auto& loadLambda = input->ChildRef(4); - - if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { + auto& saveLambda = input->ChildRef(3); + auto& loadLambda = input->ChildRef(4); + + if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << - "Save and load lambdas must be specified at the same time")); - return IGraphTransformer::TStatus::Error; - } - - if (!saveLambda->IsCallable("Void")) { - auto status = ConvertToLambda(saveLambda, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1)); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + "Save and load lambdas must be specified at the same time")); + return IGraphTransformer::TStatus::Error; + } + + if (!saveLambda->IsCallable("Void")) { + auto status = ConvertToLambda(saveLambda, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1)); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!saveLambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + auto savedStateType = saveLambda->GetTypeAnn(); - if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!loadLambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " << *loadLambda->GetTypeAnn() << " != " << *stateType)); - return IGraphTransformer::TStatus::Error; - } - } - + return IGraphTransformer::TStatus::Error; + } + } + auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Deprecated Squeeze1, use Condense1 instead."); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_YQL_DEPRECATED_FUNCTION_OR_SIGNATURE, issue); if (!ctx.Expr.AddWarning(issue)) { @@ -4307,12 +4307,12 @@ namespace { } IGraphTransformer::TStatus AggregateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureMinArgsCount(*input, 3, ctx.Expr)) { + if (!EnsureMinArgsCount(*input, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureMaxArgsCount(*input, 4, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (!EnsureMaxArgsCount(*input, 4, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } if (IsEmptyList(input->Head())) { output = input->HeadPtr(); @@ -4551,7 +4551,7 @@ namespace { } } else { auto defVal = child->Child(1)->Child(7); - if (defVal->IsCallable("Null") && !isOptional && !isHopping && input->Child(1)->ChildrenSize() == 0) { + if (defVal->IsCallable("Null") && !isOptional && !isHopping && input->Child(1)->ChildrenSize() == 0) { if (finishType->GetKind() != ETypeAnnotationKind::Null) { finishType = ctx.Expr.MakeType<TOptionalExprType>(finishType); } @@ -4566,11 +4566,11 @@ namespace { } } - auto rowType = ctx.Expr.MakeType<TStructExprType>(rowColumns); - if (!rowType->Validate(input->Pos(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + auto rowType = ctx.Expr.MakeType<TStructExprType>(rowColumns); + if (!rowType->Validate(input->Pos(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + input->SetTypeAnn(isStream ? (const TTypeAnnotationNode*)ctx.Expr.MakeType<TStreamExprType>(rowType) : (const TTypeAnnotationNode*)ctx.Expr.MakeType<TListExprType>(rowType)); @@ -5397,79 +5397,79 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); + IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); if (!EnsureMinArgsCount(*input, 5, ctx.Expr) || !EnsureMaxArgsCount(*input, 6, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + return IGraphTransformer::TStatus::Error; + } if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { return status; - } - + } + auto itemType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (!EnsureComputableType(input->Head().Pos(), *itemType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto& lambdaTimeExtractor = input->ChildRef(1); - auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + return IGraphTransformer::TStatus::Error; + } + + auto& lambdaTimeExtractor = input->ChildRef(1); + auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!lambdaTimeExtractor->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + const TTypeAnnotationNode* timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp)); - + if (!IsSameAnnotation(*lambdaTimeExtractor->GetTypeAnn(), *timeType)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaTimeExtractor->Pos()), TStringBuilder() - << "Mismatch hopping window time extractor lambda output type, expected: " + << "Mismatch hopping window time extractor lambda output type, expected: " << *timeType << ", but got: " << *lambdaTimeExtractor->GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - + return IGraphTransformer::TStatus::Error; + } + const TTypeAnnotationNode* intervalType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Interval); - - auto checkWindowParam = [&] (TExprNode::TPtr& param) -> IGraphTransformer::TStatus { + + auto checkWindowParam = [&] (TExprNode::TPtr& param) -> IGraphTransformer::TStatus { auto type = param->GetTypeAnn(); - if (type->GetKind() == ETypeAnnotationKind::Optional) { - if (param->IsCallable("Nothing")) { + if (type->GetKind() == ETypeAnnotationKind::Optional) { + if (param->IsCallable("Nothing")) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), "Hopping window parameter value cannot be evaluated")); - return IGraphTransformer::TStatus::Error; - } - type = type->Cast<TOptionalExprType>()->GetItemType(); - } - if (!IsSameAnnotation(*type, *intervalType)) { + return IGraphTransformer::TStatus::Error; + } + type = type->Cast<TOptionalExprType>()->GetItemType(); + } + if (!IsSameAnnotation(*type, *intervalType)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), TStringBuilder() - << "Mismatch hopping window parameter type, expected: " - << *intervalType << ", but got: " << *type)); - return IGraphTransformer::TStatus::Error; - } - if (!IsPureIsolatedLambda(*param)) { + << "Mismatch hopping window parameter type, expected: " + << *intervalType << ", but got: " << *type)); + return IGraphTransformer::TStatus::Error; + } + if (!IsPureIsolatedLambda(*param)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), "Parameter is not a pure expression")); - return IGraphTransformer::TStatus::Error; - } - return IGraphTransformer::TStatus::Ok; - }; - - auto convertStatus = checkWindowParam(input->ChildRef(2)); - if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { - return convertStatus; - } - convertStatus = checkWindowParam(input->ChildRef(3)); - if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { - return convertStatus; - } - convertStatus = checkWindowParam(input->ChildRef(4)); - if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { - return convertStatus; - } - + return IGraphTransformer::TStatus::Error; + } + return IGraphTransformer::TStatus::Ok; + }; + + auto convertStatus = checkWindowParam(input->ChildRef(2)); + if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { + return convertStatus; + } + convertStatus = checkWindowParam(input->ChildRef(3)); + if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { + return convertStatus; + } + convertStatus = checkWindowParam(input->ChildRef(4)); + if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { + return convertStatus; + } + if (input->ChildrenSize() == 6) { const auto& dataWatermarksNodePtr = input->ChildRef(5); if (dataWatermarksNodePtr->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Unit) { @@ -5481,17 +5481,17 @@ namespace { } input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); if (!EnsureArgsCount(*input, 11, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + return IGraphTransformer::TStatus::Error; + } + + auto& item = input->ChildRef(0); - auto& item = input->ChildRef(0); - auto& lambdaTimeExtractor = input->ChildRef(1); auto* hop = input->Child(2); auto* interval = input->Child(3); @@ -5504,26 +5504,26 @@ namespace { auto& lambdaMerge = input->ChildRef(9); auto& lambdaFinish = input->ChildRef(10); - if (!EnsureStreamType(*item, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + if (!EnsureStreamType(*item, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1); status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1)); - status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2)); - status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2)); - status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 2)); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - + status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2)); + status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2)); + status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 2)); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + auto timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp)); auto itemType = item->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType(); - - if (!EnsureStructType(input->Pos(), *itemType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + + if (!EnsureStructType(input->Pos(), *itemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -5542,99 +5542,99 @@ namespace { return IGraphTransformer::TStatus::Error; } - if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!lambdaInit->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + auto stateType = lambdaInit->GetTypeAnn(); - if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaUpdate, {itemType, stateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaUpdate, {itemType, stateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!lambdaUpdate->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: " << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, {stateType, stateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, {stateType, stateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!lambdaMerge->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *stateType)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaMerge->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: " << *stateType << ", but got: " << *lambdaMerge->GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaFinish, {stateType, timeType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaFinish, {stateType, timeType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!lambdaFinish->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } + return IGraphTransformer::TStatus::Repeat; + } if (lambdaFinish->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaFinish->Pos()), TStringBuilder() << "Expected struct type as finish lambda result type, but got: " << *lambdaMerge->GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { + return IGraphTransformer::TStatus::Error; + } + + if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << - "Save and load lambdas must be specified at the same time")); - return IGraphTransformer::TStatus::Error; - } - - if (!saveLambda->IsCallable("Void")) { - auto status = ConvertToLambda(saveLambda, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1)); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + "Save and load lambdas must be specified at the same time")); + return IGraphTransformer::TStatus::Error; + } + + if (!saveLambda->IsCallable("Void")) { + auto status = ConvertToLambda(saveLambda, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1)); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!saveLambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + auto savedStateType = saveLambda->GetTypeAnn(); - if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } if (!loadLambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " << *loadLambda->GetTypeAnn() << " != " << *stateType)); - return IGraphTransformer::TStatus::Error; - } - } - + return IGraphTransformer::TStatus::Error; + } + } + input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(lambdaFinish->GetTypeAnn())); - return IGraphTransformer::TStatus::Ok; - } - + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 13, ctx.Expr)) { diff --git a/ydb/library/yql/core/type_ann/type_ann_list.h b/ydb/library/yql/core/type_ann/type_ann_list.h index ca433faa5f..608b813fe8 100644 --- a/ydb/library/yql/core/type_ann/type_ann_list.h +++ b/ydb/library/yql/core/type_ann/type_ann_list.h @@ -104,9 +104,9 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus WinLeadLagWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WinRowNumberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WinRankWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListMinWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListSumWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); diff --git a/ydb/library/yql/core/yql_data_provider.h b/ydb/library/yql/core/yql_data_provider.h index 2c2b6ce3af..3ffbdc041d 100644 --- a/ydb/library/yql/core/yql_data_provider.h +++ b/ydb/library/yql/core/yql_data_provider.h @@ -102,7 +102,7 @@ public: //-- configuration virtual IGraphTransformer& GetConfigurationTransformer() = 0; - virtual TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) = 0; + virtual TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) = 0; virtual const THashMap<TString, TString>* GetClusterTokens() = 0; //-- discovery & rewrite @@ -152,13 +152,13 @@ public: virtual void GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) = 0; virtual IGraphTransformer& GetCallableExecutionTransformer() = 0; - //-- finalizing - virtual IGraphTransformer& GetFinalizingTransformer() = 0; + //-- finalizing + virtual IGraphTransformer& GetFinalizingTransformer() = 0; virtual bool CollectDiagnostics(NYson::TYsonWriter& writer) = 0; virtual bool GetTasksInfo(NYson::TYsonWriter& writer) = 0; virtual bool CollectStatistics(NYson::TYsonWriter& writer, bool totalOnly) = 0; virtual bool CollectDiscoveredData(NYson::TYsonWriter& writer) = 0; - + //-- plan virtual IGraphTransformer& GetPlanInfoTransformer() = 0; virtual IPlanFormatter& GetPlanFormatter() = 0; @@ -220,8 +220,8 @@ struct TDataProviderInfo { const TOperationProgressWriter& progressWriter, const TYqlOperationOptions& operationOptions, TIntrusivePtr<IRandomProvider> randomProvider, TIntrusivePtr<ITimeProvider> timeProvider)> OpenSession; - std::function<bool()> HasActiveProcesses; - + std::function<bool()> HasActiveProcesses; + std::function<void(const TString& sessionId)> CloseSession; std::function<void(const TString& sessionId)> CleanupSession; diff --git a/ydb/library/yql/core/yql_execution.cpp b/ydb/library/yql/core/yql_execution.cpp index 9a0a645327..eb138b1b4f 100644 --- a/ydb/library/yql/core/yql_execution.cpp +++ b/ydb/library/yql/core/yql_execution.cpp @@ -1,6 +1,6 @@ #include "yql_execution.h" #include "yql_expr_optimize.h" -#include "yql_opt_proposed_by_data.h" +#include "yql_opt_proposed_by_data.h" #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/yql_panic.h> @@ -45,13 +45,13 @@ public: } TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - if (FinalizingTransformer) { + if (FinalizingTransformer) { YQL_CLOG(INFO, CoreExecution) << "FinalizingTransformer, root #" << input->UniqueId(); auto status = FinalizingTransformer->Transform(input, output, ctx); YQL_CLOG(INFO, CoreExecution) << "FinalizingTransformer done, output #" << output->UniqueId() << ", status: " << status; return status; - } - + } + YQL_CLOG(INFO, CoreExecution) << "Begin, root #" << input->UniqueId(); output = input; if (RewriteSanityCheck) { @@ -82,26 +82,26 @@ public: YQL_CLOG(INFO, CoreExecution) << "Finish, output #" << output->UniqueId() << ", status: " << status; if (status != TStatus::Ok || !WithFinalize) { - return status; - } - - FinalizingTransformer = CreateCompositeFinalizingTransformer(Types); - return FinalizingTransformer->Transform(input, output, ctx); + return status; + } + + FinalizingTransformer = CreateCompositeFinalizingTransformer(Types); + return FinalizingTransformer->Transform(input, output, ctx); } NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final { - return FinalizingTransformer ? - FinalizingTransformer->GetAsyncFuture(input) : - State->Promise.GetFuture(); + return FinalizingTransformer ? + FinalizingTransformer->GetAsyncFuture(input) : + State->Promise.GetFuture(); } TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - if (FinalizingTransformer) { - return FinalizingTransformer->ApplyAsyncChanges(input, output, ctx); - } - + if (FinalizingTransformer) { + return FinalizingTransformer->ApplyAsyncChanges(input, output, ctx); + } + output = input; - + TStatus combinedStatus = TStatus::Ok; TState::TQueueType completed; @@ -630,7 +630,7 @@ private: const bool WithFinalize; TStatePtr State; TNodeOnNodeOwnedMap NewNodes; - TAutoPtr<IGraphTransformer> FinalizingTransformer; + TAutoPtr<IGraphTransformer> FinalizingTransformer; THashMap<ui32, TOperationProgress> Progresses; struct TTrackableNodeInfo diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp index 0de311687e..f2b793af8d 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.cpp +++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp @@ -180,7 +180,7 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr& .Add(1, std::move(node)) .Seal() .Build(); - + return IGraphTransformer::TStatus::Repeat; } else if ((fromSlot == EDataSlot::Yson || fromSlot == EDataSlot::Json) && to == "Yson.Node") { node = ctx.Builder(node->Pos()) @@ -206,31 +206,31 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr& .Build(); return IGraphTransformer::TStatus::Repeat; - + } else if ((GetDataTypeInfo(fromSlot).Features & (NUdf::EDataTypeFeatures::DateType | NUdf::EDataTypeFeatures::TzDateType)) && to == "DateTime2.TM") { - node = ctx.Builder(node->Pos()) - .Callable("Apply") - .Callable(0, "Udf") + node = ctx.Builder(node->Pos()) + .Callable("Apply") + .Callable(0, "Udf") .Atom(0, "DateTime2.Split", TNodeFlags::Default) - .Callable(1, "Void") - .Seal() - .Callable(2, "TupleType") - .Callable(0, "TupleType") - .Callable(0, "DataType") + .Callable(1, "Void") + .Seal() + .Callable(2, "TupleType") + .Callable(0, "TupleType") + .Callable(0, "DataType") .Atom(0, sourceType.Cast<TDataExprType>()->GetName(), TNodeFlags::Default) - .Seal() - .Seal() - .Callable(1, "StructType") - .Seal() - .Callable(2, "TupleType") - .Seal() - .Seal() - .Seal() + .Seal() + .Seal() + .Callable(1, "StructType") + .Seal() + .Callable(2, "TupleType") + .Seal() + .Seal() + .Seal() .Add(1, std::move(node)) - .Seal() - .Build(); - - return IGraphTransformer::TStatus::Repeat; + .Seal() + .Build(); + + return IGraphTransformer::TStatus::Repeat; } else if (fromSlot == EDataSlot::Json && to == "JsonNode") { node = ctx.Builder(node->Pos()) .Callable("Apply") @@ -2778,11 +2778,11 @@ bool EnsureComparableType(TPositionHandle position, const TTypeAnnotationNode& t if (!type.IsComparable()) { ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected comparable type, i.e. combination of Data, Optional, List or Tuple, but got:" << type)); - return false; - } + return false; + } return true; -} - +} + bool EnsureEquatableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx)) { return false; diff --git a/ydb/library/yql/core/yql_opt_proposed_by_data.cpp b/ydb/library/yql/core/yql_opt_proposed_by_data.cpp index 4dfaca5db4..23a912303f 100644 --- a/ydb/library/yql/core/yql_opt_proposed_by_data.cpp +++ b/ydb/library/yql/core/yql_opt_proposed_by_data.cpp @@ -367,15 +367,15 @@ TAutoPtr<IGraphTransformer> CreateTableMetadataLoader(const TTypeAnnotationConte ); } -TAutoPtr<IGraphTransformer> CreateCompositeFinalizingTransformer(const TTypeAnnotationContext& types) { - return CreateDataProposalsInspector<ESource::DataSink>( - types, - [](IDataProvider* provider) -> IGraphTransformer& { - return provider->GetFinalizingTransformer(); - } - ); -} - +TAutoPtr<IGraphTransformer> CreateCompositeFinalizingTransformer(const TTypeAnnotationContext& types) { + return CreateDataProposalsInspector<ESource::DataSink>( + types, + [](IDataProvider* provider) -> IGraphTransformer& { + return provider->GetFinalizingTransformer(); + } + ); +} + TAutoPtr<IGraphTransformer> CreatePlanInfoTransformer(const TTypeAnnotationContext& types) { return CreateDataProposalsInspector<ESource::DataSink>( types, diff --git a/ydb/library/yql/core/yql_opt_proposed_by_data.h b/ydb/library/yql/core/yql_opt_proposed_by_data.h index 2781fa7d09..0ec0bcab43 100644 --- a/ydb/library/yql/core/yql_opt_proposed_by_data.h +++ b/ydb/library/yql/core/yql_opt_proposed_by_data.h @@ -18,7 +18,7 @@ TAutoPtr<IGraphTransformer> CreateLogicalDataProposalsInspector(const TTypeAnnot TAutoPtr<IGraphTransformer> CreatePhysicalDataProposalsInspector(const TTypeAnnotationContext& types); TAutoPtr<IGraphTransformer> CreatePhysicalFinalizers(const TTypeAnnotationContext& types); TAutoPtr<IGraphTransformer> CreateTableMetadataLoader(const TTypeAnnotationContext& types); -TAutoPtr<IGraphTransformer> CreateCompositeFinalizingTransformer(const TTypeAnnotationContext& types); +TAutoPtr<IGraphTransformer> CreateCompositeFinalizingTransformer(const TTypeAnnotationContext& types); TAutoPtr<IGraphTransformer> CreatePlanInfoTransformer(const TTypeAnnotationContext& types); } diff --git a/ydb/library/yql/core/yql_opt_utils.cpp b/ydb/library/yql/core/yql_opt_utils.cpp index 9c09520e0f..69aa84a500 100644 --- a/ydb/library/yql/core/yql_opt_utils.cpp +++ b/ydb/library/yql/core/yql_opt_utils.cpp @@ -110,7 +110,7 @@ bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& struct return true; } -bool IsPassthroughFlatMap(const TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember) { +bool IsPassthroughFlatMap(const TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember) { return IsPassthroughLambda(flatmap.Lambda(), passthroughFields, analyzeJustMember); } @@ -143,7 +143,7 @@ bool IsPassthroughLambda(const TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* for (auto child : maybeStruct.Cast()) { auto tuple = child.Cast<TCoNameValueTuple>(); auto value = tuple.Value(); - if (analyzeJustMember && value.Maybe<TCoJust>()) { + if (analyzeJustMember && value.Maybe<TCoJust>()) { value = value.Cast<TCoJust>().Input(); } @@ -341,15 +341,15 @@ const TTypeAnnotationNode* GetSeqItemType(const TTypeAnnotationNode* type) { THROW yexception() << "Impossible to get item type from " << *type; } -TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name) { - for (auto& setting : settings.Children()) { - if (setting->ChildrenSize() != 0 && setting->Child(0)->Content() == name) { - return setting; - } - } - return nullptr; -} - +TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name) { + for (auto& setting : settings.Children()) { + if (setting->ChildrenSize() != 0 && setting->Child(0)->Content() == name) { + return setting; + } + } + return nullptr; +} + bool HasSetting(const TExprNode& settings, const TStringBuf& name) { return GetSetting(settings, name) != nullptr; } diff --git a/ydb/library/yql/core/yql_opt_utils.h b/ydb/library/yql/core/yql_opt_utils.h index b192fcc0d7..00635fea81 100644 --- a/ydb/library/yql/core/yql_opt_utils.h +++ b/ydb/library/yql/core/yql_opt_utils.h @@ -18,7 +18,7 @@ bool IsPredicateFlatMap(const TExprNode& node); bool IsFilterFlatMap(const NNodes::TCoLambda& lambda); bool IsListReorder(const TExprNode& node); bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& structNode); -bool IsPassthroughFlatMap(const NNodes::TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false); +bool IsPassthroughFlatMap(const NNodes::TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false); bool IsPassthroughLambda(const NNodes::TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false); bool IsTablePropsDependent(const TExprNode& node); @@ -36,12 +36,12 @@ bool IsDepended(const TExprNode& from, const TExprNode& to); bool IsEmpty(const TExprNode& node, const TTypeAnnotationContext& typeCtx); bool IsEmptyOptional(const TExprNode& node); bool IsEmptyContainer(const TExprNode& node); - + const TTypeAnnotationNode* RemoveOptionalType(const TTypeAnnotationNode* type); const TTypeAnnotationNode* RemoveAllOptionals(const TTypeAnnotationNode* type); const TTypeAnnotationNode* GetSeqItemType(const TTypeAnnotationNode* seq); -TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name); +TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name); bool HasSetting(const TExprNode& settings, const TStringBuf& name); bool HasAnySetting(const TExprNode& settings, const THashSet<TString>& names); TExprNode::TPtr RemoveSetting(const TExprNode& settings, const TStringBuf& name, TExprContext& ctx); diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h index 95ef857c25..84eeda2edb 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h @@ -27,7 +27,7 @@ struct TEvDqCompute { struct TEvChannelData : public NActors::TEventPB<TEvChannelData, NDqProto::TEvComputeChannelData, TDqComputeEvents::EvChannelData> {}; - + struct TEvChannelDataAck : public NActors::TEventPB<TEvChannelDataAck, NDqProto::TEvComputeChannelDataAck, TDqComputeEvents::EvChannelDataAck> {}; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h index 99033b83b1..deaadefb42 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h @@ -17,7 +17,7 @@ #include <ydb/library/yql/core/issue/yql_issue.h> #include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - + #include <library/cpp/actors/core/interconnect.h> #include <util/generic/size_literals.h> @@ -469,13 +469,13 @@ protected: auto& record = execEv->Record; record.SetState(State); - record.SetStatus(status); + record.SetStatus(status); record.SetTaskId(Task.GetId()); if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) { FillStats(record.MutableStats(), /* last */ true); } IssuesToMessage(issues, record.MutableIssues()); - + this->Send(ExecuterId, execEv.Release()); if (Checkpoints && State == NDqProto::COMPUTE_STATE_FINISHED) { diff --git a/ydb/library/yql/dq/actors/dq_events_ids.h b/ydb/library/yql/dq/actors/dq_events_ids.h index 622fd20da3..698377a393 100644 --- a/ydb/library/yql/dq/actors/dq_events_ids.h +++ b/ydb/library/yql/dq/actors/dq_events_ids.h @@ -19,13 +19,13 @@ struct TDqEvents { struct TDqComputeEvents { enum EDqComputeEvents { Unused0 = EventSpaceBegin(TDqEvents::ES_DQ_COMPUTE_KQP_COMPATIBLE) + 200, - EvState, - EvResumeExecution, - EvChannelData, + EvState, + EvResumeExecution, + EvChannelData, ReservedKqp_EvScanData, ReservedKqp_EvScanDataAck, EvChannelsInfo, - EvChannelDataAck, + EvChannelDataAck, ReservedKqp_EvScanError, Unused1, EvRetryChannelData, diff --git a/ydb/library/yql/dq/actors/protos/dq_events.proto b/ydb/library/yql/dq/actors/protos/dq_events.proto index 479d57c012..0e38ab3bbc 100644 --- a/ydb/library/yql/dq/actors/protos/dq_events.proto +++ b/ydb/library/yql/dq/actors/protos/dq_events.proto @@ -47,11 +47,11 @@ message TComputeActorStats { reserved 2; //optional NKqpProto.TKqpStatsRun LegacyStats = 2; }; -message TEvComputeActorState { +message TEvComputeActorState { optional uint32 State = 1; // == EComputeState optional TComputeActorStats LegacyStats = 2; - optional Ydb.StatusIds.StatusCode Status = 3; - repeated Ydb.Issue.IssueMessage Issues = 4; + optional Ydb.StatusIds.StatusCode Status = 3; + repeated Ydb.Issue.IssueMessage Issues = 4; optional uint64 TaskId = 5; optional TDqComputeActorStats Stats = 6; }; @@ -71,7 +71,7 @@ message TEvComputeChannelDataAck { optional uint64 SeqNo = 2; optional int64 FreeSpace = 3; optional bool Finish = 4; -}; +}; message TEvAbortExecution { optional Ydb.StatusIds.StatusCode StatusCode = 1; diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp index c856baa929..c179abab49 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp @@ -1,6 +1,6 @@ #include "mkql_condense.h" #include "mkql_squeeze_state.h" - + #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/mkql_node_builder.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> @@ -175,35 +175,35 @@ public: public: using TBase = TComputationValue<TValue>; - TValue( - TMemoryUsageInfo* memInfo, - NUdf::TUnboxedValue&& stream, + TValue( + TMemoryUsageInfo* memInfo, + NUdf::TUnboxedValue&& stream, const TSqueezeState& state, - TComputationContext& ctx) + TComputationContext& ctx) : TBase(memInfo) , Stream(std::move(stream)) - , Ctx(ctx) + , Ctx(ctx) , State(state) {} private: ui32 GetTraverseCount() const final { - return 1; - } - + return 1; + } + NUdf::TUnboxedValue GetTraverseItem(ui32 index) const final { - Y_UNUSED(index); - return Stream; - } - + Y_UNUSED(index); + return Stream; + } + NUdf::TUnboxedValue Save() const final { return State.Save(Ctx); - } - + } + void Load(const NUdf::TStringRef& state) final { State.Load(Ctx, state); - } - + } + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override { if (ESqueezeState::Finished == State.Stage) return NUdf::EFetchStatus::Finish; @@ -246,13 +246,13 @@ public: } const NUdf::TUnboxedValue Stream; - TComputationContext& Ctx; + TComputationContext& Ctx; TSqueezeState State; }; TCondenseWrapper( - TComputationMutables& mutables, - IComputationNode* stream, + TComputationMutables& mutables, + IComputationNode* stream, IComputationExternalNode* item, IComputationExternalNode* state, IComputationNode* outSwitch, @@ -266,9 +266,9 @@ public: : TBaseComputation(mutables) , Stream(stream) , State(item, state, outSwitch, initState, updateState, inSave, outSave, inLoad, outLoad, stateType) - { + { this->Stateless = false; - } + } NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { #ifndef MKQL_DISABLE_CODEGEN @@ -298,13 +298,13 @@ private: FetchFunc = GenerateFetch(codegen); codegen->ExportSymbol(FetchFunc); } - + void FinalizeFunctions(const NYql::NCodegen::ICodegen::TPtr& codegen) final { if (FetchFunc) { Fetch = reinterpret_cast<TFetchPtr>(codegen->GetPointerToFunction(FetchFunc)); } } - + Function* GenerateFetch(const NYql::NCodegen::ICodegen::TPtr& codegen) const { auto& module = codegen->GetModule(); auto& context = codegen->GetContext(); @@ -473,28 +473,28 @@ IComputationNode* WrapCondense(TCallable& callable, const TComputationNodeFactor } IComputationNode* WrapSqueeze(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 9, "Expected 9 args"); - + MKQL_ENSURE(callable.GetInputsCount() == 9, "Expected 9 args"); + const auto stream = LocateNode(ctx.NodeLocator, callable, 0); const auto initState = LocateNode(ctx.NodeLocator, callable, 1); const auto updateState = LocateNode(ctx.NodeLocator, callable, 4); const auto item = LocateExternalNode(ctx.NodeLocator, callable, 2); const auto state = LocateExternalNode(ctx.NodeLocator, callable, 3); - + IComputationExternalNode* inSave = nullptr; - IComputationNode* outSave = nullptr; + IComputationNode* outSave = nullptr; IComputationExternalNode* inLoad = nullptr; - IComputationNode* outLoad = nullptr; - + IComputationNode* outLoad = nullptr; + const auto hasSaveLoad = !callable.GetInput(6).GetStaticType()->IsVoid(); - if (hasSaveLoad) { - outSave = LocateNode(ctx.NodeLocator, callable, 6); - outLoad = LocateNode(ctx.NodeLocator, callable, 8); + if (hasSaveLoad) { + outSave = LocateNode(ctx.NodeLocator, callable, 6); + outLoad = LocateNode(ctx.NodeLocator, callable, 8); inSave = LocateExternalNode(ctx.NodeLocator, callable, 5); inLoad = LocateExternalNode(ctx.NodeLocator, callable, 7); - } + } const auto stateType = hasSaveLoad ? callable.GetInput(6).GetStaticType() : nullptr; - + return new TCondenseWrapper<false>(ctx.Mutables, stream, item, state, nullptr, initState, updateState, inSave, outSave, inLoad, outLoad, stateType); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp index 345cb6bd3a..2f5ac89432 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp @@ -1,6 +1,6 @@ #include "mkql_condense1.h" #include "mkql_squeeze_state.h" - + #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/mkql_node_builder.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> @@ -188,35 +188,35 @@ public: public: using TBase = TComputationValue<TValue>; - TValue( - TMemoryUsageInfo* memInfo, - NUdf::TUnboxedValue&& stream, + TValue( + TMemoryUsageInfo* memInfo, + NUdf::TUnboxedValue&& stream, const TSqueezeState& state, - TComputationContext& ctx) + TComputationContext& ctx) : TBase(memInfo) , Stream(std::move(stream)) - , Ctx(ctx) + , Ctx(ctx) , State(state) {} private: ui32 GetTraverseCount() const final { - return 1; - } - + return 1; + } + NUdf::TUnboxedValue GetTraverseItem(ui32 index) const final { - Y_UNUSED(index); - return Stream; - } - + Y_UNUSED(index); + return Stream; + } + NUdf::TUnboxedValue Save() const final { return State.Save(Ctx); - } - + } + void Load(const NUdf::TStringRef& state) final { State.Load(Ctx, state); - } - + } + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final { if (ESqueezeState::Finished == State.Stage) return NUdf::EFetchStatus::Finish; @@ -263,13 +263,13 @@ public: } const NUdf::TUnboxedValue Stream; - TComputationContext& Ctx; + TComputationContext& Ctx; TSqueezeState State; }; TCondense1Wrapper( - TComputationMutables& mutables, - IComputationNode* stream, + TComputationMutables& mutables, + IComputationNode* stream, IComputationExternalNode* item, IComputationExternalNode* state, IComputationNode* outSwitch, @@ -283,9 +283,9 @@ public: : TBaseComputation(mutables) , Stream(stream) , State(item, state, outSwitch, initState, updateState, inSave, outSave, inLoad, outLoad, stateType) - { + { this->Stateless = false; - } + } NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { #ifndef MKQL_DISABLE_CODEGEN @@ -315,13 +315,13 @@ private: FetchFunc = GenerateFetch(codegen); codegen->ExportSymbol(FetchFunc); } - + void FinalizeFunctions(const NYql::NCodegen::ICodegen::TPtr& codegen) final { if (FetchFunc) { Fetch = reinterpret_cast<TFetchPtr>(codegen->GetPointerToFunction(FetchFunc)); } } - + Function* GenerateFetch(const NYql::NCodegen::ICodegen::TPtr& codegen) const { auto& module = codegen->GetModule(); auto& context = codegen->GetContext(); @@ -504,28 +504,28 @@ IComputationNode* WrapCondense1(TCallable& callable, const TComputationNodeFacto } IComputationNode* WrapSqueeze1(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 9, "Expected 9 args"); - + MKQL_ENSURE(callable.GetInputsCount() == 9, "Expected 9 args"); + const auto stream = LocateNode(ctx.NodeLocator, callable, 0); const auto initState = LocateNode(ctx.NodeLocator, callable, 2); const auto updateState = LocateNode(ctx.NodeLocator, callable, 4); const auto item = LocateExternalNode(ctx.NodeLocator, callable, 1); const auto state = LocateExternalNode(ctx.NodeLocator, callable, 3); - + IComputationExternalNode* inSave = nullptr; - IComputationNode* outSave = nullptr; + IComputationNode* outSave = nullptr; IComputationExternalNode* inLoad = nullptr; - IComputationNode* outLoad = nullptr; - + IComputationNode* outLoad = nullptr; + const auto hasSaveLoad = !callable.GetInput(6).GetStaticType()->IsVoid(); - if (hasSaveLoad) { - outSave = LocateNode(ctx.NodeLocator, callable, 6); - outLoad = LocateNode(ctx.NodeLocator, callable, 8); + if (hasSaveLoad) { + outSave = LocateNode(ctx.NodeLocator, callable, 6); + outLoad = LocateNode(ctx.NodeLocator, callable, 8); inSave = LocateExternalNode(ctx.NodeLocator, callable, 5); inLoad = LocateExternalNode(ctx.NodeLocator, callable, 7); - } + } const auto stateType = hasSaveLoad ? callable.GetInput(6).GetStaticType() : nullptr; - + return new TCondense1Wrapper<false>(ctx.Mutables, stream, item, state, nullptr, initState, updateState, inSave, outSave, inLoad, outLoad, stateType); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp index b4b80c4c5f..57599acb64 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp @@ -36,7 +36,7 @@ #include "mkql_group.h" #include "mkql_heap.h" #include "mkql_hasitems.h" -#include "mkql_hopping.h" +#include "mkql_hopping.h" #include "mkql_if.h" #include "mkql_ifpresent.h" #include "mkql_invoke.h" diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp index 6649dbeda4..7de0ebd830 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp @@ -226,14 +226,14 @@ protected: private: ui32 GetTraverseCount() const final { - return 1; - } - + return 1; + } + NUdf::TUnboxedValue GetTraverseItem(ui32 index) const final { - Y_UNUSED(index); - return Stream; - } - + Y_UNUSED(index); + return Stream; + } + NUdf::TUnboxedValue Save() const final { return NUdf::TUnboxedValue::Zero(); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp index d1967d78ee..a9d49e109d 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp @@ -956,15 +956,15 @@ public: {} private: - ui32 GetTraverseCount() const override { - return 1; - } - - NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { - Y_UNUSED(index); - return Stream; - } - + ui32 GetTraverseCount() const override { + return 1; + } + + NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { + Y_UNUSED(index); + return Stream; + } + NUdf::TUnboxedValue Save() const override { return NUdf::TUnboxedValue::Zero(); } @@ -1077,15 +1077,15 @@ public: {} private: - ui32 GetTraverseCount() const override { - return 1; - } - - NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { - Y_UNUSED(index); - return Stream; - } - + ui32 GetTraverseCount() const override { + return 1; + } + + NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { + Y_UNUSED(index); + return Stream; + } + NUdf::TUnboxedValue Save() const override { return NUdf::TUnboxedValue::Zero(); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp index 8c98e2aa30..27d1e000ac 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp @@ -1,111 +1,111 @@ -#include "mkql_hopping.h" -#include "mkql_saveload.h" - +#include "mkql_hopping.h" +#include "mkql_saveload.h" + #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/mkql_stats_registry.h> #include <ydb/library/yql/minikql/mkql_string_util.h> - + #include <util/generic/scope.h> -namespace NKikimr { -namespace NMiniKQL { - +namespace NKikimr { +namespace NMiniKQL { + namespace { const TStatKey Hop_NewHopsCount("Hop_NewHopsCount", true); const TStatKey Hop_ThrownEventsCount("Hop_ThrownEventsCount", true); -class THoppingCoreWrapper : public TMutableComputationNode<THoppingCoreWrapper> { +class THoppingCoreWrapper : public TMutableComputationNode<THoppingCoreWrapper> { typedef TMutableComputationNode<THoppingCoreWrapper> TBaseComputation; -public: - using TSelf = THoppingCoreWrapper; - - class TStreamValue : public TComputationValue<TStreamValue> { - public: - using TBase = TComputationValue<TStreamValue>; - - TStreamValue( - TMemoryUsageInfo* memInfo, - NUdf::TUnboxedValue&& stream, +public: + using TSelf = THoppingCoreWrapper; + + class TStreamValue : public TComputationValue<TStreamValue> { + public: + using TBase = TComputationValue<TStreamValue>; + + TStreamValue( + TMemoryUsageInfo* memInfo, + NUdf::TUnboxedValue&& stream, const TSelf* self, ui64 hopTime, ui64 intervalHopCount, - ui64 delayHopCount, - TComputationContext& ctx) - : TBase(memInfo) - , Stream(std::move(stream)) - , Self(self) + ui64 delayHopCount, + TComputationContext& ctx) + : TBase(memInfo) + , Stream(std::move(stream)) + , Self(self) , HopTime(hopTime) , IntervalHopCount(intervalHopCount) , DelayHopCount(delayHopCount) , Buckets(IntervalHopCount + DelayHopCount) - , Ctx(ctx) - {} - - private: - ui32 GetTraverseCount() const override { - return 1; - } - - NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { - Y_UNUSED(index); - return Stream; - } - - NUdf::TUnboxedValue Save() const override { - MKQL_ENSURE(Ready.empty(), "Inconsistent state to save, not all elements are fetched"); - - TString out; - WriteUi32(out, Buckets.size()); - for (const auto& bucket : Buckets) { - WriteBool(out, bucket.HasValue); - if (bucket.HasValue) { - Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); + , Ctx(ctx) + {} + + private: + ui32 GetTraverseCount() const override { + return 1; + } + + NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { + Y_UNUSED(index); + return Stream; + } + + NUdf::TUnboxedValue Save() const override { + MKQL_ENSURE(Ready.empty(), "Inconsistent state to save, not all elements are fetched"); + + TString out; + WriteUi32(out, Buckets.size()); + for (const auto& bucket : Buckets) { + WriteBool(out, bucket.HasValue); + if (bucket.HasValue) { + Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); if (Self->StateType) { WriteUnboxedValue(out, Self->Packer.RefMutableObject(Ctx, false, Self->StateType), Self->OutSave->GetValue(Ctx)); } - } - } - - WriteUi64(out, HopIndex); - WriteBool(out, Started); - WriteBool(out, Finished); - + } + } + + WriteUi64(out, HopIndex); + WriteBool(out, Started); + WriteBool(out, Finished); + auto strRef = NUdf::TStringRef(out.data(), out.size()); return MakeString(strRef); - } - - void Load(const NUdf::TStringRef& state) override { - TStringBuf in(state.Data(), state.Size()); - - auto size = ReadUi32(in); - Buckets.resize(size); - for (auto& bucket : Buckets) { - bucket.HasValue = ReadBool(in); - if (bucket.HasValue) { + } + + void Load(const NUdf::TStringRef& state) override { + TStringBuf in(state.Data(), state.Size()); + + auto size = ReadUi32(in); + Buckets.resize(size); + for (auto& bucket : Buckets) { + bucket.HasValue = ReadBool(in); + if (bucket.HasValue) { if (Self->StateType) { Self->InLoad->SetValue(Ctx, ReadUnboxedValue(in, Self->Packer.RefMutableObject(Ctx, false, Self->StateType), Ctx)); } bucket.Value = Self->OutLoad->GetValue(Ctx); - } - } - - HopIndex = ReadUi64(in); - Started = ReadBool(in); - Finished = ReadBool(in); - } - - NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override { + } + } + + HopIndex = ReadUi64(in); + Started = ReadBool(in); + Finished = ReadBool(in); + } + + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override { if (!Ready.empty()) { result = std::move(Ready.front()); Ready.pop_front(); - return NUdf::EFetchStatus::Ok; - } + return NUdf::EFetchStatus::Ok; + } if (Finished) { - return NUdf::EFetchStatus::Finish; - } - + return NUdf::EFetchStatus::Finish; + } + i64 thrownEvents = 0; i64 newHops = 0; Y_DEFER { @@ -117,95 +117,95 @@ public: } }; - for (NUdf::TUnboxedValue item;;) { - if (!Ready.empty()) { - result = std::move(Ready.front()); - Ready.pop_front(); - return NUdf::EFetchStatus::Ok; - } - - const auto status = Stream.Fetch(item); - if (status != NUdf::EFetchStatus::Ok) { - if (status == NUdf::EFetchStatus::Finish) { + for (NUdf::TUnboxedValue item;;) { + if (!Ready.empty()) { + result = std::move(Ready.front()); + Ready.pop_front(); + return NUdf::EFetchStatus::Ok; + } + + const auto status = Stream.Fetch(item); + if (status != NUdf::EFetchStatus::Ok) { + if (status == NUdf::EFetchStatus::Finish) { Finished = true; - } - return status; - } - + } + return status; + } + Self->Item->SetValue(Ctx, std::move(item)); auto time = Self->OutTime->GetValue(Ctx); - if (!time) { - continue; - } - - auto hopIndex = time.Get<ui64>() / HopTime; - + if (!time) { + continue; + } + + auto hopIndex = time.Get<ui64>() / HopTime; + if (!Started) { HopIndex = hopIndex + 1; Started = true; - } - + } + while (hopIndex >= HopIndex) { auto firstBucketIndex = HopIndex % Buckets.size(); - - auto bucketIndex = firstBucketIndex; - TMaybe<NUdf::TUnboxedValue> aggregated; - - for (ui64 i = 0; i < IntervalHopCount; ++i) { - const auto& bucket = Buckets[bucketIndex]; - if (bucket.HasValue) { - if (!aggregated) { // todo: clone - Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); - Self->InLoad->SetValue(Ctx, Self->OutSave->GetValue(Ctx)); - aggregated = Self->OutLoad->GetValue(Ctx); - } else { - Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); - Self->State2->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated)); + + auto bucketIndex = firstBucketIndex; + TMaybe<NUdf::TUnboxedValue> aggregated; + + for (ui64 i = 0; i < IntervalHopCount; ++i) { + const auto& bucket = Buckets[bucketIndex]; + if (bucket.HasValue) { + if (!aggregated) { // todo: clone + Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); + Self->InLoad->SetValue(Ctx, Self->OutSave->GetValue(Ctx)); + aggregated = Self->OutLoad->GetValue(Ctx); + } else { + Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); + Self->State2->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated)); aggregated = Self->OutMerge->GetValue(Ctx); - } - } + } + } if (++bucketIndex == Buckets.size()) { - bucketIndex = 0; - } - } - + bucketIndex = 0; + } + } + auto& clearBucket = Buckets[firstBucketIndex]; - clearBucket.Value = NUdf::TUnboxedValue(); - clearBucket.HasValue = false; - - if (aggregated) { - Self->State->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated)); + clearBucket.Value = NUdf::TUnboxedValue(); + clearBucket.HasValue = false; + + if (aggregated) { + Self->State->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated)); Self->Time->SetValue(Ctx, NUdf::TUnboxedValuePod((HopIndex - DelayHopCount) * HopTime)); Ready.emplace_back(Self->OutFinish->GetValue(Ctx)); - } - + } + ++newHops; ++HopIndex; - } - - if (hopIndex + DelayHopCount + 1 >= HopIndex) { + } + + if (hopIndex + DelayHopCount + 1 >= HopIndex) { auto& bucket = Buckets[hopIndex % Buckets.size()]; - if (!bucket.HasValue) { + if (!bucket.HasValue) { bucket.Value = Self->OutInit->GetValue(Ctx); - bucket.HasValue = true; - } else { - Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); + bucket.HasValue = true; + } else { + Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); bucket.Value = Self->OutUpdate->GetValue(Ctx); - } + } } else { ++thrownEvents; - } - } - } - + } + } + } + const NUdf::TUnboxedValue Stream; const TSelf *const Self; - const ui64 HopTime; - const ui64 IntervalHopCount; - const ui64 DelayHopCount; - + const ui64 HopTime; + const ui64 IntervalHopCount; + const ui64 DelayHopCount; + struct TBucket { NUdf::TUnboxedValue Value; bool HasValue = false; @@ -216,73 +216,73 @@ public: ui64 HopIndex = 0; bool Started = false; bool Finished = false; - - TComputationContext& Ctx; - }; - - THoppingCoreWrapper( + + TComputationContext& Ctx; + }; + + THoppingCoreWrapper( TComputationMutables& mutables, - IComputationNode* stream, + IComputationNode* stream, IComputationExternalNode* item, IComputationExternalNode* state, IComputationExternalNode* state2, IComputationExternalNode* time, IComputationExternalNode* inSave, IComputationExternalNode* inLoad, - IComputationNode* outTime, - IComputationNode* outInit, - IComputationNode* outUpdate, - IComputationNode* outSave, - IComputationNode* outLoad, - IComputationNode* outMerge, - IComputationNode* outFinish, - IComputationNode* hop, - IComputationNode* interval, - IComputationNode* delay, - TType* stateType) + IComputationNode* outTime, + IComputationNode* outInit, + IComputationNode* outUpdate, + IComputationNode* outSave, + IComputationNode* outLoad, + IComputationNode* outMerge, + IComputationNode* outFinish, + IComputationNode* hop, + IComputationNode* interval, + IComputationNode* delay, + TType* stateType) : TBaseComputation(mutables) , Stream(stream) - , Item(item) - , State(state) - , State2(state2) - , Time(time) - , InSave(inSave) - , InLoad(inLoad) - , OutTime(outTime) - , OutInit(outInit) - , OutUpdate(outUpdate) - , OutSave(outSave) - , OutLoad(outLoad) - , OutMerge(outMerge) - , OutFinish(outFinish) - , Hop(hop) - , Interval(interval) - , Delay(delay) + , Item(item) + , State(state) + , State2(state2) + , Time(time) + , InSave(inSave) + , InLoad(inLoad) + , OutTime(outTime) + , OutInit(outInit) + , OutUpdate(outUpdate) + , OutSave(outSave) + , OutLoad(outLoad) + , OutMerge(outMerge) + , OutFinish(outFinish) + , Hop(hop) + , Interval(interval) + , Delay(delay) , StateType(stateType) , Packer(mutables) - { - Stateless = false; - } - + { + Stateless = false; + } + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { const auto hopTime = Hop->GetValue(ctx).Get<i64>(); const auto interval = Interval->GetValue(ctx).Get<i64>(); const auto delay = Delay->GetValue(ctx).Get<i64>(); - - // TODO: move checks from here - MKQL_ENSURE(hopTime > 0, "hop must be positive"); + + // TODO: move checks from here + MKQL_ENSURE(hopTime > 0, "hop must be positive"); MKQL_ENSURE(interval >= hopTime, "interval should be greater or equal to hop"); MKQL_ENSURE(delay >= hopTime, "delay should be greater or equal to hop"); - + const auto intervalHopCount = interval / hopTime; const auto delayHopCount = delay / hopTime; - + MKQL_ENSURE(intervalHopCount <= 100000, "too many hops in interval"); MKQL_ENSURE(delayHopCount <= 100000, "too many hops in delay"); - + return ctx.HolderFactory.Create<TStreamValue>(Stream->GetValue(ctx), this, (ui64)hopTime, (ui64)intervalHopCount, (ui64)delayHopCount, ctx); - } - + } + private: void RegisterDependencies() const final { DependsOn(Stream); @@ -302,80 +302,80 @@ private: DependsOn(Hop); DependsOn(Interval); DependsOn(Delay); - } - - IComputationNode* const Stream; - + } + + IComputationNode* const Stream; + IComputationExternalNode* const Item; IComputationExternalNode* const State; IComputationExternalNode* const State2; IComputationExternalNode* const Time; IComputationExternalNode* const InSave; IComputationExternalNode* const InLoad; - - IComputationNode* const OutTime; - IComputationNode* const OutInit; - IComputationNode* const OutUpdate; - IComputationNode* const OutSave; - IComputationNode* const OutLoad; - IComputationNode* const OutMerge; - IComputationNode* const OutFinish; - - IComputationNode* const Hop; - IComputationNode* const Interval; - IComputationNode* const Delay; - + + IComputationNode* const OutTime; + IComputationNode* const OutInit; + IComputationNode* const OutUpdate; + IComputationNode* const OutSave; + IComputationNode* const OutLoad; + IComputationNode* const OutMerge; + IComputationNode* const OutFinish; + + IComputationNode* const Hop; + IComputationNode* const Interval; + IComputationNode* const Delay; + TType* const StateType; TMutableObjectOverBoxedValue<TValuePackerBoxed> Packer; -}; - +}; + } -IComputationNode* WrapHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 17, "Expected 17 args"); - - auto hasSaveLoad = !callable.GetInput(10).GetStaticType()->IsVoid(); - +IComputationNode* WrapHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() == 17, "Expected 17 args"); + + auto hasSaveLoad = !callable.GetInput(10).GetStaticType()->IsVoid(); + IComputationExternalNode* inSave = nullptr; - IComputationNode* outSave = nullptr; + IComputationNode* outSave = nullptr; IComputationExternalNode* inLoad = nullptr; - IComputationNode* outLoad = nullptr; - - auto streamType = callable.GetInput(0).GetStaticType(); - MKQL_ENSURE(streamType->IsStream(), "Expected stream"); - - auto stream = LocateNode(ctx.NodeLocator, callable, 0); - - auto outTime = LocateNode(ctx.NodeLocator, callable, 7); - auto outInit = LocateNode(ctx.NodeLocator, callable, 8); - auto outUpdate = LocateNode(ctx.NodeLocator, callable, 9); - if (hasSaveLoad) { - outSave = LocateNode(ctx.NodeLocator, callable, 10); - outLoad = LocateNode(ctx.NodeLocator, callable, 11); - } - auto outMerge = LocateNode(ctx.NodeLocator, callable, 12); - auto outFinish = LocateNode(ctx.NodeLocator, callable, 13); - - auto hop = LocateNode(ctx.NodeLocator, callable, 14); - auto interval = LocateNode(ctx.NodeLocator, callable, 15); - auto delay = LocateNode(ctx.NodeLocator, callable, 16); - + IComputationNode* outLoad = nullptr; + + auto streamType = callable.GetInput(0).GetStaticType(); + MKQL_ENSURE(streamType->IsStream(), "Expected stream"); + + auto stream = LocateNode(ctx.NodeLocator, callable, 0); + + auto outTime = LocateNode(ctx.NodeLocator, callable, 7); + auto outInit = LocateNode(ctx.NodeLocator, callable, 8); + auto outUpdate = LocateNode(ctx.NodeLocator, callable, 9); + if (hasSaveLoad) { + outSave = LocateNode(ctx.NodeLocator, callable, 10); + outLoad = LocateNode(ctx.NodeLocator, callable, 11); + } + auto outMerge = LocateNode(ctx.NodeLocator, callable, 12); + auto outFinish = LocateNode(ctx.NodeLocator, callable, 13); + + auto hop = LocateNode(ctx.NodeLocator, callable, 14); + auto interval = LocateNode(ctx.NodeLocator, callable, 15); + auto delay = LocateNode(ctx.NodeLocator, callable, 16); + auto item = LocateExternalNode(ctx.NodeLocator, callable, 1); auto state = LocateExternalNode(ctx.NodeLocator, callable, 2); auto state2 = LocateExternalNode(ctx.NodeLocator, callable, 3); auto time = LocateExternalNode(ctx.NodeLocator, callable, 4); - if (hasSaveLoad) { + if (hasSaveLoad) { inSave = LocateExternalNode(ctx.NodeLocator, callable, 5); inLoad = LocateExternalNode(ctx.NodeLocator, callable, 6); - } - - auto stateType = hasSaveLoad ? callable.GetInput(10).GetStaticType() : nullptr; - + } + + auto stateType = hasSaveLoad ? callable.GetInput(10).GetStaticType() : nullptr; + return new THoppingCoreWrapper(ctx.Mutables, - stream, item, state, state2, time, inSave, inLoad, - outTime, outInit, outUpdate, outSave, outLoad, outMerge, outFinish, - hop, interval, delay, stateType); -} - -} -} + stream, item, state, state2, time, inSave, inLoad, + outTime, outInit, outUpdate, outSave, outLoad, outMerge, outFinish, + hop, interval, delay, stateType); +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.h b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.h index 57e69d2235..6f5950ed49 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.h @@ -1,10 +1,10 @@ -#pragma once +#pragma once #include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h> - -namespace NKikimr { -namespace NMiniKQL { - -IComputationNode* WrapHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx); - -} -} + +namespace NKikimr { +namespace NMiniKQL { + +IComputationNode* WrapHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx); + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp index 938609b22e..04997bd1f1 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp @@ -152,12 +152,12 @@ protected: private: ui32 GetTraverseCount() const final { return 1U; - } - + } + NUdf::TUnboxedValue GetTraverseItem(ui32) const final { - return Stream; - } - + return Stream; + } + NUdf::TUnboxedValue Save() const final { return NUdf::TUnboxedValuePod::Zero(); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_saveload.h b/ydb/library/yql/minikql/comp_nodes/mkql_saveload.h index 6af5b211af..0440994f8d 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_saveload.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_saveload.h @@ -1,41 +1,41 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/minikql/defs.h> #include <ydb/library/yql/minikql/pack_num.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_pack.h> - -#include <util/generic/strbuf.h> - + +#include <util/generic/strbuf.h> + #include <string_view> -namespace NKikimr { -namespace NMiniKQL { - +namespace NKikimr { +namespace NMiniKQL { + Y_FORCE_INLINE void WriteByte(TString& out, ui8 value) { out.append((char)value); } -Y_FORCE_INLINE void WriteBool(TString& out, bool value) { - out.append((char)value); -} - -Y_FORCE_INLINE void WriteUi32(TString& out, ui32 value) { - char buf[MAX_PACKED32_SIZE]; - out.AppendNoAlias(buf, Pack32(value, buf)); -} - -Y_FORCE_INLINE void WriteUi64(TString& out, ui64 value) { - char buf[MAX_PACKED64_SIZE]; - out.AppendNoAlias(buf, Pack64(value, buf)); -} - -Y_FORCE_INLINE bool ReadBool(TStringBuf& in) { +Y_FORCE_INLINE void WriteBool(TString& out, bool value) { + out.append((char)value); +} + +Y_FORCE_INLINE void WriteUi32(TString& out, ui32 value) { + char buf[MAX_PACKED32_SIZE]; + out.AppendNoAlias(buf, Pack32(value, buf)); +} + +Y_FORCE_INLINE void WriteUi64(TString& out, ui64 value) { + char buf[MAX_PACKED64_SIZE]; + out.AppendNoAlias(buf, Pack64(value, buf)); +} + +Y_FORCE_INLINE bool ReadBool(TStringBuf& in) { MKQL_ENSURE(in.size(), "Serialized state is corrupted"); bool result = (bool)*in.data(); - in.Skip(1); - return result; -} - + in.Skip(1); + return result; +} + Y_FORCE_INLINE ui8 ReadByte(TStringBuf& in) { MKQL_ENSURE(in.size(), "Serialized state is corrupted"); ui8 result = *in.data(); @@ -43,22 +43,22 @@ Y_FORCE_INLINE ui8 ReadByte(TStringBuf& in) { return result; } -Y_FORCE_INLINE ui32 ReadUi32(TStringBuf& in) { - ui32 result; +Y_FORCE_INLINE ui32 ReadUi32(TStringBuf& in) { + ui32 result; auto count = Unpack32(in.data(), in.size(), result); - MKQL_ENSURE(count, "Serialized state is corrupted"); - in.Skip(count); - return result; -} - -Y_FORCE_INLINE ui64 ReadUi64(TStringBuf& in) { - ui64 result; + MKQL_ENSURE(count, "Serialized state is corrupted"); + in.Skip(count); + return result; +} + +Y_FORCE_INLINE ui64 ReadUi64(TStringBuf& in) { + ui64 result; auto count = Unpack64(in.data(), in.size(), result); - MKQL_ENSURE(count, "Serialized state is corrupted"); - in.Skip(count); - return result; -} - + MKQL_ENSURE(count, "Serialized state is corrupted"); + in.Skip(count); + return result; +} + Y_FORCE_INLINE void WriteString(TString& out, std::string_view str) { WriteUi32(out, str.size()); out.AppendNoAlias(str.data(), str.size()); @@ -72,19 +72,19 @@ Y_FORCE_INLINE std::string_view ReadString(TStringBuf& in) { return head; } -Y_FORCE_INLINE void WriteUnboxedValue(TString& out, const TValuePacker& packer, const NUdf::TUnboxedValue& value) { - auto state = packer.Pack(value); +Y_FORCE_INLINE void WriteUnboxedValue(TString& out, const TValuePacker& packer, const NUdf::TUnboxedValue& value) { + auto state = packer.Pack(value); WriteUi32(out, state.size()); out.AppendNoAlias(state.data(), state.size()); -} - -Y_FORCE_INLINE NUdf::TUnboxedValue ReadUnboxedValue(TStringBuf& in, const TValuePacker& packer, TComputationContext& ctx) { - auto size = ReadUi32(in); +} + +Y_FORCE_INLINE NUdf::TUnboxedValue ReadUnboxedValue(TStringBuf& in, const TValuePacker& packer, TComputationContext& ctx) { + auto size = ReadUi32(in); MKQL_ENSURE(size <= in.size(), "Serialized state is corrupted"); auto value = packer.Unpack(TStringBuf(in.data(), in.data() + size), ctx.HolderFactory); - in.Skip(size); - return value; -} - -} // namespace NMiniKQL -} // namespace NKikimr + in.Skip(size); + return value; +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp index caf6dc06ac..a8650d5754 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp @@ -205,15 +205,15 @@ protected: } private: - ui32 GetTraverseCount() const override { - return 1; - } - - NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { - Y_UNUSED(index); - return Stream; - } - + ui32 GetTraverseCount() const override { + return 1; + } + + NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { + Y_UNUSED(index); + return Stream; + } + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override { if (FilterWorkFinished) { return SkipOrTake ? Stream.Fetch(result) : NUdf::EFetchStatus::Finish; diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make index 23211965bd..55838c4012 100644 --- a/ydb/library/yql/minikql/comp_nodes/ya.make +++ b/ydb/library/yql/minikql/comp_nodes/ya.make @@ -82,8 +82,8 @@ SRCS( mkql_hasitems.h mkql_heap.cpp mkql_heap.h - mkql_hopping.cpp - mkql_hopping.h + mkql_hopping.cpp + mkql_hopping.h mkql_if.cpp mkql_if.h mkql_ifpresent.cpp @@ -144,7 +144,7 @@ SRCS( mkql_round.h mkql_safe_circular_buffer.cpp mkql_safe_circular_buffer.h - mkql_saveload.h + mkql_saveload.h mkql_seq.cpp mkql_seq.h mkql_size.cpp diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h index bcd7c82578..d9feede204 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h @@ -641,7 +641,7 @@ protected: } Value* CreateGetValue(const TCodegenContext& ctx, BasicBlock*& block) const final { - if (*this->Stateless) { + if (*this->Stateless) { const auto newValue = static_cast<const TDerived*>(this)->DoGenerateGetValue(ctx, block); if (newValue->getType()->isPointerTy()) { ValueRelease(this->GetRepresentation(), newValue, ctx, block); diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp index e05b19acb9..c481a5815e 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp @@ -1,93 +1,93 @@ -#include "mkql_computation_node_graph_saveload.h" -#include "mkql_computation_node_holders.h" - +#include "mkql_computation_node_graph_saveload.h" +#include "mkql_computation_node_holders.h" + #include <ydb/library/yql/minikql/pack_num.h> #include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h> - -namespace NKikimr { -namespace NMiniKQL { - -namespace { - + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + void TraverseGraph(const NUdf::TUnboxedValue* roots, ui32 rootCount, TVector<NUdf::TUnboxedValue>& values) { THashSet<NUdf::IBoxedValue*> dedup; - - for (ui32 i = 0; i < rootCount; ++i) { - const auto& value = roots[i]; - if (!value.IsBoxed()) { - continue; - } - auto* ptr = value.AsBoxed().Get(); + + for (ui32 i = 0; i < rootCount; ++i) { + const auto& value = roots[i]; + if (!value.IsBoxed()) { + continue; + } + auto* ptr = value.AsBoxed().Get(); if (dedup.contains(ptr)) { - continue; - } - dedup.insert(ptr); - values.push_back(value); - } - - for (ui32 from = 0, to = values.size(); from != to; ++from) { + continue; + } + dedup.insert(ptr); + values.push_back(value); + } + + for (ui32 from = 0, to = values.size(); from != to; ++from) { auto current = values[from]; - auto count = current.GetTraverseCount(); - - for (ui32 i = 0; i < count; ++i) { - auto value = current.GetTraverseItem(i); - if (!value.IsBoxed()) { - continue; - } - auto* ptr = value.AsBoxed().Get(); + auto count = current.GetTraverseCount(); + + for (ui32 i = 0; i < count; ++i) { + auto value = current.GetTraverseItem(i); + if (!value.IsBoxed()) { + continue; + } + auto* ptr = value.AsBoxed().Get(); if (dedup.contains(ptr)) { - continue; - } - dedup.insert(ptr); - values.push_back(value); - ++to; - } - } -} - -} - -void SaveGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, TString& out) { - out.clear(); - out.AppendNoAlias((const char*)&hash, sizeof(hash)); - + continue; + } + dedup.insert(ptr); + values.push_back(value); + ++to; + } + } +} + +} + +void SaveGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, TString& out) { + out.clear(); + out.AppendNoAlias((const char*)&hash, sizeof(hash)); + TVector<NUdf::TUnboxedValue> values; - TraverseGraph(roots, rootCount, values); - - for (ui32 i = 0; i < values.size(); ++i) { - auto state = values[i].Save(); - auto strRef = state.AsStringRef(); - auto size = strRef.Size(); - WriteUi32(out, size); - if (size) { - out.AppendNoAlias(strRef.Data(), size); - } - } -} - -void LoadGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, const TStringBuf& in) { - TStringBuf state(in); - + TraverseGraph(roots, rootCount, values); + + for (ui32 i = 0; i < values.size(); ++i) { + auto state = values[i].Save(); + auto strRef = state.AsStringRef(); + auto size = strRef.Size(); + WriteUi32(out, size); + if (size) { + out.AppendNoAlias(strRef.Data(), size); + } + } +} + +void LoadGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, const TStringBuf& in) { + TStringBuf state(in); + MKQL_ENSURE(state.size() >= sizeof(ui64), "Serialized state is corrupted - no hash"); ui64 storedHash = *(ui64*)state.data(); - state.Skip(sizeof(storedHash)); - - MKQL_ENSURE(hash == storedHash, "Unable to load graph state, different hashes"); - + state.Skip(sizeof(storedHash)); + + MKQL_ENSURE(hash == storedHash, "Unable to load graph state, different hashes"); + TVector<NUdf::TUnboxedValue> values; - TraverseGraph(roots, rootCount, values); - - for (ui32 i = 0; i < values.size(); ++i) { - auto size = ReadUi32(state); - if (size) { + TraverseGraph(roots, rootCount, values); + + for (ui32 i = 0; i < values.size(); ++i) { + auto size = ReadUi32(state); + if (size) { MKQL_ENSURE(size <= state.size(), "Serialized state is corrupted"); values[i].Load(NUdf::TStringRef(state.data(), size)); - state.Skip(size); - } - } - + state.Skip(size); + } + } + MKQL_ENSURE(state.size() == 0, "State was not loaded correctly"); -} - -} // namespace NMiniKQL -} // namespace NKikimr +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h index 171a83fa69..55564400f2 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h @@ -1,13 +1,13 @@ -#pragma once - -#include "mkql_computation_node.h" - -namespace NKikimr { -namespace NMiniKQL { - -void SaveGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, TString& out); - -void LoadGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, const TStringBuf& in); - -} // namespace NMiniKQL -} // namespace NKikimr +#pragma once + +#include "mkql_computation_node.h" + +namespace NKikimr { +namespace NMiniKQL { + +void SaveGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, TString& out); + +void LoadGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, const TStringBuf& in); + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp index 48848db326..cc734e11d6 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp @@ -7,77 +7,77 @@ #include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> #include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> - + #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { -namespace NMiniKQL { - -namespace { - TIntrusivePtr<IRandomProvider> CreateRandomProvider() { - return CreateDeterministicRandomProvider(1); - } - - TIntrusivePtr<ITimeProvider> CreateTimeProvider() { + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + TIntrusivePtr<IRandomProvider> CreateRandomProvider() { + return CreateDeterministicRandomProvider(1); + } + + TIntrusivePtr<ITimeProvider> CreateTimeProvider() { return CreateDeterministicTimeProvider(10000000); - } - - TComputationNodeFactory GetAuxCallableFactory() { - return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { - if (callable.GetType()->GetName() == "OneYieldStream") { - return new TExternalComputationNode(ctx.Mutables); - } - - return GetBuiltinFactory()(callable, ctx); - }; - } - - struct TSetup { - TSetup(TScopedAlloc& alloc) - : Alloc(alloc) - { - FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); - RandomProvider = CreateRandomProvider(); - TimeProvider = CreateTimeProvider(); - - Env.Reset(new TTypeEnvironment(Alloc)); - PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); - } - + } + + TComputationNodeFactory GetAuxCallableFactory() { + return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (callable.GetType()->GetName() == "OneYieldStream") { + return new TExternalComputationNode(ctx.Mutables); + } + + return GetBuiltinFactory()(callable, ctx); + }; + } + + struct TSetup { + TSetup(TScopedAlloc& alloc) + : Alloc(alloc) + { + FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + RandomProvider = CreateRandomProvider(); + TimeProvider = CreateTimeProvider(); + + Env.Reset(new TTypeEnvironment(Alloc)); + PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); + } + THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { - Explorer.Walk(pgm.GetNode(), *Env); - TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(), + Explorer.Walk(pgm.GetNode(), *Env); + TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(), FunctionRegistry.Get(), NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi); - Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); + Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider); - return Pattern->Clone(compOpts); - } - - TIntrusivePtr<IFunctionRegistry> FunctionRegistry; - TIntrusivePtr<IRandomProvider> RandomProvider; - TIntrusivePtr<ITimeProvider> TimeProvider; - - TScopedAlloc& Alloc; - THolder<TTypeEnvironment> Env; - THolder<TProgramBuilder> PgmBuilder; - - TExploringNodeVisitor Explorer; + return Pattern->Clone(compOpts); + } + + TIntrusivePtr<IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + + TScopedAlloc& Alloc; + THolder<TTypeEnvironment> Env; + THolder<TProgramBuilder> PgmBuilder; + + TExploringNodeVisitor Explorer; IComputationPattern::TPtr Pattern; - }; - - struct TStreamWithYield : public NUdf::TBoxedValue { + }; + + struct TStreamWithYield : public NUdf::TBoxedValue { TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index) - : Items(items) - , YieldPos(yieldPos) - , Index(index) - {} - - private: + : Items(items) + , YieldPos(yieldPos) + , Index(index) + {} + + private: TUnboxedValueVector Items; - ui32 YieldPos; - ui32 Index; - + ui32 YieldPos; + ui32 Index; + ui32 GetTraverseCount() const override { return 0; } @@ -91,290 +91,290 @@ namespace { } NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final { - if (Index >= Items.size()) { - return NUdf::EFetchStatus::Finish; - } - if (Index == YieldPos) { - return NUdf::EFetchStatus::Yield; - } - result = Items[Index++]; - return NUdf::EFetchStatus::Ok; - } - }; -} - + if (Index >= Items.size()) { + return NUdf::EFetchStatus::Finish; + } + if (Index == YieldPos) { + return NUdf::EFetchStatus::Yield; + } + result = Items[Index++]; + return NUdf::EFetchStatus::Ok; + } + }; +} + Y_UNIT_TEST_SUITE(TMiniKQLSaveLoadTest) { Y_UNIT_TEST(TestSqueezeSaveLoad) { - TScopedAlloc alloc; - + TScopedAlloc alloc; + const std::vector<ui32> items = {2, 3, 4, 5, 6, 7, 8}; - - auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> { - TProgramBuilder& pgmBuilder = *setup.PgmBuilder; - - auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); - auto streamType = pgmBuilder.NewStreamType(dataType); - - TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType); - auto streamNode = inStream.Build(); - - auto pgmReturn = pgmBuilder.Squeeze( - TRuntimeNode(streamNode, false), - pgmBuilder.NewDataLiteral<ui32>(1), - [&](TRuntimeNode item, TRuntimeNode state) { - return pgmBuilder.Add(item, state); - }, - [](TRuntimeNode state) { - return state; - }, - [](TRuntimeNode state) { - return state; - }); - + + auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); + auto streamType = pgmBuilder.NewStreamType(dataType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType); + auto streamNode = inStream.Build(); + + auto pgmReturn = pgmBuilder.Squeeze( + TRuntimeNode(streamNode, false), + pgmBuilder.NewDataLiteral<ui32>(1), + [&](TRuntimeNode item, TRuntimeNode state) { + return pgmBuilder.Add(item, state); + }, + [](TRuntimeNode state) { + return state; + }, + [](TRuntimeNode state) { + return state; + }); + TUnboxedValueVector streamItems; - for (auto item : items) { - streamItems.push_back(NUdf::TUnboxedValuePod(item)); - } - - auto graph = setup.BuildGraph(pgmReturn, {streamNode}); - auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); + for (auto item : items) { + streamItems.push_back(NUdf::TUnboxedValuePod(item)); + } + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); - return graph; - }; - - for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) { - TSetup setup1(alloc); - auto graph1 = buildGraph(setup1, yieldPos, 0); - + return graph; + }; + + for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) { + TSetup setup1(alloc); + auto graph1 = buildGraph(setup1, yieldPos, 0); + auto root1 = graph1->GetValue(); - NUdf::TUnboxedValue res; + NUdf::TUnboxedValue res; auto status = root1.Fetch(res); - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); - - TString graphState; - SaveGraphState(&root1, 1, 0ULL, graphState); - - TSetup setup2(alloc); - auto graph2 = buildGraph(setup2, -1, yieldPos); - - auto root2 = graph2->GetValue(); + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); + + TString graphState; + SaveGraphState(&root1, 1, 0ULL, graphState); + + TSetup setup2(alloc); + auto graph2 = buildGraph(setup2, -1, yieldPos); + + auto root2 = graph2->GetValue(); LoadGraphState(&root2, 1, 0ULL, graphState); - + status = root2.Fetch(res); - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok); - UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36); - + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok); + UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36); + status = root2.Fetch(res); - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); - } - } - + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + } + } + Y_UNIT_TEST(TestSqueeze1SaveLoad) { - TScopedAlloc alloc; - + TScopedAlloc alloc; + const std::vector<ui32> items = {1, 2, 3, 4, 5, 6, 7, 8}; - - auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> { - TProgramBuilder& pgmBuilder = *setup.PgmBuilder; - - auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); - auto streamType = pgmBuilder.NewStreamType(dataType); - - TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType); - auto streamNode = inStream.Build(); - - auto pgmReturn = pgmBuilder.Squeeze1( - TRuntimeNode(streamNode, false), - [](TRuntimeNode item) { - return item; - }, - [&](TRuntimeNode item, TRuntimeNode state) { - return pgmBuilder.Add(item, state); - }, - [](TRuntimeNode state) { - return state; - }, - [](TRuntimeNode state) { - return state; - }); - + + auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); + auto streamType = pgmBuilder.NewStreamType(dataType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType); + auto streamNode = inStream.Build(); + + auto pgmReturn = pgmBuilder.Squeeze1( + TRuntimeNode(streamNode, false), + [](TRuntimeNode item) { + return item; + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pgmBuilder.Add(item, state); + }, + [](TRuntimeNode state) { + return state; + }, + [](TRuntimeNode state) { + return state; + }); + TUnboxedValueVector streamItems; - for (auto item : items) { - streamItems.push_back(NUdf::TUnboxedValuePod(item)); - } - - auto graph = setup.BuildGraph(pgmReturn, {streamNode}); - auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); + for (auto item : items) { + streamItems.push_back(NUdf::TUnboxedValuePod(item)); + } + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); - return graph; - }; - - for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) { - TSetup setup1(alloc); - auto graph1 = buildGraph(setup1, yieldPos, 0); - + return graph; + }; + + for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) { + TSetup setup1(alloc); + auto graph1 = buildGraph(setup1, yieldPos, 0); + auto root1 = graph1->GetValue(); - NUdf::TUnboxedValue res; + NUdf::TUnboxedValue res; auto status = root1.Fetch(res); - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); - - TString graphState; - SaveGraphState(&root1, 1, 0ULL, graphState); - - TSetup setup2(alloc); - auto graph2 = buildGraph(setup2, -1, yieldPos); - - auto root2 = graph2->GetValue(); + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); + + TString graphState; + SaveGraphState(&root1, 1, 0ULL, graphState); + + TSetup setup2(alloc); + auto graph2 = buildGraph(setup2, -1, yieldPos); + + auto root2 = graph2->GetValue(); LoadGraphState(&root2, 1, 0ULL, graphState); - + status = root2.Fetch(res); - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok); - UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36); - + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok); + UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36); + status = root2.Fetch(res); - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); - } - } - + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + } + } + Y_UNIT_TEST(TestHoppingSaveLoad) { - TScopedAlloc alloc; - + TScopedAlloc alloc; + const std::vector<std::pair<i64, ui32>> items = { - {1, 2}, - {2, 3}, - {15, 4}, - {23, 6}, - {24, 5}, - {25, 7}, - {40, 2}, - {47, 1}, - {51, 6}, - {59, 2}, - {85, 8}, - {55, 1000}, - {200, 0} - }; - - auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> { - TProgramBuilder& pgmBuilder = *setup.PgmBuilder; - - auto structType = pgmBuilder.NewEmptyStructType(); + {1, 2}, + {2, 3}, + {15, 4}, + {23, 6}, + {24, 5}, + {25, 7}, + {40, 2}, + {47, 1}, + {51, 6}, + {59, 2}, + {85, 8}, + {55, 1000}, + {200, 0} + }; + + auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto structType = pgmBuilder.NewEmptyStructType(); structType = pgmBuilder.NewStructType(structType, "time", - pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id)); + pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id)); structType = pgmBuilder.NewStructType(structType, "sum", - pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); - auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time"); - auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum"); - - auto inStreamType = pgmBuilder.NewStreamType(structType); - - TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType); - auto streamNode = inStream.Build(); - - ui64 hop = 10, interval = 30, delay = 20; - - auto pgmReturn = pgmBuilder.HoppingCore( - TRuntimeNode(streamNode, false), - [&](TRuntimeNode item) { // timeExtractor + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time"); + auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum"); + + auto inStreamType = pgmBuilder.NewStreamType(structType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType); + auto streamNode = inStream.Build(); + + ui64 hop = 10, interval = 30, delay = 20; + + auto pgmReturn = pgmBuilder.HoppingCore( + TRuntimeNode(streamNode, false), + [&](TRuntimeNode item) { // timeExtractor return pgmBuilder.Member(item, "time"); - }, - [&](TRuntimeNode item) { // init + }, + [&](TRuntimeNode item) { // init std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("sum", pgmBuilder.Member(item, "sum")); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode item, TRuntimeNode state) { // update - auto add = pgmBuilder.AggrAdd( + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode item, TRuntimeNode state) { // update + auto add = pgmBuilder.AggrAdd( pgmBuilder.Member(item, "sum"), pgmBuilder.Member(state, "sum")); std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("sum", add); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode state) { // save + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state) { // save return pgmBuilder.Member(state, "sum"); - }, - [&](TRuntimeNode savedState) { // load + }, + [&](TRuntimeNode savedState) { // load std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("sum", savedState); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode state1, TRuntimeNode state2) { // merge - auto add = pgmBuilder.AggrAdd( + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state1, TRuntimeNode state2) { // merge + auto add = pgmBuilder.AggrAdd( pgmBuilder.Member(state1, "sum"), pgmBuilder.Member(state2, "sum")); std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("sum", add); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode state, TRuntimeNode time) { // finish - Y_UNUSED(time); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state, TRuntimeNode time) { // finish + Y_UNUSED(time); std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("sum", pgmBuilder.Member(state, "sum")); - return pgmBuilder.NewStruct(members); - }, - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))) // delay - ); - - auto graph = setup.BuildGraph(pgmReturn, {streamNode}); - + return pgmBuilder.NewStruct(members); + }, + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))) // delay + ); + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + TUnboxedValueVector streamItems; - for (size_t i = 0; i < items.size(); ++i) { - NUdf::TUnboxedValue* itemsPtr; - auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(2, itemsPtr); - itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items[i].first); - itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items[i].second); - streamItems.push_back(std::move(structValues)); - } - - auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); + for (size_t i = 0; i < items.size(); ++i) { + NUdf::TUnboxedValue* itemsPtr; + auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(2, itemsPtr); + itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items[i].first); + itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items[i].second); + streamItems.push_back(std::move(structValues)); + } + + auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); - return graph; - }; - - for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) { + return graph; + }; + + for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) { std::vector<ui32> result; - - TSetup setup1(alloc); - auto graph1 = buildGraph(setup1, yieldPos, 0); - auto root1 = graph1->GetValue(); - - NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok; - while (status == NUdf::EFetchStatus::Ok) { - NUdf::TUnboxedValue val; - status = root1.Fetch(val); - if (status == NUdf::EFetchStatus::Ok) { - result.push_back(val.GetElement(0).Get<ui32>()); - } - } - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); - - TString graphState; - SaveGraphState(&root1, 1, 0ULL, graphState); - - TSetup setup2(alloc); - auto graph2 = buildGraph(setup2, -1, yieldPos); - auto root2 = graph2->GetValue(); + + TSetup setup1(alloc); + auto graph1 = buildGraph(setup1, yieldPos, 0); + auto root1 = graph1->GetValue(); + + NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok) { + NUdf::TUnboxedValue val; + status = root1.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + result.push_back(val.GetElement(0).Get<ui32>()); + } + } + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); + + TString graphState; + SaveGraphState(&root1, 1, 0ULL, graphState); + + TSetup setup2(alloc); + auto graph2 = buildGraph(setup2, -1, yieldPos); + auto root2 = graph2->GetValue(); LoadGraphState(&root2, 1, 0ULL, graphState); - - status = NUdf::EFetchStatus::Ok; - while (status == NUdf::EFetchStatus::Ok) { - NUdf::TUnboxedValue val; - status = root2.Fetch(val); - if (status == NUdf::EFetchStatus::Ok) { - result.push_back(val.GetElement(0).Get<ui32>()); - } - } - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); - + + status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok) { + NUdf::TUnboxedValue val; + status = root2.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + result.push_back(val.GetElement(0).Get<ui32>()); + } + } + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + const std::vector<ui32> resultCompare = {5, 9, 27, 22, 21, 11, 11, 8, 8, 8, 8}; - UNIT_ASSERT_EQUAL(result, resultCompare); - } - } -} - -} // namespace NMiniKQL -} // namespace NKikimr + UNIT_ASSERT_EQUAL(result, resultCompare); + } + } +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h index 1fd7a9bde8..4486d6f43a 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h @@ -56,11 +56,11 @@ inline int CompareValues(NUdf::EDataSlot type, cmp = 1; } else { - cmp = NUdf::CompareValues(type, lhs, rhs); + cmp = NUdf::CompareValues(type, lhs, rhs); } } else { - cmp = NUdf::CompareValues(type, lhs, rhs); + cmp = NUdf::CompareValues(type, lhs, rhs); } if (!asc) { diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h index 843d838817..0a63ca9d34 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h @@ -155,10 +155,10 @@ private: } void PrepareStageOne() final { - if (!Stateless) { + if (!Stateless) { Stateless = std::accumulate(this->Dependencies.cbegin(), this->Dependencies.cend(), 0, std::bind(std::plus<i32>(), std::placeholders::_1, std::bind(&IComputationNode::GetDependencyWeight, std::placeholders::_2))) <= 1; - } + } } void PrepareStageTwo() final {} @@ -960,27 +960,27 @@ private: return NUdf::EFetchStatus::Finish; } - ui32 GetTraverseCount() const override { + ui32 GetTraverseCount() const override { ThrowNotSupported(__func__); - return 0; - } - - NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { - Y_UNUSED(index); + return 0; + } + + NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { + Y_UNUSED(index); ThrowNotSupported(__func__); - return {}; - } - - NUdf::TUnboxedValue Save() const override { + return {}; + } + + NUdf::TUnboxedValue Save() const override { ThrowNotSupported(__func__); - return NUdf::TUnboxedValue::Zero(); - } - - void Load(const NUdf::TStringRef& state) override { - Y_UNUSED(state); + return NUdf::TUnboxedValue::Zero(); + } + + void Load(const NUdf::TStringRef& state) override { + Y_UNUSED(state); ThrowNotSupported(__func__); - } - + } + void Push(const NUdf::TUnboxedValuePod& value) override { Y_UNUSED(value); ThrowNotSupported(__func__); diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp index 60fab95239..625a269eaf 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp @@ -389,7 +389,7 @@ std::pair<ui32, bool> TValuePacker::SkipEmbeddedLength(TStringBuf& buf) { return {length, emptySingleOptional}; } -NUdf::TUnboxedValue TValuePacker::Unpack(TStringBuf buf, const THolderFactory& holderFactory) const { +NUdf::TUnboxedValue TValuePacker::Unpack(TStringBuf buf, const THolderFactory& holderFactory) const { auto pair = SkipEmbeddedLength(buf); ui32 length = pair.first; bool emptySingleOptional = pair.second; @@ -417,7 +417,7 @@ NUdf::TUnboxedValue TValuePacker::Unpack(TStringBuf buf, const THolderFactory& h } NUdf::TUnboxedValue TValuePacker::UnpackImpl(const TType* type, TStringBuf& buf, ui32 topLength, - const THolderFactory& holderFactory) const + const THolderFactory& holderFactory) const { switch (type->GetKind()) { case TType::EKind::Void: @@ -592,7 +592,7 @@ NUdf::TUnboxedValue TValuePacker::UnpackImpl(const TType* type, TStringBuf& buf, } } -TStringBuf TValuePacker::Pack(const NUdf::TUnboxedValuePod& value) const { +TStringBuf TValuePacker::Pack(const NUdf::TUnboxedValuePod& value) const { OptionalUsageMask.Reset(); const size_t lengthReserve = sizeof(ui32); Buffer.Proceed(lengthReserve + OptionalMaskReserve); @@ -643,7 +643,7 @@ TStringBuf TValuePacker::Pack(const NUdf::TUnboxedValuePod& value) const { return TStringBuf(Buffer.Data() + delta, len - delta); } -void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const { +void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const { switch (type->GetKind()) { case TType::EKind::Void: break; diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h index f9473ebff1..c69f4467a3 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h @@ -36,12 +36,12 @@ public: TValuePacker(const TValuePacker& other); // Returned buffer is temporary and should be copied before next Pack() call - TStringBuf Pack(const NUdf::TUnboxedValuePod& value) const; - NUdf::TUnboxedValue Unpack(TStringBuf buf, const THolderFactory& holderFactory) const; + TStringBuf Pack(const NUdf::TUnboxedValuePod& value) const; + NUdf::TUnboxedValue Unpack(TStringBuf buf, const THolderFactory& holderFactory) const; private: - void PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const; - NUdf::TUnboxedValue UnpackImpl(const TType* type, TStringBuf& buf, ui32 topLength, const THolderFactory& holderFactory) const; + void PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const; + NUdf::TUnboxedValue UnpackImpl(const TType* type, TStringBuf& buf, ui32 topLength, const THolderFactory& holderFactory) const; static TProperties ScanTypeProperties(const TType* type); static bool HasOptionalFields(const TType* type); // Returns length and empty single optional flag @@ -54,12 +54,12 @@ private: #endif const bool Stable; const TType* Type; - // TODO: real thread safety with external state - mutable TBuffer Buffer; + // TODO: real thread safety with external state + mutable TBuffer Buffer; TProperties Properties; - mutable size_t OptionalMaskReserve; - mutable NDetails::TOptionalUsageMask OptionalUsageMask; - mutable TPlainContainerCache TopStruct; + mutable size_t OptionalMaskReserve; + mutable NDetails::TOptionalUsageMask OptionalUsageMask; + mutable TPlainContainerCache TopStruct; mutable TVector<TVector<std::pair<NUdf::TUnboxedValue, NUdf::TUnboxedValue>>> DictBuffers; mutable TVector<TVector<std::tuple<NUdf::TUnboxedValue, NUdf::TUnboxedValue, NUdf::TUnboxedValue>>> EncodedDictBuffers; TPackFunction PackFunc = nullptr; diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder.cpp b/ydb/library/yql/minikql/computation/mkql_value_builder.cpp index c27a241428..ad47661715 100644 --- a/ydb/library/yql/minikql/computation/mkql_value_builder.cpp +++ b/ydb/library/yql/minikql/computation/mkql_value_builder.cpp @@ -131,24 +131,24 @@ NUdf::IDictValueBuilder::TPtr TDefaultValueBuilder::NewDict(const NUdf::TType* d return HolderFactory_.NewDict(dictType, flags); } -bool TDefaultValueBuilder::MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const { - return ::NKikimr::NMiniKQL::MakeDate(year, month, day, value); -} - -bool TDefaultValueBuilder::SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const { - return ::NKikimr::NMiniKQL::SplitDate(value, year, month, day); -} - -bool TDefaultValueBuilder::MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, ui16 tzId) const -{ - return ::NKikimr::NMiniKQL::MakeTzDatetime(year, month, day, hour, minute, second, value, tzId); -} - -bool TDefaultValueBuilder::SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, ui16 tzId) const -{ - return ::NKikimr::NMiniKQL::SplitTzDatetime(value, year, month, day, hour, minute, second, tzId); -} - +bool TDefaultValueBuilder::MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const { + return ::NKikimr::NMiniKQL::MakeDate(year, month, day, value); +} + +bool TDefaultValueBuilder::SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const { + return ::NKikimr::NMiniKQL::SplitDate(value, year, month, day); +} + +bool TDefaultValueBuilder::MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, ui16 tzId) const +{ + return ::NKikimr::NMiniKQL::MakeTzDatetime(year, month, day, hour, minute, second, value, tzId); +} + +bool TDefaultValueBuilder::SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, ui16 tzId) const +{ + return ::NKikimr::NMiniKQL::SplitTzDatetime(value, year, month, day, hour, minute, second, tzId); +} + bool TDefaultValueBuilder::FullSplitDate(ui16 value, ui32& year, ui32& month, ui32& day, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek, ui16 tzId) const { ui32 unusedWeekOfYearIso8601 = 0; @@ -171,20 +171,20 @@ bool TDefaultValueBuilder::FullSplitDatetime2(ui32 value, ui32& year, ui32& mont return ::NKikimr::NMiniKQL::SplitTzDatetime(value, year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, tzId); } -bool TDefaultValueBuilder::EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const { +bool TDefaultValueBuilder::EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const { ui32 unusedWeekOfYearIso8601 = 0; return ::NKikimr::NMiniKQL::EnrichDate(date, dayOfYear, weekOfYear, unusedWeekOfYearIso8601, dayOfWeek); -} - +} + bool TDefaultValueBuilder::EnrichDate2(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek) const { return ::NKikimr::NMiniKQL::EnrichDate(date, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek); } -bool TDefaultValueBuilder::GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui16 tzId, i32& value) const -{ - return ::NKikimr::NMiniKQL::GetTimezoneShift(year, month, day, hour, minute, second, tzId, value); -} - +bool TDefaultValueBuilder::GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui16 tzId, i32& value) const +{ + return ::NKikimr::NMiniKQL::GetTimezoneShift(year, month, day, hour, minute, second, tzId, value); +} + const NUdf::TSourcePosition* TDefaultValueBuilder::CalleePosition() const { return *CalleePositionPtr_; } diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder.h b/ydb/library/yql/minikql/computation/mkql_value_builder.h index eea453b6b1..b293b263a3 100644 --- a/ydb/library/yql/minikql/computation/mkql_value_builder.h +++ b/ydb/library/yql/minikql/computation/mkql_value_builder.h @@ -61,17 +61,17 @@ public: NUdf::TFlatArrayBlockPtr NewFlatArrayBlock(ui32 count) const; NUdf::TSingleBlockPtr NewSingleBlock(const NUdf::TUnboxedValue& value) const; - bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const final; - bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const final; + bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const final; + bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const final; - bool MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, ui16 tzId = 0) const final; - bool SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, ui16 tzId = 0) const final; + bool MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, ui16 tzId = 0) const final; + bool SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, ui16 tzId = 0) const final; + + bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const final; + + // in minutes + bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui16 tzId, i32& value) const final; - bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const final; - - // in minutes - bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui16 tzId, i32& value) const final; - bool FullSplitDate(ui16 value, ui32& year, ui32& month, ui32& day, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek, ui16 timezoneId = 0) const final; bool FullSplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, diff --git a/ydb/library/yql/minikql/computation/presort.cpp b/ydb/library/yql/minikql/computation/presort.cpp index 25f0a4806c..26b96a60aa 100644 --- a/ydb/library/yql/minikql/computation/presort.cpp +++ b/ydb/library/yql/minikql/computation/presort.cpp @@ -1,510 +1,510 @@ -#include "presort.h" +#include "presort.h" #include "mkql_computation_node_holders.h" #include <ydb/library/yql/minikql/defs.h> #include <ydb/library/yql/minikql/mkql_string_util.h> - + #include <ydb/library/yql/utils/swap_bytes.h> #include <ydb/library/yql/public/decimal/yql_decimal_serialize.h> - -#include <util/system/unaligned_mem.h> + +#include <util/system/unaligned_mem.h> #include <util/string/builder.h> - -namespace NKikimr { -namespace NMiniKQL { - -namespace NDetail { - + +namespace NKikimr { +namespace NMiniKQL { + +namespace NDetail { + using NYql::SwapBytes; -Y_FORCE_INLINE -void EnsureInputSize(TStringBuf& input, size_t size) { - MKQL_ENSURE(input.size() >= size, "premature end of input"); -} - -template <bool Desc> -Y_FORCE_INLINE -void EncodeBool(TVector<ui8>& output, bool value) { - output.push_back(Desc ? 0xFF ^ ui8(value) : ui8(value)); -} - -template <bool Desc> -Y_FORCE_INLINE -bool DecodeBool(TStringBuf& input) { - EnsureInputSize(input, 1); - auto result = Desc ? bool(0xFF ^ ui8(input[0])) : bool(input[0]); - input.Skip(1); - return result; -} - -template <typename TUnsigned, bool Desc> -Y_FORCE_INLINE -void EncodeUnsigned(TVector<ui8>& output, TUnsigned value) { - constexpr size_t size = sizeof(TUnsigned); - - if (Desc) { - value = ~value; - } - - output.resize(output.size() + size); - WriteUnaligned<TUnsigned>(output.end() - size, SwapBytes(value)); -} - -template <typename TUnsigned, bool Desc> -Y_FORCE_INLINE -TUnsigned DecodeUnsigned(TStringBuf& input) { - constexpr size_t size = sizeof(TUnsigned); - - EnsureInputSize(input, size); +Y_FORCE_INLINE +void EnsureInputSize(TStringBuf& input, size_t size) { + MKQL_ENSURE(input.size() >= size, "premature end of input"); +} + +template <bool Desc> +Y_FORCE_INLINE +void EncodeBool(TVector<ui8>& output, bool value) { + output.push_back(Desc ? 0xFF ^ ui8(value) : ui8(value)); +} + +template <bool Desc> +Y_FORCE_INLINE +bool DecodeBool(TStringBuf& input) { + EnsureInputSize(input, 1); + auto result = Desc ? bool(0xFF ^ ui8(input[0])) : bool(input[0]); + input.Skip(1); + return result; +} + +template <typename TUnsigned, bool Desc> +Y_FORCE_INLINE +void EncodeUnsigned(TVector<ui8>& output, TUnsigned value) { + constexpr size_t size = sizeof(TUnsigned); + + if (Desc) { + value = ~value; + } + + output.resize(output.size() + size); + WriteUnaligned<TUnsigned>(output.end() - size, SwapBytes(value)); +} + +template <typename TUnsigned, bool Desc> +Y_FORCE_INLINE +TUnsigned DecodeUnsigned(TStringBuf& input) { + constexpr size_t size = sizeof(TUnsigned); + + EnsureInputSize(input, size); auto value = ReadUnaligned<TUnsigned>(input.data()); - input.Skip(size); - - value = SwapBytes(value); - if (Desc) { - value = ~value; - } - return value; -} - -template <typename TSigned, bool Desc> -Y_FORCE_INLINE -void EncodeSigned(TVector<ui8>& output, TSigned value) { + input.Skip(size); + + value = SwapBytes(value); + if (Desc) { + value = ~value; + } + return value; +} + +template <typename TSigned, bool Desc> +Y_FORCE_INLINE +void EncodeSigned(TVector<ui8>& output, TSigned value) { using TUnsigned = std::make_unsigned_t<TSigned>; - constexpr size_t size = sizeof(TUnsigned); - constexpr TUnsigned shift = TUnsigned(1) << (size * 8 - 1); - - EncodeUnsigned<TUnsigned, Desc>(output, TUnsigned(value) + shift); -} - -template <typename TSigned, bool Desc> -Y_FORCE_INLINE -TSigned DecodeSigned(TStringBuf& input) { + constexpr size_t size = sizeof(TUnsigned); + constexpr TUnsigned shift = TUnsigned(1) << (size * 8 - 1); + + EncodeUnsigned<TUnsigned, Desc>(output, TUnsigned(value) + shift); +} + +template <typename TSigned, bool Desc> +Y_FORCE_INLINE +TSigned DecodeSigned(TStringBuf& input) { using TUnsigned = std::make_unsigned_t<TSigned>; - constexpr size_t size = sizeof(TUnsigned); - constexpr TUnsigned shift = TUnsigned(1) << (size * 8 - 1); - - return TSigned(DecodeUnsigned<TUnsigned, Desc>(input) - shift); -} - -enum class EFPCode : ui8 { - NegInf = 0, - Neg = 1, - Zero = 2, - Pos = 3, - PosInf = 4, - Nan = 5 -}; - -template <typename TFloat> -struct TFloatToInteger {}; - -template <> -struct TFloatToInteger<float> { - using TType = ui32; -}; - -template <> -struct TFloatToInteger<double> { - using TType = ui64; -}; - -static_assert(std::numeric_limits<float>::is_iec559, "float type is not iec559(ieee754)"); -static_assert(std::numeric_limits<double>::is_iec559, "double type is not iec559(ieee754)"); - -template <typename TFloat, bool Desc> -Y_FORCE_INLINE -void EncodeFloating(TVector<ui8>& output, TFloat value) { - using TInteger = typename TFloatToInteger<TFloat>::TType; - EFPCode code; - - switch (std::fpclassify(value)) { - case FP_NORMAL: - case FP_SUBNORMAL: { - auto integer = ReadUnaligned<TInteger>(&value); - if (value < 0) { - integer = ~integer; - code = EFPCode::Neg; - } else { - code = EFPCode::Pos; - } - output.push_back(Desc ? 0xFF ^ ui8(code) : ui8(code)); - EncodeUnsigned<TInteger, Desc>(output, integer); - return; - } - case FP_ZERO: - code = EFPCode::Zero; - break; - case FP_INFINITE: - code = value < 0 ? EFPCode::NegInf : EFPCode::PosInf; - break; - default: - code = EFPCode::Nan; - break; - } - output.push_back(Desc ? 0xFF ^ ui8(code) : ui8(code)); -} - -template <typename TFloat, bool Desc> -Y_FORCE_INLINE -TFloat DecodeFloating(TStringBuf& input) { - using TInteger = typename TFloatToInteger<TFloat>::TType; - - EnsureInputSize(input, 1); - auto code = EFPCode(Desc ? 0xFF ^ input[0] : input[0]); - input.Skip(1); - - bool negative; - switch (code) { - case EFPCode::Zero: - return 0; - case EFPCode::NegInf: - return -std::numeric_limits<TFloat>::infinity(); - case EFPCode::PosInf: - return std::numeric_limits<TFloat>::infinity(); - case EFPCode::Nan: - return std::numeric_limits<TFloat>::quiet_NaN(); - case EFPCode::Neg: - negative = true; - break; - case EFPCode::Pos: - negative = false; - break; - default: - MKQL_ENSURE(false, "floating point data is corrupted"); - } - - auto integer = DecodeUnsigned<TInteger, Desc>(input); - if (negative) { - integer = ~integer; - } - - return ReadUnaligned<TFloat>(&integer); -} - -constexpr ui8 BlockCode = 0x1F; -constexpr size_t BlockSize = 15; -constexpr size_t BlockSizeUi64 = BlockSize / 8 + 1; - -template <bool Desc> -Y_FORCE_INLINE -void EncodeString(TVector<ui8>& output, TStringBuf value) { - size_t part = 0; - - while (!value.empty()) { - union { - ui8 buffer[BlockSize + 1]; - ui64 buffer64[BlockSizeUi64]; - }; - - part = std::min(value.size(), BlockSize); - if (part == BlockSize) { - std::memcpy(buffer + 1, value.data(), BlockSize); - } else { - for (size_t i = 0; i < BlockSizeUi64; ++i) { - buffer64[i] = 0; - } - std::memcpy(buffer + 1, value.data(), part); - } - value.Skip(part); - - buffer[0] = BlockCode; - - if (Desc) { - for (size_t i = 0; i < BlockSizeUi64; ++i) { - buffer64[i] ^= std::numeric_limits<ui64>::max(); - } - } - - output.insert(output.end(), buffer, buffer + BlockSize + 1); - } - - auto lastLength = ui8(part); - output.push_back(Desc ? 0xFF ^ lastLength : lastLength); -} - -template <bool Desc> -Y_FORCE_INLINE -TStringBuf DecodeString(TStringBuf& input, TVector<ui8>& value) { - EnsureInputSize(input, 1); - ui8 code = Desc ? 0xFF ^ input[0] : input[0]; - input.Skip(1); - - if (code != BlockCode) { - MKQL_ENSURE(code == 0, TStringBuilder() << "unknown string block code: " << code); - return TStringBuf(); - } - - while (code == BlockCode) { - union { - ui8 buffer[BlockSize + 1]; - ui64 buffer64[BlockSizeUi64]; - }; - - EnsureInputSize(input, BlockSize + 1); - std::memcpy(buffer, input.data(), BlockSize + 1); - input.Skip(BlockSize + 1); - - if (Desc) { - for (size_t i = 0; i < BlockSizeUi64; ++i) { - buffer64[i] ^= std::numeric_limits<ui64>::max(); - } - } - - value.insert(value.end(), buffer, buffer + BlockSize); - code = buffer[BlockSize]; - } - - auto begin = (const char*)value.begin(); - auto end = (const char*)value.end() - BlockSize + code; - return TStringBuf(begin, end - begin); -} - -constexpr size_t UuidSize = 16; - -template <bool Desc> -Y_FORCE_INLINE -void EncodeUuid(TVector<ui8>& output, const char* data) { - output.resize(output.size() + UuidSize); - auto ptr = output.end() - UuidSize; - - if (Desc) { - for (size_t i = 0; i < UuidSize; ++i) { - *ptr++ = ui8(*data++) ^ 0xFF; - } - } else { - std::memcpy(ptr, data, UuidSize); - } -} - -template <bool Desc> -Y_FORCE_INLINE -TStringBuf DecodeUuid(TStringBuf& input, TVector<ui8>& value) { - EnsureInputSize(input, UuidSize); + constexpr size_t size = sizeof(TUnsigned); + constexpr TUnsigned shift = TUnsigned(1) << (size * 8 - 1); + + return TSigned(DecodeUnsigned<TUnsigned, Desc>(input) - shift); +} + +enum class EFPCode : ui8 { + NegInf = 0, + Neg = 1, + Zero = 2, + Pos = 3, + PosInf = 4, + Nan = 5 +}; + +template <typename TFloat> +struct TFloatToInteger {}; + +template <> +struct TFloatToInteger<float> { + using TType = ui32; +}; + +template <> +struct TFloatToInteger<double> { + using TType = ui64; +}; + +static_assert(std::numeric_limits<float>::is_iec559, "float type is not iec559(ieee754)"); +static_assert(std::numeric_limits<double>::is_iec559, "double type is not iec559(ieee754)"); + +template <typename TFloat, bool Desc> +Y_FORCE_INLINE +void EncodeFloating(TVector<ui8>& output, TFloat value) { + using TInteger = typename TFloatToInteger<TFloat>::TType; + EFPCode code; + + switch (std::fpclassify(value)) { + case FP_NORMAL: + case FP_SUBNORMAL: { + auto integer = ReadUnaligned<TInteger>(&value); + if (value < 0) { + integer = ~integer; + code = EFPCode::Neg; + } else { + code = EFPCode::Pos; + } + output.push_back(Desc ? 0xFF ^ ui8(code) : ui8(code)); + EncodeUnsigned<TInteger, Desc>(output, integer); + return; + } + case FP_ZERO: + code = EFPCode::Zero; + break; + case FP_INFINITE: + code = value < 0 ? EFPCode::NegInf : EFPCode::PosInf; + break; + default: + code = EFPCode::Nan; + break; + } + output.push_back(Desc ? 0xFF ^ ui8(code) : ui8(code)); +} + +template <typename TFloat, bool Desc> +Y_FORCE_INLINE +TFloat DecodeFloating(TStringBuf& input) { + using TInteger = typename TFloatToInteger<TFloat>::TType; + + EnsureInputSize(input, 1); + auto code = EFPCode(Desc ? 0xFF ^ input[0] : input[0]); + input.Skip(1); + + bool negative; + switch (code) { + case EFPCode::Zero: + return 0; + case EFPCode::NegInf: + return -std::numeric_limits<TFloat>::infinity(); + case EFPCode::PosInf: + return std::numeric_limits<TFloat>::infinity(); + case EFPCode::Nan: + return std::numeric_limits<TFloat>::quiet_NaN(); + case EFPCode::Neg: + negative = true; + break; + case EFPCode::Pos: + negative = false; + break; + default: + MKQL_ENSURE(false, "floating point data is corrupted"); + } + + auto integer = DecodeUnsigned<TInteger, Desc>(input); + if (negative) { + integer = ~integer; + } + + return ReadUnaligned<TFloat>(&integer); +} + +constexpr ui8 BlockCode = 0x1F; +constexpr size_t BlockSize = 15; +constexpr size_t BlockSizeUi64 = BlockSize / 8 + 1; + +template <bool Desc> +Y_FORCE_INLINE +void EncodeString(TVector<ui8>& output, TStringBuf value) { + size_t part = 0; + + while (!value.empty()) { + union { + ui8 buffer[BlockSize + 1]; + ui64 buffer64[BlockSizeUi64]; + }; + + part = std::min(value.size(), BlockSize); + if (part == BlockSize) { + std::memcpy(buffer + 1, value.data(), BlockSize); + } else { + for (size_t i = 0; i < BlockSizeUi64; ++i) { + buffer64[i] = 0; + } + std::memcpy(buffer + 1, value.data(), part); + } + value.Skip(part); + + buffer[0] = BlockCode; + + if (Desc) { + for (size_t i = 0; i < BlockSizeUi64; ++i) { + buffer64[i] ^= std::numeric_limits<ui64>::max(); + } + } + + output.insert(output.end(), buffer, buffer + BlockSize + 1); + } + + auto lastLength = ui8(part); + output.push_back(Desc ? 0xFF ^ lastLength : lastLength); +} + +template <bool Desc> +Y_FORCE_INLINE +TStringBuf DecodeString(TStringBuf& input, TVector<ui8>& value) { + EnsureInputSize(input, 1); + ui8 code = Desc ? 0xFF ^ input[0] : input[0]; + input.Skip(1); + + if (code != BlockCode) { + MKQL_ENSURE(code == 0, TStringBuilder() << "unknown string block code: " << code); + return TStringBuf(); + } + + while (code == BlockCode) { + union { + ui8 buffer[BlockSize + 1]; + ui64 buffer64[BlockSizeUi64]; + }; + + EnsureInputSize(input, BlockSize + 1); + std::memcpy(buffer, input.data(), BlockSize + 1); + input.Skip(BlockSize + 1); + + if (Desc) { + for (size_t i = 0; i < BlockSizeUi64; ++i) { + buffer64[i] ^= std::numeric_limits<ui64>::max(); + } + } + + value.insert(value.end(), buffer, buffer + BlockSize); + code = buffer[BlockSize]; + } + + auto begin = (const char*)value.begin(); + auto end = (const char*)value.end() - BlockSize + code; + return TStringBuf(begin, end - begin); +} + +constexpr size_t UuidSize = 16; + +template <bool Desc> +Y_FORCE_INLINE +void EncodeUuid(TVector<ui8>& output, const char* data) { + output.resize(output.size() + UuidSize); + auto ptr = output.end() - UuidSize; + + if (Desc) { + for (size_t i = 0; i < UuidSize; ++i) { + *ptr++ = ui8(*data++) ^ 0xFF; + } + } else { + std::memcpy(ptr, data, UuidSize); + } +} + +template <bool Desc> +Y_FORCE_INLINE +TStringBuf DecodeUuid(TStringBuf& input, TVector<ui8>& value) { + EnsureInputSize(input, UuidSize); auto data = input.data(); - input.Skip(UuidSize); - - value.resize(UuidSize); - auto ptr = value.begin(); - - if (Desc) { - for (size_t i = 0; i < UuidSize; ++i) { - *ptr++ = ui8(*data++) ^ 0xFF; - } - } else { - std::memcpy(ptr, data, UuidSize); - } - - return TStringBuf((const char*)value.begin(), (const char*)value.end()); -} - -template <typename TUnsigned, bool Desc> -Y_FORCE_INLINE -void EncodeTzUnsigned(TVector<ui8>& output, TUnsigned value, ui16 tzId) { - constexpr size_t size = sizeof(TUnsigned); - - if (Desc) { - value = ~value; - tzId = ~tzId; - } - - output.resize(output.size() + size + sizeof(ui16)); - WriteUnaligned<TUnsigned>(output.end() - size - sizeof(ui16), SwapBytes(value)); - WriteUnaligned<ui16>(output.end() - sizeof(ui16), SwapBytes(tzId)); -} - -template <typename TUnsigned, bool Desc> -Y_FORCE_INLINE -void DecodeTzUnsigned(TStringBuf& input, TUnsigned& value, ui16& tzId) { - constexpr size_t size = sizeof(TUnsigned); - - EnsureInputSize(input, size + sizeof(ui16)); + input.Skip(UuidSize); + + value.resize(UuidSize); + auto ptr = value.begin(); + + if (Desc) { + for (size_t i = 0; i < UuidSize; ++i) { + *ptr++ = ui8(*data++) ^ 0xFF; + } + } else { + std::memcpy(ptr, data, UuidSize); + } + + return TStringBuf((const char*)value.begin(), (const char*)value.end()); +} + +template <typename TUnsigned, bool Desc> +Y_FORCE_INLINE +void EncodeTzUnsigned(TVector<ui8>& output, TUnsigned value, ui16 tzId) { + constexpr size_t size = sizeof(TUnsigned); + + if (Desc) { + value = ~value; + tzId = ~tzId; + } + + output.resize(output.size() + size + sizeof(ui16)); + WriteUnaligned<TUnsigned>(output.end() - size - sizeof(ui16), SwapBytes(value)); + WriteUnaligned<ui16>(output.end() - sizeof(ui16), SwapBytes(tzId)); +} + +template <typename TUnsigned, bool Desc> +Y_FORCE_INLINE +void DecodeTzUnsigned(TStringBuf& input, TUnsigned& value, ui16& tzId) { + constexpr size_t size = sizeof(TUnsigned); + + EnsureInputSize(input, size + sizeof(ui16)); auto v = ReadUnaligned<TUnsigned>(input.data()); auto t = ReadUnaligned<ui16>(input.data() + size); - input.Skip(size + sizeof(ui16)); - - if (Desc) { - value = ~SwapBytes(v); - tzId = ~SwapBytes(t); - } else { - value = SwapBytes(v); - tzId = SwapBytes(t); - } -} - + input.Skip(size + sizeof(ui16)); + + if (Desc) { + value = ~SwapBytes(v); + tzId = ~SwapBytes(t); + } else { + value = SwapBytes(v); + tzId = SwapBytes(t); + } +} + constexpr size_t DecimalSize = sizeof(NYql::NDecimal::TInt128); - -template <bool Desc> -Y_FORCE_INLINE + +template <bool Desc> +Y_FORCE_INLINE void EncodeDecimal(TVector<ui8>& output, NYql::NDecimal::TInt128 value) { - output.resize(output.size() + DecimalSize); - auto ptr = reinterpret_cast<char*>(output.end() - DecimalSize); + output.resize(output.size() + DecimalSize); + auto ptr = reinterpret_cast<char*>(output.end() - DecimalSize); output.resize(output.size() + NYql::NDecimal::Serialize(Desc ? -value : value, ptr) - DecimalSize); -} - -template <bool Desc> -Y_FORCE_INLINE +} + +template <bool Desc> +Y_FORCE_INLINE NYql::NDecimal::TInt128 DecodeDecimal(TStringBuf& input) { MKQL_ENSURE(input.size() > 0U && input.size() <= DecimalSize, "premature end of input"); const auto des = NYql::NDecimal::Deserialize(input.data()); input.Skip(des.second); return Desc ? -des.first : des.first; -} - - -template <bool Desc> -Y_FORCE_INLINE -void Encode(TVector<ui8>& output, NUdf::EDataSlot slot, const NUdf::TUnboxedValuePod& value) { - switch (slot) { - - case NUdf::EDataSlot::Bool: - EncodeBool<Desc>(output, value.Get<bool>()); - break; - case NUdf::EDataSlot::Int8: - EncodeSigned<i8, Desc>(output, value.Get<i8>()); - break; - case NUdf::EDataSlot::Uint8: - EncodeUnsigned<ui8, Desc>(output, value.Get<ui8>()); - break; - case NUdf::EDataSlot::Int16: - EncodeSigned<i16, Desc>(output, value.Get<i16>()); - break; - case NUdf::EDataSlot::Uint16: - case NUdf::EDataSlot::Date: - EncodeUnsigned<ui16, Desc>(output, value.Get<ui16>()); - break; - case NUdf::EDataSlot::Int32: - EncodeSigned<i32, Desc>(output, value.Get<i32>()); - break; - case NUdf::EDataSlot::Uint32: - case NUdf::EDataSlot::Datetime: - EncodeUnsigned<ui32, Desc>(output, value.Get<ui32>()); - break; - case NUdf::EDataSlot::Int64: - case NUdf::EDataSlot::Interval: - EncodeSigned<i64, Desc>(output, value.Get<i64>()); - break; - case NUdf::EDataSlot::Uint64: - case NUdf::EDataSlot::Timestamp: - EncodeUnsigned<ui64, Desc>(output, value.Get<ui64>()); - break; - case NUdf::EDataSlot::Double: - EncodeFloating<double, Desc>(output, value.Get<double>()); - break; - case NUdf::EDataSlot::Float: - EncodeFloating<float, Desc>(output, value.Get<float>()); - break; +} + + +template <bool Desc> +Y_FORCE_INLINE +void Encode(TVector<ui8>& output, NUdf::EDataSlot slot, const NUdf::TUnboxedValuePod& value) { + switch (slot) { + + case NUdf::EDataSlot::Bool: + EncodeBool<Desc>(output, value.Get<bool>()); + break; + case NUdf::EDataSlot::Int8: + EncodeSigned<i8, Desc>(output, value.Get<i8>()); + break; + case NUdf::EDataSlot::Uint8: + EncodeUnsigned<ui8, Desc>(output, value.Get<ui8>()); + break; + case NUdf::EDataSlot::Int16: + EncodeSigned<i16, Desc>(output, value.Get<i16>()); + break; + case NUdf::EDataSlot::Uint16: + case NUdf::EDataSlot::Date: + EncodeUnsigned<ui16, Desc>(output, value.Get<ui16>()); + break; + case NUdf::EDataSlot::Int32: + EncodeSigned<i32, Desc>(output, value.Get<i32>()); + break; + case NUdf::EDataSlot::Uint32: + case NUdf::EDataSlot::Datetime: + EncodeUnsigned<ui32, Desc>(output, value.Get<ui32>()); + break; + case NUdf::EDataSlot::Int64: + case NUdf::EDataSlot::Interval: + EncodeSigned<i64, Desc>(output, value.Get<i64>()); + break; + case NUdf::EDataSlot::Uint64: + case NUdf::EDataSlot::Timestamp: + EncodeUnsigned<ui64, Desc>(output, value.Get<ui64>()); + break; + case NUdf::EDataSlot::Double: + EncodeFloating<double, Desc>(output, value.Get<double>()); + break; + case NUdf::EDataSlot::Float: + EncodeFloating<float, Desc>(output, value.Get<float>()); + break; case NUdf::EDataSlot::DyNumber: - case NUdf::EDataSlot::String: - case NUdf::EDataSlot::Utf8: { - auto stringRef = value.AsStringRef(); - EncodeString<Desc>(output, TStringBuf(stringRef.Data(), stringRef.Size())); - break; - } - case NUdf::EDataSlot::Uuid: - EncodeUuid<Desc>(output, value.AsStringRef().Data()); - break; - case NUdf::EDataSlot::TzDate: - EncodeTzUnsigned<ui16, Desc>(output, value.Get<ui16>(), value.GetTimezoneId()); - break; - case NUdf::EDataSlot::TzDatetime: - EncodeTzUnsigned<ui32, Desc>(output, value.Get<ui32>(), value.GetTimezoneId()); - break; - case NUdf::EDataSlot::TzTimestamp: - EncodeTzUnsigned<ui64, Desc>(output, value.Get<ui64>(), value.GetTimezoneId()); - break; - case NUdf::EDataSlot::Decimal: - EncodeDecimal<Desc>(output, value.GetInt128()); - break; - - default: - MKQL_ENSURE(false, TStringBuilder() << "unknown data slot for presort encoding: " << slot); - } -} - -template <bool Desc> -Y_FORCE_INLINE + case NUdf::EDataSlot::String: + case NUdf::EDataSlot::Utf8: { + auto stringRef = value.AsStringRef(); + EncodeString<Desc>(output, TStringBuf(stringRef.Data(), stringRef.Size())); + break; + } + case NUdf::EDataSlot::Uuid: + EncodeUuid<Desc>(output, value.AsStringRef().Data()); + break; + case NUdf::EDataSlot::TzDate: + EncodeTzUnsigned<ui16, Desc>(output, value.Get<ui16>(), value.GetTimezoneId()); + break; + case NUdf::EDataSlot::TzDatetime: + EncodeTzUnsigned<ui32, Desc>(output, value.Get<ui32>(), value.GetTimezoneId()); + break; + case NUdf::EDataSlot::TzTimestamp: + EncodeTzUnsigned<ui64, Desc>(output, value.Get<ui64>(), value.GetTimezoneId()); + break; + case NUdf::EDataSlot::Decimal: + EncodeDecimal<Desc>(output, value.GetInt128()); + break; + + default: + MKQL_ENSURE(false, TStringBuilder() << "unknown data slot for presort encoding: " << slot); + } +} + +template <bool Desc> +Y_FORCE_INLINE NUdf::TUnboxedValue Decode(TStringBuf& input, NUdf::EDataSlot slot, TVector<ui8>& buffer) -{ - switch (slot) { - - case NUdf::EDataSlot::Bool: - return NUdf::TUnboxedValuePod(DecodeBool<Desc>(input)); - - case NUdf::EDataSlot::Int8: - return NUdf::TUnboxedValuePod(DecodeSigned<i8, Desc>(input)); - - case NUdf::EDataSlot::Uint8: - return NUdf::TUnboxedValuePod(DecodeUnsigned<ui8, Desc>(input)); - - case NUdf::EDataSlot::Int16: - return NUdf::TUnboxedValuePod(DecodeSigned<i16, Desc>(input)); - - case NUdf::EDataSlot::Uint16: - case NUdf::EDataSlot::Date: - return NUdf::TUnboxedValuePod(DecodeUnsigned<ui16, Desc>(input)); - - case NUdf::EDataSlot::Int32: - return NUdf::TUnboxedValuePod(DecodeSigned<i32, Desc>(input)); - - case NUdf::EDataSlot::Uint32: - case NUdf::EDataSlot::Datetime: - return NUdf::TUnboxedValuePod(DecodeUnsigned<ui32, Desc>(input)); - - case NUdf::EDataSlot::Int64: - case NUdf::EDataSlot::Interval: - return NUdf::TUnboxedValuePod(DecodeSigned<i64, Desc>(input)); - - case NUdf::EDataSlot::Uint64: - case NUdf::EDataSlot::Timestamp: - return NUdf::TUnboxedValuePod(DecodeUnsigned<ui64, Desc>(input)); - - case NUdf::EDataSlot::Double: - return NUdf::TUnboxedValuePod(DecodeFloating<double, Desc>(input)); - - case NUdf::EDataSlot::Float: - return NUdf::TUnboxedValuePod(DecodeFloating<float, Desc>(input)); - +{ + switch (slot) { + + case NUdf::EDataSlot::Bool: + return NUdf::TUnboxedValuePod(DecodeBool<Desc>(input)); + + case NUdf::EDataSlot::Int8: + return NUdf::TUnboxedValuePod(DecodeSigned<i8, Desc>(input)); + + case NUdf::EDataSlot::Uint8: + return NUdf::TUnboxedValuePod(DecodeUnsigned<ui8, Desc>(input)); + + case NUdf::EDataSlot::Int16: + return NUdf::TUnboxedValuePod(DecodeSigned<i16, Desc>(input)); + + case NUdf::EDataSlot::Uint16: + case NUdf::EDataSlot::Date: + return NUdf::TUnboxedValuePod(DecodeUnsigned<ui16, Desc>(input)); + + case NUdf::EDataSlot::Int32: + return NUdf::TUnboxedValuePod(DecodeSigned<i32, Desc>(input)); + + case NUdf::EDataSlot::Uint32: + case NUdf::EDataSlot::Datetime: + return NUdf::TUnboxedValuePod(DecodeUnsigned<ui32, Desc>(input)); + + case NUdf::EDataSlot::Int64: + case NUdf::EDataSlot::Interval: + return NUdf::TUnboxedValuePod(DecodeSigned<i64, Desc>(input)); + + case NUdf::EDataSlot::Uint64: + case NUdf::EDataSlot::Timestamp: + return NUdf::TUnboxedValuePod(DecodeUnsigned<ui64, Desc>(input)); + + case NUdf::EDataSlot::Double: + return NUdf::TUnboxedValuePod(DecodeFloating<double, Desc>(input)); + + case NUdf::EDataSlot::Float: + return NUdf::TUnboxedValuePod(DecodeFloating<float, Desc>(input)); + case NUdf::EDataSlot::DyNumber: - case NUdf::EDataSlot::String: - case NUdf::EDataSlot::Utf8: - buffer.clear(); + case NUdf::EDataSlot::String: + case NUdf::EDataSlot::Utf8: + buffer.clear(); return MakeString(NUdf::TStringRef(DecodeString<Desc>(input, buffer))); - - case NUdf::EDataSlot::Uuid: - buffer.clear(); - return MakeString(NUdf::TStringRef(DecodeUuid<Desc>(input, buffer))); - - case NUdf::EDataSlot::TzDate: { - ui16 date; - ui16 tzId; - DecodeTzUnsigned<ui16, Desc>(input, date, tzId); - NUdf::TUnboxedValuePod value(date); - value.SetTimezoneId(tzId); - return value; - } - case NUdf::EDataSlot::TzDatetime: { - ui32 datetime; - ui16 tzId; - DecodeTzUnsigned<ui32, Desc>(input, datetime, tzId); - NUdf::TUnboxedValuePod value(datetime); - value.SetTimezoneId(tzId); - return value; - } - case NUdf::EDataSlot::TzTimestamp: { - ui64 timestamp; - ui16 tzId; - DecodeTzUnsigned<ui64, Desc>(input, timestamp, tzId); - NUdf::TUnboxedValuePod value(timestamp); - value.SetTimezoneId(tzId); - return value; - } - case NUdf::EDataSlot::Decimal: - return NUdf::TUnboxedValuePod(DecodeDecimal<Desc>(input)); - - default: - MKQL_ENSURE(false, TStringBuilder() << "unknown data slot for presort decoding: " << slot); - } -} - + + case NUdf::EDataSlot::Uuid: + buffer.clear(); + return MakeString(NUdf::TStringRef(DecodeUuid<Desc>(input, buffer))); + + case NUdf::EDataSlot::TzDate: { + ui16 date; + ui16 tzId; + DecodeTzUnsigned<ui16, Desc>(input, date, tzId); + NUdf::TUnboxedValuePod value(date); + value.SetTimezoneId(tzId); + return value; + } + case NUdf::EDataSlot::TzDatetime: { + ui32 datetime; + ui16 tzId; + DecodeTzUnsigned<ui32, Desc>(input, datetime, tzId); + NUdf::TUnboxedValuePod value(datetime); + value.SetTimezoneId(tzId); + return value; + } + case NUdf::EDataSlot::TzTimestamp: { + ui64 timestamp; + ui16 tzId; + DecodeTzUnsigned<ui64, Desc>(input, timestamp, tzId); + NUdf::TUnboxedValuePod value(timestamp); + value.SetTimezoneId(tzId); + return value; + } + case NUdf::EDataSlot::Decimal: + return NUdf::TUnboxedValuePod(DecodeDecimal<Desc>(input)); + + default: + MKQL_ENSURE(false, TStringBuilder() << "unknown data slot for presort decoding: " << slot); + } +} + struct TDictItem { TString KeyBuffer; NUdf::TUnboxedValue Payload; @@ -716,80 +716,80 @@ NUdf::TUnboxedValue DecodeImpl(TType* type, TStringBuf& input, const THolderFact } } -} // NDetail - -void TPresortCodec::AddType(NUdf::EDataSlot slot, bool isOptional, bool isDesc) { - Types.push_back({slot, isOptional, isDesc}); -} - -void TPresortEncoder::Start() { - Output.clear(); - Current = 0; -} - -void TPresortEncoder::Start(TStringBuf prefix) { - Output.clear(); +} // NDetail + +void TPresortCodec::AddType(NUdf::EDataSlot slot, bool isOptional, bool isDesc) { + Types.push_back({slot, isOptional, isDesc}); +} + +void TPresortEncoder::Start() { + Output.clear(); + Current = 0; +} + +void TPresortEncoder::Start(TStringBuf prefix) { + Output.clear(); auto data = reinterpret_cast<const ui8*>(prefix.data()); Output.insert(Output.begin(), data, data + prefix.size()); - Current = 0; -} - -void TPresortEncoder::Encode(const NUdf::TUnboxedValuePod& value) { - auto& type = Types[Current++]; - - if (type.IsDesc) { - if (type.IsOptional) { - auto hasValue = (bool)value; - NDetail::EncodeBool<true>(Output, hasValue); - if (!hasValue) { - return; - } - } - NDetail::Encode<true>(Output, type.Slot, value); - } else { - if (type.IsOptional) { - auto hasValue = (bool)value; - NDetail::EncodeBool<false>(Output, hasValue); - if (!hasValue) { - return; - } - } - NDetail::Encode<false>(Output, type.Slot, value); - } -} - -TStringBuf TPresortEncoder::Finish() { - MKQL_ENSURE(Current == Types.size(), "not all fields were encoded"); - return TStringBuf((const char*)Output.data(), Output.size()); -} - - -void TPresortDecoder::Start(TStringBuf input) { - Input = input; - Current = 0; -} - -NUdf::TUnboxedValue TPresortDecoder::Decode() { - auto& type = Types[Current++]; - - if (type.IsDesc) { - if (type.IsOptional && !NDetail::DecodeBool<true>(Input)) { - return NUdf::TUnboxedValuePod(); - } + Current = 0; +} + +void TPresortEncoder::Encode(const NUdf::TUnboxedValuePod& value) { + auto& type = Types[Current++]; + + if (type.IsDesc) { + if (type.IsOptional) { + auto hasValue = (bool)value; + NDetail::EncodeBool<true>(Output, hasValue); + if (!hasValue) { + return; + } + } + NDetail::Encode<true>(Output, type.Slot, value); + } else { + if (type.IsOptional) { + auto hasValue = (bool)value; + NDetail::EncodeBool<false>(Output, hasValue); + if (!hasValue) { + return; + } + } + NDetail::Encode<false>(Output, type.Slot, value); + } +} + +TStringBuf TPresortEncoder::Finish() { + MKQL_ENSURE(Current == Types.size(), "not all fields were encoded"); + return TStringBuf((const char*)Output.data(), Output.size()); +} + + +void TPresortDecoder::Start(TStringBuf input) { + Input = input; + Current = 0; +} + +NUdf::TUnboxedValue TPresortDecoder::Decode() { + auto& type = Types[Current++]; + + if (type.IsDesc) { + if (type.IsOptional && !NDetail::DecodeBool<true>(Input)) { + return NUdf::TUnboxedValuePod(); + } return NDetail::Decode<true>(Input, type.Slot, Buffer); - } else { - if (type.IsOptional && !NDetail::DecodeBool<false>(Input)) { - return NUdf::TUnboxedValuePod(); - } + } else { + if (type.IsOptional && !NDetail::DecodeBool<false>(Input)) { + return NUdf::TUnboxedValuePod(); + } return NDetail::Decode<false>(Input, type.Slot, Buffer); - } -} - -void TPresortDecoder::Finish() { - MKQL_ENSURE(Current == Types.size(), "not all fields were decoded"); - MKQL_ENSURE(Input.empty(), "buffer is not empty"); -} - + } +} + +void TPresortDecoder::Finish() { + MKQL_ENSURE(Current == Types.size(), "not all fields were decoded"); + MKQL_ENSURE(Input.empty(), "buffer is not empty"); +} + TGenericPresortEncoder::TGenericPresortEncoder(TType* type) : Type(type) {} @@ -825,5 +825,5 @@ NUdf::TUnboxedValue TGenericPresortEncoder::Decode(TStringBuf buf, bool desc, co } } -} // NMiniKQL -} // NKikimr +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/computation/presort.h b/ydb/library/yql/minikql/computation/presort.h index 8bce45e520..b90f54343b 100644 --- a/ydb/library/yql/minikql/computation/presort.h +++ b/ydb/library/yql/minikql/computation/presort.h @@ -1,57 +1,57 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/library/yql/public/udf/udf_value.h> - -#include <util/generic/vector.h> - -namespace NKikimr { -namespace NMiniKQL { - -class TPresortCodec { -public: - TPresortCodec() = default; - - struct TTypeInfo { - NUdf::EDataSlot Slot; - bool IsOptional; - bool IsDesc; - }; - - void AddType(NUdf::EDataSlot slot, bool isOptional = false, bool isDesc = false); - -protected: - size_t Current = 0; - TVector<TTypeInfo> Types; -}; - -class TPresortEncoder : public TPresortCodec { -public: - TPresortEncoder() = default; - - void Start(); - void Start(TStringBuf prefix); - void Encode(const NUdf::TUnboxedValuePod& value); - TStringBuf Finish(); // user must copy - -private: - TVector<ui8> Output; -}; - -class TPresortDecoder : public TPresortCodec { -public: + +#include <util/generic/vector.h> + +namespace NKikimr { +namespace NMiniKQL { + +class TPresortCodec { +public: + TPresortCodec() = default; + + struct TTypeInfo { + NUdf::EDataSlot Slot; + bool IsOptional; + bool IsDesc; + }; + + void AddType(NUdf::EDataSlot slot, bool isOptional = false, bool isDesc = false); + +protected: + size_t Current = 0; + TVector<TTypeInfo> Types; +}; + +class TPresortEncoder : public TPresortCodec { +public: + TPresortEncoder() = default; + + void Start(); + void Start(TStringBuf prefix); + void Encode(const NUdf::TUnboxedValuePod& value); + TStringBuf Finish(); // user must copy + +private: + TVector<ui8> Output; +}; + +class TPresortDecoder : public TPresortCodec { +public: TPresortDecoder() = default; - - void Start(TStringBuf input); - NUdf::TUnboxedValue Decode(); - void Finish(); - -private: - TVector<ui8> Buffer; - TStringBuf Input; -}; - + + void Start(TStringBuf input); + NUdf::TUnboxedValue Decode(); + void Finish(); + +private: + TVector<ui8> Buffer; + TStringBuf Input; +}; + class THolderFactory; class TGenericPresortEncoder { @@ -65,5 +65,5 @@ private: TVector<ui8> Buffer; }; -} // NMiniKQL -} // NKikimr +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/computation/presort_ut.cpp b/ydb/library/yql/minikql/computation/presort_ut.cpp index b95af1d50e..4adf957ca1 100644 --- a/ydb/library/yql/minikql/computation/presort_ut.cpp +++ b/ydb/library/yql/minikql/computation/presort_ut.cpp @@ -1,411 +1,411 @@ -#include "presort.h" - +#include "presort.h" + #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> - + #include <library/cpp/testing/unittest/registar.h> - -#include <util/string/hex.h> - + +#include <util/string/hex.h> + using namespace std::literals::string_view_literals; -namespace NKikimr { -namespace NMiniKQL { - -namespace { - -#define TYPE_MAP(XX) \ - XX(bool, Bool) \ - XX(ui8, Uint8) \ - XX(ui16, Uint16) \ - XX(ui32, Uint32) \ - XX(ui64, Uint64) \ - XX(i8, Int8) \ - XX(i16, Int16) \ - XX(i32, Int32) \ - XX(i64, Int64) \ - XX(float, Float) \ - XX(double, Double) - -#define ALL_VALUES(xType, xName) \ - xType xName; - -struct TSimpleTypes { - TYPE_MAP(ALL_VALUES) -}; -#undef ALL_VALUES - -#define ADD_TYPE(xType, xName) \ - codec.AddType(NUdf::EDataSlot::xName, isOptional, isDesc); - -void AddTypes(TPresortCodec& codec, bool isOptional, bool isDesc) { - TYPE_MAP(ADD_TYPE) -} -#undef ADD_TYPE - -#define ENCODE(xType, xName) \ - encoder.Encode(NUdf::TUnboxedValuePod(values.xName)); - -TStringBuf Encode(NKikimr::NMiniKQL::TPresortEncoder& encoder, const TSimpleTypes& values) { - encoder.Start(); - TYPE_MAP(ENCODE) - return encoder.Finish(); -} -#undef ENCODE - -#define DECODE(xType, xName) \ - UNIT_ASSERT_EQUAL(decoder.Decode().Get<xType>(), values.xName); - -void Decode(TPresortDecoder& decoder, TStringBuf input, const TSimpleTypes& values) { - decoder.Start(input); - TYPE_MAP(DECODE) - decoder.Finish(); -} -#undef DECODE - -#undef TYPE_MAP - -struct TPresortTest { - TScopedAlloc Alloc; - TMemoryUsageInfo MemInfo; - - TPresortTest() - : MemInfo("Memory") - {} - - template <typename T> - void ValidateEncoding(bool isDesc, T value, const TString& hex) { - TPresortEncoder encoder; - encoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc); - +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +#define TYPE_MAP(XX) \ + XX(bool, Bool) \ + XX(ui8, Uint8) \ + XX(ui16, Uint16) \ + XX(ui32, Uint32) \ + XX(ui64, Uint64) \ + XX(i8, Int8) \ + XX(i16, Int16) \ + XX(i32, Int32) \ + XX(i64, Int64) \ + XX(float, Float) \ + XX(double, Double) + +#define ALL_VALUES(xType, xName) \ + xType xName; + +struct TSimpleTypes { + TYPE_MAP(ALL_VALUES) +}; +#undef ALL_VALUES + +#define ADD_TYPE(xType, xName) \ + codec.AddType(NUdf::EDataSlot::xName, isOptional, isDesc); + +void AddTypes(TPresortCodec& codec, bool isOptional, bool isDesc) { + TYPE_MAP(ADD_TYPE) +} +#undef ADD_TYPE + +#define ENCODE(xType, xName) \ + encoder.Encode(NUdf::TUnboxedValuePod(values.xName)); + +TStringBuf Encode(NKikimr::NMiniKQL::TPresortEncoder& encoder, const TSimpleTypes& values) { + encoder.Start(); + TYPE_MAP(ENCODE) + return encoder.Finish(); +} +#undef ENCODE + +#define DECODE(xType, xName) \ + UNIT_ASSERT_EQUAL(decoder.Decode().Get<xType>(), values.xName); + +void Decode(TPresortDecoder& decoder, TStringBuf input, const TSimpleTypes& values) { + decoder.Start(input); + TYPE_MAP(DECODE) + decoder.Finish(); +} +#undef DECODE + +#undef TYPE_MAP + +struct TPresortTest { + TScopedAlloc Alloc; + TMemoryUsageInfo MemInfo; + + TPresortTest() + : MemInfo("Memory") + {} + + template <typename T> + void ValidateEncoding(bool isDesc, T value, const TString& hex) { + TPresortEncoder encoder; + encoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc); + TPresortDecoder decoder; - decoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc); - - encoder.Start(); - encoder.Encode(NUdf::TUnboxedValuePod(value)); - auto bytes = encoder.Finish(); - - UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); - - decoder.Start(bytes); - auto decoded = decoder.Decode().Get<T>(); - decoder.Finish(); - - UNIT_ASSERT_EQUAL(decoded, value); - }; - - template <NUdf::EDataSlot Slot> - void ValidateEncoding(bool isDesc, TStringBuf value, const TString& hex) { - TPresortEncoder encoder; - encoder.AddType(Slot, false, isDesc); - + decoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc); + + encoder.Start(); + encoder.Encode(NUdf::TUnboxedValuePod(value)); + auto bytes = encoder.Finish(); + + UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); + + decoder.Start(bytes); + auto decoded = decoder.Decode().Get<T>(); + decoder.Finish(); + + UNIT_ASSERT_EQUAL(decoded, value); + }; + + template <NUdf::EDataSlot Slot> + void ValidateEncoding(bool isDesc, TStringBuf value, const TString& hex) { + TPresortEncoder encoder; + encoder.AddType(Slot, false, isDesc); + TPresortDecoder decoder; - decoder.AddType(Slot, false, isDesc); - - encoder.Start(); + decoder.AddType(Slot, false, isDesc); + + encoder.Start(); encoder.Encode(NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(value)))); - auto bytes = encoder.Finish(); - - UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); - - decoder.Start(bytes); - auto uv = decoder.Decode(); - decoder.Finish(); - - auto stringRef = uv.AsStringRef(); - auto decoded = TStringBuf(stringRef.Data(), stringRef.Size()); - - UNIT_ASSERT_EQUAL(decoded, value); - } - - template <NUdf::EDataSlot Slot, typename T> - void ValidateEncoding(bool isDesc, const std::pair<T, ui16>& value, const TString& hex) { - TPresortEncoder encoder; - encoder.AddType(Slot, false, isDesc); - - TPresortDecoder decoder; - decoder.AddType(Slot, false, isDesc); - - NUdf::TUnboxedValuePod uv(value.first); - uv.SetTimezoneId(value.second); - - encoder.Start(); - encoder.Encode(uv); - auto bytes = encoder.Finish(); - - UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); - - decoder.Start(bytes); - auto decoded = decoder.Decode(); - decoder.Finish(); - - UNIT_ASSERT_EQUAL(decoded.Get<T>(), value.first); - UNIT_ASSERT_EQUAL(decoded.GetTimezoneId(), value.second); - }; - + auto bytes = encoder.Finish(); + + UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); + + decoder.Start(bytes); + auto uv = decoder.Decode(); + decoder.Finish(); + + auto stringRef = uv.AsStringRef(); + auto decoded = TStringBuf(stringRef.Data(), stringRef.Size()); + + UNIT_ASSERT_EQUAL(decoded, value); + } + + template <NUdf::EDataSlot Slot, typename T> + void ValidateEncoding(bool isDesc, const std::pair<T, ui16>& value, const TString& hex) { + TPresortEncoder encoder; + encoder.AddType(Slot, false, isDesc); + + TPresortDecoder decoder; + decoder.AddType(Slot, false, isDesc); + + NUdf::TUnboxedValuePod uv(value.first); + uv.SetTimezoneId(value.second); + + encoder.Start(); + encoder.Encode(uv); + auto bytes = encoder.Finish(); + + UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); + + decoder.Start(bytes); + auto decoded = decoder.Decode(); + decoder.Finish(); + + UNIT_ASSERT_EQUAL(decoded.Get<T>(), value.first); + UNIT_ASSERT_EQUAL(decoded.GetTimezoneId(), value.second); + }; + void ValidateEncoding(bool isDesc, NYql::NDecimal::TInt128 value, const TString& hex) { - TPresortEncoder encoder; - encoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc); - - TPresortDecoder decoder; - decoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc); - - encoder.Start(); - encoder.Encode(NUdf::TUnboxedValuePod(value)); - auto bytes = encoder.Finish(); - - UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); - - decoder.Start(bytes); - auto decoded = decoder.Decode().GetInt128(); - decoder.Finish(); - - UNIT_ASSERT_EQUAL(decoded, value); - }; - - template <typename T> - void ValidateEncoding(const TVector<T>& values) { - for (auto& value : values) { - ValidateEncoding(false, std::get<0>(value), std::get<1>(value)); - ValidateEncoding(true, std::get<0>(value), std::get<2>(value)); - } - } - - template <NUdf::EDataSlot Slot, typename T> - void ValidateEncoding(const TVector<T>& values) { - for (auto& value : values) { - ValidateEncoding<Slot>(false, std::get<0>(value), std::get<1>(value)); - ValidateEncoding<Slot>(true, std::get<0>(value), std::get<2>(value)); - } - } -}; - -} - + TPresortEncoder encoder; + encoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc); + + TPresortDecoder decoder; + decoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc); + + encoder.Start(); + encoder.Encode(NUdf::TUnboxedValuePod(value)); + auto bytes = encoder.Finish(); + + UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); + + decoder.Start(bytes); + auto decoded = decoder.Decode().GetInt128(); + decoder.Finish(); + + UNIT_ASSERT_EQUAL(decoded, value); + }; + + template <typename T> + void ValidateEncoding(const TVector<T>& values) { + for (auto& value : values) { + ValidateEncoding(false, std::get<0>(value), std::get<1>(value)); + ValidateEncoding(true, std::get<0>(value), std::get<2>(value)); + } + } + + template <NUdf::EDataSlot Slot, typename T> + void ValidateEncoding(const TVector<T>& values) { + for (auto& value : values) { + ValidateEncoding<Slot>(false, std::get<0>(value), std::get<1>(value)); + ValidateEncoding<Slot>(true, std::get<0>(value), std::get<2>(value)); + } + } +}; + +} + Y_UNIT_TEST_SUITE(TPresortCodecTest) { - + Y_UNIT_TEST(SimpleTypes) { - TPresortTest test; - - TSimpleTypes values = {false, 1u, 2u, 3u, 4u, 5, 6, 7, 8, 9.f, 10.0}; - - auto validateSimpleTypes = [&] (bool isOptional, bool isDesc) { - TPresortEncoder encoder; - AddTypes(encoder, isOptional, isDesc); - + TPresortTest test; + + TSimpleTypes values = {false, 1u, 2u, 3u, 4u, 5, 6, 7, 8, 9.f, 10.0}; + + auto validateSimpleTypes = [&] (bool isOptional, bool isDesc) { + TPresortEncoder encoder; + AddTypes(encoder, isOptional, isDesc); + TPresortDecoder decoder; - AddTypes(decoder, isOptional, isDesc); - - auto bytes = Encode(encoder, values); - Decode(decoder, bytes, values); - }; - - validateSimpleTypes(false, false); - validateSimpleTypes(false, true); - validateSimpleTypes(true, false); - validateSimpleTypes(true, true); -} - - + AddTypes(decoder, isOptional, isDesc); + + auto bytes = Encode(encoder, values); + Decode(decoder, bytes, values); + }; + + validateSimpleTypes(false, false); + validateSimpleTypes(false, true); + validateSimpleTypes(true, false); + validateSimpleTypes(true, true); +} + + Y_UNIT_TEST(Bool) { - const TVector<std::tuple<bool, TString, TString>> values = { - {false, "00", "FF"}, - {true, "01", "FE"} - }; - TPresortTest().ValidateEncoding(values); -} - + const TVector<std::tuple<bool, TString, TString>> values = { + {false, "00", "FF"}, + {true, "01", "FE"} + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Int8) { - const TVector<std::tuple<i8, TString, TString>> values = { - {-0x80, "00", "FF"}, - {-1, "7F", "80"}, - {0, "80", "7F"}, - {1, "81", "7E"}, - {0x7F, "FF", "00"} - }; - TPresortTest().ValidateEncoding(values); -} - + const TVector<std::tuple<i8, TString, TString>> values = { + {-0x80, "00", "FF"}, + {-1, "7F", "80"}, + {0, "80", "7F"}, + {1, "81", "7E"}, + {0x7F, "FF", "00"} + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Uint8) { - const TVector<std::tuple<ui8, TString, TString>> values = { - {0u, "00", "FF"}, - {0x80u, "80", "7F"}, - {0xFFu, "FF", "00"}, - }; - TPresortTest().ValidateEncoding(values); -} - + const TVector<std::tuple<ui8, TString, TString>> values = { + {0u, "00", "FF"}, + {0x80u, "80", "7F"}, + {0xFFu, "FF", "00"}, + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Int16) { - const TVector<std::tuple<i16, TString, TString>> values = { - {-0x8000, "0000", "FFFF"}, - {-1, "7FFF", "8000"}, - {0, "8000", "7FFF"}, - {1, "8001", "7FFE"}, - {0x7FFF, "FFFF", "0000"} - }; - TPresortTest().ValidateEncoding(values); -} - + const TVector<std::tuple<i16, TString, TString>> values = { + {-0x8000, "0000", "FFFF"}, + {-1, "7FFF", "8000"}, + {0, "8000", "7FFF"}, + {1, "8001", "7FFE"}, + {0x7FFF, "FFFF", "0000"} + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Uint16) { - const TVector<std::tuple<ui16, TString, TString>> values = { - {0, "0000", "FFFF"}, - {0x8000u, "8000", "7FFF"}, - {0xFFFFu, "FFFF", "0000"}, - }; - TPresortTest().ValidateEncoding(values); -} - + const TVector<std::tuple<ui16, TString, TString>> values = { + {0, "0000", "FFFF"}, + {0x8000u, "8000", "7FFF"}, + {0xFFFFu, "FFFF", "0000"}, + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Int32) { - const TVector<std::tuple<i32, TString, TString>> values = { - {-0x80000000, "00000000", "FFFFFFFF"}, - {-1, "7FFFFFFF", "80000000"}, - {0, "80000000", "7FFFFFFF"}, - {1, "80000001", "7FFFFFFE"}, - {0x7FFFFFFF, "FFFFFFFF", "00000000"} - }; - TPresortTest().ValidateEncoding(values); -} - + const TVector<std::tuple<i32, TString, TString>> values = { + {-0x80000000, "00000000", "FFFFFFFF"}, + {-1, "7FFFFFFF", "80000000"}, + {0, "80000000", "7FFFFFFF"}, + {1, "80000001", "7FFFFFFE"}, + {0x7FFFFFFF, "FFFFFFFF", "00000000"} + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Uint32) { - const TVector<std::tuple<ui32, TString, TString>> values = { - {0u, "00000000", "FFFFFFFF"}, - {0x80000000u, "80000000", "7FFFFFFF"}, - {0xFFFFFFFFu, "FFFFFFFF", "00000000"}, - }; - TPresortTest().ValidateEncoding(values); -} - + const TVector<std::tuple<ui32, TString, TString>> values = { + {0u, "00000000", "FFFFFFFF"}, + {0x80000000u, "80000000", "7FFFFFFF"}, + {0xFFFFFFFFu, "FFFFFFFF", "00000000"}, + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Int64) { - const TVector<std::tuple<i64, TString, TString>> values = { - {-0x8000000000000000, "0000000000000000", "FFFFFFFFFFFFFFFF"}, - {-1, "7FFFFFFFFFFFFFFF", "8000000000000000"}, - {0, "8000000000000000", "7FFFFFFFFFFFFFFF"}, - {1, "8000000000000001", "7FFFFFFFFFFFFFFE"}, - {0x7FFFFFFFFFFFFFFF, "FFFFFFFFFFFFFFFF", "0000000000000000"} - }; - TPresortTest().ValidateEncoding(values); -} - + const TVector<std::tuple<i64, TString, TString>> values = { + {-0x8000000000000000, "0000000000000000", "FFFFFFFFFFFFFFFF"}, + {-1, "7FFFFFFFFFFFFFFF", "8000000000000000"}, + {0, "8000000000000000", "7FFFFFFFFFFFFFFF"}, + {1, "8000000000000001", "7FFFFFFFFFFFFFFE"}, + {0x7FFFFFFFFFFFFFFF, "FFFFFFFFFFFFFFFF", "0000000000000000"} + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Uint64) { - const TVector<std::tuple<ui64, TString, TString>> values = { - {0u, "0000000000000000", "FFFFFFFFFFFFFFFF"}, - {0x8000000000000000u, "8000000000000000", "7FFFFFFFFFFFFFFF"}, - {0xFFFFFFFFFFFFFFFFu, "FFFFFFFFFFFFFFFF", "0000000000000000"} - }; - TPresortTest().ValidateEncoding(values); -} - + const TVector<std::tuple<ui64, TString, TString>> values = { + {0u, "0000000000000000", "FFFFFFFFFFFFFFFF"}, + {0x8000000000000000u, "8000000000000000", "7FFFFFFFFFFFFFFF"}, + {0xFFFFFFFFFFFFFFFFu, "FFFFFFFFFFFFFFFF", "0000000000000000"} + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Float) { - using TLimits = std::numeric_limits<float>; - - const TVector<std::tuple<float, TString, TString>> values = { - {-TLimits::infinity(), "00", "FF"}, - {-TLimits::max(), "0100800000", "FEFF7FFFFF"}, - {-1.f, "01407FFFFF", "FEBF800000"}, - {-TLimits::min(), "017F7FFFFF", "FE80800000"}, - {-TLimits::min()/8.f, "017FEFFFFF", "FE80100000"}, - {0.f, "02", "FD"}, - {TLimits::min()/8.f, "0300100000", "FCFFEFFFFF"}, - {TLimits::min(), "0300800000", "FCFF7FFFFF"}, - {1.f, "033F800000", "FCC07FFFFF"}, - {TLimits::max(), "037F7FFFFF", "FC80800000"}, - {TLimits::infinity(), "04", "FB"}, - }; - TPresortTest().ValidateEncoding(values); -} - + using TLimits = std::numeric_limits<float>; + + const TVector<std::tuple<float, TString, TString>> values = { + {-TLimits::infinity(), "00", "FF"}, + {-TLimits::max(), "0100800000", "FEFF7FFFFF"}, + {-1.f, "01407FFFFF", "FEBF800000"}, + {-TLimits::min(), "017F7FFFFF", "FE80800000"}, + {-TLimits::min()/8.f, "017FEFFFFF", "FE80100000"}, + {0.f, "02", "FD"}, + {TLimits::min()/8.f, "0300100000", "FCFFEFFFFF"}, + {TLimits::min(), "0300800000", "FCFF7FFFFF"}, + {1.f, "033F800000", "FCC07FFFFF"}, + {TLimits::max(), "037F7FFFFF", "FC80800000"}, + {TLimits::infinity(), "04", "FB"}, + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(Double) { - using TLimits = std::numeric_limits<double>; - - const TVector<std::tuple<double, TString, TString>> values = { - {-TLimits::infinity(), "00", "FF"}, - {-TLimits::max(), "010010000000000000", "FEFFEFFFFFFFFFFFFF"}, - {-1., "01400FFFFFFFFFFFFF", "FEBFF0000000000000"}, - {-TLimits::min(), "017FEFFFFFFFFFFFFF", "FE8010000000000000"}, - {-TLimits::min()/8., "017FFDFFFFFFFFFFFF", "FE8002000000000000"}, - {0., "02", "FD"}, - {TLimits::min()/8., "030002000000000000", "FCFFFDFFFFFFFFFFFF"}, - {TLimits::min(), "030010000000000000", "FCFFEFFFFFFFFFFFFF"}, - {1., "033FF0000000000000", "FCC00FFFFFFFFFFFFF"}, - {TLimits::max(), "037FEFFFFFFFFFFFFF", "FC8010000000000000"}, - {TLimits::infinity(), "04", "FB"}, - }; - TPresortTest().ValidateEncoding(values); -} - + using TLimits = std::numeric_limits<double>; + + const TVector<std::tuple<double, TString, TString>> values = { + {-TLimits::infinity(), "00", "FF"}, + {-TLimits::max(), "010010000000000000", "FEFFEFFFFFFFFFFFFF"}, + {-1., "01400FFFFFFFFFFFFF", "FEBFF0000000000000"}, + {-TLimits::min(), "017FEFFFFFFFFFFFFF", "FE8010000000000000"}, + {-TLimits::min()/8., "017FFDFFFFFFFFFFFF", "FE8002000000000000"}, + {0., "02", "FD"}, + {TLimits::min()/8., "030002000000000000", "FCFFFDFFFFFFFFFFFF"}, + {TLimits::min(), "030010000000000000", "FCFFEFFFFFFFFFFFFF"}, + {1., "033FF0000000000000", "FCC00FFFFFFFFFFFFF"}, + {TLimits::max(), "037FEFFFFFFFFFFFFF", "FC8010000000000000"}, + {TLimits::infinity(), "04", "FB"}, + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(String) { - const TVector<std::tuple<TStringBuf, TString, TString>> values = { + const TVector<std::tuple<TStringBuf, TString, TString>> values = { {TStringBuf(""), "00", "FF"}, {"\x00"sv, "1F00000000000000000000000000000001", - "E0FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + "E0FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, {"\x01", "1F01000000000000000000000000000001", - "E0FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + "E0FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, {"0", "1F30000000000000000000000000000001", - "E0CFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + "E0CFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, {"0123", "1F30313233000000000000000000000004", - "E0CFCECDCCFFFFFFFFFFFFFFFFFFFFFFFB"}, + "E0CFCECDCCFFFFFFFFFFFFFFFFFFFFFFFB"}, {"0123456789abcde", "1F3031323334353637383961626364650F", - "E0CFCECDCCCBCAC9C8C7C69E9D9C9B9AF0"}, + "E0CFCECDCCCBCAC9C8C7C69E9D9C9B9AF0"}, {"a", "1F61000000000000000000000000000001", - "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, {"a\x00"sv, "1F61000000000000000000000000000002", - "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD"}, + "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD"}, {"abc", "1F61626300000000000000000000000003", - "E09E9D9CFFFFFFFFFFFFFFFFFFFFFFFFFC"}, + "E09E9D9CFFFFFFFFFFFFFFFFFFFFFFFFFC"}, {"b", "1F62000000000000000000000000000001", - "E09DFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, - }; - TPresortTest().ValidateEncoding<NUdf::EDataSlot::String>(values); -} - -Y_UNIT_TEST(Uuid) { - const TVector<std::tuple<TStringBuf, TString, TString>> values = { + "E09DFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::String>(values); +} + +Y_UNIT_TEST(Uuid) { + const TVector<std::tuple<TStringBuf, TString, TString>> values = { {"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"sv, - "00000000000000000000000000000000", - "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"}, + "00000000000000000000000000000000", + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"}, {"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"sv, - "00000000000000000000000000000001", - "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + "00000000000000000000000000000001", + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", - "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", - "00000000000000000000000000000000"}, - }; - TPresortTest().ValidateEncoding<NUdf::EDataSlot::Uuid>(values); -} - -Y_UNIT_TEST(TzDate) { - const TVector<std::tuple<std::pair<ui16, ui16>, TString, TString>> values = { - {{0u, 0u}, "00000000", "FFFFFFFF"}, - {{0u, 1u}, "00000001", "FFFFFFFE"}, - {{1u, 0u}, "00010000", "FFFEFFFF"}, - {{NUdf::MAX_DATE, 0u}, "C2090000", "3DF6FFFF"}, - }; - TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDate>(values); -} - -Y_UNIT_TEST(TzDatetime) { - const TVector<std::tuple<std::pair<ui32, ui16>, TString, TString>> values = { - {{0u, 0u}, "000000000000", "FFFFFFFFFFFF"}, - {{0u, 1u}, "000000000001", "FFFFFFFFFFFE"}, - {{1u, 0u}, "000000010000", "FFFFFFFEFFFF"}, - {{NUdf::MAX_DATETIME, 0u}, "FFCEDD800000", "0031227FFFFF"}, - }; - TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDatetime>(values); -} - -Y_UNIT_TEST(TzTimestamp) { - const TVector<std::tuple<std::pair<ui64, ui16>, TString, TString>> values = { - {{0u, 0u}, "00000000000000000000", "FFFFFFFFFFFFFFFFFFFF"}, - {{0u, 1u}, "00000000000000000001", "FFFFFFFFFFFFFFFFFFFE"}, - {{1u, 0u}, "00000000000000010000", "FFFFFFFFFFFFFFFEFFFF"}, - {{NUdf::MAX_TIMESTAMP, 0u}, "000F3F52435260000000", "FFF0C0ADBCAD9FFFFFFF"}, - }; - TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzTimestamp>(values); -} - -Y_UNIT_TEST(Decimal) { + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + "00000000000000000000000000000000"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::Uuid>(values); +} + +Y_UNIT_TEST(TzDate) { + const TVector<std::tuple<std::pair<ui16, ui16>, TString, TString>> values = { + {{0u, 0u}, "00000000", "FFFFFFFF"}, + {{0u, 1u}, "00000001", "FFFFFFFE"}, + {{1u, 0u}, "00010000", "FFFEFFFF"}, + {{NUdf::MAX_DATE, 0u}, "C2090000", "3DF6FFFF"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDate>(values); +} + +Y_UNIT_TEST(TzDatetime) { + const TVector<std::tuple<std::pair<ui32, ui16>, TString, TString>> values = { + {{0u, 0u}, "000000000000", "FFFFFFFFFFFF"}, + {{0u, 1u}, "000000000001", "FFFFFFFFFFFE"}, + {{1u, 0u}, "000000010000", "FFFFFFFEFFFF"}, + {{NUdf::MAX_DATETIME, 0u}, "FFCEDD800000", "0031227FFFFF"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDatetime>(values); +} + +Y_UNIT_TEST(TzTimestamp) { + const TVector<std::tuple<std::pair<ui64, ui16>, TString, TString>> values = { + {{0u, 0u}, "00000000000000000000", "FFFFFFFFFFFFFFFFFFFF"}, + {{0u, 1u}, "00000000000000000001", "FFFFFFFFFFFFFFFFFFFE"}, + {{1u, 0u}, "00000000000000010000", "FFFFFFFFFFFFFFFEFFFF"}, + {{NUdf::MAX_TIMESTAMP, 0u}, "000F3F52435260000000", "FFF0C0ADBCAD9FFFFFFF"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzTimestamp>(values); +} + +Y_UNIT_TEST(Decimal) { const TVector<std::tuple<NYql::NDecimal::TInt128, TString, TString>> values = { {-NYql::NDecimal::Nan(), "00", @@ -428,10 +428,10 @@ Y_UNIT_TEST(Decimal) { {NYql::NDecimal::Nan(), "FF", "00"}, - }; - TPresortTest().ValidateEncoding(values); -} - + }; + TPresortTest().ValidateEncoding(values); +} + Y_UNIT_TEST(GenericVoid) { TScopedAlloc alloc; TTypeEnvironment env(alloc); @@ -440,8 +440,8 @@ Y_UNIT_TEST(GenericVoid) { TGenericPresortEncoder encoder(type); auto buf = encoder.Encode(value, false); UNIT_ASSERT_NO_DIFF(buf, TStringBuf("")); -} - +} + Y_UNIT_TEST(GenericBool) { TScopedAlloc alloc; TTypeEnvironment env(alloc); @@ -642,5 +642,5 @@ Y_UNIT_TEST(GenericDict) { } -} // NMiniKQL -} // NKikimr +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/computation/ut/ya.make b/ydb/library/yql/minikql/computation/ut/ya.make index d59d23d2ec..ab27689c7b 100644 --- a/ydb/library/yql/minikql/computation/ut/ya.make +++ b/ydb/library/yql/minikql/computation/ut/ya.make @@ -20,7 +20,7 @@ SRCS( mkql_computation_node_pack_ut.cpp mkql_computation_node_list_ut.cpp mkql_computation_node_dict_ut.cpp - mkql_computation_node_graph_saveload_ut.cpp + mkql_computation_node_graph_saveload_ut.cpp mkql_validate_ut.cpp mkql_value_builder_ut.cpp presort_ut.cpp diff --git a/ydb/library/yql/minikql/computation/ya.make b/ydb/library/yql/minikql/computation/ya.make index 6bd282409d..ae419b3b23 100644 --- a/ydb/library/yql/minikql/computation/ya.make +++ b/ydb/library/yql/minikql/computation/ya.make @@ -13,8 +13,8 @@ SRCS( mkql_computation_node_codegen.h mkql_computation_node_codegen.cpp mkql_computation_node_graph.cpp - mkql_computation_node_graph_saveload.cpp - mkql_computation_node_graph_saveload.h + mkql_computation_node_graph_saveload.cpp + mkql_computation_node_graph_saveload.h mkql_computation_node_holders.cpp mkql_computation_node_impl.h mkql_computation_node_impl.cpp diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index 7717ac582a..47ae6e4c8f 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -654,26 +654,26 @@ TRuntimeNode TProgramBuilder::Squeeze(TRuntimeNode stream, TRuntimeNode state, const auto newState = handler(itemArg, stateNodeArg); MKQL_ENSURE(newState.GetStaticType()->IsSameType(*state.GetStaticType()), "State type is changed by the handler"); - TRuntimeNode saveArg, outSave, loadArg, outLoad; - + TRuntimeNode saveArg, outSave, loadArg, outLoad; + if (save && load) { outSave = save(saveArg = Arg(state.GetStaticType())); outLoad = load(loadArg = Arg(outSave.GetStaticType())); - MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*state.GetStaticType()), "Loaded type is changed by the load handler"); - } else { - saveArg = outSave = loadArg = outLoad = NewVoid(); - } - + MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*state.GetStaticType()), "Loaded type is changed by the load handler"); + } else { + saveArg = outSave = loadArg = outLoad = NewVoid(); + } + TCallableBuilder callableBuilder(Env, __func__, TStreamType::Create(state.GetStaticType(), Env)); callableBuilder.Add(stream); callableBuilder.Add(state); callableBuilder.Add(itemArg); callableBuilder.Add(stateNodeArg); callableBuilder.Add(newState); - callableBuilder.Add(saveArg); - callableBuilder.Add(outSave); - callableBuilder.Add(loadArg); - callableBuilder.Add(outLoad); + callableBuilder.Add(saveArg); + callableBuilder.Add(outSave); + callableBuilder.Add(loadArg); + callableBuilder.Add(outLoad); return TRuntimeNode(callableBuilder.Build(), false); } @@ -694,26 +694,26 @@ TRuntimeNode TProgramBuilder::Squeeze1(TRuntimeNode stream, const TUnaryLambda& const auto newState = handler(itemArg, stateNodeArg); MKQL_ENSURE(newState.GetStaticType()->IsSameType(*initState.GetStaticType()), "State type is changed by the handler"); - TRuntimeNode saveArg, outSave, loadArg, outLoad; - + TRuntimeNode saveArg, outSave, loadArg, outLoad; + if (save && load) { outSave = save(saveArg = Arg(initState.GetStaticType())); outLoad = load(loadArg = Arg(outSave.GetStaticType())); - MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*initState.GetStaticType()), "Loaded type is changed by the load handler"); - } else { - saveArg = outSave = loadArg = outLoad = NewVoid(); - } - + MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*initState.GetStaticType()), "Loaded type is changed by the load handler"); + } else { + saveArg = outSave = loadArg = outLoad = NewVoid(); + } + TCallableBuilder callableBuilder(Env, __func__, NewStreamType(newState.GetStaticType())); callableBuilder.Add(stream); callableBuilder.Add(itemArg); callableBuilder.Add(initState); callableBuilder.Add(stateNodeArg); callableBuilder.Add(newState); - callableBuilder.Add(saveArg); - callableBuilder.Add(outSave); - callableBuilder.Add(loadArg); - callableBuilder.Add(outLoad); + callableBuilder.Add(saveArg); + callableBuilder.Add(outSave); + callableBuilder.Add(loadArg); + callableBuilder.Add(outLoad); return TRuntimeNode(callableBuilder.Build(), false); } @@ -4589,7 +4589,7 @@ TRuntimeNode TProgramBuilder::WideChopper(TRuntimeNode flow, const TWideLambda& return TRuntimeNode(callableBuilder.Build(), false); } -TRuntimeNode TProgramBuilder::HoppingCore(TRuntimeNode list, +TRuntimeNode TProgramBuilder::HoppingCore(TRuntimeNode list, const TUnaryLambda& timeExtractor, const TUnaryLambda& init, const TBinaryLambda& update, @@ -4597,69 +4597,69 @@ TRuntimeNode TProgramBuilder::HoppingCore(TRuntimeNode list, const TUnaryLambda& load, const TBinaryLambda& merge, const TBinaryLambda& finish, - TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay) -{ - auto streamType = AS_TYPE(TStreamType, list); - auto itemType = AS_TYPE(TStructType, streamType->GetItemType()); - auto timestampType = TOptionalType::Create(TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, Env), Env); - + TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay) +{ + auto streamType = AS_TYPE(TStreamType, list); + auto itemType = AS_TYPE(TStructType, streamType->GetItemType()); + auto timestampType = TOptionalType::Create(TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, Env), Env); + TRuntimeNode itemArg = Arg(itemType); - - auto outTime = timeExtractor(itemArg); - auto outStateInit = init(itemArg); - - auto stateType = outStateInit.GetStaticType(); + + auto outTime = timeExtractor(itemArg); + auto outStateInit = init(itemArg); + + auto stateType = outStateInit.GetStaticType(); TRuntimeNode stateArg = Arg(stateType); - - auto outStateUpdate = update(itemArg, stateArg); - - auto hasSaveLoad = (bool)save; - TRuntimeNode saveArg, outSave, loadArg, outLoad; - if (hasSaveLoad) { + + auto outStateUpdate = update(itemArg, stateArg); + + auto hasSaveLoad = (bool)save; + TRuntimeNode saveArg, outSave, loadArg, outLoad; + if (hasSaveLoad) { saveArg = Arg(stateType); - outSave = save(saveArg); - + outSave = save(saveArg); + loadArg = Arg(outSave.GetStaticType()); - outLoad = load(loadArg); - - MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*stateType), "Loaded type is changed by the load handler"); - } else { - saveArg = outSave = loadArg = outLoad = NewVoid(); - } - + outLoad = load(loadArg); + + MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*stateType), "Loaded type is changed by the load handler"); + } else { + saveArg = outSave = loadArg = outLoad = NewVoid(); + } + TRuntimeNode state2Arg = Arg(stateType); - TRuntimeNode timeArg = Arg(timestampType); - - auto outStateMerge = merge(stateArg, state2Arg); - auto outItemFinish = finish(stateArg, timeArg); - - auto finishType = outItemFinish.GetStaticType(); - MKQL_ENSURE(finishType->IsStruct(), "Expected struct type as finish lambda output"); - - auto resultType = TStreamType::Create(outItemFinish.GetStaticType(), Env); - + TRuntimeNode timeArg = Arg(timestampType); + + auto outStateMerge = merge(stateArg, state2Arg); + auto outItemFinish = finish(stateArg, timeArg); + + auto finishType = outItemFinish.GetStaticType(); + MKQL_ENSURE(finishType->IsStruct(), "Expected struct type as finish lambda output"); + + auto resultType = TStreamType::Create(outItemFinish.GetStaticType(), Env); + TCallableBuilder callableBuilder(Env, __func__, resultType); - callableBuilder.Add(list); - callableBuilder.Add(itemArg); - callableBuilder.Add(stateArg); - callableBuilder.Add(state2Arg); - callableBuilder.Add(timeArg); - callableBuilder.Add(saveArg); - callableBuilder.Add(loadArg); - callableBuilder.Add(outTime); - callableBuilder.Add(outStateInit); - callableBuilder.Add(outStateUpdate); - callableBuilder.Add(outSave); - callableBuilder.Add(outLoad); - callableBuilder.Add(outStateMerge); - callableBuilder.Add(outItemFinish); - callableBuilder.Add(hop); - callableBuilder.Add(interval); - callableBuilder.Add(delay); - - return TRuntimeNode(callableBuilder.Build(), false); -} - + callableBuilder.Add(list); + callableBuilder.Add(itemArg); + callableBuilder.Add(stateArg); + callableBuilder.Add(state2Arg); + callableBuilder.Add(timeArg); + callableBuilder.Add(saveArg); + callableBuilder.Add(loadArg); + callableBuilder.Add(outTime); + callableBuilder.Add(outStateInit); + callableBuilder.Add(outStateUpdate); + callableBuilder.Add(outSave); + callableBuilder.Add(outLoad); + callableBuilder.Add(outStateMerge); + callableBuilder.Add(outItemFinish); + callableBuilder.Add(hop); + callableBuilder.Add(interval); + callableBuilder.Add(delay); + + return TRuntimeNode(callableBuilder.Build(), false); +} + TRuntimeNode TProgramBuilder::MultiHoppingCore(TRuntimeNode list, const TUnaryLambda& keyExtractor, const TUnaryLambda& timeExtractor, diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h index 9939e2d291..10f1ad7ccf 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.h +++ b/ydb/library/yql/minikql/mkql_program_builder.h @@ -416,7 +416,7 @@ public: const TBinaryLambda& groupSwitch, const TUnaryLambda& keyExtractor, const TUnaryLambda& handler = {}); - TRuntimeNode HoppingCore(TRuntimeNode list, + TRuntimeNode HoppingCore(TRuntimeNode list, const TUnaryLambda& timeExtractor, const TUnaryLambda& init, const TBinaryLambda& update, @@ -424,7 +424,7 @@ public: const TUnaryLambda& load, const TBinaryLambda& merge, const TBinaryLambda& finish, - TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay); + TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay); TRuntimeNode MultiHoppingCore(TRuntimeNode list, const TUnaryLambda& keyExtractor, const TUnaryLambda& timeExtractor, diff --git a/ydb/library/yql/minikql/mkql_type_builder.cpp b/ydb/library/yql/minikql/mkql_type_builder.cpp index e8b3f8bcb9..4aecb564bf 100644 --- a/ydb/library/yql/minikql/mkql_type_builder.cpp +++ b/ydb/library/yql/minikql/mkql_type_builder.cpp @@ -3,7 +3,7 @@ #include "mkql_node_builder.h" #include <ydb/library/yql/public/udf/udf_type_ops.h> - + #include <library/cpp/containers/stack_vector/stack_vec.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h> @@ -598,13 +598,13 @@ private: TVector<NMiniKQL::TArgInfo>& Args_; }; -////////////////////////////////////////////////////////////////////////////// -// THash -////////////////////////////////////////////////////////////////////////////// - -struct TTypeNotSupported : public yexception -{}; - +////////////////////////////////////////////////////////////////////////////// +// THash +////////////////////////////////////////////////////////////////////////////// + +struct TTypeNotSupported : public yexception +{}; + class TEmptyHash final : public NUdf::IHash { public: ui64 Hash(NUdf::TUnboxedValuePod value) const override { @@ -613,35 +613,35 @@ public: } }; -template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool> -class THash; - -template <NUdf::EDataSlot Slot> -class THash<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::IHash { -public: - ui64 Hash(NUdf::TUnboxedValuePod value) const override { +template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool> +class THash; + +template <NUdf::EDataSlot Slot> +class THash<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::IHash { +public: + ui64 Hash(NUdf::TUnboxedValuePod value) const override { return NUdf::GetValueHash<Slot>(std::move(value)); - } -}; - -template <> -class THash<NMiniKQL::TType::EKind::Optional> final : public NUdf::IHash { -public: + } +}; + +template <> +class THash<NMiniKQL::TType::EKind::Optional> final : public NUdf::IHash { +public: explicit THash(const NMiniKQL::TType* type) : Hash_(MakeHashImpl(static_cast<const NMiniKQL::TOptionalType*>(type)->GetItemType())) {} - - ui64 Hash(NUdf::TUnboxedValuePod value) const override { - if (!value) { - return 0; - } + + ui64 Hash(NUdf::TUnboxedValuePod value) const override { + if (!value) { + return 0; + } return CombineHashes(ui64(1), Hash_->Hash(value.GetOptionalValue())); - } - -private: + } + +private: const NUdf::IHash::TPtr Hash_; -}; - +}; + template <> class THash<NMiniKQL::TType::EKind::List> final : public NUdf::IHash { public: @@ -711,9 +711,9 @@ private: NUdf::IHash::TPtr PayloadHash_; }; -class TVectorHash : public NUdf::IHash { -public: - ui64 Hash(NUdf::TUnboxedValuePod value) const override { +class TVectorHash : public NUdf::IHash { +public: + ui64 Hash(NUdf::TUnboxedValuePod value) const override { ui64 result = 0ULL; auto elements = value.GetElements(); if (elements) { @@ -727,39 +727,39 @@ public: } } - return result; - } - -protected: + return result; + } + +protected: std::vector<NUdf::IHash::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::IHash::TPtr>> Hash_; -}; - -template <> -class THash<NMiniKQL::TType::EKind::Tuple> final : public TVectorHash { -public: - explicit THash(const NMiniKQL::TType* type) { - auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type); - auto count = tupleType->GetElementsCount(); - Hash_.reserve(count); - for (ui32 i = 0; i < count; ++i) { - Hash_.push_back(MakeHashImpl(tupleType->GetElementType(i))); - } - } -}; - -template <> -class THash<NMiniKQL::TType::EKind::Struct> final : public TVectorHash { -public: - explicit THash(const NMiniKQL::TType* type) { - auto structType = static_cast<const NMiniKQL::TStructType*>(type); - auto count = structType->GetMembersCount(); - Hash_.reserve(count); - for (ui32 i = 0; i < count; ++i) { - Hash_.push_back(MakeHashImpl(structType->GetMemberType(i))); - } - } -}; - +}; + +template <> +class THash<NMiniKQL::TType::EKind::Tuple> final : public TVectorHash { +public: + explicit THash(const NMiniKQL::TType* type) { + auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type); + auto count = tupleType->GetElementsCount(); + Hash_.reserve(count); + for (ui32 i = 0; i < count; ++i) { + Hash_.push_back(MakeHashImpl(tupleType->GetElementType(i))); + } + } +}; + +template <> +class THash<NMiniKQL::TType::EKind::Struct> final : public TVectorHash { +public: + explicit THash(const NMiniKQL::TType* type) { + auto structType = static_cast<const NMiniKQL::TStructType*>(type); + auto count = structType->GetMembersCount(); + Hash_.reserve(count); + for (ui32 i = 0; i < count; ++i) { + Hash_.push_back(MakeHashImpl(structType->GetMemberType(i))); + } + } +}; + template <> class THash<NMiniKQL::TType::EKind::Variant> final : public NUdf::IHash { public: @@ -793,9 +793,9 @@ private: std::vector<NUdf::IHash::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::IHash::TPtr>> Hash_; }; -////////////////////////////////////////////////////////////////////////////// -// TEquate -////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////// +// TEquate +////////////////////////////////////////////////////////////////////////////// class TEmptyEquate final : public NUdf::IEquate { public: bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { @@ -806,42 +806,42 @@ public: }; -template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool> -class TEquate; - -template <NUdf::EDataSlot Slot> -class TEquate<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::IEquate { -public: - bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { +template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool> +class TEquate; + +template <NUdf::EDataSlot Slot> +class TEquate<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::IEquate { +public: + bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { return NUdf::EquateValues<Slot>(std::move(lhs), std::move(rhs)); - } -}; - -template <> -class TEquate<NMiniKQL::TType::EKind::Optional> final : public NUdf::IEquate { -public: + } +}; + +template <> +class TEquate<NMiniKQL::TType::EKind::Optional> final : public NUdf::IEquate { +public: explicit TEquate(const NMiniKQL::TType* type) : Equate_(MakeEquateImpl(static_cast<const NMiniKQL::TOptionalType*>(type)->GetItemType())) {} - - bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { - if (!lhs) { - if (!rhs) { - return true; - } - return false; - } else { - if (!rhs) { - return false; - } + + bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { + if (!lhs) { + if (!rhs) { + return true; + } + return false; + } else { + if (!rhs) { + return false; + } return Equate_->Equals(lhs.GetOptionalValue(), rhs.GetOptionalValue()); - } - } - -private: + } + } + +private: const NUdf::IEquate::TPtr Equate_; -}; - +}; + template <> class TEquate<NMiniKQL::TType::EKind::List> final : public NUdf::IEquate { public: @@ -944,49 +944,49 @@ private: }; -class TVectorEquate : public NUdf::IEquate { -public: - bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { - for (size_t i = 0; i < Equate_.size(); ++i) { +class TVectorEquate : public NUdf::IEquate { +public: + bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { + for (size_t i = 0; i < Equate_.size(); ++i) { if (!Equate_[i]->Equals( static_cast<const NUdf::TUnboxedValuePod&>(lhs.GetElement(i)), static_cast<const NUdf::TUnboxedValuePod&>(rhs.GetElement(i)))) { - return false; - } - } - return true; - } - -protected: + return false; + } + } + return true; + } + +protected: std::vector<NUdf::IEquate::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::IEquate::TPtr>> Equate_; -}; - -template <> -class TEquate<NMiniKQL::TType::EKind::Tuple> final : public TVectorEquate { -public: - explicit TEquate(const NMiniKQL::TType* type) { - auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type); - auto count = tupleType->GetElementsCount(); - Equate_.reserve(count); - for (ui32 i = 0; i < count; ++i) { - Equate_.push_back(MakeEquateImpl(tupleType->GetElementType(i))); - } - } -}; - -template <> -class TEquate<NMiniKQL::TType::EKind::Struct> final : public TVectorEquate { -public: - explicit TEquate(const NMiniKQL::TType* type) { - auto structType = static_cast<const NMiniKQL::TStructType*>(type); - auto count = structType->GetMembersCount(); - Equate_.reserve(count); - for (ui32 i = 0; i < count; ++i) { - Equate_.push_back(MakeEquateImpl(structType->GetMemberType(i))); - } - } -}; - +}; + +template <> +class TEquate<NMiniKQL::TType::EKind::Tuple> final : public TVectorEquate { +public: + explicit TEquate(const NMiniKQL::TType* type) { + auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type); + auto count = tupleType->GetElementsCount(); + Equate_.reserve(count); + for (ui32 i = 0; i < count; ++i) { + Equate_.push_back(MakeEquateImpl(tupleType->GetElementType(i))); + } + } +}; + +template <> +class TEquate<NMiniKQL::TType::EKind::Struct> final : public TVectorEquate { +public: + explicit TEquate(const NMiniKQL::TType* type) { + auto structType = static_cast<const NMiniKQL::TStructType*>(type); + auto count = structType->GetMembersCount(); + Equate_.reserve(count); + for (ui32 i = 0; i < count; ++i) { + Equate_.push_back(MakeEquateImpl(structType->GetMemberType(i))); + } + } +}; + template <> class TEquate<NMiniKQL::TType::EKind::Variant> final : public NUdf::IEquate { public: @@ -1027,9 +1027,9 @@ private: std::vector<NUdf::IEquate::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::IEquate::TPtr>> Equate_; }; -////////////////////////////////////////////////////////////////////////////// -// TCompare -////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////// +// TCompare +////////////////////////////////////////////////////////////////////////////// class TEmptyCompare final : public NUdf::ICompare { public: bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { @@ -1045,92 +1045,92 @@ public: } }; -template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool> -class TCompare; - -template <NUdf::EDataSlot Slot> -class TCompare<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::ICompare { -public: - bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { +template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool> +class TCompare; + +template <NUdf::EDataSlot Slot> +class TCompare<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::ICompare { +public: + bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { return NUdf::CompareValues<Slot>(std::move(lhs), std::move(rhs)) < 0; - } - - int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { + } + + int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { return NUdf::CompareValues<Slot>(std::move(lhs), std::move(rhs)); - } -}; - -template <> -class TCompare<NMiniKQL::TType::EKind::Optional> final : public NUdf::ICompare { -public: + } +}; + +template <> +class TCompare<NMiniKQL::TType::EKind::Optional> final : public NUdf::ICompare { +public: explicit TCompare(const NMiniKQL::TType* type) : Compare_(MakeCompareImpl(static_cast<const NMiniKQL::TOptionalType*>(type)->GetItemType())) {} - - bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { - if (!lhs) { - if (!rhs) { - return false; - } - return true; - } else { - if (!rhs) { - return false; - } + + bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { + if (!lhs) { + if (!rhs) { + return false; + } + return true; + } else { + if (!rhs) { + return false; + } return Compare_->Less(lhs.GetOptionalValue(), rhs.GetOptionalValue()); - } - } - - int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { - if (!lhs) { - if (!rhs) { - return 0; - } - return -1; - } else { - if (!rhs) { - return 1; - } + } + } + + int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { + if (!lhs) { + if (!rhs) { + return 0; + } + return -1; + } else { + if (!rhs) { + return 1; + } return Compare_->Compare(lhs.GetOptionalValue(), rhs.GetOptionalValue()); - } - } - -private: + } + } + +private: const NUdf::ICompare::TPtr Compare_; -}; - -template <> -class TCompare<NMiniKQL::TType::EKind::Tuple> final : public NUdf::ICompare { -public: +}; + +template <> +class TCompare<NMiniKQL::TType::EKind::Tuple> final : public NUdf::ICompare { +public: explicit TCompare(const NMiniKQL::TType* type) { - auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type); - auto count = tupleType->GetElementsCount(); - Compare_.reserve(count); - for (ui32 i = 0; i < count; ++i) { + auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type); + auto count = tupleType->GetElementsCount(); + Compare_.reserve(count); + for (ui32 i = 0; i < count; ++i) { Compare_.push_back(MakeCompareImpl(tupleType->GetElementType(i))); - } - } - - bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { - return Compare(lhs, rhs) < 0; - } - - int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { - for (size_t i = 0; i < Compare_.size(); ++i) { + } + } + + bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { + return Compare(lhs, rhs) < 0; + } + + int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { + for (size_t i = 0; i < Compare_.size(); ++i) { auto cmp = Compare_[i]->Compare( static_cast<const NUdf::TUnboxedValuePod&>(lhs.GetElement(i)), static_cast<const NUdf::TUnboxedValuePod&>(rhs.GetElement(i))); - if (cmp != 0) { - return cmp; - } - } - return 0; - } - -private: + if (cmp != 0) { + return cmp; + } + } + return 0; + } + +private: std::vector<NUdf::ICompare::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::ICompare::TPtr>> Compare_; -}; - +}; + template <> class TCompare<NMiniKQL::TType::EKind::Variant> final : public NUdf::ICompare { public: @@ -1517,36 +1517,36 @@ NUdf::TSourcePosition TFunctionTypeInfoBuilder::GetSourcePosition() { return Pos_; } -NUdf::IHash::TPtr TFunctionTypeInfoBuilder::MakeHash(const NUdf::TType* type) { - try { - auto mkqlType = static_cast<const NMiniKQL::TType*>(type); - return MakeHashImpl(mkqlType); - } catch (const TTypeNotSupported& ex) { - SetError(TStringBuf(ex.what())); - return nullptr; - } -} - -NUdf::IEquate::TPtr TFunctionTypeInfoBuilder::MakeEquate(const NUdf::TType* type) { - try { - auto mkqlType = static_cast<const NMiniKQL::TType*>(type); - return MakeEquateImpl(mkqlType); - } catch (const TTypeNotSupported& ex) { - SetError(TStringBuf(ex.what())); - return nullptr; - } -} - -NUdf::ICompare::TPtr TFunctionTypeInfoBuilder::MakeCompare(const NUdf::TType* type) { - try { - auto mkqlType = static_cast<const NMiniKQL::TType*>(type); +NUdf::IHash::TPtr TFunctionTypeInfoBuilder::MakeHash(const NUdf::TType* type) { + try { + auto mkqlType = static_cast<const NMiniKQL::TType*>(type); + return MakeHashImpl(mkqlType); + } catch (const TTypeNotSupported& ex) { + SetError(TStringBuf(ex.what())); + return nullptr; + } +} + +NUdf::IEquate::TPtr TFunctionTypeInfoBuilder::MakeEquate(const NUdf::TType* type) { + try { + auto mkqlType = static_cast<const NMiniKQL::TType*>(type); + return MakeEquateImpl(mkqlType); + } catch (const TTypeNotSupported& ex) { + SetError(TStringBuf(ex.what())); + return nullptr; + } +} + +NUdf::ICompare::TPtr TFunctionTypeInfoBuilder::MakeCompare(const NUdf::TType* type) { + try { + auto mkqlType = static_cast<const NMiniKQL::TType*>(type); return MakeCompareImpl(mkqlType); - } catch (const TTypeNotSupported& ex) { - SetError(TStringBuf(ex.what())); - return nullptr; - } -} - + } catch (const TTypeNotSupported& ex) { + SetError(TStringBuf(ex.what())); + return nullptr; + } +} + NUdf::ETypeKind TTypeInfoHelper::GetTypeKind(const NUdf::TType* type) const { if (!type) { return NUdf::ETypeKind::Unknown; diff --git a/ydb/library/yql/minikql/mkql_type_builder.h b/ydb/library/yql/minikql/mkql_type_builder.h index 4a9a280218..a93fb41cee 100644 --- a/ydb/library/yql/minikql/mkql_type_builder.h +++ b/ydb/library/yql/minikql/mkql_type_builder.h @@ -95,10 +95,10 @@ public: NUdf::TScopedProbe GetScopedProbe(const NUdf::TStringRef& name) override; NUdf::TSourcePosition GetSourcePosition() override; - NUdf::IHash::TPtr MakeHash(const NUdf::TType* type) override; - NUdf::IEquate::TPtr MakeEquate(const NUdf::TType* type) override; - NUdf::ICompare::TPtr MakeCompare(const NUdf::TType* type) override; - + NUdf::IHash::TPtr MakeHash(const NUdf::TType* type) override; + NUdf::IEquate::TPtr MakeEquate(const NUdf::TType* type) override; + NUdf::ICompare::TPtr MakeCompare(const NUdf::TType* type) override; + NUdf::TType* Decimal(ui8 precision, ui8 scale) const override; NUdf::IFunctionTypeInfoBuilder7& IRImplementationImpl( diff --git a/ydb/library/yql/minikql/mkql_type_ops.cpp b/ydb/library/yql/minikql/mkql_type_ops.cpp index 1962eab8e9..d559bb108c 100644 --- a/ydb/library/yql/minikql/mkql_type_ops.cpp +++ b/ydb/library/yql/minikql/mkql_type_ops.cpp @@ -667,32 +667,32 @@ public: TDateTable() { ui32 prevYear = NUdf::MIN_YEAR - 1; YearsOffsets_[0] = 0; - + ui32 dayOfYear = 365; ui32 dayOfWeek = 2; ui32 weekOfYear = 52; ui32 weekOfYearIso8601 = 1; - + for (ui16 date = 0; date < Days_.size(); ++date) { ui32 year, month, day; Y_VERIFY(SplitDateUncached(date, year, month, day)); - - ++dayOfYear; - if (++dayOfWeek > 7) { - dayOfWeek = 1; - ++weekOfYear; + + ++dayOfYear; + if (++dayOfWeek > 7) { + dayOfWeek = 1; + ++weekOfYear; if ((month == 12 && day >= 29) || (month == 1 && day <= 4)) { weekOfYearIso8601 = 1; } else { ++weekOfYearIso8601; } - } - + } + if (year > prevYear) { YearsOffsets_[year - NUdf::MIN_YEAR] = date; prevYear = year; - dayOfYear = 1; - weekOfYear = 1; + dayOfYear = 1; + weekOfYear = 1; } Days_[date] = TDayInfo{month, day, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek}; @@ -705,9 +705,9 @@ public: } year = NUdf::MIN_YEAR + std::distance(YearsOffsets_.cbegin(), std::upper_bound(YearsOffsets_.cbegin(), YearsOffsets_.cend(), value)) - 1; - auto& info = Days_[value]; - month = info.Month; - day = info.Day; + auto& info = Days_[value]; + month = info.Month; + day = info.Day; return true; } @@ -726,14 +726,14 @@ public: const auto ptr = YearsOffsets_.data() + year - NUdf::MIN_YEAR; const auto begin = Days_.cbegin() + ptr[0]; const auto end = year == NUdf::MAX_YEAR ? Days_.cend() : Days_.cbegin() + ptr[1]; - + // search valid month/day in this year const auto target = PackMonthDay(month, day); const auto it = std::lower_bound(begin, end, target, [](const TDayInfo& info, ui16 y) { - return PackMonthDay(info.Month, info.Day) < y; + return PackMonthDay(info.Month, info.Day) < y; }); - if (Y_UNLIKELY(it == end || PackMonthDay(it->Month, it->Day) != target)) { + if (Y_UNLIKELY(it == end || PackMonthDay(it->Month, it->Day) != target)) { return false; } @@ -757,17 +757,17 @@ public: bool EnrichByOffset(ui16 value, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek) const { if (Y_UNLIKELY(value >= Days_.size())) { - return false; - } - - auto& info = Days_[value]; - dayOfYear = info.DayOfYear; - weekOfYear = info.WeekOfYear; + return false; + } + + auto& info = Days_[value]; + dayOfYear = info.DayOfYear; + weekOfYear = info.WeekOfYear; weekOfYearIso8601 = info.WeekOfYearIso8601; - dayOfWeek = info.DayOfWeek; - return true; - } - + dayOfWeek = info.DayOfWeek; + return true; + } + bool EnrichDate(ui16 value, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek) const { return EnrichByOffset(++value, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek); } @@ -781,15 +781,15 @@ public: } private: - struct TDayInfo { - ui32 Month : 4; - ui32 Day : 5; - ui32 DayOfYear : 9; - ui32 WeekOfYear : 6; + struct TDayInfo { + ui32 Month : 4; + ui32 Day : 5; + ui32 DayOfYear : 9; + ui32 WeekOfYear : 6; ui32 WeekOfYearIso8601: 6; - ui32 DayOfWeek : 3; - }; - + ui32 DayOfWeek : 3; + }; + std::array<ui16, NUdf::MAX_YEAR - NUdf::MIN_YEAR + 1> YearsOffsets_; // start of linear date for each year std::array<TDayInfo, NUdf::MAX_DATE + 2> Days_; // packed info for each date }; @@ -853,44 +853,44 @@ bool SplitInterval(i64 value, bool& sign, ui32& day, ui32& hour, ui32& min, ui32 return true; } -bool MakeTzDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui32& value, ui16 tzId) { - if (tzId) { - const auto& tz = Singleton<TTimezones>()->GetZone(tzId); - cctz::civil_second cs(year, month, day, hour, min, sec); +bool MakeTzDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui32& value, ui16 tzId) { + if (tzId) { + const auto& tz = Singleton<TTimezones>()->GetZone(tzId); + cctz::civil_second cs(year, month, day, hour, min, sec); auto utcSeconds = cctz::TimePointToUnixSeconds(tz.lookup(cs).pre); if (utcSeconds < 0 || utcSeconds >= (std::int_fast64_t) NUdf::MAX_DATETIME) { - return false; - } - - value = (ui32)utcSeconds; - return true; - - } else { - ui16 date; - ui32 time; - if (!MakeDate(year, month, day, date)) { - return false; - } - if (!MakeTime(hour, min, sec, time)) { - return false; - } - value = date * 86400u + time; - return true; - } -} - -bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui16 tzId) { - if (tzId) { - if (value >= NUdf::MAX_DATETIME) { - return false; - } - ToLocalTime(value, tzId, year, month, day, hour, min, sec); - return true; - } else { - return SplitDatetime(value, year, month, day, hour, min, sec); - } -} - + return false; + } + + value = (ui32)utcSeconds; + return true; + + } else { + ui16 date; + ui32 time; + if (!MakeDate(year, month, day, date)) { + return false; + } + if (!MakeTime(hour, min, sec, time)) { + return false; + } + value = date * 86400u + time; + return true; + } +} + +bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui16 tzId) { + if (tzId) { + if (value >= NUdf::MAX_DATETIME) { + return false; + } + ToLocalTime(value, tzId, year, month, day, hour, min, sec); + return true; + } else { + return SplitDatetime(value, year, month, day, hour, min, sec); + } +} + bool SplitTzDate(ui16 value, ui32& year, ui32& month, ui32& day, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 tzId) { if (tzId) { if (value >= NUdf::MAX_DATE) { @@ -931,55 +931,55 @@ bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek) { return TDateTable::Instance().EnrichDate(date, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek); -} - -bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui16 tzId, i32& value) { - if (!tzId) { - value = 0; - return true; - } - - const auto& tz = Singleton<TTimezones>()->GetZone(tzId); - cctz::civil_second cs(year, month, day, hour, min, sec); - value = tz.lookup(tz.lookup(cs).pre).offset / 60; - return true; -} - -namespace { - -bool FromLocalTimeValidated(ui16 tzId, ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value) { - if (year < NUdf::MIN_YEAR - 1 || year > NUdf::MAX_YEAR) { - return false; - } else if (year == NUdf::MIN_YEAR - 1) { - if (month != 12 || day != 31) { - return false; - } - } else if (year == NUdf::MAX_YEAR) { - if (month != 1 || day != 1) { - return false; - } - } - - const auto& tz = Singleton<TTimezones>()->GetZone(tzId); - - cctz::civil_second sec(year, month, day, hour, minute, second); +} + +bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui16 tzId, i32& value) { + if (!tzId) { + value = 0; + return true; + } + + const auto& tz = Singleton<TTimezones>()->GetZone(tzId); + cctz::civil_second cs(year, month, day, hour, min, sec); + value = tz.lookup(tz.lookup(cs).pre).offset / 60; + return true; +} + +namespace { + +bool FromLocalTimeValidated(ui16 tzId, ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value) { + if (year < NUdf::MIN_YEAR - 1 || year > NUdf::MAX_YEAR) { + return false; + } else if (year == NUdf::MIN_YEAR - 1) { + if (month != 12 || day != 31) { + return false; + } + } else if (year == NUdf::MAX_YEAR) { + if (month != 1 || day != 1) { + return false; + } + } + + const auto& tz = Singleton<TTimezones>()->GetZone(tzId); + + cctz::civil_second sec(year, month, day, hour, minute, second); if ((ui32)sec.year() != year || (ui32)sec.month() != month || (ui32)sec.day() != day || (ui32)sec.hour() != hour || (ui32)sec.minute() != minute || (ui32)sec.second() != second) { // normalized return false; } - const auto absoluteSeconds = std::chrono::system_clock::to_time_t(tz.lookup(sec).pre); - if (absoluteSeconds < 0 || (ui32)absoluteSeconds >= NUdf::MAX_DATETIME) { - return false; - } - - value = (ui32)absoluteSeconds; - return true; -} - -} // namespace - + const auto absoluteSeconds = std::chrono::system_clock::to_time_t(tz.lookup(sec).pre); + if (absoluteSeconds < 0 || (ui32)absoluteSeconds >= NUdf::MAX_DATETIME) { + return false; + } + + value = (ui32)absoluteSeconds; + return true; +} + +} // namespace + ui32 ParseNumber(ui32& pos, NUdf::TStringRef buf, ui32& value) { value = 0; ui32 count = 0; @@ -1157,12 +1157,12 @@ NUdf::TUnboxedValuePod ParseTzDate(NUdf::TStringRef str) { return NUdf::TUnboxedValuePod(); } - ui32 absoluteSeconds; + ui32 absoluteSeconds; if (!FromLocalTimeValidated(*tzId, year, month, day, 0, 0, 0, absoluteSeconds)) { return NUdf::TUnboxedValuePod(); } - ui16 value = absoluteSeconds / 86400u; + ui16 value = absoluteSeconds / 86400u; NUdf::TUnboxedValuePod out(value); out.SetTimezoneId(*tzId); return out; @@ -1276,12 +1276,12 @@ NUdf::TUnboxedValuePod ParseTzDatetime(NUdf::TStringRef str) { return NUdf::TUnboxedValuePod(); } - ui32 absoluteSeconds; - if (!FromLocalTimeValidated(*tzId, year, month, day, hour, minute, second, absoluteSeconds)) { + ui32 absoluteSeconds; + if (!FromLocalTimeValidated(*tzId, year, month, day, hour, minute, second, absoluteSeconds)) { return NUdf::TUnboxedValuePod(); } - ui32 value = absoluteSeconds; + ui32 value = absoluteSeconds; NUdf::TUnboxedValuePod out(value); out.SetTimezoneId(*tzId); return out; @@ -1449,8 +1449,8 @@ NUdf::TUnboxedValuePod ParseTzTimestamp(NUdf::TStringRef str) { } } - ui32 absoluteSeconds; - if (!FromLocalTimeValidated(*tzId, year, month, day, hour, minute, second, absoluteSeconds)) { + ui32 absoluteSeconds; + if (!FromLocalTimeValidated(*tzId, year, month, day, hour, minute, second, absoluteSeconds)) { return NUdf::TUnboxedValuePod(); } @@ -2013,7 +2013,7 @@ ui32 FromLocalTime(ui16 tzId, ui32 year, ui32 month, ui32 day, ui32 hour, ui32 m return absoluteSeconds; } - + void SerializeTzDate(ui16 date, ui16 tzId, IOutputStream& out) { date = SwapBytes(date); tzId = SwapBytes(tzId); diff --git a/ydb/library/yql/minikql/mkql_type_ops.h b/ydb/library/yql/minikql/mkql_type_ops.h index 93b5e88a56..8ff9b258dd 100644 --- a/ydb/library/yql/minikql/mkql_type_ops.h +++ b/ydb/library/yql/minikql/mkql_type_ops.h @@ -43,11 +43,11 @@ bool SplitInterval(i64 value, bool& sign, ui32& day, ui32& hour, ui32& min, ui32 bool SplitTzDate(ui16 value, ui32& year, ui32& month, ui32& day, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 tzId); bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 tzId); -bool MakeTzDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui32& value, ui16 tzId); -bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui16 tzId); +bool MakeTzDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui32& value, ui16 tzId); +bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui16 tzId); bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek); -bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui16 tzId, i32& value); - +bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui16 tzId, i32& value); + ui16 InitTimezones(); bool IsValidTimezoneId(ui16 id); TMaybe<ui16> FindTimezoneId(TStringBuf ianaName); @@ -55,10 +55,10 @@ ui16 GetTimezoneId(TStringBuf ianaName); TMaybe<TStringBuf> FindTimezoneIANAName(ui16 id); TStringBuf GetTimezoneIANAName(ui16 id); std::vector<ui16> GetTzBlackList(); - + void ToLocalTime(ui32 utcSeconds, ui16 tzId, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec); ui32 FromLocalTime(ui16 tzId, ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec); - + void SerializeTzDate(ui16 date, ui16 tzId, IOutputStream& out); void SerializeTzDatetime(ui32 datetime, ui16 tzId, IOutputStream& out); void SerializeTzTimestamp(ui64 timestamp, ui16 tzId, IOutputStream& out); diff --git a/ydb/library/yql/minikql/perf/presort/presort.cpp b/ydb/library/yql/minikql/perf/presort/presort.cpp index 278250ba19..cfe4ddfb8f 100644 --- a/ydb/library/yql/minikql/perf/presort/presort.cpp +++ b/ydb/library/yql/minikql/perf/presort/presort.cpp @@ -4,298 +4,298 @@ #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/public/udf/udf_types.h> - + #include <library/cpp/presort/presort.h> - -#include <util/random/random.h> -#include <util/datetime/cputimer.h> + +#include <util/random/random.h> +#include <util/datetime/cputimer.h> #include <util/string/builder.h> - -using namespace NKikimr; -using namespace NKikimr::NMiniKQL; - -namespace { - -struct TSettings { - ui32 Index; - bool IsOptional; - NKikimr::NUdf::EDataSlot Slot; -}; - -template <bool Desc> -struct TPresortOps : public NPresort::TResultOps { - const TVector<TSettings>& Settings; - NUdf::TUnboxedValue* Items; - - size_t Current = 0; - - TPresortOps( - const TVector<TSettings>& settings, - NUdf::TUnboxedValue* items) + +using namespace NKikimr; +using namespace NKikimr::NMiniKQL; + +namespace { + +struct TSettings { + ui32 Index; + bool IsOptional; + NKikimr::NUdf::EDataSlot Slot; +}; + +template <bool Desc> +struct TPresortOps : public NPresort::TResultOps { + const TVector<TSettings>& Settings; + NUdf::TUnboxedValue* Items; + + size_t Current = 0; + + TPresortOps( + const TVector<TSettings>& settings, + NUdf::TUnboxedValue* items) : Settings(settings) - , Items(items) - {} - - void Encode(IOutputStream& out) { - for (const auto& setting : Settings) { - auto& value = Items[setting.Index]; - - switch (setting.Slot) { - case NUdf::EDataSlot::Bool: - NPresort::EncodeUnsignedInt(out, value.template Get<bool>(), Desc); - break; - case NUdf::EDataSlot::Uint8: - NPresort::EncodeUnsignedInt(out, value.template Get<ui8>(), Desc); - break; - case NUdf::EDataSlot::Uint16: - case NUdf::EDataSlot::Date: - NPresort::EncodeUnsignedInt(out, value.template Get<ui16>(), Desc); - break; - case NUdf::EDataSlot::Uint32: - case NUdf::EDataSlot::Datetime: - NPresort::EncodeUnsignedInt(out, value.template Get<ui32>(), Desc); - break; - case NUdf::EDataSlot::Uint64: - case NUdf::EDataSlot::Timestamp: - NPresort::EncodeUnsignedInt(out, value.template Get<ui64>(), Desc); - break; - case NUdf::EDataSlot::Int8: - NPresort::EncodeSignedInt(out, value.template Get<i8>(), Desc); - break; - case NUdf::EDataSlot::Int16: - NPresort::EncodeSignedInt(out, value.template Get<i16>(), Desc); - break; - case NUdf::EDataSlot::Int32: - NPresort::EncodeSignedInt(out, value.template Get<i32>(), Desc); - break; - case NUdf::EDataSlot::Int64: - case NUdf::EDataSlot::Interval: - NPresort::EncodeSignedInt(out, value.template Get<i64>(), Desc); - break; - case NUdf::EDataSlot::Float: - NPresort::EncodeFloating(out, value.template Get<float>(), Desc); - break; - case NUdf::EDataSlot::Double: - NPresort::EncodeFloating(out, value.template Get<double>(), Desc); - break; - case NUdf::EDataSlot::String: - case NUdf::EDataSlot::Utf8: { - auto strRef = value.AsStringRef(); - NPresort::EncodeString(out, TStringBuf(strRef.Data(), strRef.Size()), Desc); - break; - } - default: - MKQL_ENSURE(false, TStringBuilder() << "Unknown slot: " << setting.Slot); - } - } - } - - void SetError(const TString& err) { - MKQL_ENSURE(false, TStringBuilder() << "Presort decoding error: " << err); - } - - void SetUnsignedInt(ui64 value) { - const auto& setting = Settings[Current++]; - switch (setting.Slot) { - case NUdf::EDataSlot::Bool: - Items[setting.Index] = NUdf::TUnboxedValuePod(value != 0); - break; - case NUdf::EDataSlot::Uint8: - Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui8>(value)); - break; - case NUdf::EDataSlot::Uint16: - case NUdf::EDataSlot::Date: - Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui16>(value)); - break; - case NUdf::EDataSlot::Uint32: - case NUdf::EDataSlot::Datetime: - Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui32>(value)); - break; - case NUdf::EDataSlot::Uint64: - case NUdf::EDataSlot::Timestamp: - Items[setting.Index] = NUdf::TUnboxedValuePod(value); - break; - default: - MKQL_ENSURE(false, TStringBuilder() << "Unknown slot: " << setting.Slot); - } - } - - void SetSignedInt(i64 value) { - const auto& setting = Settings[Current++]; - switch (setting.Slot) { - case NUdf::EDataSlot::Int8: - Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i8>(value)); - break; - case NUdf::EDataSlot::Int16: - Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i16>(value)); - break; - case NUdf::EDataSlot::Int32: - Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i32>(value)); - break; - case NUdf::EDataSlot::Int64: - case NUdf::EDataSlot::Interval: - Items[setting.Index] = NUdf::TUnboxedValuePod(value); - break; - default: - MKQL_ENSURE(false, "Unknown slot: " << setting.Slot); - } - } - - void SetFloat(float value) { - Items[Settings[Current++].Index] = NUdf::TUnboxedValuePod(value); - } - - void SetDouble(double value) { - Items[Settings[Current++].Index] = NUdf::TUnboxedValuePod(value); - } - - void SetString(const TString& value) { + , Items(items) + {} + + void Encode(IOutputStream& out) { + for (const auto& setting : Settings) { + auto& value = Items[setting.Index]; + + switch (setting.Slot) { + case NUdf::EDataSlot::Bool: + NPresort::EncodeUnsignedInt(out, value.template Get<bool>(), Desc); + break; + case NUdf::EDataSlot::Uint8: + NPresort::EncodeUnsignedInt(out, value.template Get<ui8>(), Desc); + break; + case NUdf::EDataSlot::Uint16: + case NUdf::EDataSlot::Date: + NPresort::EncodeUnsignedInt(out, value.template Get<ui16>(), Desc); + break; + case NUdf::EDataSlot::Uint32: + case NUdf::EDataSlot::Datetime: + NPresort::EncodeUnsignedInt(out, value.template Get<ui32>(), Desc); + break; + case NUdf::EDataSlot::Uint64: + case NUdf::EDataSlot::Timestamp: + NPresort::EncodeUnsignedInt(out, value.template Get<ui64>(), Desc); + break; + case NUdf::EDataSlot::Int8: + NPresort::EncodeSignedInt(out, value.template Get<i8>(), Desc); + break; + case NUdf::EDataSlot::Int16: + NPresort::EncodeSignedInt(out, value.template Get<i16>(), Desc); + break; + case NUdf::EDataSlot::Int32: + NPresort::EncodeSignedInt(out, value.template Get<i32>(), Desc); + break; + case NUdf::EDataSlot::Int64: + case NUdf::EDataSlot::Interval: + NPresort::EncodeSignedInt(out, value.template Get<i64>(), Desc); + break; + case NUdf::EDataSlot::Float: + NPresort::EncodeFloating(out, value.template Get<float>(), Desc); + break; + case NUdf::EDataSlot::Double: + NPresort::EncodeFloating(out, value.template Get<double>(), Desc); + break; + case NUdf::EDataSlot::String: + case NUdf::EDataSlot::Utf8: { + auto strRef = value.AsStringRef(); + NPresort::EncodeString(out, TStringBuf(strRef.Data(), strRef.Size()), Desc); + break; + } + default: + MKQL_ENSURE(false, TStringBuilder() << "Unknown slot: " << setting.Slot); + } + } + } + + void SetError(const TString& err) { + MKQL_ENSURE(false, TStringBuilder() << "Presort decoding error: " << err); + } + + void SetUnsignedInt(ui64 value) { + const auto& setting = Settings[Current++]; + switch (setting.Slot) { + case NUdf::EDataSlot::Bool: + Items[setting.Index] = NUdf::TUnboxedValuePod(value != 0); + break; + case NUdf::EDataSlot::Uint8: + Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui8>(value)); + break; + case NUdf::EDataSlot::Uint16: + case NUdf::EDataSlot::Date: + Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui16>(value)); + break; + case NUdf::EDataSlot::Uint32: + case NUdf::EDataSlot::Datetime: + Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui32>(value)); + break; + case NUdf::EDataSlot::Uint64: + case NUdf::EDataSlot::Timestamp: + Items[setting.Index] = NUdf::TUnboxedValuePod(value); + break; + default: + MKQL_ENSURE(false, TStringBuilder() << "Unknown slot: " << setting.Slot); + } + } + + void SetSignedInt(i64 value) { + const auto& setting = Settings[Current++]; + switch (setting.Slot) { + case NUdf::EDataSlot::Int8: + Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i8>(value)); + break; + case NUdf::EDataSlot::Int16: + Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i16>(value)); + break; + case NUdf::EDataSlot::Int32: + Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i32>(value)); + break; + case NUdf::EDataSlot::Int64: + case NUdf::EDataSlot::Interval: + Items[setting.Index] = NUdf::TUnboxedValuePod(value); + break; + default: + MKQL_ENSURE(false, "Unknown slot: " << setting.Slot); + } + } + + void SetFloat(float value) { + Items[Settings[Current++].Index] = NUdf::TUnboxedValuePod(value); + } + + void SetDouble(double value) { + Items[Settings[Current++].Index] = NUdf::TUnboxedValuePod(value); + } + + void SetString(const TString& value) { Items[Settings[Current++].Index] = MakeString(NUdf::TStringRef(value.data(), value.size())); - } - - void SetOptional(bool) {} -}; - -template <typename T> + } + + void SetOptional(bool) {} +}; + +template <typename T> NUdf::TUnboxedValue RandomValue() { - return NUdf::TUnboxedValuePod(RandomNumber<T>()); -} - -template <> + return NUdf::TUnboxedValuePod(RandomNumber<T>()); +} + +template <> NUdf::TUnboxedValue RandomValue<char*>() { - auto length = RandomNumber<ui64>(64); + auto length = RandomNumber<ui64>(64); return MakeStringNotFilled(length); -} - -template <typename T, NUdf::EDataSlot Slot, bool Desc> -std::pair<ui64, ui64> MeasureOld() { - constexpr size_t count = 1000; - constexpr size_t rowCount = 100000; - - TScopedAlloc alloc; - TMemoryUsageInfo memInfo("Memory"); - - TVector<NUdf::TUnboxedValue> values; - TVector<TSettings> settings; - - for (ui32 i = 0; i < count; ++i) { +} + +template <typename T, NUdf::EDataSlot Slot, bool Desc> +std::pair<ui64, ui64> MeasureOld() { + constexpr size_t count = 1000; + constexpr size_t rowCount = 100000; + + TScopedAlloc alloc; + TMemoryUsageInfo memInfo("Memory"); + + TVector<NUdf::TUnboxedValue> values; + TVector<TSettings> settings; + + for (ui32 i = 0; i < count; ++i) { values.push_back(RandomValue<T>()); - settings.push_back({i, false, NUdf::TDataType<T>::Slot}); - } - - TSimpleTimer timer; - - TStringStream stream; - for (size_t n = 0; n < rowCount; ++n) { - stream.clear(); + settings.push_back({i, false, NUdf::TDataType<T>::Slot}); + } + + TSimpleTimer timer; + + TStringStream stream; + for (size_t n = 0; n < rowCount; ++n) { + stream.clear(); TPresortOps<Desc> ops{settings, values.begin()}; - ops.Encode(stream); - } - auto encodeTime = timer.Get().MicroSeconds(); - - auto rowSize = stream.Str().size(); - Cerr << "row size " << rowSize << ", row count " << rowCount << Endl; - Cerr << "encoding " << rowSize * rowCount * 1000000 / encodeTime << " bytes per sec (" - << encodeTime << " us)" << Endl; - - timer.Reset(); - for (size_t n = 0; n < rowCount; ++n) { + ops.Encode(stream); + } + auto encodeTime = timer.Get().MicroSeconds(); + + auto rowSize = stream.Str().size(); + Cerr << "row size " << rowSize << ", row count " << rowCount << Endl; + Cerr << "encoding " << rowSize * rowCount * 1000000 / encodeTime << " bytes per sec (" + << encodeTime << " us)" << Endl; + + timer.Reset(); + for (size_t n = 0; n < rowCount; ++n) { TPresortOps<Desc> ops{settings, values.begin()}; - auto str = stream.Str(); - NPresort::Decode(ops, TStringBuf(str.data(), str.size())); - } - auto decodeTime = timer.Get().MicroSeconds(); - - Cerr << "decoding " << rowSize * rowCount * 1000000 / decodeTime << " bytes per sec (" - << decodeTime << " us)" << Endl; - Cerr << Endl; - - return std::make_pair(encodeTime, decodeTime); -} - -template <typename T, NUdf::EDataSlot Slot, bool Desc> -std::pair<ui64, ui64> MeasureNew() { - constexpr size_t count = 1000; - constexpr size_t rowCount = 100000; - - TScopedAlloc alloc; - TMemoryUsageInfo memInfo("Memory"); - - TVector<NUdf::TUnboxedValuePod> values; - TPresortEncoder encoder; + auto str = stream.Str(); + NPresort::Decode(ops, TStringBuf(str.data(), str.size())); + } + auto decodeTime = timer.Get().MicroSeconds(); + + Cerr << "decoding " << rowSize * rowCount * 1000000 / decodeTime << " bytes per sec (" + << decodeTime << " us)" << Endl; + Cerr << Endl; + + return std::make_pair(encodeTime, decodeTime); +} + +template <typename T, NUdf::EDataSlot Slot, bool Desc> +std::pair<ui64, ui64> MeasureNew() { + constexpr size_t count = 1000; + constexpr size_t rowCount = 100000; + + TScopedAlloc alloc; + TMemoryUsageInfo memInfo("Memory"); + + TVector<NUdf::TUnboxedValuePod> values; + TPresortEncoder encoder; TPresortDecoder decoder; - - for (size_t i = 0; i < count; ++i) { + + for (size_t i = 0; i < count; ++i) { values.push_back(RandomValue<T>()); - encoder.AddType(Slot, false, Desc); - decoder.AddType(Slot, false, Desc); - } - - TSimpleTimer timer; - TStringBuf buffer; - - for (size_t n = 0; n < rowCount; ++n) { - encoder.Start(); - for (size_t i = 0; i < count; ++i) { - encoder.Encode(values[i]); - } - buffer = encoder.Finish(); - } - auto encodeTime = timer.Get().MicroSeconds(); - - auto rowSize = buffer.size(); - - Cerr << "row size " << rowSize << ", row count " << rowCount << Endl; - Cerr << "encoding " << rowSize * rowCount * 1000000 / encodeTime << " bytes per sec (" - << encodeTime << " us)" << Endl; - - timer.Reset(); - for (size_t n = 0; n < rowCount; ++n) { - decoder.Start(buffer); - for (size_t i = 0; i < count; ++i) { - decoder.Decode(); - } - encoder.Finish(); - } - auto decodeTime = timer.Get().MicroSeconds(); - - Cerr << "decoding " << rowSize * rowCount * 1000000 / decodeTime << " bytes per sec (" - << decodeTime << " us)" << Endl; - Cerr << Endl; - - return std::make_pair(encodeTime, decodeTime); -} - -template <typename T, NUdf::EDataSlot Slot, bool Desc> -void Compare() { - auto newTimes = MeasureNew<T, Slot, Desc>(); - auto oldTimes = MeasureOld<T, Slot, Desc>(); - - Cerr << "encoding speedup " << (double)oldTimes.first / (double)newTimes.first << Endl; - Cerr << "decoding speedup " << (double)oldTimes.second / (double)newTimes.second << Endl; - Cerr << "--------" << Endl << Endl; -} - -template <typename T, NUdf::EDataSlot Slot> -void CompareType(const char* type) { - Cerr << type << Endl; - Compare<T, Slot, false>(); - - Cerr << type << " desc" << Endl; - Compare<T, Slot, true>(); -} - -} - -int main(int, char**) { - CompareType<bool, NUdf::EDataSlot::Bool>("bool"); - CompareType<ui8, NUdf::EDataSlot::Uint8>("ui8"); - CompareType<ui16, NUdf::EDataSlot::Uint16>("ui16"); - CompareType<ui32, NUdf::EDataSlot::Uint32>("ui32"); - CompareType<ui64, NUdf::EDataSlot::Uint64>("ui64"); - CompareType<float, NUdf::EDataSlot::Float>("float"); - CompareType<double, NUdf::EDataSlot::Double>("double"); - CompareType<char*, NUdf::EDataSlot::String>("string"); - - return 0; -} + encoder.AddType(Slot, false, Desc); + decoder.AddType(Slot, false, Desc); + } + + TSimpleTimer timer; + TStringBuf buffer; + + for (size_t n = 0; n < rowCount; ++n) { + encoder.Start(); + for (size_t i = 0; i < count; ++i) { + encoder.Encode(values[i]); + } + buffer = encoder.Finish(); + } + auto encodeTime = timer.Get().MicroSeconds(); + + auto rowSize = buffer.size(); + + Cerr << "row size " << rowSize << ", row count " << rowCount << Endl; + Cerr << "encoding " << rowSize * rowCount * 1000000 / encodeTime << " bytes per sec (" + << encodeTime << " us)" << Endl; + + timer.Reset(); + for (size_t n = 0; n < rowCount; ++n) { + decoder.Start(buffer); + for (size_t i = 0; i < count; ++i) { + decoder.Decode(); + } + encoder.Finish(); + } + auto decodeTime = timer.Get().MicroSeconds(); + + Cerr << "decoding " << rowSize * rowCount * 1000000 / decodeTime << " bytes per sec (" + << decodeTime << " us)" << Endl; + Cerr << Endl; + + return std::make_pair(encodeTime, decodeTime); +} + +template <typename T, NUdf::EDataSlot Slot, bool Desc> +void Compare() { + auto newTimes = MeasureNew<T, Slot, Desc>(); + auto oldTimes = MeasureOld<T, Slot, Desc>(); + + Cerr << "encoding speedup " << (double)oldTimes.first / (double)newTimes.first << Endl; + Cerr << "decoding speedup " << (double)oldTimes.second / (double)newTimes.second << Endl; + Cerr << "--------" << Endl << Endl; +} + +template <typename T, NUdf::EDataSlot Slot> +void CompareType(const char* type) { + Cerr << type << Endl; + Compare<T, Slot, false>(); + + Cerr << type << " desc" << Endl; + Compare<T, Slot, true>(); +} + +} + +int main(int, char**) { + CompareType<bool, NUdf::EDataSlot::Bool>("bool"); + CompareType<ui8, NUdf::EDataSlot::Uint8>("ui8"); + CompareType<ui16, NUdf::EDataSlot::Uint16>("ui16"); + CompareType<ui32, NUdf::EDataSlot::Uint32>("ui32"); + CompareType<ui64, NUdf::EDataSlot::Uint64>("ui64"); + CompareType<float, NUdf::EDataSlot::Float>("float"); + CompareType<double, NUdf::EDataSlot::Double>("double"); + CompareType<char*, NUdf::EDataSlot::String>("string"); + + return 0; +} diff --git a/ydb/library/yql/minikql/perf/presort/ya.make b/ydb/library/yql/minikql/perf/presort/ya.make index f30b3c4ba2..148198f13d 100644 --- a/ydb/library/yql/minikql/perf/presort/ya.make +++ b/ydb/library/yql/minikql/perf/presort/ya.make @@ -1,20 +1,20 @@ -PROGRAM() - +PROGRAM() + OWNER(monster) - -PEERDIR( + +PEERDIR( library/cpp/presort ydb/library/yql/minikql ydb/library/yql/minikql/computation ydb/library/yql/minikql/invoke_builtins ydb/library/yql/public/udf ydb/library/yql/public/udf/service/exception_policy -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - presort.cpp -) - -END() +SRCS( + presort.cpp +) + +END() diff --git a/ydb/library/yql/mount/lib/yql/aggregate.yql b/ydb/library/yql/mount/lib/yql/aggregate.yql index 049132c1b3..a1c231f339 100644 --- a/ydb/library/yql/mount/lib/yql/aggregate.yql +++ b/ydb/library/yql/mount/lib/yql/aggregate.yql @@ -217,117 +217,117 @@ (return (AggregationTraits (ListItemType list_type) init update save load merge finish (Null))) )))) -# list_type:type count:ui32 -# doesn't support optional values -(let set_traits_factory_raw (lambda '(list_type count) (block '( - (let value_type (ListItemType list_type)) - (let serialized_type (TupleType (DataType 'Uint32) (ListType value_type))) - - (let UdfSetCreate (Udf 'Set.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type))) - - (let resource_type (TypeOf (Apply UdfSetCreate (InstanceOf value_type) (Uint32 '0)))) - - (let UdfSetAddValue (Udf 'Set.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type))) - (let UdfSetMerge (Udf 'Set.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type))) - (let UdfSetSerialize (Udf 'Set.Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type))) - (let UdfSetDeserialize (Udf 'Set.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type))) - (let UdfSetGetResult (Udf 'Set.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type))) - - (let init (lambda '(value parent) (NamedApply UdfSetCreate '(value count) (AsStruct) (DependsOn parent)))) - (let update (lambda '(value state parent) (NamedApply UdfSetAddValue '(state value) (AsStruct) (DependsOn parent)))) - (let save (lambda '(state) (Apply UdfSetSerialize state))) - (let load (lambda '(state) (Apply UdfSetDeserialize state))) - (let merge (lambda '(one two) (Apply UdfSetMerge one two))) - (let finish (lambda '(state) (Apply UdfSetGetResult state))) - +# list_type:type count:ui32 +# doesn't support optional values +(let set_traits_factory_raw (lambda '(list_type count) (block '( + (let value_type (ListItemType list_type)) + (let serialized_type (TupleType (DataType 'Uint32) (ListType value_type))) + + (let UdfSetCreate (Udf 'Set.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type))) + + (let resource_type (TypeOf (Apply UdfSetCreate (InstanceOf value_type) (Uint32 '0)))) + + (let UdfSetAddValue (Udf 'Set.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type))) + (let UdfSetMerge (Udf 'Set.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type))) + (let UdfSetSerialize (Udf 'Set.Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type))) + (let UdfSetDeserialize (Udf 'Set.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type))) + (let UdfSetGetResult (Udf 'Set.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type))) + + (let init (lambda '(value parent) (NamedApply UdfSetCreate '(value count) (AsStruct) (DependsOn parent)))) + (let update (lambda '(value state parent) (NamedApply UdfSetAddValue '(state value) (AsStruct) (DependsOn parent)))) + (let save (lambda '(state) (Apply UdfSetSerialize state))) + (let load (lambda '(state) (Apply UdfSetDeserialize state))) + (let merge (lambda '(one two) (Apply UdfSetMerge one two))) + (let finish (lambda '(state) (Apply UdfSetGetResult state))) + (return (AggregationTraits value_type init update save load merge finish (EmptyList))) -)))) - +)))) + # list_type:type n:ui32 buffer:ui32 # doesn't support optional values (let topfreq_traits_factory_raw (lambda '(list_type n buffer) (block '( (let value_type (ListItemType list_type)) - (let vector_element_type (TupleType (DataType 'Uint64) value_type)) - (let serialized_type (TupleType (DataType 'Uint32) (DataType 'Uint32) (ListType vector_element_type))) - - (let UdfTopFreqCreate (Udf 'TopFreq.TopFreq_Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type))) - - (let resource_type (TypeOf (Apply UdfTopFreqCreate (InstanceOf value_type) (Uint32 '1)))) - - (let UdfTopFreqAddValue (Udf 'TopFreq.TopFreq_AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type))) - (let UdfTopFreqMerge (Udf 'TopFreq.TopFreq_Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type))) - (let UdfTopFreqSerialize (Udf 'TopFreq.TopFreq_Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type))) - (let UdfTopFreqDeserialize (Udf 'TopFreq.TopFreq_Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type))) - (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type))) - + (let vector_element_type (TupleType (DataType 'Uint64) value_type)) + (let serialized_type (TupleType (DataType 'Uint32) (DataType 'Uint32) (ListType vector_element_type))) + + (let UdfTopFreqCreate (Udf 'TopFreq.TopFreq_Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type))) + + (let resource_type (TypeOf (Apply UdfTopFreqCreate (InstanceOf value_type) (Uint32 '1)))) + + (let UdfTopFreqAddValue (Udf 'TopFreq.TopFreq_AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type))) + (let UdfTopFreqMerge (Udf 'TopFreq.TopFreq_Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type))) + (let UdfTopFreqSerialize (Udf 'TopFreq.TopFreq_Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type))) + (let UdfTopFreqDeserialize (Udf 'TopFreq.TopFreq_Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type))) + (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type))) + (let init (lambda '(value parent) (NamedApply UdfTopFreqCreate '(value buffer) (AsStruct) (DependsOn parent)))) (let update (lambda '(value state parent) (NamedApply UdfTopFreqAddValue '(state value) (AsStruct) (DependsOn parent)))) (let save (lambda '(state) (Apply UdfTopFreqSerialize state))) (let load (lambda '(state) (Apply UdfTopFreqDeserialize state))) (let merge (lambda '(one two) (Apply UdfTopFreqMerge one two))) (let finish (lambda '(state) (Apply UdfTopFreqGet state n))) - + (return (AggregationTraits value_type init update save load merge finish (EmptyList))) )))) -# list_type:type extractor:lambda count:ui32 is_top:atom -# doesn't support optional values -(let top_traits_factory_raw (lambda '(list_type extractor count is_top) (block '( - (let value_type (ListItemType list_type)) - (let serialized_type (TupleType (DataType 'Uint32) (ListType value_type))) - (let type_config (Combine '0 is_top)) - - (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type) type_config)) - - (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf value_type) (Uint32 '0)))) - - (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type) type_config)) - (let UdfTopMerge (Udf 'Top.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type) type_config)) - (let UdfTopSerialize (Udf 'Top.Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config)) - (let UdfTopDeserialize (Udf 'Top.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type) type_config)) - (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config)) - - (let init (lambda '(value parent) (NamedApply UdfTopCreate '(value count) (AsStruct) (DependsOn parent)))) - (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state value) (AsStruct) (DependsOn parent)))) - (let save (lambda '(state) (Apply UdfTopSerialize state))) - (let load (lambda '(state) (Apply UdfTopDeserialize state))) - (let merge (lambda '(one two) (Apply UdfTopMerge one two))) - (let finish (lambda '(state) (Apply UdfTopGetResult state))) - +# list_type:type extractor:lambda count:ui32 is_top:atom +# doesn't support optional values +(let top_traits_factory_raw (lambda '(list_type extractor count is_top) (block '( + (let value_type (ListItemType list_type)) + (let serialized_type (TupleType (DataType 'Uint32) (ListType value_type))) + (let type_config (Combine '0 is_top)) + + (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type) type_config)) + + (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf value_type) (Uint32 '0)))) + + (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type) type_config)) + (let UdfTopMerge (Udf 'Top.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type) type_config)) + (let UdfTopSerialize (Udf 'Top.Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config)) + (let UdfTopDeserialize (Udf 'Top.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type) type_config)) + (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config)) + + (let init (lambda '(value parent) (NamedApply UdfTopCreate '(value count) (AsStruct) (DependsOn parent)))) + (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state value) (AsStruct) (DependsOn parent)))) + (let save (lambda '(state) (Apply UdfTopSerialize state))) + (let load (lambda '(state) (Apply UdfTopDeserialize state))) + (let merge (lambda '(one two) (Apply UdfTopMerge one two))) + (let finish (lambda '(state) (Apply UdfTopGetResult state))) + (return (AggregationTraits value_type init update save load merge finish (EmptyList))) -)))) - -# list_type:type key_extractor:lambda payload_extractor:lambda count:ui32 is_top:atom -# doesn't support optional values -(let top_by_traits_factory_raw (lambda '(list_type key_extractor payload_extractor count is_top) (block '( - (let value_type (ListItemType list_type)) - (let key_type (TypeOf (Apply key_extractor (InstanceOf value_type)))) - (let payload_type (TypeOf (Apply payload_extractor (InstanceOf value_type)))) - (let key_payload_type (TupleType key_type payload_type)) - (let serialized_type (TupleType (DataType 'Uint32) (ListType key_payload_type))) - (let type_config (Combine '1 is_top)) - - (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType key_type payload_type (DataType 'Uint32)) (StructType) key_payload_type) type_config)) - - (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf key_type) (InstanceOf payload_type) (Uint32 '0)))) - - (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type key_type payload_type) (StructType) key_payload_type) type_config)) - (let UdfTopMerge (Udf 'Top.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) key_payload_type) type_config)) - (let UdfTopSerialize (Udf 'Top.Serialize (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config)) - (let UdfTopDeserialize (Udf 'Top.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) key_payload_type) type_config)) - (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config)) - - (let init (lambda '(value parent) (NamedApply UdfTopCreate '((Apply key_extractor value) (Apply payload_extractor value) count) (AsStruct) (DependsOn parent)))) - (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state (Apply key_extractor value) (Apply payload_extractor value)) (AsStruct) (DependsOn parent)))) - (let save (lambda '(state) (Apply UdfTopSerialize state))) - (let load (lambda '(state) (Apply UdfTopDeserialize state))) - (let merge (lambda '(one two) (Apply UdfTopMerge one two))) - (let finish (lambda '(state) (Apply UdfTopGetResult state))) - +)))) + +# list_type:type key_extractor:lambda payload_extractor:lambda count:ui32 is_top:atom +# doesn't support optional values +(let top_by_traits_factory_raw (lambda '(list_type key_extractor payload_extractor count is_top) (block '( + (let value_type (ListItemType list_type)) + (let key_type (TypeOf (Apply key_extractor (InstanceOf value_type)))) + (let payload_type (TypeOf (Apply payload_extractor (InstanceOf value_type)))) + (let key_payload_type (TupleType key_type payload_type)) + (let serialized_type (TupleType (DataType 'Uint32) (ListType key_payload_type))) + (let type_config (Combine '1 is_top)) + + (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType key_type payload_type (DataType 'Uint32)) (StructType) key_payload_type) type_config)) + + (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf key_type) (InstanceOf payload_type) (Uint32 '0)))) + + (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type key_type payload_type) (StructType) key_payload_type) type_config)) + (let UdfTopMerge (Udf 'Top.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) key_payload_type) type_config)) + (let UdfTopSerialize (Udf 'Top.Serialize (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config)) + (let UdfTopDeserialize (Udf 'Top.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) key_payload_type) type_config)) + (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config)) + + (let init (lambda '(value parent) (NamedApply UdfTopCreate '((Apply key_extractor value) (Apply payload_extractor value) count) (AsStruct) (DependsOn parent)))) + (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state (Apply key_extractor value) (Apply payload_extractor value)) (AsStruct) (DependsOn parent)))) + (let save (lambda '(state) (Apply UdfTopSerialize state))) + (let load (lambda '(state) (Apply UdfTopDeserialize state))) + (let merge (lambda '(one two) (Apply UdfTopMerge one two))) + (let finish (lambda '(state) (Apply UdfTopGetResult state))) + (return (AggregationTraits value_type init update save load merge finish (List (ListType payload_type)))) -)))) - - +)))) + + # list_type:type n:double # doesn't support optional values (let percentile_traits_factory_raw (lambda '(list_type n) (block '( @@ -578,10 +578,10 @@ # support optional values (let percentile_traits_factory_opt (lambda '(list_type n) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply percentile_traits_factory_raw list_type n))))) -# list_type:type count:ui32 -# support optional values +# list_type:type count:ui32 +# support optional values (let set_traits_factory_opt (lambda '(list_type count) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply set_traits_factory_raw list_type count))))) - + # list_type:type n:ui32 buffer:ui32 # support optional values (let topfreq_traits_factory_opt (lambda '(list_type n buffer) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply topfreq_traits_factory_raw list_type n buffer))))) @@ -655,10 +655,10 @@ # support optional values (let percentile_traits_factory (lambda '(list_type extractor n) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply percentile_traits_factory_opt list_type n))))) -# list_type:type extractor:lambda count:ui32 -# support optional values -(let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count))))) - +# list_type:type extractor:lambda count:ui32 +# support optional values +(let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count))))) + # list_type:type extractor:lambda n:ui32 buffer:ui32 # support optional values (let topfreq_traits_factory (lambda '(list_type extractor n buffer) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply topfreq_traits_factory_opt list_type n buffer))))) @@ -689,10 +689,10 @@ (let compare_traits_factory_list_raw (lambda '(list_type compare first second limit) (block '( (let key_type (TypeOf (Apply first (InstanceOf (ListItemType list_type))))) (let init (lambda '(row) '((AsList (Apply second row)) (Apply first row)))) - (let update (lambda '(row state) (If (AggrEquals (Apply first row) (Nth state '1)) '((Take (Insert (Nth state '0) (Apply second row)) limit) (Nth state '1)) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state)))) + (let update (lambda '(row state) (If (AggrEquals (Apply first row) (Nth state '1)) '((Take (Insert (Nth state '0) (Apply second row)) limit) (Nth state '1)) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state)))) (let save (lambda '(state) state)) (let load (lambda '(state) state)) - (let merge (lambda '(one two) (If (AggrEquals (Nth one '1) (Nth two '1)) '((Take (Extend (Nth one '0) (Nth two '0)) limit) (Nth one '1)) (If (Apply compare (Nth one '1) (Nth two '1)) one two)))) + (let merge (lambda '(one two) (If (AggrEquals (Nth one '1) (Nth two '1)) '((Take (Extend (Nth one '0) (Nth two '0)) limit) (Nth one '1)) (If (Apply compare (Nth one '1) (Nth two '1)) one two)))) (let finish (lambda '(state) (Nth state '0))) (return (AggregationTraits (ListItemType list_type) init update save load merge finish (Null))) )))) @@ -849,8 +849,8 @@ ))) # deprecated -(let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit))) -(let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit))) +(let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit))) +(let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit))) (let min_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess one two)) first second (Void)))) (let max_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess two one)) first second (Void)))) @@ -867,27 +867,27 @@ (let pl (Apply payload value)) (return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl))))))))))) -(let top_bottom_traits_factory (lambda '(list_type extractor count is_top) - (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) +(let top_bottom_traits_factory (lambda '(list_type extractor count is_top) + (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) - (Apply top_traits_factory_raw list_type extractor count is_top))))))) - -(let top_traits_factory (lambda '(list_type extractor count) - (Apply top_bottom_traits_factory list_type extractor count '1))) - -(let bottom_traits_factory (lambda '(list_type extractor count) - (Apply top_bottom_traits_factory list_type extractor count '0))) - -(let top_bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count is_top) + (Apply top_traits_factory_raw list_type extractor count is_top))))))) + +(let top_traits_factory (lambda '(list_type extractor count) + (Apply top_bottom_traits_factory list_type extractor count '1))) + +(let bottom_traits_factory (lambda '(list_type extractor count) + (Apply top_bottom_traits_factory list_type extractor count '0))) + +(let top_bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count is_top) (Apply list_by_traits_factory_opt_parent (lambda '(list_type key_extractor payload_extractor) - (Apply top_by_traits_factory_raw list_type key_extractor payload_extractor count is_top)) list_type key_extractor payload_extractor))) - -(let top_by_traits_factory (lambda '(list_type key_extractor payload_extractor count) - (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '1))) - -(let bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count) - (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '0))) - + (Apply top_by_traits_factory_raw list_type key_extractor payload_extractor count is_top)) list_type key_extractor payload_extractor))) + +(let top_by_traits_factory (lambda '(list_type key_extractor payload_extractor count) + (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '1))) + +(let bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count) + (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '0))) + (export min_traits_factory) (export max_traits_factory) (export sum_traits_factory) @@ -928,12 +928,12 @@ (export histogram_logarithmic_traits_factory) (export udaf_traits_factory) (export percentile_traits_factory) -(export set_traits_factory) +(export set_traits_factory) (export topfreq_traits_factory) (export hyperloglog_traits_factory) -(export top_traits_factory) -(export bottom_traits_factory) -(export top_by_traits_factory) -(export bottom_by_traits_factory) +(export top_traits_factory) +(export bottom_traits_factory) +(export top_by_traits_factory) +(export bottom_by_traits_factory) ) diff --git a/ydb/library/yql/mount/lib/yql/window.yql b/ydb/library/yql/mount/lib/yql/window.yql index 627d31a153..371f4c687c 100644 --- a/ydb/library/yql/mount/lib/yql/window.yql +++ b/ydb/library/yql/mount/lib/yql/window.yql @@ -224,89 +224,89 @@ # list_type:type n:ui32 buffer:ui32 # doesn't support optional values -(let set_traits_factory_raw (lambda '(list_type count) (block '( - (let value_type (ListItemType list_type)) - - (let UdfSetCreate (Udf 'Set.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type))) - - (let resource_type (TypeOf (Apply UdfSetCreate (InstanceOf value_type) (Uint32 '0)))) - - (let UdfSetAddValue (Udf 'Set.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type))) - (let UdfSetGetResult (Udf 'Set.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type))) - - (let init (lambda '(value parent) (NamedApply UdfSetCreate '(value count) (AsStruct) (DependsOn parent)))) - (let update (lambda '(value state parent) (NamedApply UdfSetAddValue '(state value) (AsStruct) (DependsOn parent)))) - (let shift (lambda '(value state) (Void))) - (let current (lambda '(state) (Apply UdfSetGetResult state))) - +(let set_traits_factory_raw (lambda '(list_type count) (block '( + (let value_type (ListItemType list_type)) + + (let UdfSetCreate (Udf 'Set.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type))) + + (let resource_type (TypeOf (Apply UdfSetCreate (InstanceOf value_type) (Uint32 '0)))) + + (let UdfSetAddValue (Udf 'Set.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type))) + (let UdfSetGetResult (Udf 'Set.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type))) + + (let init (lambda '(value parent) (NamedApply UdfSetCreate '(value count) (AsStruct) (DependsOn parent)))) + (let update (lambda '(value state parent) (NamedApply UdfSetAddValue '(state value) (AsStruct) (DependsOn parent)))) + (let shift (lambda '(value state) (Void))) + (let current (lambda '(state) (Apply UdfSetGetResult state))) + (return (WindowTraits value_type init update shift current (EmptyList))) -)))) - -# list_type:type n:ui32 buffer:ui32 -# doesn't support optional values +)))) + +# list_type:type n:ui32 buffer:ui32 +# doesn't support optional values (let topfreq_traits_factory_raw (lambda '(list_type n buffer) (block '( (let value_type (ListItemType list_type)) - - (let UdfTopFreqCreate (Udf 'TopFreq.TopFreq_Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type))) - - (let resource_type (TypeOf (Apply UdfTopFreqCreate (InstanceOf value_type) (Uint32 '0)))) - - (let UdfTopFreqAddValue (Udf 'TopFreq.TopFreq_AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type))) - (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type))) - + + (let UdfTopFreqCreate (Udf 'TopFreq.TopFreq_Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type))) + + (let resource_type (TypeOf (Apply UdfTopFreqCreate (InstanceOf value_type) (Uint32 '0)))) + + (let UdfTopFreqAddValue (Udf 'TopFreq.TopFreq_AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type))) + (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type))) + (let init (lambda '(value parent) (NamedApply UdfTopFreqCreate '(value buffer) (AsStruct) (DependsOn parent)))) (let update (lambda '(value state parent) (NamedApply UdfTopFreqAddValue '(state value) (AsStruct) (DependsOn parent)))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) (Apply UdfTopFreqGet state n))) - + (return (WindowTraits value_type init update shift current (EmptyList))) )))) -# list_type:type extractor:lambda count:ui32 is_top:atom -# doesn't support optional values -(let top_traits_factory_raw (lambda '(list_type extractor count is_top) (block '( - (let value_type (ListItemType list_type)) - (let type_config (Combine '0 is_top)) - - (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type) type_config)) - - (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf value_type) (Uint32 '0)))) - - (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type) type_config)) - (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config)) - - (let init (lambda '(value parent) (NamedApply UdfTopCreate '(value count) (AsStruct) (DependsOn parent)))) - (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state value) (AsStruct) (DependsOn parent)))) - (let shift (lambda '(value state) (Void))) - (let current (lambda '(state) (Apply UdfTopGetResult state))) - +# list_type:type extractor:lambda count:ui32 is_top:atom +# doesn't support optional values +(let top_traits_factory_raw (lambda '(list_type extractor count is_top) (block '( + (let value_type (ListItemType list_type)) + (let type_config (Combine '0 is_top)) + + (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type) type_config)) + + (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf value_type) (Uint32 '0)))) + + (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type) type_config)) + (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config)) + + (let init (lambda '(value parent) (NamedApply UdfTopCreate '(value count) (AsStruct) (DependsOn parent)))) + (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state value) (AsStruct) (DependsOn parent)))) + (let shift (lambda '(value state) (Void))) + (let current (lambda '(state) (Apply UdfTopGetResult state))) + (return (WindowTraits value_type init update shift current (EmptyList))) -)))) - -# list_type:type key_extractor:lambda payload_extractor:lambda count:ui32 is_top:atom -# doesn't support optional values -(let top_by_traits_factory_raw (lambda '(list_type key_extractor payload_extractor count is_top) (block '( - (let value_type (ListItemType list_type)) - (let key_type (TypeOf (Apply key_extractor (InstanceOf value_type)))) - (let payload_type (TypeOf (Apply payload_extractor (InstanceOf value_type)))) - (let key_payload_type (TupleType key_type payload_type)) - (let type_config (Combine '1 is_top)) - - (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType key_type payload_type (DataType 'Uint32)) (StructType) key_payload_type) type_config)) - - (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf key_type) (InstanceOf payload_type) (Uint32 '0)))) - - (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type key_type payload_type) (StructType) key_payload_type) type_config)) - (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config)) - - (let init (lambda '(value parent) (NamedApply UdfTopCreate '((Apply key_extractor value) (Apply payload_extractor value) count) (AsStruct) (DependsOn parent)))) - (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state (Apply key_extractor value) (Apply payload_extractor value)) (AsStruct) (DependsOn parent)))) - (let shift (lambda '(value state) (Void))) - (let current (lambda '(state) (Apply UdfTopGetResult state))) - +)))) + +# list_type:type key_extractor:lambda payload_extractor:lambda count:ui32 is_top:atom +# doesn't support optional values +(let top_by_traits_factory_raw (lambda '(list_type key_extractor payload_extractor count is_top) (block '( + (let value_type (ListItemType list_type)) + (let key_type (TypeOf (Apply key_extractor (InstanceOf value_type)))) + (let payload_type (TypeOf (Apply payload_extractor (InstanceOf value_type)))) + (let key_payload_type (TupleType key_type payload_type)) + (let type_config (Combine '1 is_top)) + + (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType key_type payload_type (DataType 'Uint32)) (StructType) key_payload_type) type_config)) + + (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf key_type) (InstanceOf payload_type) (Uint32 '0)))) + + (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type key_type payload_type) (StructType) key_payload_type) type_config)) + (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config)) + + (let init (lambda '(value parent) (NamedApply UdfTopCreate '((Apply key_extractor value) (Apply payload_extractor value) count) (AsStruct) (DependsOn parent)))) + (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state (Apply key_extractor value) (Apply payload_extractor value)) (AsStruct) (DependsOn parent)))) + (let shift (lambda '(value state) (Void))) + (let current (lambda '(state) (Apply UdfTopGetResult state))) + (return (WindowTraits value_type init update shift current (List (ListType payload_type)))) -)))) - +)))) + # list_type:type n:double # doesn't support optional values (let percentile_traits_factory_raw (lambda '(list_type n) (block '( @@ -550,10 +550,10 @@ # support optional values (let percentile_traits_factory_opt (lambda '(list_type n) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply percentile_traits_factory_raw list_type n))))) -# list_type:type count:ui32 -# support optional values +# list_type:type count:ui32 +# support optional values (let set_traits_factory_opt (lambda '(list_type count) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply set_traits_factory_raw list_type count))))) - + # list_type:type n:ui32 buffer:ui32 # support optional values (let topfreq_traits_factory_opt (lambda '(list_type n buffer) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply topfreq_traits_factory_raw list_type n buffer))))) @@ -630,10 +630,10 @@ # support optional values (let percentile_traits_factory (lambda '(list_type extractor n) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply percentile_traits_factory_opt list_type n))))) -# list_type:type extractor:lambda count:ui32 -# support optional values -(let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count))))) - +# list_type:type extractor:lambda count:ui32 +# support optional values +(let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count))))) + # list_type:type extractor:lambda n:ui32 buffer:ui32 # support optional values (let topfreq_traits_factory (lambda '(list_type extractor n buffer) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply topfreq_traits_factory_opt list_type n buffer))))) @@ -662,7 +662,7 @@ (let compare_traits_factory_list_raw (lambda '(list_type compare first second limit) (block '( (let key_type (TypeOf (Apply first (InstanceOf (ListItemType list_type))))) (let init (lambda '(row) '((AsList (Apply second row)) (Apply first row)))) - (let update (lambda '(row state) (If (AggrEquals (Apply first row) (Nth state '1)) '((Take (Insert (Nth state '0) (Apply second row)) limit) (Nth state '1)) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state)))) + (let update (lambda '(row state) (If (AggrEquals (Apply first row) (Nth state '1)) '((Take (Insert (Nth state '0) (Apply second row)) limit) (Nth state '1)) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state)))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) (Nth state '0))) (return (WindowTraits (ListItemType list_type) init update shift current (Null))) @@ -798,18 +798,18 @@ (let histogram_linear_traits_factory (lambda '(list_type value binsize minimum maximum) (Apply double_traits_factory_opt_parent (lambda '(list_type value binsize) (Apply histogram_linear_traits_factory_raw list_type value binsize minimum maximum)) list_type value binsize))) (let histogram_logarithmic_traits_factory (lambda '(list_type value binsize minimum maximum) (Apply double_traits_factory_opt_parent (lambda '(list_type value binsize) (Apply histogram_logarithmic_traits_factory_raw list_type value binsize minimum maximum)) list_type value binsize))) -(let compare_by_traits_factory_opt (lambda '(list_type compare first second limit) (Apply double_traits_factory_opt (lambda '(list_type first second) (Apply compare_traits_factory_raw list_type compare first second limit)) list_type first second))) -(let list_compare_by_traits_factory_opt (lambda '(list_type compare first second limit) (Apply double_traits_factory_opt (lambda '(list_type first second) (Apply compare_traits_factory_list_raw list_type compare first second limit)) list_type first second))) +(let compare_by_traits_factory_opt (lambda '(list_type compare first second limit) (Apply double_traits_factory_opt (lambda '(list_type first second) (Apply compare_traits_factory_raw list_type compare first second limit)) list_type first second))) +(let list_compare_by_traits_factory_opt (lambda '(list_type compare first second limit) (Apply double_traits_factory_opt (lambda '(list_type first second) (Apply compare_traits_factory_list_raw list_type compare first second limit)) list_type first second))) #deprecated (let compare_by_traits_factory (lambda '(list_type compare first second limit) (IfType limit (VoidType) - (lambda '() (Apply compare_by_traits_factory_opt list_type compare first second limit)) - (lambda '() (Apply list_compare_by_traits_factory_opt list_type compare first second limit)) + (lambda '() (Apply compare_by_traits_factory_opt list_type compare first second limit)) + (lambda '() (Apply list_compare_by_traits_factory_opt list_type compare first second limit)) ))) #deprecated -(let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit))) -(let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit))) +(let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit))) +(let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit))) (let min_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess one two)) first second (Void)))) (let max_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess two one)) first second (Void)))) @@ -832,27 +832,27 @@ (let udaf_traits_factory (lambda '(list_type extractor init update merge finish save load defval) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply udaf_traits_factory_opt list_type init update finish defval))))) -(let top_bottom_traits_factory (lambda '(list_type extractor count is_top) - (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) +(let top_bottom_traits_factory (lambda '(list_type extractor count is_top) + (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) - (Apply top_traits_factory_raw list_type extractor count is_top))))))) - -(let top_traits_factory (lambda '(list_type extractor count) - (Apply top_bottom_traits_factory list_type extractor count '1))) - -(let bottom_traits_factory (lambda '(list_type extractor count) - (Apply top_bottom_traits_factory list_type extractor count '0))) - -(let top_bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count is_top) + (Apply top_traits_factory_raw list_type extractor count is_top))))))) + +(let top_traits_factory (lambda '(list_type extractor count) + (Apply top_bottom_traits_factory list_type extractor count '1))) + +(let bottom_traits_factory (lambda '(list_type extractor count) + (Apply top_bottom_traits_factory list_type extractor count '0))) + +(let top_bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count is_top) (Apply list_by_traits_factory_opt_parent (lambda '(list_type key_extractor payload_extractor) - (Apply top_by_traits_factory_raw list_type key_extractor payload_extractor count is_top)) list_type key_extractor payload_extractor))) - -(let top_by_traits_factory (lambda '(list_type key_extractor payload_extractor count) - (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '1))) - -(let bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count) - (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '0))) - + (Apply top_by_traits_factory_raw list_type key_extractor payload_extractor count is_top)) list_type key_extractor payload_extractor))) + +(let top_by_traits_factory (lambda '(list_type key_extractor payload_extractor count) + (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '1))) + +(let bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count) + (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '0))) + (export min_traits_factory) (export max_traits_factory) (export sum_traits_factory) @@ -893,7 +893,7 @@ (export histogram_logarithmic_traits_factory) (export udwf_traits_factory) (export percentile_traits_factory) -(export set_traits_factory) +(export set_traits_factory) (export topfreq_traits_factory) (export hyperloglog_traits_factory) (export rank_traits_factory) @@ -903,9 +903,9 @@ (export first_value_ignore_nulls_traits_factory) (export last_value_ignore_nulls_traits_factory) (export udaf_traits_factory) -(export top_traits_factory) -(export bottom_traits_factory) -(export top_by_traits_factory) -(export bottom_by_traits_factory) +(export top_traits_factory) +(export bottom_traits_factory) +(export top_by_traits_factory) +(export bottom_by_traits_factory) ) diff --git a/ydb/library/yql/providers/common/codec/yql_codec.cpp b/ydb/library/yql/providers/common/codec/yql_codec.cpp index dea9c6350a..7587e188fa 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.cpp +++ b/ydb/library/yql/providers/common/codec/yql_codec.cpp @@ -51,22 +51,22 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& auto dataType = AS_TYPE(TDataType, type); switch (dataType->GetSchemeType()) { case NUdf::TDataType<bool>::Id: - writer.OnBooleanScalar(value.Get<bool>()); + writer.OnBooleanScalar(value.Get<bool>()); return; case NUdf::TDataType<i32>::Id: - writer.OnInt64Scalar(value.Get<i32>()); + writer.OnInt64Scalar(value.Get<i32>()); return; case NUdf::TDataType<ui32>::Id: - writer.OnUint64Scalar(value.Get<ui32>()); + writer.OnUint64Scalar(value.Get<ui32>()); return; case NUdf::TDataType<i64>::Id: - writer.OnInt64Scalar(value.Get<i64>()); + writer.OnInt64Scalar(value.Get<i64>()); return; case NUdf::TDataType<ui64>::Id: - writer.OnUint64Scalar(value.Get<ui64>()); + writer.OnUint64Scalar(value.Get<ui64>()); return; case NUdf::TDataType<ui8>::Id: - writer.OnUint64Scalar(value.Get<ui8>()); + writer.OnUint64Scalar(value.Get<ui8>()); return; case NUdf::TDataType<i8>::Id: writer.OnInt64Scalar(value.Get<i8>()); diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp index a2817179ae..dd1f162d78 100644 --- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp +++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp @@ -1568,10 +1568,10 @@ TMkqlCommonCallableCompiler::TShared::TShared() { AddCallable("HoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto stream = MkqlBuildExpr(node.Head(), ctx); - + const auto timeExtractor = [&](TRuntimeNode item) { return MkqlBuildLambda(*node.Child(1), ctx, {item}); - }; + }; const auto hop = MkqlBuildExpr(*node.Child(2), ctx); const auto interval = MkqlBuildExpr(*node.Child(3), ctx); const auto delay = MkqlBuildExpr(*node.Child(4), ctx); @@ -1581,24 +1581,24 @@ TMkqlCommonCallableCompiler::TShared::TShared() { }; const auto update = [&](TRuntimeNode item, TRuntimeNode state) { return MkqlBuildLambda(*node.Child(6), ctx, {item, state}); - }; + }; const auto save = node.Child(3)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) { return MkqlBuildLambda(*node.Child(7), ctx, {state}); - }; + }; const auto load = node.Child(4)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) { return MkqlBuildLambda(*node.Child(8), ctx, {state}); - }; + }; const auto merge = [&](TRuntimeNode state1, TRuntimeNode state2) { return MkqlBuildLambda(*node.Child(9), ctx, {state1, state2}); - }; + }; const auto finish = [&](TRuntimeNode state, TRuntimeNode time) { return MkqlBuildLambda(*node.Child(10), ctx, {state, time}); - }; - + }; + return ctx.ProgramBuilder.HoppingCore( stream, timeExtractor, init, update, save, load, merge, finish, hop, interval, delay); - }); - + }); + AddCallable("MultiHoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto stream = MkqlBuildExpr(node.Head(), ctx); diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index 1f6b9de339..8561fe2536 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -112,7 +112,7 @@ message TKikimrClusterConfig { repeated THostPort MessageBus = 3; optional TKikimrGrpcData Grpc = 4; optional string Database = 5; - optional uint32 TvmId = 6 [default = 0]; + optional uint32 TvmId = 6 [default = 0]; optional string Token = 7; optional string ProxyUrl = 8; optional string Location = 9; @@ -132,7 +132,7 @@ message TKikimrMvpProxyConfig { optional string Host = 1; optional uint32 Port = 2; optional string BasePath = 3; - optional uint32 TvmId = 4 [default = 0]; + optional uint32 TvmId = 4 [default = 0]; } message TKikimrMvpGatewayConfig { @@ -187,13 +187,13 @@ message TClickHouseGatewayConfig { repeated TAttr DefaultSettings = 2; } -///////////////////////////// RTMR ///////////////////////////// +///////////////////////////// RTMR ///////////////////////////// enum EYfArtifactType { AT_NONE = 0; AT_FILE = 1; AT_LAYER = 2; }; - + enum ERtmrOperationType { OT_NONE = 0; OT_LF_PARSE = 1; @@ -218,16 +218,16 @@ message TRtmrPqConsumerInfo { optional string Consumer = 2; } -message TRtmrClusterConfig { - optional string Name = 1; - optional bool Default = 2 [default = false]; - optional string Cluster = 3; - optional string RemoteName = 4; - optional string YfCluster = 5; - optional string S3TokenPath = 6; - optional string S3FileCachePath = 7; - optional uint32 TvmId = 8 [default = 0]; - optional string TvmSecretPath = 9; +message TRtmrClusterConfig { + optional string Name = 1; + optional bool Default = 2 [default = false]; + optional string Cluster = 3; + optional string RemoteName = 4; + optional string YfCluster = 5; + optional string S3TokenPath = 6; + optional string S3FileCachePath = 7; + optional uint32 TvmId = 8 [default = 0]; + optional string TvmSecretPath = 9; repeated string Users = 10; optional bool UploadViaYfClient = 11 [default = false]; optional string MdsTorrentUrl = 12; @@ -236,20 +236,20 @@ message TRtmrClusterConfig { repeated TRtmrPqConsumerInfo PqConsumerMapping = 15; optional uint32 MaxPqPartitions = 16; optional uint32 PreviewCollectTimeoutMs = 17; - repeated TAttr Settings = 101; -} - -message TRtmrGatewayConfig { - repeated TRtmrClusterConfig ClusterMapping = 1; - repeated TAttr DefaultSettings = 2; - optional string YqlRtmrDynLib = 3; // path to libyql-dynlib.so + repeated TAttr Settings = 101; +} + +message TRtmrGatewayConfig { + repeated TRtmrClusterConfig ClusterMapping = 1; + repeated TAttr DefaultSettings = 2; + optional string YqlRtmrDynLib = 3; // path to libyql-dynlib.so optional bool UseFakeYfUpload = 4 [default = false]; // use in tests only to speedup upload repeated string Artifacts = 5; /// default artifacts to put into every function repeated TRtmrOperationArtifactsInfo CommonArtifacts = 6; // common artifacts for all clusters optional uint32 MaxPqPartitions = 7 [default = 10]; optional uint32 PreviewCollectTimeoutMs = 8 [default = 2000]; -} - +} + ///////////////////////////// Pq ////////////////////////////// message TPqClusterConfig { @@ -331,9 +331,9 @@ message TS3GatewayConfig { repeated TAttr DefaultSettings = 100; } -///////////////////////////// Solomon ///////////////////////////// - -message TSolomonClusterConfig { +///////////////////////////// Solomon ///////////////////////////// + +message TSolomonClusterConfig { enum ESolomonClusterType { SCT_UNSPECIFIED = 0; SCT_SOLOMON = 1; @@ -345,23 +345,23 @@ message TSolomonClusterConfig { required string Cluster = 2; // FolderId for YC } - optional string Name = 1; - optional string Cluster = 2; + optional string Name = 1; + optional string Cluster = 2; optional bool UseSsl = 3; optional ESolomonClusterType ClusterType = 4 [default = SCT_SOLOMON]; optional string Token = 5; optional string ServiceAccountId = 6; optional string ServiceAccountIdSignature = 7; optional TShardPath Path = 8; - - repeated TAttr Settings = 100; -} - -message TSolomonGatewayConfig { - repeated TSolomonClusterConfig ClusterMapping = 1; - repeated TAttr DefaultSettings = 2; -} - + + repeated TAttr Settings = 100; +} + +message TSolomonGatewayConfig { + repeated TSolomonClusterConfig ClusterMapping = 1; + repeated TAttr DefaultSettings = 2; +} + message TFileStorageAdditionalConfig { repeated string AllowedUrlPatterns = 1; } @@ -439,11 +439,11 @@ message TGatewaysConfig { optional TYtGatewayConfig Yt = 1; optional TKikimrGatewayConfig Kikimr = 2; optional TClickHouseGatewayConfig ClickHouse = 3; - optional TRtmrGatewayConfig Rtmr = 4; + optional TRtmrGatewayConfig Rtmr = 4; optional TKikimrMvpGatewayConfig KikimrMvp = 5; optional TStatGatewayConfig Stat = 6; optional TChytGatewayConfig Chyt = 7; - optional TSolomonGatewayConfig Solomon = 8; + optional TSolomonGatewayConfig Solomon = 8; optional TFileStorageAdditionalConfig Fs = 9; optional TYqlCoreConfig YqlCore = 10; optional TPostgresqlGatewayConfig Postgresql = 11; diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp index 26cfc79ccd..8de3d57fd3 100644 --- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp +++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp @@ -82,12 +82,12 @@ IGraphTransformer& TDataProviderBase::GetConfigurationTransformer() { return NullTransformer_; } -TExprNode::TPtr TDataProviderBase::GetClusterInfo(const TString& cluster, TExprContext& ctx) { - Y_UNUSED(cluster); - Y_UNUSED(ctx); - return {}; -} - +TExprNode::TPtr TDataProviderBase::GetClusterInfo(const TString& cluster, TExprContext& ctx) { + Y_UNUSED(cluster); + Y_UNUSED(ctx); + return {}; +} + const THashMap<TString, TString>* TDataProviderBase::GetClusterTokens() { return nullptr; } diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h index bdef7492b9..242a39877e 100644 --- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h +++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h @@ -43,7 +43,7 @@ public: bool Initialize(TExprContext& ctx) override; IGraphTransformer& GetConfigurationTransformer() override; - TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override; + TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override; const THashMap<TString, TString>* GetClusterTokens() override; IGraphTransformer& GetIODiscoveryTransformer() override; IGraphTransformer& GetEpochsTransformer() override; diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index ae524b0d99..aa782ee93f 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -152,8 +152,8 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { TMaybeNode<TCoAtom> mode; TMaybeNode<TExprList> columns; TMaybeNode<TCoAtomList> primaryKey; - TMaybeNode<TCoAtomList> partitionBy; - TMaybeNode<TCoNameValueTupleList> orderBy; + TMaybeNode<TCoAtomList> partitionBy; + TMaybeNode<TCoNameValueTupleList> orderBy; TMaybeNode<TCoLambda> filter; TMaybeNode<TCoLambda> update; TVector<TCoNameValueTuple> other; @@ -176,12 +176,12 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { } else if (name == "primarykey") { YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>()); primaryKey = tuple.Value().Cast<TCoAtomList>(); - } else if (name == "partitionby") { - YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>()); - partitionBy = tuple.Value().Cast<TCoAtomList>(); - } else if (name == "orderby") { - YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); - orderBy = tuple.Value().Cast<TCoNameValueTupleList>(); + } else if (name == "partitionby") { + YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>()); + partitionBy = tuple.Value().Cast<TCoAtomList>(); + } else if (name == "orderby") { + YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); + orderBy = tuple.Value().Cast<TCoNameValueTupleList>(); } else if (name == "filter") { YQL_ENSURE(tuple.Value().Maybe<TCoLambda>()); filter = tuple.Value().Cast<TCoLambda>(); @@ -270,8 +270,8 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { ret.Mode = mode; ret.Columns = columns; ret.PrimaryKey = primaryKey; - ret.PartitionBy = partitionBy; - ret.OrderBy = orderBy; + ret.PartitionBy = partitionBy; + ret.OrderBy = orderBy; ret.Filter = filter; ret.Update = update; ret.Indexes = idx; diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h index e562c40ce7..9e80cfd135 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.h +++ b/ydb/library/yql/providers/common/provider/yql_provider.h @@ -36,8 +36,8 @@ struct TWriteTableSettings { NNodes::TMaybeNode<NNodes::TCoAtom> Mode; NNodes::TMaybeNode<NNodes::TExprList> Columns; NNodes::TMaybeNode<NNodes::TCoAtomList> PrimaryKey; - NNodes::TMaybeNode<NNodes::TCoAtomList> PartitionBy; - NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> OrderBy; + NNodes::TMaybeNode<NNodes::TCoAtomList> PartitionBy; + NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> OrderBy; NNodes::TMaybeNode<NNodes::TCoLambda> Filter; NNodes::TMaybeNode<NNodes::TCoLambda> Update; NNodes::TMaybeNode<NNodes::TCoIndexList> Indexes; diff --git a/ydb/library/yql/providers/config/yql_config_provider.cpp b/ydb/library/yql/providers/config/yql_config_provider.cpp index 086fcce08c..6952e96cf0 100644 --- a/ydb/library/yql/providers/config/yql_config_provider.cpp +++ b/ydb/library/yql/providers/config/yql_config_provider.cpp @@ -167,7 +167,7 @@ namespace { return (node.Child(1)->Child(0)->Content() == ConfigProviderName); } - bool CanParse(const TExprNode& node) override { + bool CanParse(const TExprNode& node) override { if (ConfigProviderFunctions().contains(node.Content()) || node.Content() == ConfigureName) { @@ -246,8 +246,8 @@ namespace { }); return *ConfigurationTransformer; - } - + } + IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { Y_UNUSED(instantOnly); if (!TypeAnnotationTransformer) { diff --git a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp index 5834937736..13bfd5e0f7 100644 --- a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp +++ b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp @@ -891,7 +891,7 @@ namespace { return true; } - bool CanParse(const TExprNode& node) override { + bool CanParse(const TExprNode& node) override { return ResultProviderFunctions().contains(node.Content()) || node.Content() == ConfigureName; } diff --git a/ydb/library/yql/providers/solomon/expr_nodes/ya.make b/ydb/library/yql/providers/solomon/expr_nodes/ya.make index d893ae13b0..ea3a97d449 100644 --- a/ydb/library/yql/providers/solomon/expr_nodes/ya.make +++ b/ydb/library/yql/providers/solomon/expr_nodes/ya.make @@ -1,38 +1,38 @@ -LIBRARY() - -OWNER( - monster - g:yql -) - -SRCS( - yql_solomon_expr_nodes.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:yql +) + +SRCS( + yql_solomon_expr_nodes.cpp +) + +PEERDIR( ydb/library/yql/core/expr_nodes ydb/library/yql/providers/common/provider -) - -SRCDIR( +) + +SRCDIR( ydb/library/yql/core/expr_nodes_gen -) - -RUN_PROGRAM( +) + +RUN_PROGRAM( ydb/library/yql/core/expr_nodes_gen/gen - yql_expr_nodes_gen.jnj - yql_solomon_expr_nodes.json - yql_solomon_expr_nodes.gen.h - yql_solomon_expr_nodes.decl.inl.h - yql_solomon_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_solomon_expr_nodes.json - OUT yql_solomon_expr_nodes.gen.h - OUT yql_solomon_expr_nodes.decl.inl.h - OUT yql_solomon_expr_nodes.defs.inl.h - OUTPUT_INCLUDES + yql_expr_nodes_gen.jnj + yql_solomon_expr_nodes.json + yql_solomon_expr_nodes.gen.h + yql_solomon_expr_nodes.decl.inl.h + yql_solomon_expr_nodes.defs.inl.h + IN yql_expr_nodes_gen.jnj + IN yql_solomon_expr_nodes.json + OUT yql_solomon_expr_nodes.gen.h + OUT yql_solomon_expr_nodes.decl.inl.h + OUT yql_solomon_expr_nodes.defs.inl.h + OUTPUT_INCLUDES ${ARCADIA_ROOT}/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h -) - -END() + ${ARCADIA_ROOT}/util/generic/hash_set.h +) + +END() diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.cpp b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.cpp index 8aa5adc03a..e5b4a6ec05 100644 --- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.cpp +++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.cpp @@ -1 +1 @@ -#include "yql_solomon_expr_nodes.h" +#include "yql_solomon_expr_nodes.h" diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h index 741cf7fca4..7787041994 100644 --- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h +++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h @@ -1,66 +1,66 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.gen.h> - -namespace NYql { -namespace NNodes { - + +namespace NYql { +namespace NNodes { + #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.decl.inl.h> - -class TSoDataSource: public NGenerated::TSoDataSourceStub<TExprBase, TCallable, TCoAtom> { -public: - explicit TSoDataSource(const TExprNode* node) - : TSoDataSourceStub(node) - { - } - - explicit TSoDataSource(const TExprNode::TPtr& node) - : TSoDataSourceStub(node) - { - } - - static bool Match(const TExprNode* node) { - if (!TSoDataSourceStub::Match(node)) { - return false; - } - - if (node->Child(0)->Content() != SolomonProviderName) { - return false; - } - - return true; - } -}; - - -class TSoDataSink: public NGenerated::TSoDataSinkStub<TExprBase, TCallable, TCoAtom> { -public: - explicit TSoDataSink(const TExprNode* node) - : TSoDataSinkStub(node) - { - } - - explicit TSoDataSink(const TExprNode::TPtr& node) - : TSoDataSinkStub(node) - { - } - - static bool Match(const TExprNode* node) { - if (!TSoDataSinkStub::Match(node)) { - return false; - } - - if (node->Child(0)->Content() != SolomonProviderName) { - return false; - } - - return true; - } -}; - + +class TSoDataSource: public NGenerated::TSoDataSourceStub<TExprBase, TCallable, TCoAtom> { +public: + explicit TSoDataSource(const TExprNode* node) + : TSoDataSourceStub(node) + { + } + + explicit TSoDataSource(const TExprNode::TPtr& node) + : TSoDataSourceStub(node) + { + } + + static bool Match(const TExprNode* node) { + if (!TSoDataSourceStub::Match(node)) { + return false; + } + + if (node->Child(0)->Content() != SolomonProviderName) { + return false; + } + + return true; + } +}; + + +class TSoDataSink: public NGenerated::TSoDataSinkStub<TExprBase, TCallable, TCoAtom> { +public: + explicit TSoDataSink(const TExprNode* node) + : TSoDataSinkStub(node) + { + } + + explicit TSoDataSink(const TExprNode::TPtr& node) + : TSoDataSinkStub(node) + { + } + + static bool Match(const TExprNode* node) { + if (!TSoDataSinkStub::Match(node)) { + return false; + } + + if (node->Child(0)->Content() != SolomonProviderName) { + return false; + } + + return true; + } +}; + #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.defs.inl.h> - -} // namespace NNodes -} // namespace NYql + +} // namespace NNodes +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json index 61305c9fee..d064daa1b4 100644 --- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json +++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json @@ -1,68 +1,68 @@ -{ - "NodeRootType": "TExprBase", - "NodeBuilderBase": "TNodeBuilderBase", - "ListBuilderBase": "TListBuilderBase", - "FreeArgCallableBase": "TFreeArgCallable", - "FreeArgBuilderBase": "TFreeArgCallableBuilderBase", - "Nodes": [ - { - "Name": "TSoDataSource", - "Base": "TCallable", - "Definition": "Custom", - "Match": {"Type": "Callable", "Name": "DataSource"}, - "Children": [ - {"Index": 0, "Name": "Category", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"} - ] - }, - { - "Name": "TSoDataSink", - "Base": "TCallable", - "Definition": "Custom", - "Match": {"Type": "Callable", "Name": "DataSink"}, - "Children": [ - {"Index": 0, "Name": "Category", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"} - ] - }, - { - "Name": "TSoRead", - "Base": "TFreeArgCallable", - "Match": {"Type": "Callable", "Name": "Read!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSource", "Type": "TSoDataSource"} - ] - }, - { - "Name": "TSoWrite", - "Base": "TFreeArgCallable", - "Match": {"Type": "Callable", "Name": "Write!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TSoDataSink"} - ] - }, - { - "Name": "TSoReadShardMeta", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "SoReadShardMeta!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSource", "Type": "TSoDataSource"}, - {"Index": 2, "Name": "Shard", "Type": "TCoAtom"} - ] - }, - { - "Name": "TSoWriteToShard", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "SoWriteToShard!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TSoDataSink"}, - {"Index": 2, "Name": "Shard", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Input", "Type": "TExprBase"} - ] +{ + "NodeRootType": "TExprBase", + "NodeBuilderBase": "TNodeBuilderBase", + "ListBuilderBase": "TListBuilderBase", + "FreeArgCallableBase": "TFreeArgCallable", + "FreeArgBuilderBase": "TFreeArgCallableBuilderBase", + "Nodes": [ + { + "Name": "TSoDataSource", + "Base": "TCallable", + "Definition": "Custom", + "Match": {"Type": "Callable", "Name": "DataSource"}, + "Children": [ + {"Index": 0, "Name": "Category", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"} + ] + }, + { + "Name": "TSoDataSink", + "Base": "TCallable", + "Definition": "Custom", + "Match": {"Type": "Callable", "Name": "DataSink"}, + "Children": [ + {"Index": 0, "Name": "Category", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"} + ] + }, + { + "Name": "TSoRead", + "Base": "TFreeArgCallable", + "Match": {"Type": "Callable", "Name": "Read!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSource", "Type": "TSoDataSource"} + ] + }, + { + "Name": "TSoWrite", + "Base": "TFreeArgCallable", + "Match": {"Type": "Callable", "Name": "Write!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TSoDataSink"} + ] + }, + { + "Name": "TSoReadShardMeta", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "SoReadShardMeta!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSource", "Type": "TSoDataSource"}, + {"Index": 2, "Name": "Shard", "Type": "TCoAtom"} + ] + }, + { + "Name": "TSoWriteToShard", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "SoWriteToShard!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TSoDataSink"}, + {"Index": 2, "Name": "Shard", "Type": "TCoAtom"}, + {"Index": 3, "Name": "Input", "Type": "TExprBase"} + ] }, { "Name": "TSoShard", @@ -85,6 +85,6 @@ {"Index": 1, "Name": "Settings", "Type": "TCoNameValueTupleList"}, {"Index": 2, "Name": "Token", "Type": "TCoSecureParam", "Optional": true} ] - } - ] -} + } + ] +} diff --git a/ydb/library/yql/providers/solomon/gateway/ya.make b/ydb/library/yql/providers/solomon/gateway/ya.make index 62fb6780eb..a5d0b49c19 100644 --- a/ydb/library/yql/providers/solomon/gateway/ya.make +++ b/ydb/library/yql/providers/solomon/gateway/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - monster - g:yql -) - -SRCS( - yql_solomon_gateway.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + monster + g:yql +) + +SRCS( + yql_solomon_gateway.cpp +) + +PEERDIR( ydb/library/yql/providers/common/gateway ydb/library/yql/providers/solomon/provider -) - -END() +) + +END() diff --git a/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.cpp b/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.cpp index 7ea343a160..ea9119c9fd 100644 --- a/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.cpp +++ b/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.cpp @@ -1,58 +1,58 @@ -#include "yql_solomon_gateway.h" - +#include "yql_solomon_gateway.h" + #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> - -namespace NYql { - -using namespace NThreading; - -class TSolomonGateway : public ISolomonGateway { -public: - TSolomonGateway(const TSolomonGatewayConfig& config) - { - for (const auto& item: config.GetClusterMapping()) { - Clusters_[item.GetName()] = item; - } - } - - NThreading::TFuture<TGetMetaResult> GetMeta(const TGetMetaRequest& request) const override { - Y_UNUSED(request); - try { - TGetMetaResult result; - auto meta = MakeIntrusive<TSolomonMetadata>(); - result.Meta = meta; - - // TODO: actual meta - meta->DoesExist = true; - - result.SetSuccess(); - return MakeFuture(result); - - } catch (const yexception& e) { - return MakeFuture(NCommon::ResultFromException<TGetMetaResult>(e)); - - } catch (...) { - return MakeFuture(NCommon::ResultFromError<TGetMetaResult>(CurrentExceptionMessage())); - } - } - - TMaybe<TSolomonClusterConfig> GetClusterConfig(const TStringBuf cluster) const override { - if (Clusters_.contains(cluster)) { - return Clusters_.find(cluster)->second; - } - return Nothing(); - } - - bool HasCluster(const TStringBuf cluster) const override { - return Clusters_.contains(cluster); - } - -private: - THashMap<TString, TSolomonClusterConfig> Clusters_; -}; - -ISolomonGateway::TPtr CreateSolomonGateway(const TSolomonGatewayConfig& config) { - return new TSolomonGateway(config); -} - -} // namespace NYql + +namespace NYql { + +using namespace NThreading; + +class TSolomonGateway : public ISolomonGateway { +public: + TSolomonGateway(const TSolomonGatewayConfig& config) + { + for (const auto& item: config.GetClusterMapping()) { + Clusters_[item.GetName()] = item; + } + } + + NThreading::TFuture<TGetMetaResult> GetMeta(const TGetMetaRequest& request) const override { + Y_UNUSED(request); + try { + TGetMetaResult result; + auto meta = MakeIntrusive<TSolomonMetadata>(); + result.Meta = meta; + + // TODO: actual meta + meta->DoesExist = true; + + result.SetSuccess(); + return MakeFuture(result); + + } catch (const yexception& e) { + return MakeFuture(NCommon::ResultFromException<TGetMetaResult>(e)); + + } catch (...) { + return MakeFuture(NCommon::ResultFromError<TGetMetaResult>(CurrentExceptionMessage())); + } + } + + TMaybe<TSolomonClusterConfig> GetClusterConfig(const TStringBuf cluster) const override { + if (Clusters_.contains(cluster)) { + return Clusters_.find(cluster)->second; + } + return Nothing(); + } + + bool HasCluster(const TStringBuf cluster) const override { + return Clusters_.contains(cluster); + } + +private: + THashMap<TString, TSolomonClusterConfig> Clusters_; +}; + +ISolomonGateway::TPtr CreateSolomonGateway(const TSolomonGatewayConfig& config) { + return new TSolomonGateway(config); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h b/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h index 4b82832e03..2319e7d844 100644 --- a/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h +++ b/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h> - -namespace NYql { - -class TSolomonGatewayConfig; - -ISolomonGateway::TPtr CreateSolomonGateway(const TSolomonGatewayConfig& config); - -} // namespace NYql + +namespace NYql { + +class TSolomonGatewayConfig; + +ISolomonGateway::TPtr CreateSolomonGateway(const TSolomonGatewayConfig& config); + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/ya.make b/ydb/library/yql/providers/solomon/provider/ya.make index f81eb7ba1f..b80c01bdb7 100644 --- a/ydb/library/yql/providers/solomon/provider/ya.make +++ b/ydb/library/yql/providers/solomon/provider/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - -OWNER( - monster - g:yql -) - -SRCS( - yql_solomon_config.cpp +LIBRARY() + +OWNER( + monster + g:yql +) + +SRCS( + yql_solomon_config.cpp yql_solomon_datasink_execution.cpp yql_solomon_datasink_type_ann.cpp yql_solomon_datasink.cpp @@ -14,13 +14,13 @@ SRCS( yql_solomon_datasource_type_ann.cpp yql_solomon_datasource.cpp yql_solomon_dq_integration.cpp - yql_solomon_io_discovery.cpp - yql_solomon_load_meta.cpp + yql_solomon_io_discovery.cpp + yql_solomon_load_meta.cpp yql_solomon_physical_optimize.cpp yql_solomon_provider.cpp -) - -PEERDIR( +) + +PEERDIR( library/cpp/actors/protos ydb/library/yql/dq/expr_nodes ydb/library/yql/providers/common/config @@ -32,8 +32,8 @@ PEERDIR( ydb/library/yql/providers/solomon/expr_nodes ydb/library/yql/providers/solomon/proto ydb/library/yql/dq/opt -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp index 02760151a1..fe00223f7c 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp @@ -1,15 +1,15 @@ -#include "yql_solomon_config.h" - -namespace NYql { - -using namespace NCommon; - -TSolomonConfiguration::TSolomonConfiguration() -{ -} - -TSolomonSettings::TConstPtr TSolomonConfiguration::Snapshot() const { +#include "yql_solomon_config.h" + +namespace NYql { + +using namespace NCommon; + +TSolomonConfiguration::TSolomonConfiguration() +{ +} + +TSolomonSettings::TConstPtr TSolomonConfiguration::Snapshot() const { return std::make_shared<const TSolomonSettings>(*this); -} - -} // NYql +} + +} // NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h index 4c932f947a..941b2adc30 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h @@ -1,50 +1,50 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h> #include <ydb/library/yql/providers/common/config/yql_dispatch.h> #include <ydb/library/yql/providers/common/config/yql_setting.h> #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> - -namespace NYql { - -struct TSolomonSettings { + +namespace NYql { + +struct TSolomonSettings { using TConstPtr = std::shared_ptr<const TSolomonSettings>; -}; - -struct TSolomonConfiguration - : public TSolomonSettings - , public NCommon::TSettingDispatcher -{ - using TPtr = TIntrusivePtr<TSolomonConfiguration>; - - TSolomonConfiguration(); - TSolomonConfiguration(const TSolomonConfiguration&) = delete; - - template <typename TProtoConfig> +}; + +struct TSolomonConfiguration + : public TSolomonSettings + , public NCommon::TSettingDispatcher +{ + using TPtr = TIntrusivePtr<TSolomonConfiguration>; + + TSolomonConfiguration(); + TSolomonConfiguration(const TSolomonConfiguration&) = delete; + + template <typename TProtoConfig> void Init(const TProtoConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx) - { - TVector<TString> clusters(Reserve(config.ClusterMappingSize())); - for (auto& cluster: config.GetClusterMapping()) { - clusters.push_back(cluster.GetName()); + { + TVector<TString> clusters(Reserve(config.ClusterMappingSize())); + for (auto& cluster: config.GetClusterMapping()) { + clusters.push_back(cluster.GetName()); ClusterConfigs[cluster.GetName()] = cluster; const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_solomon", cluster.GetToken()); Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); - } - - this->SetValidClusters(clusters); - - this->Dispatch(config.GetDefaultSettings()); - for (auto& cluster: config.GetClusterMapping()) { - this->Dispatch(cluster.GetName(), cluster.GetSettings()); - } - this->FreezeDefaults(); - } - - TSolomonSettings::TConstPtr Snapshot() const; + } + + this->SetValidClusters(clusters); + + this->Dispatch(config.GetDefaultSettings()); + for (auto& cluster: config.GetClusterMapping()) { + this->Dispatch(cluster.GetName(), cluster.GetSettings()); + } + this->FreezeDefaults(); + } + + TSolomonSettings::TConstPtr Snapshot() const; THashMap<TString, TSolomonClusterConfig> ClusterConfigs; THashMap<TString, TString> Tokens; -}; - -} // NYql +}; + +} // NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp index b88154b579..b09fd381a0 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp @@ -1,180 +1,180 @@ -#include "yql_solomon_provider_impl.h" - +#include "yql_solomon_provider_impl.h" + #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h> - + #include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h> #include <ydb/library/yql/providers/common/provider/yql_provider.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> - + #include <ydb/library/yql/utils/log/log.h> - -namespace NYql { - -using namespace NNodes; - -class TSolomonDataSink : public TDataProviderBase { -public: - TSolomonDataSink(TSolomonState::TPtr state) - : State_ {state} - , TypeAnnotationTransformer_(CreateSolomonDataSinkTypeAnnotationTransformer(State_)) - , ExecutionTransformer_(CreateSolomonDataSinkExecTransformer(State_)) + +namespace NYql { + +using namespace NNodes; + +class TSolomonDataSink : public TDataProviderBase { +public: + TSolomonDataSink(TSolomonState::TPtr state) + : State_ {state} + , TypeAnnotationTransformer_(CreateSolomonDataSinkTypeAnnotationTransformer(State_)) + , ExecutionTransformer_(CreateSolomonDataSinkExecTransformer(State_)) , PhysicalOptProposalTransformer_(CreateSoPhysicalOptProposalTransformer(State_)) - { - } - - TStringBuf GetName() const override { - return SolomonProviderName; - } - - TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override { - auto config = State_->Gateway->GetClusterConfig(cluster); - if (!config) { - return {}; - } - + { + } + + TStringBuf GetName() const override { + return SolomonProviderName; + } + + TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override { + auto config = State_->Gateway->GetClusterConfig(cluster); + if (!config) { + return {}; + } + TPositionHandle pos; - return Build<TCoAtom>(ctx, pos) - .Value(config->GetCluster()) - .Done().Ptr(); - } - + return Build<TCoAtom>(ctx, pos) + .Value(config->GetCluster()) + .Done().Ptr(); + } + const THashMap<TString, TString>* GetClusterTokens() override { return &State_->Configuration->Tokens; } - IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { - Y_UNUSED(instantOnly); - return *TypeAnnotationTransformer_; - } - - IGraphTransformer& GetCallableExecutionTransformer() override { - return *ExecutionTransformer_; - } - + IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { + Y_UNUSED(instantOnly); + return *TypeAnnotationTransformer_; + } + + IGraphTransformer& GetCallableExecutionTransformer() override { + return *ExecutionTransformer_; + } + IGraphTransformer& GetPhysicalOptProposalTransformer() override { return *PhysicalOptProposalTransformer_; } - bool CanParse(const TExprNode& node) override { - if (node.IsCallable(TCoWrite::CallableName())) { - return TSoDataSink::Match(node.Child(1)); - } - - return TypeAnnotationTransformer_->CanParse(node); - } - + bool CanParse(const TExprNode& node) override { + if (node.IsCallable(TCoWrite::CallableName())) { + return TSoDataSink::Match(node.Child(1)); + } + + return TypeAnnotationTransformer_->CanParse(node); + } + void FillModifyCallables(THashSet<TStringBuf>& callables) override { callables.insert(TSoWriteToShard::CallableName()); } - bool CanExecute(const TExprNode& node) override { - return ExecutionTransformer_->CanExec(node); - } - - bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override { - if (node.IsCallable(TCoDataSink::CallableName())) { - if (node.Child(0)->Content() == SolomonProviderName) { - auto clusterName = node.Child(1)->Content(); - if (!State_->Gateway->HasCluster(clusterName)) { + bool CanExecute(const TExprNode& node) override { + return ExecutionTransformer_->CanExec(node); + } + + bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override { + if (node.IsCallable(TCoDataSink::CallableName())) { + if (node.Child(0)->Content() == SolomonProviderName) { + auto clusterName = node.Child(1)->Content(); + if (!State_->Gateway->HasCluster(clusterName)) { ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << - "Unknown cluster name: " << clusterName - )); - return false; - } - cluster = clusterName; - return true; - } - } + "Unknown cluster name: " << clusterName + )); + return false; + } + cluster = clusterName; + return true; + } + } ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Solomon DataSink parameters")); - return false; - } - - TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override { - auto maybeWrite = TMaybeNode<TSoWrite>(node); - YQL_ENSURE(maybeWrite); - - YQL_CLOG(INFO, ProviderSolomon) << "RewriteIO"; - - auto write = maybeWrite.Cast(); - auto& key = write.Arg(2).Ref(); + return false; + } + + TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override { + auto maybeWrite = TMaybeNode<TSoWrite>(node); + YQL_ENSURE(maybeWrite); + + YQL_CLOG(INFO, ProviderSolomon) << "RewriteIO"; + + auto write = maybeWrite.Cast(); + auto& key = write.Arg(2).Ref(); if (!key.IsCallable(TStringBuf("Key"))) { ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), TStringBuf("Expected key"))); - return {}; - } - if (key.ChildrenSize() < 1) { + return {}; + } + if (key.ChildrenSize() < 1) { ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), TStringBuf("Key must have at least one component"))); - return {}; - } - - auto tagName = key.Child(0)->Child(0)->Content(); + return {}; + } + + auto tagName = key.Child(0)->Child(0)->Content(); if (tagName != TStringBuf("table")) { ctx.AddError(TIssue(ctx.GetPosition(key.Child(0)->Pos()), - TStringBuilder() << "Unexpected tag: " << tagName)); - return {}; - } - - const TExprNode* nameNode = key.Child(0)->Child(1); - if (!nameNode->IsCallable("String")) { + TStringBuilder() << "Unexpected tag: " << tagName)); + return {}; + } + + const TExprNode* nameNode = key.Child(0)->Child(1); + if (!nameNode->IsCallable("String")) { ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), "Expected String as table key")); - return {}; - } - - if (!EnsureArgsCount(*nameNode, 1, ctx)) { - return {}; - } - - const TExprNode* tablePath = nameNode->Child(0); - if (!EnsureAtom(*tablePath, ctx)) { - return {}; - } - if (tablePath->Content().empty()) { + return {}; + } + + if (!EnsureArgsCount(*nameNode, 1, ctx)) { + return {}; + } + + const TExprNode* tablePath = nameNode->Child(0); + if (!EnsureAtom(*tablePath, ctx)) { + return {}; + } + if (tablePath->Content().empty()) { ctx.AddError(TIssue(ctx.GetPosition(tablePath->Pos()), "Table name must not be empty")); - return {}; - } - - return Build<TSoWriteToShard>(ctx, write.Pos()) - .World(write.World()) - .DataSink(write.DataSink()) - .Shard<TCoAtom>().Value(tablePath->Content()).Build() - .Input(write.Arg(3)) - .Done().Ptr(); - } - - void GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) override { - if (CanExecute(node)) { - children.push_back(node.ChildPtr(0)); - } - } - - bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override { - Y_UNUSED(compact); - if (CanExecute(node)) { - children.push_back(node.ChildPtr(0)); - return true; - } - return false; - } - - TString GetProviderPath(const TExprNode& node) override { - return TStringBuilder() << SolomonProviderName << '.' << node.Child(1)->Content(); - } - + return {}; + } + + return Build<TSoWriteToShard>(ctx, write.Pos()) + .World(write.World()) + .DataSink(write.DataSink()) + .Shard<TCoAtom>().Value(tablePath->Content()).Build() + .Input(write.Arg(3)) + .Done().Ptr(); + } + + void GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) override { + if (CanExecute(node)) { + children.push_back(node.ChildPtr(0)); + } + } + + bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override { + Y_UNUSED(compact); + if (CanExecute(node)) { + children.push_back(node.ChildPtr(0)); + return true; + } + return false; + } + + TString GetProviderPath(const TExprNode& node) override { + return TStringBuilder() << SolomonProviderName << '.' << node.Child(1)->Content(); + } + IDqIntegration* GetDqIntegration() override { return State_->IsRtmrMode() ? nullptr : State_->DqIntegration.Get(); } -private: - TSolomonState::TPtr State_; - - THolder<TVisitorTransformerBase> TypeAnnotationTransformer_; - THolder<TExecTransformerBase> ExecutionTransformer_; +private: + TSolomonState::TPtr State_; + + THolder<TVisitorTransformerBase> TypeAnnotationTransformer_; + THolder<TExecTransformerBase> ExecutionTransformer_; THolder<IGraphTransformer> PhysicalOptProposalTransformer_; -}; - - -TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state) { - return new TSolomonDataSink(state); -} - -} // namespace NYql +}; + + +TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state) { + return new TSolomonDataSink(state); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp index 9f6048eabd..e3048498c7 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp @@ -1,31 +1,31 @@ -#include "yql_solomon_provider_impl.h" - +#include "yql_solomon_provider_impl.h" + #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> #include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h> #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h> - + #include <ydb/library/yql/providers/common/provider/yql_provider.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h> #include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h> #include <ydb/library/yql/utils/log/log.h> - + #include <util/string/split.h> -namespace NYql { - -using namespace NNodes; - -class TSolomonDataSinkExecTransformer : public TExecTransformerBase { -public: +namespace NYql { + +using namespace NNodes; + +class TSolomonDataSinkExecTransformer : public TExecTransformerBase { +public: explicit TSolomonDataSinkExecTransformer(TSolomonState::TPtr state) - : State_(state) - { + : State_(state) + { AddHandler({TCoCommit::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleCommit)); AddHandler({TSoWriteToShard::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleSoWriteToShard)); - } - + } + TStatusCallbackPair HandleCommit(const TExprNode::TPtr& input, TExprContext& ctx) { if (!State_->IsRtmrMode() && TDqQuery::Match(input->Child(TCoCommit::idx_World))) { return DelegateExecutionToDqProvider(input->ChildPtr(TCoCommit::idx_World), input, ctx); @@ -42,7 +42,7 @@ public: return DelegateExecutionToDqProvider(input->ChildPtr(TSoWriteToShard::idx_Input), input, ctx); } -private: +private: TStatusCallbackPair DelegateExecutionToDqProvider( const TExprNode::TPtr& input, const TExprNode::TPtr& originInput, @@ -128,11 +128,11 @@ private: } private: - TSolomonState::TPtr State_; -}; - -THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state) { + TSolomonState::TPtr State_; +}; + +THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state) { return THolder(new TSolomonDataSinkExecTransformer(state)); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp index cd66a622c9..dc9ea45a8f 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp @@ -1,42 +1,42 @@ -#include "yql_solomon_provider_impl.h" - +#include "yql_solomon_provider_impl.h" + #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h> - -namespace NYql { - -using namespace NNodes; - -class TSolomonDataSinkTypeAnnotationTransformer : public TVisitorTransformerBase { -public: - TSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state) - : TVisitorTransformerBase(true) - , State_(state) - { - using TSelf = TSolomonDataSinkTypeAnnotationTransformer; - AddHandler({TSoWriteToShard::CallableName()}, Hndl(&TSelf::HandleWriteToShard)); + +namespace NYql { + +using namespace NNodes; + +class TSolomonDataSinkTypeAnnotationTransformer : public TVisitorTransformerBase { +public: + TSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state) + : TVisitorTransformerBase(true) + , State_(state) + { + using TSelf = TSolomonDataSinkTypeAnnotationTransformer; + AddHandler({TSoWriteToShard::CallableName()}, Hndl(&TSelf::HandleWriteToShard)); AddHandler({TSoShard::CallableName()}, Hndl(&TSelf::HandleSoShard)); - AddHandler({TCoCommit::CallableName()}, Hndl(&TSelf::HandleCommit)); - } - -private: - TStatus HandleWriteToShard(TExprBase input, TExprContext& ctx) { - if (!EnsureArgsCount(input.Ref(), 4, ctx)) { - return TStatus::Error; - } - TSoWriteToShard write = input.Cast<TSoWriteToShard>(); - if (!EnsureWorldType(write.World().Ref(), ctx)) { - return TStatus::Error; - } - if (!EnsureSpecificDataSink(write.DataSink().Ref(), SolomonProviderName, ctx)) { - return TStatus::Error; - } - if (!EnsureAtom(write.Shard().Ref(), ctx)) { - return TStatus::Error; - } - input.Ptr()->SetTypeAnn(write.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - + AddHandler({TCoCommit::CallableName()}, Hndl(&TSelf::HandleCommit)); + } + +private: + TStatus HandleWriteToShard(TExprBase input, TExprContext& ctx) { + if (!EnsureArgsCount(input.Ref(), 4, ctx)) { + return TStatus::Error; + } + TSoWriteToShard write = input.Cast<TSoWriteToShard>(); + if (!EnsureWorldType(write.World().Ref(), ctx)) { + return TStatus::Error; + } + if (!EnsureSpecificDataSink(write.DataSink().Ref(), SolomonProviderName, ctx)) { + return TStatus::Error; + } + if (!EnsureAtom(write.Shard().Ref(), ctx)) { + return TStatus::Error; + } + input.Ptr()->SetTypeAnn(write.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + TStatus HandleSoShard(TExprBase input, TExprContext& ctx) { YQL_ENSURE(!State_->IsRtmrMode(), "SoShard can't be used in rtmr mode"); @@ -70,18 +70,18 @@ private: return TStatus::Ok; } - TStatus HandleCommit(TExprBase input, TExprContext& ctx) { - Y_UNUSED(ctx); - auto commit = input.Cast<TCoCommit>(); - input.Ptr()->SetTypeAnn(commit.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - - TSolomonState::TPtr State_; -}; - -THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state) { + TStatus HandleCommit(TExprBase input, TExprContext& ctx) { + Y_UNUSED(ctx); + auto commit = input.Cast<TCoCommit>(); + input.Ptr()->SetTypeAnn(commit.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + + TSolomonState::TPtr State_; +}; + +THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state) { return THolder(new TSolomonDataSinkTypeAnnotationTransformer(state)); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp index 4952cdd68e..31ceb1701c 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp @@ -1,110 +1,110 @@ -#include "yql_solomon_provider_impl.h" - +#include "yql_solomon_provider_impl.h" + #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h> - + #include <ydb/library/yql/providers/common/provider/yql_provider.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h> #include <ydb/library/yql/providers/common/config/yql_configuration_transformer.h> - + #include <ydb/library/yql/utils/log/log.h> - -namespace NYql { - -using namespace NNodes; - -class TSolomonDataSource : public TDataProviderBase { -public: - TSolomonDataSource(TSolomonState::TPtr state) - : State_(state) - , ConfigurationTransformer_(NCommon::CreateProviderConfigurationTransformer( + +namespace NYql { + +using namespace NNodes; + +class TSolomonDataSource : public TDataProviderBase { +public: + TSolomonDataSource(TSolomonState::TPtr state) + : State_(state) + , ConfigurationTransformer_(NCommon::CreateProviderConfigurationTransformer( State_->Configuration, *State_->Types, TString{SolomonProviderName})) - , IODiscoveryTransformer_(CreateSolomonIODiscoveryTransformer(State_)) - , LoadMetaDataTransformer_(CreateSolomonLoadTableMetadataTransformer(State_)) - , TypeAnnotationTransformer_(CreateSolomonDataSourceTypeAnnotationTransformer(State_)) - , ExecutionTransformer_(CreateSolomonDataSourceExecTransformer(State_)) - { - } - - TStringBuf GetName() const override { - return SolomonProviderName; - } - - IGraphTransformer& GetConfigurationTransformer() override { - return *ConfigurationTransformer_; - } - -// IGraphTransformer& GetIODiscoveryTransformer() override { -// return *IODiscoveryTransformer_; -// } - -// IGraphTransformer& GetLoadTableMetadataTransformer() override { -// return *LoadMetaDataTransformer_; -// } - - IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { - Y_UNUSED(instantOnly); - return *TypeAnnotationTransformer_; - } - - IGraphTransformer& GetCallableExecutionTransformer() override { - return *ExecutionTransformer_; - } - - bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override { - if (node.IsCallable(TCoDataSource::CallableName())) { - if (node.Child(0)->Content() == SolomonProviderName) { - auto clusterName = node.Child(1)->Content(); - if (!State_->Gateway->HasCluster(clusterName)) { + , IODiscoveryTransformer_(CreateSolomonIODiscoveryTransformer(State_)) + , LoadMetaDataTransformer_(CreateSolomonLoadTableMetadataTransformer(State_)) + , TypeAnnotationTransformer_(CreateSolomonDataSourceTypeAnnotationTransformer(State_)) + , ExecutionTransformer_(CreateSolomonDataSourceExecTransformer(State_)) + { + } + + TStringBuf GetName() const override { + return SolomonProviderName; + } + + IGraphTransformer& GetConfigurationTransformer() override { + return *ConfigurationTransformer_; + } + +// IGraphTransformer& GetIODiscoveryTransformer() override { +// return *IODiscoveryTransformer_; +// } + +// IGraphTransformer& GetLoadTableMetadataTransformer() override { +// return *LoadMetaDataTransformer_; +// } + + IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { + Y_UNUSED(instantOnly); + return *TypeAnnotationTransformer_; + } + + IGraphTransformer& GetCallableExecutionTransformer() override { + return *ExecutionTransformer_; + } + + bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override { + if (node.IsCallable(TCoDataSource::CallableName())) { + if (node.Child(0)->Content() == SolomonProviderName) { + auto clusterName = node.Child(1)->Content(); + if (!State_->Gateway->HasCluster(clusterName)) { ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << - "Unknown cluster name: " << clusterName)); - return false; - } - cluster = clusterName; - return true; - } - } + "Unknown cluster name: " << clusterName)); + return false; + } + cluster = clusterName; + return true; + } + } ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Solomon DataSource parameters")); - return false; - } - - bool CanParse(const TExprNode& node) override { - if (node.IsCallable(TCoRead::CallableName())) { - return TSoDataSource::Match(node.Child(1)); - } - return TypeAnnotationTransformer_->CanParse(node); - } - - bool CanExecute(const TExprNode& node) override { - return ExecutionTransformer_->CanExec(node); - } - - bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override { - Y_UNUSED(node); - Y_UNUSED(syncList); - canRef = false; - return false; - } - - TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override { - Y_UNUSED(ctx); - YQL_CLOG(INFO, ProviderSolomon) << "RewriteIO"; - return node; - } - -private: - TSolomonState::TPtr State_; - - THolder<IGraphTransformer> ConfigurationTransformer_; - THolder<IGraphTransformer> IODiscoveryTransformer_; - THolder<IGraphTransformer> LoadMetaDataTransformer_; - THolder<TVisitorTransformerBase> TypeAnnotationTransformer_; - THolder<TExecTransformerBase> ExecutionTransformer_; -}; - -TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state) { - return new TSolomonDataSource(state); -} - -} // namespace NYql + return false; + } + + bool CanParse(const TExprNode& node) override { + if (node.IsCallable(TCoRead::CallableName())) { + return TSoDataSource::Match(node.Child(1)); + } + return TypeAnnotationTransformer_->CanParse(node); + } + + bool CanExecute(const TExprNode& node) override { + return ExecutionTransformer_->CanExec(node); + } + + bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override { + Y_UNUSED(node); + Y_UNUSED(syncList); + canRef = false; + return false; + } + + TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override { + Y_UNUSED(ctx); + YQL_CLOG(INFO, ProviderSolomon) << "RewriteIO"; + return node; + } + +private: + TSolomonState::TPtr State_; + + THolder<IGraphTransformer> ConfigurationTransformer_; + THolder<IGraphTransformer> IODiscoveryTransformer_; + THolder<IGraphTransformer> LoadMetaDataTransformer_; + THolder<TVisitorTransformerBase> TypeAnnotationTransformer_; + THolder<TExecTransformerBase> ExecutionTransformer_; +}; + +TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state) { + return new TSolomonDataSource(state); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_execution.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_execution.cpp index d18bba5089..c381baf708 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_execution.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_execution.cpp @@ -1,22 +1,22 @@ -#include "yql_solomon_provider_impl.h" - -namespace NYql { - -using namespace NNodes; - -class TSolomonDataSourceExecTransformer : public TExecTransformerBase { -public: - TSolomonDataSourceExecTransformer(TSolomonState::TPtr state) - : State_(state) - { - } - -private: - TSolomonState::TPtr State_; -}; - -THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonState::TPtr state) { +#include "yql_solomon_provider_impl.h" + +namespace NYql { + +using namespace NNodes; + +class TSolomonDataSourceExecTransformer : public TExecTransformerBase { +public: + TSolomonDataSourceExecTransformer(TSolomonState::TPtr state) + : State_(state) + { + } + +private: + TSolomonState::TPtr State_; +}; + +THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonState::TPtr state) { return THolder(new TSolomonDataSourceExecTransformer(state)); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp index f7b49f3cdb..f6a0ce8003 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp @@ -1,25 +1,25 @@ -#include "yql_solomon_provider_impl.h" - +#include "yql_solomon_provider_impl.h" + #include <ydb/library/yql/providers/common/provider/yql_provider.h> - -namespace NYql { - -using namespace NNodes; - -class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { -public: - TSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state) - : TVisitorTransformerBase(true) - , State_(state) - { - } - -private: - TSolomonState::TPtr State_; -}; - -THolder<TVisitorTransformerBase> CreateSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state) { + +namespace NYql { + +using namespace NNodes; + +class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { +public: + TSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state) + : TVisitorTransformerBase(true) + , State_(state) + { + } + +private: + TSolomonState::TPtr State_; +}; + +THolder<TVisitorTransformerBase> CreateSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state) { return THolder(new TSolomonDataSourceTypeAnnotationTransformer(state)); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h index f775c4a909..574bf79235 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h @@ -1,39 +1,39 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/common/gateway/yql_provider_gateway.h> #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> - + #include <library/cpp/threading/future/future.h> - -namespace NYql { - -struct TSolomonMetadata : public TThrRefBase -{ - using TPtr = TIntrusivePtr<TSolomonMetadata>; - - bool DoesExist = false; -}; - -class ISolomonGateway : public TThrRefBase { -public: - using TPtr = TIntrusivePtr<ISolomonGateway>; - - struct TGetMetaRequest { - TString Cluster; - TString Name; - }; - - struct TGetMetaResult : public NCommon::TOperationResult { - TSolomonMetadata::TPtr Meta; - }; - - virtual ~ISolomonGateway() = default; - - virtual NThreading::TFuture<TGetMetaResult> GetMeta( - const TGetMetaRequest& request) const = 0; - - virtual TMaybe<TSolomonClusterConfig> GetClusterConfig(const TStringBuf cluster) const = 0; - virtual bool HasCluster(const TStringBuf cluster) const = 0; -}; - -} // namespace NYql + +namespace NYql { + +struct TSolomonMetadata : public TThrRefBase +{ + using TPtr = TIntrusivePtr<TSolomonMetadata>; + + bool DoesExist = false; +}; + +class ISolomonGateway : public TThrRefBase { +public: + using TPtr = TIntrusivePtr<ISolomonGateway>; + + struct TGetMetaRequest { + TString Cluster; + TString Name; + }; + + struct TGetMetaResult : public NCommon::TOperationResult { + TSolomonMetadata::TPtr Meta; + }; + + virtual ~ISolomonGateway() = default; + + virtual NThreading::TFuture<TGetMetaResult> GetMeta( + const TGetMetaRequest& request) const = 0; + + virtual TMaybe<TSolomonClusterConfig> GetClusterConfig(const TStringBuf cluster) const = 0; + virtual bool HasCluster(const TStringBuf cluster) const = 0; +}; + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp index 15445b916e..880811beee 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp @@ -1,79 +1,79 @@ -#include "yql_solomon_provider_impl.h" - +#include "yql_solomon_provider_impl.h" + #include <ydb/library/yql/providers/common/provider/yql_provider.h> #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h> - -/* + +/* #include <ydb/library/yql/core/yql_expr_optimize.h> -#include <yql/library/statface_client/client.h> - -#include <util/generic/hash_set.h> -*/ - -namespace NYql { - -using namespace NNodes; - -class TSolomonIODiscoveryTransformer : public TSyncTransformerBase { -public: - TSolomonIODiscoveryTransformer(TSolomonState::TPtr state) - : State_(state) - { - } - - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override { - output = input; - - if (ctx.Step.IsDone(ctx.Step.DiscoveryIO)) { - return TStatus::Ok; - } - - TVector<TIssue> issues; - - auto status = OptimizeExpr(input, output, [] (const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { - if (auto maybeWrite = TMaybeNode<TSoWrite>(node)) { - if (!maybeWrite.DataSink()) { - return node; - } - auto write = maybeWrite.Cast(); - - if (!EnsureArgsCount(write.Ref(), 5, ctx)) { - return {}; - } - - return Build<TSoWrite>(ctx, write.Pos()) - .World(write.World()) - .DataSink(write.DataSink()) - .FreeArgs() - .Add<TCoAtom>() - .Value("") - .Build() - .Add(write.Arg(3)) - .Build() - .Done().Ptr(); - - } else if (TMaybeNode<TSoRead>(node).DataSource()) { - return node; - } - return node; - }, ctx, TOptimizeExprSettings {nullptr}); - - if (issues) { - for (const auto& issue: issues) { - ctx.AddError(issue); - } - status = status.Combine(TStatus::Error); - } - - return status; - } - -private: - TSolomonState::TPtr State_; -}; - -THolder<IGraphTransformer> CreateSolomonIODiscoveryTransformer(TSolomonState::TPtr state) { +#include <yql/library/statface_client/client.h> + +#include <util/generic/hash_set.h> +*/ + +namespace NYql { + +using namespace NNodes; + +class TSolomonIODiscoveryTransformer : public TSyncTransformerBase { +public: + TSolomonIODiscoveryTransformer(TSolomonState::TPtr state) + : State_(state) + { + } + + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override { + output = input; + + if (ctx.Step.IsDone(ctx.Step.DiscoveryIO)) { + return TStatus::Ok; + } + + TVector<TIssue> issues; + + auto status = OptimizeExpr(input, output, [] (const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { + if (auto maybeWrite = TMaybeNode<TSoWrite>(node)) { + if (!maybeWrite.DataSink()) { + return node; + } + auto write = maybeWrite.Cast(); + + if (!EnsureArgsCount(write.Ref(), 5, ctx)) { + return {}; + } + + return Build<TSoWrite>(ctx, write.Pos()) + .World(write.World()) + .DataSink(write.DataSink()) + .FreeArgs() + .Add<TCoAtom>() + .Value("") + .Build() + .Add(write.Arg(3)) + .Build() + .Done().Ptr(); + + } else if (TMaybeNode<TSoRead>(node).DataSource()) { + return node; + } + return node; + }, ctx, TOptimizeExprSettings {nullptr}); + + if (issues) { + for (const auto& issue: issues) { + ctx.AddError(issue); + } + status = status.Combine(TStatus::Error); + } + + return status; + } + +private: + TSolomonState::TPtr State_; +}; + +THolder<IGraphTransformer> CreateSolomonIODiscoveryTransformer(TSolomonState::TPtr state) { return THolder(new TSolomonIODiscoveryTransformer(state)); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp index 8c15dc4de2..42324bb3de 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp @@ -1,45 +1,45 @@ -#include "yql_solomon_provider_impl.h" - -namespace NYql { - -using namespace NNodes; - -class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase { -public: - TSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state) - : State_(state) - { - } - - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - Y_UNUSED(input); - Y_UNUSED(output); - - if (ctx.Step.IsDone(TExprStep::LoadTablesMetadata)) { - return TStatus::Ok; - } - - return TStatus::Async; - } - - NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final { - Y_UNUSED(input); - return AsyncFuture_; - } - - TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { - output = input; - - return TStatus::Ok; - } - -private: - TSolomonState::TPtr State_; - NThreading::TFuture<void> AsyncFuture_; -}; - -THolder<IGraphTransformer> CreateSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state) { +#include "yql_solomon_provider_impl.h" + +namespace NYql { + +using namespace NNodes; + +class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase { +public: + TSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state) + : State_(state) + { + } + + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + Y_UNUSED(input); + Y_UNUSED(output); + + if (ctx.Step.IsDone(TExprStep::LoadTablesMetadata)) { + return TStatus::Ok; + } + + return TStatus::Async; + } + + NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final { + Y_UNUSED(input); + return AsyncFuture_; + } + + TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { + output = input; + + return TStatus::Ok; + } + +private: + TSolomonState::TPtr State_; + NThreading::TFuture<void> AsyncFuture_; +}; + +THolder<IGraphTransformer> CreateSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state) { return THolder(new TSolomonLoadTableMetadataTransformer(state)); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp index 1ff28bab3c..d84f1502a6 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp @@ -1,47 +1,47 @@ -#include "yql_solomon_provider.h" - +#include "yql_solomon_provider.h" + #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h> - -namespace NYql { - + +namespace NYql { + TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode) { return [gateway, supportRtmrMode] ( - const TString& userName, - const TString& sessionId, - const TGatewaysConfig* gatewaysConfig, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - TIntrusivePtr<IRandomProvider> randomProvider, - TIntrusivePtr<TTypeAnnotationContext> typeCtx, - const TOperationProgressWriter& progressWriter, - const TYqlOperationOptions& operationOptions) - { - Y_UNUSED(sessionId); - Y_UNUSED(userName); - Y_UNUSED(functionRegistry); - Y_UNUSED(randomProvider); - Y_UNUSED(progressWriter); - Y_UNUSED(operationOptions); - - auto solomonState = MakeIntrusive<TSolomonState>(); - + const TString& userName, + const TString& sessionId, + const TGatewaysConfig* gatewaysConfig, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + TIntrusivePtr<IRandomProvider> randomProvider, + TIntrusivePtr<TTypeAnnotationContext> typeCtx, + const TOperationProgressWriter& progressWriter, + const TYqlOperationOptions& operationOptions) + { + Y_UNUSED(sessionId); + Y_UNUSED(userName); + Y_UNUSED(functionRegistry); + Y_UNUSED(randomProvider); + Y_UNUSED(progressWriter); + Y_UNUSED(operationOptions); + + auto solomonState = MakeIntrusive<TSolomonState>(); + solomonState->SupportRtmrMode = supportRtmrMode; - solomonState->Types = typeCtx.Get(); - solomonState->Gateway = gateway; + solomonState->Types = typeCtx.Get(); + solomonState->Gateway = gateway; solomonState->DqIntegration = CreateSolomonDqIntegration(solomonState); - if (gatewaysConfig) { + if (gatewaysConfig) { solomonState->Configuration->Init(gatewaysConfig->GetSolomon(), typeCtx); - } - - TDataProviderInfo info; - + } + + TDataProviderInfo info; + info.Names.insert({TString{SolomonProviderName}}); - info.Source = CreateSolomonDataSource(solomonState); - info.Sink = CreateSolomonDataSink(solomonState); - - return info; - }; -} - -} // namespace NYql + info.Source = CreateSolomonDataSource(solomonState); + info.Sink = CreateSolomonDataSink(solomonState); + + return info; + }; +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h index ef4e42b3a1..7f4a88a131 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h @@ -1,16 +1,16 @@ -#pragma once - -#include "yql_solomon_gateway.h" -#include "yql_solomon_config.h" - +#pragma once + +#include "yql_solomon_gateway.h" +#include "yql_solomon_config.h" + #include <ydb/library/yql/core/yql_data_provider.h> - -namespace NYql { - -struct TSolomonState : public TThrRefBase -{ - using TPtr = TIntrusivePtr<TSolomonState>; - + +namespace NYql { + +struct TSolomonState : public TThrRefBase +{ + using TPtr = TIntrusivePtr<TSolomonState>; + public: bool IsRtmrMode() const { return SupportRtmrMode; @@ -19,15 +19,15 @@ public: public: bool SupportRtmrMode = true; - ISolomonGateway::TPtr Gateway; - TTypeAnnotationContext* Types = nullptr; - TSolomonConfiguration::TPtr Configuration = MakeIntrusive<TSolomonConfiguration>(); + ISolomonGateway::TPtr Gateway; + TTypeAnnotationContext* Types = nullptr; + TSolomonConfiguration::TPtr Configuration = MakeIntrusive<TSolomonConfiguration>(); THolder<IDqIntegration> DqIntegration; -}; - +}; + TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode = true); - -TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state); -TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state); - -} // namespace NYql + +TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state); +TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state); + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h index 6517605d7b..c1f48a5b54 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h @@ -1,24 +1,24 @@ -#pragma once - -#include "yql_solomon_provider.h" - +#pragma once + +#include "yql_solomon_provider.h" + #include <ydb/library/yql/core/yql_graph_transformer.h> #include <ydb/library/yql/providers/common/transform/yql_exec.h> #include <ydb/library/yql/providers/common/transform/yql_visit.h> - -#include <util/generic/ptr.h> - -namespace NYql { - -THolder<IGraphTransformer> CreateSolomonIODiscoveryTransformer(TSolomonState::TPtr state); -THolder<IGraphTransformer> CreateSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state); - -THolder<TVisitorTransformerBase> CreateSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state); -THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonState::TPtr state); - -THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state); -THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state); - + +#include <util/generic/ptr.h> + +namespace NYql { + +THolder<IGraphTransformer> CreateSolomonIODiscoveryTransformer(TSolomonState::TPtr state); +THolder<IGraphTransformer> CreateSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state); + +THolder<TVisitorTransformerBase> CreateSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state); +THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonState::TPtr state); + +THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state); +THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state); + THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state); -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/ya.make b/ydb/library/yql/providers/solomon/ya.make index d5f7e727f9..579c088007 100644 --- a/ydb/library/yql/providers/solomon/ya.make +++ b/ydb/library/yql/providers/solomon/ya.make @@ -1,7 +1,7 @@ -RECURSE( +RECURSE( async_io - expr_nodes - gateway + expr_nodes + gateway proto - provider -) + provider +) diff --git a/ydb/library/yql/public/udf/udf_helpers.h b/ydb/library/yql/public/udf/udf_helpers.h index b1c11f78e6..2be9443559 100644 --- a/ydb/library/yql/public/udf/udf_helpers.h +++ b/ydb/library/yql/public/udf/udf_helpers.h @@ -211,7 +211,7 @@ namespace NUdf { namespace NYql { namespace NUdf { -template<bool CheckOptional, const char* TFuncName, template<class> class TFunc, typename... TUserTypes> +template<bool CheckOptional, const char* TFuncName, template<class> class TFunc, typename... TUserTypes> class TUserDataTypeFuncFactory : public ::NYql::NUdf::TBoxedValue { public: typedef bool TTypeAwareMarker; @@ -223,37 +223,37 @@ public: } template<typename TUserType> - static bool DeclareSignatureImpl( - TDataTypeId typeId, + static bool DeclareSignatureImpl( + TDataTypeId typeId, ::NYql::NUdf::TType* userType, ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, - bool typesOnly) - { + bool typesOnly) + { if (TDataType<TUserType>::Id != typeId) { return false; } - TFunc<TUserType>::DeclareSignature(userType, builder, typesOnly); + TFunc<TUserType>::DeclareSignature(userType, builder, typesOnly); return true; } template<typename TUserType, typename THead, typename... TTail> - static bool DeclareSignatureImpl( - TDataTypeId typeId, + static bool DeclareSignatureImpl( + TDataTypeId typeId, ::NYql::NUdf::TType* userType, ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, - bool typesOnly) - { - if (DeclareSignatureImpl<TUserType>(typeId, userType, builder, typesOnly)) { + bool typesOnly) + { + if (DeclareSignatureImpl<TUserType>(typeId, userType, builder, typesOnly)) { return true; } - return DeclareSignatureImpl<THead, TTail...>(typeId, userType, builder, typesOnly); + return DeclareSignatureImpl<THead, TTail...>(typeId, userType, builder, typesOnly); } static bool DeclareSignature( const ::NYql::NUdf::TStringRef& name, ::NYql::NUdf::TType* userType, ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, - bool typesOnly) + bool typesOnly) { if (Name() != name) { // the only case when we return false @@ -265,37 +265,37 @@ public: return true; } - auto typeHelper = builder.TypeInfoHelper(); - auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); - if (!userTypeInspector || userTypeInspector.GetElementsCount() < 1) { - builder.SetError("Missing or invalid user type"); + auto typeHelper = builder.TypeInfoHelper(); + auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); + if (!userTypeInspector || userTypeInspector.GetElementsCount() < 1) { + builder.SetError("Missing or invalid user type"); + return true; + } + + auto argsTypeInspector = TTupleTypeInspector(*typeHelper, userTypeInspector.GetElementType(0)); + if (!argsTypeInspector || argsTypeInspector.GetElementsCount() < 1) { + builder.SetError("Missing or invalid user type arguments"); + return true; + } + + auto argType = argsTypeInspector.GetElementType(0); + if (CheckOptional) { + TOptionalTypeInspector optionalTypeInspector(*typeHelper, argType); + if (optionalTypeInspector) { + argType = optionalTypeInspector.GetItemType(); + } + } + + TDataTypeInspector dataTypeInspector(*typeHelper, argType); + if (!dataTypeInspector) { + builder.SetError("User type must be a data type"); return true; } - auto argsTypeInspector = TTupleTypeInspector(*typeHelper, userTypeInspector.GetElementType(0)); - if (!argsTypeInspector || argsTypeInspector.GetElementsCount() < 1) { - builder.SetError("Missing or invalid user type arguments"); - return true; - } - - auto argType = argsTypeInspector.GetElementType(0); - if (CheckOptional) { - TOptionalTypeInspector optionalTypeInspector(*typeHelper, argType); - if (optionalTypeInspector) { - argType = optionalTypeInspector.GetItemType(); - } - } - - TDataTypeInspector dataTypeInspector(*typeHelper, argType); - if (!dataTypeInspector) { - builder.SetError("User type must be a data type"); - return true; - } - - builder.UserType(userType); - - auto typeId = dataTypeInspector.GetTypeId(); - if (!DeclareSignatureImpl<TUserTypes...>(typeId, userType, builder, typesOnly)) { + builder.UserType(userType); + + auto typeId = dataTypeInspector.GetTypeId(); + if (!DeclareSignatureImpl<TUserTypes...>(typeId, userType, builder, typesOnly)) { TStringBuilder sb; sb << "User type " << NYql::NUdf::GetDataTypeInfo(NYql::NUdf::GetDataSlot(typeId)).Name << " is not supported"; builder.SetError(sb); diff --git a/ydb/library/yql/public/udf/udf_type_builder.h b/ydb/library/yql/public/udf/udf_type_builder.h index 83f289429b..8bfc73a449 100644 --- a/ydb/library/yql/public/udf/udf_type_builder.h +++ b/ydb/library/yql/public/udf/udf_type_builder.h @@ -4,7 +4,7 @@ #include "udf_types.h" #include "udf_ptr.h" #include "udf_string_ref.h" -#include "udf_value.h" +#include "udf_value.h" #include <type_traits> @@ -342,73 +342,73 @@ private: UDF_ASSERT_TYPE_SIZE(IFunctionArgTypesBuilder, 16); ////////////////////////////////////////////////////////////////////////////// -// IRefCounted -////////////////////////////////////////////////////////////////////////////// -class IRefCounted { -public: - virtual ~IRefCounted() = default; - - inline void Ref() noexcept { - Refs_++; - } - - inline void UnRef() noexcept { - Y_VERIFY_DEBUG(Refs_ > 0); - if (--Refs_ == 0) { - delete this; - } - } - -private: - ui32 Refs_ = 0; - ui32 Reserved_ = 0; - - void Unused() { - Y_UNUSED(Reserved_); - } -}; - -UDF_ASSERT_TYPE_SIZE(IRefCounted, 16); - -////////////////////////////////////////////////////////////////////////////// -// IHash -////////////////////////////////////////////////////////////////////////////// -class IHash : public IRefCounted { -public: - using TPtr = TRefCountedPtr<IHash>; - - virtual ui64 Hash(TUnboxedValuePod value) const = 0; -}; - -UDF_ASSERT_TYPE_SIZE(IHash, 16); - -////////////////////////////////////////////////////////////////////////////// -// IEquate -////////////////////////////////////////////////////////////////////////////// -class IEquate : public IRefCounted { -public: - using TPtr = TRefCountedPtr<IEquate>; - - virtual bool Equals(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0; -}; - -UDF_ASSERT_TYPE_SIZE(IEquate, 16); - -////////////////////////////////////////////////////////////////////////////// -// ICompare -////////////////////////////////////////////////////////////////////////////// -class ICompare : public IRefCounted { -public: - using TPtr = TRefCountedPtr<ICompare>; - - virtual bool Less(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0; - // signed difference, for strings may not be just -1/0/1 - virtual int Compare(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0; -}; - -UDF_ASSERT_TYPE_SIZE(ICompare, 16); - -////////////////////////////////////////////////////////////////////////////// +// IRefCounted +////////////////////////////////////////////////////////////////////////////// +class IRefCounted { +public: + virtual ~IRefCounted() = default; + + inline void Ref() noexcept { + Refs_++; + } + + inline void UnRef() noexcept { + Y_VERIFY_DEBUG(Refs_ > 0); + if (--Refs_ == 0) { + delete this; + } + } + +private: + ui32 Refs_ = 0; + ui32 Reserved_ = 0; + + void Unused() { + Y_UNUSED(Reserved_); + } +}; + +UDF_ASSERT_TYPE_SIZE(IRefCounted, 16); + +////////////////////////////////////////////////////////////////////////////// +// IHash +////////////////////////////////////////////////////////////////////////////// +class IHash : public IRefCounted { +public: + using TPtr = TRefCountedPtr<IHash>; + + virtual ui64 Hash(TUnboxedValuePod value) const = 0; +}; + +UDF_ASSERT_TYPE_SIZE(IHash, 16); + +////////////////////////////////////////////////////////////////////////////// +// IEquate +////////////////////////////////////////////////////////////////////////////// +class IEquate : public IRefCounted { +public: + using TPtr = TRefCountedPtr<IEquate>; + + virtual bool Equals(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0; +}; + +UDF_ASSERT_TYPE_SIZE(IEquate, 16); + +////////////////////////////////////////////////////////////////////////////// +// ICompare +////////////////////////////////////////////////////////////////////////////// +class ICompare : public IRefCounted { +public: + using TPtr = TRefCountedPtr<ICompare>; + + virtual bool Less(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0; + // signed difference, for strings may not be just -1/0/1 + virtual int Compare(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0; +}; + +UDF_ASSERT_TYPE_SIZE(ICompare, 16); + +////////////////////////////////////////////////////////////////////////////// // IFunctionTypeInfoBuilder ////////////////////////////////////////////////////////////////////////////// namespace NImpl { @@ -516,15 +516,15 @@ public: virtual TSourcePosition GetSourcePosition() = 0; }; #endif - -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 10) + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 10) class IFunctionTypeInfoBuilder5: public IFunctionTypeInfoBuilder4 { public: - virtual IHash::TPtr MakeHash(const TType* type) = 0; - virtual IEquate::TPtr MakeEquate(const TType* type) = 0; - virtual ICompare::TPtr MakeCompare(const TType* type) = 0; + virtual IHash::TPtr MakeHash(const TType* type) = 0; + virtual IEquate::TPtr MakeEquate(const TType* type) = 0; + virtual ICompare::TPtr MakeCompare(const TType* type) = 0; }; -#endif +#endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 13) class IFunctionTypeInfoBuilder6: public IFunctionTypeInfoBuilder5 { diff --git a/ydb/library/yql/public/udf/udf_type_ops.h b/ydb/library/yql/public/udf/udf_type_ops.h index bf7f6a310b..4b2446c8fd 100644 --- a/ydb/library/yql/public/udf/udf_type_ops.h +++ b/ydb/library/yql/public/udf/udf_type_ops.h @@ -1,567 +1,567 @@ -#pragma once - -#include "udf_types.h" -#include "udf_value.h" - +#pragma once + +#include "udf_types.h" +#include "udf_value.h" + #include <util/generic/hash.h> -#include <util/digest/numeric.h> - +#include <util/digest/numeric.h> + namespace NYql { -namespace NUdf { - +namespace NUdf { + using THashType = ui64; -template <EDataSlot Type> +template <EDataSlot Type> inline THashType GetValueHash(const TUnboxedValuePod& value); inline THashType GetValueHash(EDataSlot type, const TUnboxedValuePod& value); - -template <EDataSlot Type> + +template <EDataSlot Type> inline int CompareValues(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs); inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs); - -template <EDataSlot Type> + +template <EDataSlot Type> inline bool EquateValues(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs); inline bool EquateValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs); - -template <EDataSlot Type> -struct TUnboxedValueHash { + +template <EDataSlot Type> +struct TUnboxedValueHash { std::size_t operator()(const TUnboxedValuePod& value) const { - return static_cast<std::size_t>(GetValueHash<Type>(value)); - } -}; - -template <EDataSlot Type> -struct TUnboxedValueEquals { + return static_cast<std::size_t>(GetValueHash<Type>(value)); + } +}; + +template <EDataSlot Type> +struct TUnboxedValueEquals { bool operator()(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) const { - return EquateValues<Type>(lhs, rhs); - } -}; - -// hash - -template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> + return EquateValues<Type>(lhs, rhs); + } +}; + +// hash + +template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> inline THashType GetIntegerHash(const TUnboxedValuePod& value) { - return std::hash<T>()(value.Get<T>()); -} - -template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr> + return std::hash<T>()(value.Get<T>()); +} + +template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr> inline THashType GetFloatHash(const TUnboxedValuePod& value) { - const auto x = value.Get<T>(); + const auto x = value.Get<T>(); return std::isunordered(x, x) ? ~0ULL : std::hash<T>()(x); -} - +} + inline THashType GetStringHash(const TUnboxedValuePod& value) { return THash<TStringBuf>{}(value.AsStringRef()); -} - -template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> +} + +template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> inline THashType GetTzIntegerHash(const TUnboxedValuePod& value) { return CombineHashes(std::hash<T>()(value.Get<T>()), std::hash<ui16>()(value.GetTimezoneId())); -} - -template <> +} + +template <> inline THashType GetValueHash<EDataSlot::Bool>(const TUnboxedValuePod& value) { - return std::hash<bool>()(value.Get<bool>()); -} - -template <> + return std::hash<bool>()(value.Get<bool>()); +} + +template <> inline THashType GetValueHash<EDataSlot::Int8>(const TUnboxedValuePod& value) { - return GetIntegerHash<i8>(value); -} - -template <> + return GetIntegerHash<i8>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Uint8>(const TUnboxedValuePod& value) { - return GetIntegerHash<ui8>(value); -} - -template <> + return GetIntegerHash<ui8>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Int16>(const TUnboxedValuePod& value) { - return GetIntegerHash<i16>(value); -} - -template <> + return GetIntegerHash<i16>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Uint16>(const TUnboxedValuePod& value) { - return GetIntegerHash<ui16>(value); -} - -template <> + return GetIntegerHash<ui16>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Int32>(const TUnboxedValuePod& value) { - return GetIntegerHash<i32>(value); -} - -template <> + return GetIntegerHash<i32>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Uint32>(const TUnboxedValuePod& value) { - return GetIntegerHash<ui32>(value); -} - -template <> + return GetIntegerHash<ui32>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Int64>(const TUnboxedValuePod& value) { - return GetIntegerHash<i64>(value); -} - -template <> + return GetIntegerHash<i64>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Uint64>(const TUnboxedValuePod& value) { - return GetIntegerHash<ui64>(value); -} - -template <> + return GetIntegerHash<ui64>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Float>(const TUnboxedValuePod& value) { - return GetFloatHash<float>(value); -} - -template <> + return GetFloatHash<float>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Double>(const TUnboxedValuePod& value) { - return GetFloatHash<double>(value); -} - -template <> + return GetFloatHash<double>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::String>(const TUnboxedValuePod& value) { - return GetStringHash(value); -} - -template <> + return GetStringHash(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Utf8>(const TUnboxedValuePod& value) { - return GetStringHash(value); -} - -template <> + return GetStringHash(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Uuid>(const TUnboxedValuePod& value) { - return GetStringHash(value); -} - -template <> + return GetStringHash(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Yson>(const TUnboxedValuePod&) { - Y_FAIL("Yson isn't hashable."); -} - -template <> + Y_FAIL("Yson isn't hashable."); +} + +template <> inline THashType GetValueHash<EDataSlot::Json>(const TUnboxedValuePod&) { - Y_FAIL("Json isn't hashable."); -} - -template <> + Y_FAIL("Json isn't hashable."); +} + +template <> inline THashType GetValueHash<EDataSlot::JsonDocument>(const TUnboxedValuePod&) { Y_FAIL("JsonDocument isn't hashable."); } template <> inline THashType GetValueHash<EDataSlot::Date>(const TUnboxedValuePod& value) { - return GetIntegerHash<ui16>(value); -} - -template <> + return GetIntegerHash<ui16>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Datetime>(const TUnboxedValuePod& value) { - return GetIntegerHash<ui32>(value); -} - -template <> + return GetIntegerHash<ui32>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Timestamp>(const TUnboxedValuePod& value) { - return GetIntegerHash<ui64>(value); -} - -template <> + return GetIntegerHash<ui64>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Interval>(const TUnboxedValuePod& value) { - return GetIntegerHash<i64>(value); -} - -template <> + return GetIntegerHash<i64>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::TzDate>(const TUnboxedValuePod& value) { - return GetTzIntegerHash<ui16>(value); -} - -template <> + return GetTzIntegerHash<ui16>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::TzDatetime>(const TUnboxedValuePod& value) { - return GetTzIntegerHash<ui32>(value); -} - -template <> + return GetTzIntegerHash<ui32>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::TzTimestamp>(const TUnboxedValuePod& value) { - return GetTzIntegerHash<ui64>(value); -} - -template <> + return GetTzIntegerHash<ui64>(value); +} + +template <> inline THashType GetValueHash<EDataSlot::Decimal>(const TUnboxedValuePod& value) { const auto pair = NYql::NDecimal::MakePair(value.GetInt128()); return CombineHashes(pair.first, pair.second); -} - +} + template <> inline THashType GetValueHash<EDataSlot::DyNumber>(const TUnboxedValuePod& value) { return GetStringHash(value); } inline THashType GetValueHash(EDataSlot type, const TUnboxedValuePod& value) { -#define HASH_TYPE(slot, ...) \ - case EDataSlot::slot: \ - return GetValueHash<EDataSlot::slot>(value); - - switch (type) { - UDF_TYPE_ID_MAP(HASH_TYPE) - } - -#undef HASH_TYPE - - Y_ENSURE(false, "Incorrect data slot: " << (ui32)type); -} - -// compare - -template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> +#define HASH_TYPE(slot, ...) \ + case EDataSlot::slot: \ + return GetValueHash<EDataSlot::slot>(value); + + switch (type) { + UDF_TYPE_ID_MAP(HASH_TYPE) + } + +#undef HASH_TYPE + + Y_ENSURE(false, "Incorrect data slot: " << (ui32)type); +} + +// compare + +template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> inline int CompareIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - const auto x = lhs.Get<T>(); - const auto y = rhs.Get<T>(); - return (x == y) ? 0 : (x < y ? -1 : 1); -} - -template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr> + const auto x = lhs.Get<T>(); + const auto y = rhs.Get<T>(); + return (x == y) ? 0 : (x < y ? -1 : 1); +} + +template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr> inline int CompareFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - const auto x = lhs.Get<T>(); - const auto y = rhs.Get<T>(); - if (std::isunordered(x, y)) { - const auto xn = std::isnan(x); - const auto yn = std::isnan(y); - return (xn == yn) ? 0 : xn ? 1 : -1; - } - return (x == y) ? 0 : (x < y ? -1 : 1); -} - + const auto x = lhs.Get<T>(); + const auto y = rhs.Get<T>(); + if (std::isunordered(x, y)) { + const auto xn = std::isnan(x); + const auto yn = std::isnan(y); + return (xn == yn) ? 0 : xn ? 1 : -1; + } + return (x == y) ? 0 : (x < y ? -1 : 1); +} + inline int CompareStrings(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { const TStringBuf lhsBuf = lhs.AsStringRef(); const TStringBuf rhsBuf = rhs.AsStringRef(); return lhsBuf.compare(rhsBuf); -} - -template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> +} + +template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> inline int CompareTzIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - const auto x = lhs.Get<T>(); - const auto y = rhs.Get<T>(); - if (x < y) { - return -1; - } - - if (x > y) { - return 1; - } - - const auto tx = lhs.GetTimezoneId(); - const auto ty = rhs.GetTimezoneId(); - return (tx == ty) ? 0 : (tx < ty ? -1 : 1); -} - -template <> + const auto x = lhs.Get<T>(); + const auto y = rhs.Get<T>(); + if (x < y) { + return -1; + } + + if (x > y) { + return 1; + } + + const auto tx = lhs.GetTimezoneId(); + const auto ty = rhs.GetTimezoneId(); + return (tx == ty) ? 0 : (tx < ty ? -1 : 1); +} + +template <> inline int CompareValues<EDataSlot::Bool>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - const auto x = lhs.Get<bool>(); - const auto y = rhs.Get<bool>(); - return x == y ? 0 : (!x ? -1 : 1); -} - -template <> + const auto x = lhs.Get<bool>(); + const auto y = rhs.Get<bool>(); + return x == y ? 0 : (!x ? -1 : 1); +} + +template <> inline int CompareValues<EDataSlot::Int8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<i8>(lhs, rhs); -} - -template <> + return CompareIntegers<i8>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Uint8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<ui8>(lhs, rhs); -} - -template <> + return CompareIntegers<ui8>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Int16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<i16>(lhs, rhs); -} - -template <> + return CompareIntegers<i16>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Uint16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<ui16>(lhs, rhs); -} - -template <> + return CompareIntegers<ui16>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Int32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<i32>(lhs, rhs); -} - -template <> + return CompareIntegers<i32>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Uint32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<ui32>(lhs, rhs); -} - -template <> + return CompareIntegers<ui32>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Int64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<i64>(lhs, rhs); -} - -template <> + return CompareIntegers<i64>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Uint64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<ui64>(lhs, rhs); -} - -template <> + return CompareIntegers<ui64>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Float>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareFloats<float>(lhs, rhs); -} - -template <> + return CompareFloats<float>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Double>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareFloats<double>(lhs, rhs); -} - -template <> + return CompareFloats<double>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::String>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareStrings(lhs, rhs); -} - -template <> + return CompareStrings(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Utf8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareStrings(lhs, rhs); -} - -template <> + return CompareStrings(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Uuid>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareStrings(lhs, rhs); -} - -template <> + return CompareStrings(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Yson>(const TUnboxedValuePod&, const TUnboxedValuePod&) { - Y_FAIL("Yson isn't comparable."); -} - -template <> + Y_FAIL("Yson isn't comparable."); +} + +template <> inline int CompareValues<EDataSlot::Json>(const TUnboxedValuePod&, const TUnboxedValuePod&) { - Y_FAIL("Json isn't comparable."); -} - -template <> + Y_FAIL("Json isn't comparable."); +} + +template <> inline int CompareValues<EDataSlot::JsonDocument>(const TUnboxedValuePod&, const TUnboxedValuePod&) { Y_FAIL("JsonDocument isn't comparable."); } template <> inline int CompareValues<EDataSlot::Date>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<ui16>(lhs, rhs); -} - -template <> + return CompareIntegers<ui16>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Datetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<ui32>(lhs, rhs); -} - -template <> + return CompareIntegers<ui32>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Timestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<ui64>(lhs, rhs); -} - -template <> + return CompareIntegers<ui64>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Interval>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareIntegers<i64>(lhs, rhs); -} - -template <> + return CompareIntegers<i64>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::TzDate>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareTzIntegers<ui16>(lhs, rhs); -} - -template <> + return CompareTzIntegers<ui16>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::TzDatetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareTzIntegers<ui32>(lhs, rhs); -} - -template <> + return CompareTzIntegers<ui32>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::TzTimestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return CompareTzIntegers<ui64>(lhs, rhs); -} - -template <> + return CompareTzIntegers<ui64>(lhs, rhs); +} + +template <> inline int CompareValues<EDataSlot::Decimal>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - const auto x = lhs.GetInt128(); - const auto y = rhs.GetInt128(); - return x == y ? 0 : (x < y ? -1 : 1); -} - + const auto x = lhs.GetInt128(); + const auto y = rhs.GetInt128(); + return x == y ? 0 : (x < y ? -1 : 1); +} + template <> inline int CompareValues<EDataSlot::DyNumber>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { return CompareStrings(lhs, rhs); } inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { -#define COMPARE_TYPE(slot, ...) \ - case EDataSlot::slot: \ - return CompareValues<EDataSlot::slot>(lhs, rhs); - - switch (type) { - UDF_TYPE_ID_MAP(COMPARE_TYPE) - } - -#undef COMPARE_TYPE - Y_ENSURE(false, "Incorrect data slot: " << (ui32)type); -} - -// equate - -template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> +#define COMPARE_TYPE(slot, ...) \ + case EDataSlot::slot: \ + return CompareValues<EDataSlot::slot>(lhs, rhs); + + switch (type) { + UDF_TYPE_ID_MAP(COMPARE_TYPE) + } + +#undef COMPARE_TYPE + Y_ENSURE(false, "Incorrect data slot: " << (ui32)type); +} + +// equate + +template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> inline bool EquateIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return lhs.Get<T>() == rhs.Get<T>(); -} - -template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr> + return lhs.Get<T>() == rhs.Get<T>(); +} + +template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr> inline bool EquateFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - const auto x = lhs.Get<T>(); - const auto y = rhs.Get<T>(); - return std::isunordered(x, y) ? std::isnan(x) == std::isnan(y) : x == y; -} - + const auto x = lhs.Get<T>(); + const auto y = rhs.Get<T>(); + return std::isunordered(x, y) ? std::isnan(x) == std::isnan(y) : x == y; +} + inline bool EquateStrings(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - const auto& lhsBuf = lhs.AsStringRef(); - const auto& rhsBuf = rhs.AsStringRef(); + const auto& lhsBuf = lhs.AsStringRef(); + const auto& rhsBuf = rhs.AsStringRef(); return lhsBuf == rhsBuf; -} - -template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> +} + +template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> inline bool EquateTzIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return lhs.Get<T>() == rhs.Get<T>() && lhs.GetTimezoneId() == rhs.GetTimezoneId(); -} - -template <> + return lhs.Get<T>() == rhs.Get<T>() && lhs.GetTimezoneId() == rhs.GetTimezoneId(); +} + +template <> inline bool EquateValues<EDataSlot::Bool>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<bool>(lhs, rhs); -} - -template <> + return EquateIntegers<bool>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Int8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<i8>(lhs, rhs); -} - -template <> + return EquateIntegers<i8>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Uint8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<ui8>(lhs, rhs); -} - -template <> + return EquateIntegers<ui8>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Int16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<i16>(lhs, rhs); -} - -template <> + return EquateIntegers<i16>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Uint16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<ui16>(lhs, rhs); -} - -template <> + return EquateIntegers<ui16>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Int32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<i32>(lhs, rhs); -} - -template <> + return EquateIntegers<i32>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Uint32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<ui32>(lhs, rhs); -} - -template <> + return EquateIntegers<ui32>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Int64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<i64>(lhs, rhs); -} - -template <> + return EquateIntegers<i64>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Uint64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<ui64>(lhs, rhs); -} - -template <> + return EquateIntegers<ui64>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Float>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateFloats<float>(lhs, rhs); -} - -template <> + return EquateFloats<float>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Double>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateFloats<double>(lhs, rhs); -} - -template <> + return EquateFloats<double>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::String>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateStrings(lhs, rhs); -} - -template <> + return EquateStrings(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Utf8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateStrings(lhs, rhs); -} - -template <> + return EquateStrings(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Uuid>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateStrings(lhs, rhs); -} - -template <> + return EquateStrings(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Yson>(const TUnboxedValuePod&, const TUnboxedValuePod&) { - Y_FAIL("Yson isn't comparable."); -} - -template <> + Y_FAIL("Yson isn't comparable."); +} + +template <> inline bool EquateValues<EDataSlot::Json>(const TUnboxedValuePod&, const TUnboxedValuePod&) { - Y_FAIL("Json isn't comparable."); -} - -template <> + Y_FAIL("Json isn't comparable."); +} + +template <> inline bool EquateValues<EDataSlot::JsonDocument>(const TUnboxedValuePod&, const TUnboxedValuePod&) { Y_FAIL("JsonDocument isn't comparable."); } template <> inline bool EquateValues<EDataSlot::Date>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<ui16>(lhs, rhs); -} - -template <> + return EquateIntegers<ui16>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Datetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<ui32>(lhs, rhs); -} - -template <> + return EquateIntegers<ui32>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Timestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<ui64>(lhs, rhs); -} - -template <> + return EquateIntegers<ui64>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Interval>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateIntegers<i64>(lhs, rhs); -} - -template <> + return EquateIntegers<i64>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::TzDate>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateTzIntegers<ui16>(lhs, rhs); -} - -template <> + return EquateTzIntegers<ui16>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::TzDatetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateTzIntegers<ui32>(lhs, rhs); -} - -template <> + return EquateTzIntegers<ui32>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::TzTimestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return EquateTzIntegers<ui64>(lhs, rhs); -} - -template <> + return EquateTzIntegers<ui64>(lhs, rhs); +} + +template <> inline bool EquateValues<EDataSlot::Decimal>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { - return lhs.GetInt128() == rhs.GetInt128(); -} - + return lhs.GetInt128() == rhs.GetInt128(); +} + template <> inline bool EquateValues<EDataSlot::DyNumber>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { return EquateStrings(lhs, rhs); } inline bool EquateValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) { -#define EQUATE_TYPE(slot, ...) \ - case EDataSlot::slot: \ - return EquateValues<EDataSlot::slot>(lhs, rhs); - - switch (type) { - UDF_TYPE_ID_MAP(EQUATE_TYPE) - } - -#undef EQUATE_TYPE - Y_ENSURE(false, "Incorrect data slot: " << (ui32)type); -} - -} // namespace NUdf +#define EQUATE_TYPE(slot, ...) \ + case EDataSlot::slot: \ + return EquateValues<EDataSlot::slot>(lhs, rhs); + + switch (type) { + UDF_TYPE_ID_MAP(EQUATE_TYPE) + } + +#undef EQUATE_TYPE + Y_ENSURE(false, "Incorrect data slot: " << (ui32)type); +} + +} // namespace NUdf } // namespace NYql diff --git a/ydb/library/yql/public/udf/udf_value.h b/ydb/library/yql/public/udf/udf_value.h index 807a0b7629..705ac41133 100644 --- a/ydb/library/yql/public/udf/udf_value.h +++ b/ydb/library/yql/public/udf/udf_value.h @@ -127,7 +127,7 @@ private: virtual bool NextPair(TUnboxedValue& key, TUnboxedValue& payload) = 0; virtual void Apply(IApplyContext& context) const = 0; - + public: // reference counting inline void Ref() noexcept; @@ -147,19 +147,19 @@ private: const ui8 Reserved_ = 0; }; -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) class IBoxedValue2 : public IBoxedValue1 { friend struct TBoxedValueAccessor; friend class TBlock; private: - // Save/Load state - virtual ui32 GetTraverseCount() const = 0; - virtual TUnboxedValue GetTraverseItem(ui32 index) const = 0; - virtual TUnboxedValue Save() const = 0; - virtual void Load(const TStringRef& state) = 0; + // Save/Load state + virtual ui32 GetTraverseCount() const = 0; + virtual TUnboxedValue GetTraverseItem(ui32 index) const = 0; + virtual TUnboxedValue Save() const = 0; + virtual void Load(const TStringRef& state) = 0; }; -#endif - +#endif + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11) class IBoxedValue3 : public IBoxedValue2 { friend struct TBoxedValueAccessor; @@ -391,7 +391,7 @@ UDF_ASSERT_TYPE_SIZE(TFlatDataBlockPtr, 8); struct TBoxedValueAccessor { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 19) - + #define METHOD_MAP(xx) \ xx(HasFastListLength) \ xx(GetListLength) \ @@ -421,10 +421,10 @@ struct TBoxedValueAccessor xx(Skip) \ xx(Next) \ xx(NextPair) \ - xx(Apply) \ - xx(GetTraverseCount) \ - xx(GetTraverseItem) \ - xx(Save) \ + xx(Apply) \ + xx(GetTraverseCount) \ + xx(GetTraverseItem) \ + xx(Save) \ xx(Load) \ xx(Push) \ xx(IsSortedDict) \ @@ -547,43 +547,43 @@ struct TBoxedValueAccessor xx(GetTraverseCount) \ xx(GetTraverseItem) \ xx(Save) \ - xx(Load) - -#else - -#define METHOD_MAP(xx) \ - xx(HasFastListLength) \ - xx(GetListLength) \ - xx(GetEstimatedListLength) \ - xx(GetListIterator) \ - xx(GetListRepresentation) \ - xx(ReverseListImpl) \ - xx(SkipListImpl) \ - xx(TakeListImpl) \ - xx(ToIndexDictImpl) \ - xx(GetDictLength) \ - xx(GetDictIterator) \ - xx(GetKeysIterator) \ - xx(GetPayloadsIterator) \ - xx(Contains) \ - xx(Lookup) \ - xx(GetElement) \ - xx(GetElements) \ - xx(Run) \ - xx(GetResourceTag) \ - xx(GetResource) \ - xx(HasListItems) \ - xx(HasDictItems) \ - xx(GetVariantIndex) \ - xx(GetVariantItem) \ - xx(Fetch) \ - xx(Skip) \ - xx(Next) \ - xx(NextPair) \ + xx(Load) + +#else + +#define METHOD_MAP(xx) \ + xx(HasFastListLength) \ + xx(GetListLength) \ + xx(GetEstimatedListLength) \ + xx(GetListIterator) \ + xx(GetListRepresentation) \ + xx(ReverseListImpl) \ + xx(SkipListImpl) \ + xx(TakeListImpl) \ + xx(ToIndexDictImpl) \ + xx(GetDictLength) \ + xx(GetDictIterator) \ + xx(GetKeysIterator) \ + xx(GetPayloadsIterator) \ + xx(Contains) \ + xx(Lookup) \ + xx(GetElement) \ + xx(GetElements) \ + xx(Run) \ + xx(GetResourceTag) \ + xx(GetResource) \ + xx(HasListItems) \ + xx(HasDictItems) \ + xx(GetVariantIndex) \ + xx(GetVariantItem) \ + xx(Fetch) \ + xx(Skip) \ + xx(Next) \ + xx(NextPair) \ xx(Apply) -#endif - +#endif + enum class EMethod : ui32 { #define MAP_HANDLER(xx) xx, METHOD_MAP(MAP_HANDLER) @@ -657,14 +657,14 @@ struct TBoxedValueAccessor static inline bool NextPair(IBoxedValue& value, TUnboxedValue& key, TUnboxedValue& payload); static inline void Apply(IBoxedValue& value, IApplyContext& context); - -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) - // Save/Load state - static inline ui32 GetTraverseCount(const IBoxedValue& value); - static inline TUnboxedValue GetTraverseItem(const IBoxedValue& value, ui32 index); - static inline TUnboxedValue Save(const IBoxedValue& value); - static inline void Load(IBoxedValue& value, const TStringRef& state); -#endif + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) + // Save/Load state + static inline ui32 GetTraverseCount(const IBoxedValue& value); + static inline TUnboxedValue GetTraverseItem(const IBoxedValue& value, ui32 index); + static inline TUnboxedValue Save(const IBoxedValue& value); + static inline void Load(IBoxedValue& value, const TStringRef& state); +#endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11) static inline void Push(IBoxedValue& value, const TUnboxedValuePod& data); @@ -741,15 +741,15 @@ private: bool NextPair(TUnboxedValue& key, TUnboxedValue& payload) override; void Apply(IApplyContext& context) const override; - -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) - // Save/Load state - ui32 GetTraverseCount() const override; - TUnboxedValue GetTraverseItem(ui32 index) const override; - TUnboxedValue Save() const override; - void Load(const TStringRef& state) override; -#endif - + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) + // Save/Load state + ui32 GetTraverseCount() const override; + TUnboxedValue GetTraverseItem(ui32 index) const override; + TUnboxedValue Save() const override; + void Load(const TStringRef& state) override; +#endif + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11) void Push(const TUnboxedValuePod& value) override; #endif @@ -919,14 +919,14 @@ public: inline void Apply(IApplyContext& context) const; -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) - // Save/Load state - inline ui32 GetTraverseCount() const; - inline TUnboxedValue GetTraverseItem(ui32 index) const; - inline TUnboxedValue Save() const; - inline void Load(const TStringRef& state); -#endif - +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) + // Save/Load state + inline ui32 GetTraverseCount() const; + inline TUnboxedValue GetTraverseItem(ui32 index) const; + inline TUnboxedValue Save() const; + inline void Load(const TStringRef& state); +#endif + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 12) inline bool IsSortedDict() const; #endif diff --git a/ydb/library/yql/public/udf/udf_value_builder.h b/ydb/library/yql/public/udf/udf_value_builder.h index 1a161d366c..096f735a2b 100644 --- a/ydb/library/yql/public/udf/udf_value_builder.h +++ b/ydb/library/yql/public/udf/udf_value_builder.h @@ -45,21 +45,21 @@ class IDateBuilder1 public: virtual ~IDateBuilder1() = default; - virtual bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const = 0; - virtual bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const = 0; + virtual bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const = 0; + virtual bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const = 0; - virtual bool MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, - ui16 timezoneId = 0) const = 0; - virtual bool SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, - ui16 timezoneId = 0) const = 0; + virtual bool MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, + ui16 timezoneId = 0) const = 0; + virtual bool SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, + ui16 timezoneId = 0) const = 0; // deprecated - virtual bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const = 0; - - // in minutes - virtual bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, - ui16 timezoneId, i32& value) const = 0; - + virtual bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const = 0; + + // in minutes + virtual bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, + ui16 timezoneId, i32& value) const = 0; + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT < UDF_ABI_COMPATIBILITY_VERSION(2, 23) virtual void Unused7() const = 0; virtual void Unused8() const = 0; diff --git a/ydb/library/yql/public/udf/udf_value_inl.h b/ydb/library/yql/public/udf/udf_value_inl.h index 028eef0cec..eb2db28e18 100644 --- a/ydb/library/yql/public/udf/udf_value_inl.h +++ b/ydb/library/yql/public/udf/udf_value_inl.h @@ -263,28 +263,28 @@ inline void TBoxedValueAccessor::Apply(IBoxedValue& value, IApplyContext& contex return value.Apply(context); } -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) -inline ui32 TBoxedValueAccessor::GetTraverseCount(const IBoxedValue& value) { - Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3))); - return value.GetTraverseCount(); -} - -inline TUnboxedValue TBoxedValueAccessor::GetTraverseItem(const IBoxedValue& value, ui32 index) { - Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3))); - return value.GetTraverseItem(index); -} - -inline TUnboxedValue TBoxedValueAccessor::Save(const IBoxedValue& value) { - Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3))); - return value.Save(); -} - -inline void TBoxedValueAccessor::Load(IBoxedValue& value, const TStringRef& state) { - Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3))); - value.Load(state); -} -#endif - +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) +inline ui32 TBoxedValueAccessor::GetTraverseCount(const IBoxedValue& value) { + Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3))); + return value.GetTraverseCount(); +} + +inline TUnboxedValue TBoxedValueAccessor::GetTraverseItem(const IBoxedValue& value, ui32 index) { + Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3))); + return value.GetTraverseItem(index); +} + +inline TUnboxedValue TBoxedValueAccessor::Save(const IBoxedValue& value) { + Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3))); + return value.Save(); +} + +inline void TBoxedValueAccessor::Load(IBoxedValue& value, const TStringRef& state) { + Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3))); + value.Load(state); +} +#endif + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11) inline void TBoxedValueAccessor::Push(IBoxedValue& value, const TUnboxedValuePod& data) { Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 11))); @@ -624,28 +624,28 @@ inline void TUnboxedValuePod::Apply(IApplyContext& context) const { return TBoxedValueAccessor::Apply(*Raw.Boxed.Value, context); } -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) -inline ui32 TUnboxedValuePod::GetTraverseCount() const { - UDF_VERIFY(IsBoxed(), "Value is not boxed"); - return TBoxedValueAccessor::GetTraverseCount(*Raw.Boxed.Value); -} - -inline TUnboxedValue TUnboxedValuePod::GetTraverseItem(ui32 index) const { - UDF_VERIFY(IsBoxed(), "Value is not boxed"); - return TBoxedValueAccessor::GetTraverseItem(*Raw.Boxed.Value, index); -} - -inline TUnboxedValue TUnboxedValuePod::Save() const { - UDF_VERIFY(IsBoxed(), "Value is not boxed"); - return TBoxedValueAccessor::Save(*Raw.Boxed.Value); -} - -inline void TUnboxedValuePod::Load(const TStringRef& state) { - UDF_VERIFY(IsBoxed(), "Value is not boxed"); - return TBoxedValueAccessor::Load(*Raw.Boxed.Value, state); -} -#endif - +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) +inline ui32 TUnboxedValuePod::GetTraverseCount() const { + UDF_VERIFY(IsBoxed(), "Value is not boxed"); + return TBoxedValueAccessor::GetTraverseCount(*Raw.Boxed.Value); +} + +inline TUnboxedValue TUnboxedValuePod::GetTraverseItem(ui32 index) const { + UDF_VERIFY(IsBoxed(), "Value is not boxed"); + return TBoxedValueAccessor::GetTraverseItem(*Raw.Boxed.Value, index); +} + +inline TUnboxedValue TUnboxedValuePod::Save() const { + UDF_VERIFY(IsBoxed(), "Value is not boxed"); + return TBoxedValueAccessor::Save(*Raw.Boxed.Value); +} + +inline void TUnboxedValuePod::Load(const TStringRef& state) { + UDF_VERIFY(IsBoxed(), "Value is not boxed"); + return TBoxedValueAccessor::Load(*Raw.Boxed.Value, state); +} +#endif + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 12) inline bool TUnboxedValuePod::IsSortedDict() const { UDF_VERIFY(IsBoxed(), "Value is not boxed"); diff --git a/ydb/library/yql/public/udf/udf_value_ut.cpp b/ydb/library/yql/public/udf/udf_value_ut.cpp index 0dc8bfee78..8d9708c7b1 100644 --- a/ydb/library/yql/public/udf/udf_value_ut.cpp +++ b/ydb/library/yql/public/udf/udf_value_ut.cpp @@ -144,12 +144,12 @@ Y_UNIT_TEST_SUITE(TUdfValue) { UNIT_ASSERT_VALUES_EQUAL(28, METHOD_INDEX(Next)); UNIT_ASSERT_VALUES_EQUAL(29, METHOD_INDEX(NextPair)); UNIT_ASSERT_VALUES_EQUAL(30, METHOD_INDEX(Apply)); -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) - UNIT_ASSERT_VALUES_EQUAL(31, METHOD_INDEX(GetTraverseCount)); - UNIT_ASSERT_VALUES_EQUAL(32, METHOD_INDEX(GetTraverseItem)); - UNIT_ASSERT_VALUES_EQUAL(33, METHOD_INDEX(Save)); - UNIT_ASSERT_VALUES_EQUAL(34, METHOD_INDEX(Load)); -#endif +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) + UNIT_ASSERT_VALUES_EQUAL(31, METHOD_INDEX(GetTraverseCount)); + UNIT_ASSERT_VALUES_EQUAL(32, METHOD_INDEX(GetTraverseItem)); + UNIT_ASSERT_VALUES_EQUAL(33, METHOD_INDEX(Save)); + UNIT_ASSERT_VALUES_EQUAL(34, METHOD_INDEX(Load)); +#endif #undef METHOD_INDEX } } diff --git a/ydb/library/yql/sql/v0/SQL.g b/ydb/library/yql/sql/v0/SQL.g index 6a20a39c12..3b616b9c9f 100644 --- a/ydb/library/yql/sql/v0/SQL.g +++ b/ydb/library/yql/sql/v0/SQL.g @@ -213,7 +213,7 @@ reduce_core: opt_set_quantifier: (ALL | DISTINCT)?; select_core: - (FROM join_source)? SELECT STREAM? opt_set_quantifier result_column (COMMA result_column)* (WITHOUT column_list)? (FROM join_source)? (WHERE expr)? + (FROM join_source)? SELECT STREAM? opt_set_quantifier result_column (COMMA result_column)* (WITHOUT column_list)? (FROM join_source)? (WHERE expr)? group_by_clause? (HAVING expr)? window_clause? order_by_clause? ; @@ -229,7 +229,7 @@ grouping_element: | cube_list | grouping_sets_specification //empty_grouping_set inside smart_parenthesis - | hopping_window_specification + | hopping_window_specification ; /// expect column (named column), or parenthesis list columns, or expression (named expression), or list expression @@ -242,8 +242,8 @@ cube_list: CUBE LPAREN ordinary_grouping_set_list RPAREN; /// SQL2003 grouping_set_list == grouping_element_list grouping_sets_specification: GROUPING SETS LPAREN grouping_element_list RPAREN; -hopping_window_specification: HOP LPAREN expr COMMA expr COMMA expr COMMA expr RPAREN; - +hopping_window_specification: HOP LPAREN expr COMMA expr COMMA expr COMMA expr RPAREN; + result_column: opt_id_prefix ASTERISK | expr (AS id_or_string)? @@ -306,14 +306,14 @@ alter_table_add_column: ADD COLUMN? column_schema (COMMA ADD COLUMN? column_sche alter_table_drop_column: DROP COLUMN? id; column_schema: id_schema flex_type (NOT? NULL)?; -column_order_by_specification: id (ASC | DESC)?; - -table_constraint: - PRIMARY KEY LPAREN id (COMMA id)* RPAREN - | PARTITION BY LPAREN id (COMMA id)* RPAREN - | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN -; - +column_order_by_specification: id (ASC | DESC)?; + +table_constraint: + PRIMARY KEY LPAREN id (COMMA id)* RPAREN + | PARTITION BY LPAREN id (COMMA id)* RPAREN + | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN +; + drop_table_stmt: DROP TABLE (IF EXISTS)? simple_table_ref; define_action_or_subquery_stmt: DEFINE (ACTION|SUBQUERY) bind_parameter LPAREN bind_parameter_list? RPAREN AS define_action_or_subquery_body END DEFINE; define_action_or_subquery_body: (sql_stmt_core SEMI)* SEMI?; @@ -435,7 +435,7 @@ keyword_expr_uncompat: | EXISTS | FROM | FULL - | HOP + | HOP | NOT | NULL | PROCESS @@ -772,7 +772,7 @@ GLOB: G L O B; GROUP: G R O U P; GROUPING: G R O U P I N G; HAVING: H A V I N G; -HOP: H O P; +HOP: H O P; IF: I F; IGNORE: I G N O R E; ILIKE: I L I K E; @@ -848,7 +848,7 @@ SEMI_JOIN: S E M I; SET: S E T; SETS: S E T S; SUBQUERY: S U B Q U E R Y; -STREAM: S T R E A M; +STREAM: S T R E A M; SYMBOLS: S Y M B O L S; SYSTEM: S Y S T E M; TABLE: T A B L E; diff --git a/ydb/library/yql/sql/v0/aggregation.cpp b/ydb/library/yql/sql/v0/aggregation.cpp index ad2b3e12e5..6440604881 100644 --- a/ydb/library/yql/sql/v0/aggregation.cpp +++ b/ydb/library/yql/sql/v0/aggregation.cpp @@ -260,8 +260,8 @@ private: apply = L(apply, Limit); } - std::vector<ui32> GetFactoryColumnIndices() const final { - return {1u, 0u}; + std::vector<ui32> GetFactoryColumnIndices() const final { + return {1u, 0u}; } bool DoInit(TContext& ctx, ISource* src) final { @@ -338,8 +338,8 @@ private: return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload), BuildLambda(Pos, Y("row"), Predicate)); } - std::vector<ui32> GetFactoryColumnIndices() const final { - return {0u, 1u}; + std::vector<ui32> GetFactoryColumnIndices() const final { + return {0u, 1u}; } bool DoInit(TContext& ctx, ISource* src) final { @@ -517,8 +517,8 @@ private: apply = L(apply, Intervals); } - std::vector<ui32> GetFactoryColumnIndices() const final { - return {0u, 1u}; + std::vector<ui32> GetFactoryColumnIndices() const final { + return {0u, 1u}; } bool DoInit(TContext& ctx, ISource* src) final { @@ -862,118 +862,118 @@ TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& nam return new TTopFreqFactory(pos, name, factory, aggMode); } -template <bool HasKey> -class TTopAggregationFactory final : public TAggregationFactory { -public: - TTopAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) +template <bool HasKey> +class TTopAggregationFactory final : public TAggregationFactory { +public: + TTopAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) + : TAggregationFactory(pos, name, factory, aggMode) , FakeSource(BuildFakeSource(pos)) - {} - -private: - bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { - ui32 adjustArgsCount = isFactory ? 1 : (HasKey ? 3 : 2); - if (exprs.size() != adjustArgsCount) { - ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " - << adjustArgsCount << " arguments, given: " << exprs.size(); - return false; - } - - if (!isFactory) { - Payload = exprs[0]; - if (HasKey) { - Key = exprs[1]; - } - } - - Count = exprs.back(); - - if (!isFactory) { + {} + +private: + bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { + ui32 adjustArgsCount = isFactory ? 1 : (HasKey ? 3 : 2); + if (exprs.size() != adjustArgsCount) { + ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " + << adjustArgsCount << " arguments, given: " << exprs.size(); + return false; + } + + if (!isFactory) { + Payload = exprs[0]; + if (HasKey) { + Key = exprs[1]; + } + } + + Count = exprs.back(); + + if (!isFactory) { Name = src->MakeLocalName(Name); - } - - if (!Init(ctx, src)) { - return false; - } - - if (!isFactory) { - node.Add("Member", "row", Q(Name)); - } - - return true; - } - - TNodePtr DoClone() const final { - return new TTopAggregationFactory(Pos, Name, Func, AggMode); - } - - TNodePtr GetApply(const TNodePtr& type) const final { - TNodePtr apply; - if (HasKey) { - apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload)); - } else { - apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload)); - } - AddFactoryArguments(apply); - return apply; - } - - void AddFactoryArguments(TNodePtr& apply) const final { - apply = L(apply, Count); - } - - std::vector<ui32> GetFactoryColumnIndices() const final { - if (HasKey) { - return {1u, 0u}; - } else { - return {0u}; - } - } - - bool DoInit(TContext& ctx, ISource* src) final { + } + + if (!Init(ctx, src)) { + return false; + } + + if (!isFactory) { + node.Add("Member", "row", Q(Name)); + } + + return true; + } + + TNodePtr DoClone() const final { + return new TTopAggregationFactory(Pos, Name, Func, AggMode); + } + + TNodePtr GetApply(const TNodePtr& type) const final { + TNodePtr apply; + if (HasKey) { + apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload)); + } else { + apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload)); + } + AddFactoryArguments(apply); + return apply; + } + + void AddFactoryArguments(TNodePtr& apply) const final { + apply = L(apply, Count); + } + + std::vector<ui32> GetFactoryColumnIndices() const final { + if (HasKey) { + return {1u, 0u}; + } else { + return {0u}; + } + } + + bool DoInit(TContext& ctx, ISource* src) final { ctx.PushBlockShortcuts(); if (!Count->Init(ctx, FakeSource.Get())) { - return false; - } + return false; + } Count = ctx.GroundBlockShortcutsForExpr(Count); - - if (!Payload) { - return true; - } - - if (HasKey) { - ctx.PushBlockShortcuts(); - if (!Key->Init(ctx, src)) { - return false; - } - Key = ctx.GroundBlockShortcutsForExpr(Key); - } - - ctx.PushBlockShortcuts(); - if (!Payload->Init(ctx, src)) { - return false; - } - Payload = ctx.GroundBlockShortcutsForExpr(Payload); - - if ((HasKey && Key->IsAggregated()) || (!HasKey && Payload->IsAggregated())) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; - return false; - } - return true; - } - + + if (!Payload) { + return true; + } + + if (HasKey) { + ctx.PushBlockShortcuts(); + if (!Key->Init(ctx, src)) { + return false; + } + Key = ctx.GroundBlockShortcutsForExpr(Key); + } + + ctx.PushBlockShortcuts(); + if (!Payload->Init(ctx, src)) { + return false; + } + Payload = ctx.GroundBlockShortcutsForExpr(Payload); + + if ((HasKey && Key->IsAggregated()) || (!HasKey && Payload->IsAggregated())) { + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; + return false; + } + return true; + } + TSourcePtr FakeSource; - TNodePtr Key, Payload, Count; -}; - -template <bool HasKey> -TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TTopAggregationFactory<HasKey>(pos, name, factory, aggMode); -} - -template TAggregationPtr BuildTopFactoryAggregation<false>(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); -template TAggregationPtr BuildTopFactoryAggregation<true >(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); - + TNodePtr Key, Payload, Count; +}; + +template <bool HasKey> +TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { + return new TTopAggregationFactory<HasKey>(pos, name, factory, aggMode); +} + +template TAggregationPtr BuildTopFactoryAggregation<false>(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); +template TAggregationPtr BuildTopFactoryAggregation<true >(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); + class TCountDistinctEstimateAggregationFactory final : public TAggregationFactory { public: TCountDistinctEstimateAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) diff --git a/ydb/library/yql/sql/v0/builtin.cpp b/ydb/library/yql/sql/v0/builtin.cpp index 2b6b241a72..a0945f9d05 100644 --- a/ydb/library/yql/sql/v0/builtin.cpp +++ b/ydb/library/yql/sql/v0/builtin.cpp @@ -140,14 +140,14 @@ public: auto factory = Aggr->AggregationTraitsFactory(); auto apply = Y("Apply", factory, Y("ListType", "type")); - - auto columnIndices = Aggr->GetFactoryColumnIndices(); - if (columnIndices.size() == 1) { + + auto columnIndices = Aggr->GetFactoryColumnIndices(); + if (columnIndices.size() == 1) { apply = L(apply, "extractor"); } else { // make several extractors from main that returns a tuple - for (ui32 arg = 0; arg < columnIndices.size(); ++arg) { - auto partial = BuildLambda(Pos, Y("row"), Y("Nth", Y("Apply", "extractor", "row"), Q(ToString(columnIndices[arg])))); + for (ui32 arg = 0; arg < columnIndices.size(); ++arg) { + auto partial = BuildLambda(Pos, Y("row"), Y("Nth", Y("Apply", "extractor", "row"), Q(ToString(columnIndices[arg])))); apply = L(apply, partial); } } @@ -778,12 +778,12 @@ public: } auto aliasNode = BuildFileNameArgument(Args[1]->GetPos(), Args[1]); OpName = "Apply"; - Args[0] = Y("Udf", Q("File.ByLines"), Y("Void"), - Y("TupleType", - Y("TupleType", Y("DataType", dataTypeStringNode)), - Y("StructType"), - Y("TupleType"))); - + Args[0] = Y("Udf", Q("File.ByLines"), Y("Void"), + Y("TupleType", + Y("TupleType", Y("DataType", dataTypeStringNode)), + Y("StructType"), + Y("TupleType"))); + Args[1] = Y("FilePath", aliasNode); return TCallNode::DoInit(ctx, src); } @@ -1579,47 +1579,47 @@ private: TString Mode; }; -template <bool IsStart> -class THoppingTime final: public TAstListNode { -public: - THoppingTime(TPosition pos, const TVector<TNodePtr>& args = {}) - : TAstListNode(pos) +template <bool IsStart> +class THoppingTime final: public TAstListNode { +public: + THoppingTime(TPosition pos, const TVector<TNodePtr>& args = {}) + : TAstListNode(pos) { Y_UNUSED(args); } - -private: - TNodePtr DoClone() const override { - return new THoppingTime(GetPos()); - } - - bool DoInit(TContext& ctx, ISource* src) override { - Y_UNUSED(ctx); - - auto window = src->GetHoppingWindowSpec(); - if (!window) { - ctx.Error(Pos) << "No hopping window parameters in aggregation"; - return false; - } - - Nodes.clear(); - - if (!IsStart) { - Add("Member", "row", Q("_yql_time")); - return true; - } - - Add("Sub", - Y("Member", "row", Q("_yql_time")), - window->Interval); - return true; - } - - void DoUpdateState() const override { - State.Set(ENodeState::Aggregated, true); - } -}; - + +private: + TNodePtr DoClone() const override { + return new THoppingTime(GetPos()); + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(ctx); + + auto window = src->GetHoppingWindowSpec(); + if (!window) { + ctx.Error(Pos) << "No hopping window parameters in aggregation"; + return false; + } + + Nodes.clear(); + + if (!IsStart) { + Add("Member", "row", Q("_yql_time")); + return true; + } + + Add("Sub", + Y("Member", "row", Q("_yql_time")), + window->Interval); + return true; + } + + void DoUpdateState() const override { + State.Set(ENodeState::Aggregated, true); + } +}; + class TInvalidBuiltin final: public INode { public: TInvalidBuiltin(TPosition pos, const TString& info) @@ -1655,8 +1655,8 @@ enum EAggrFuncTypeCallback { LINEAR_HISTOGRAM, PERCENTILE, TOPFREQ, - TOP, - TOP_BY, + TOP, + TOP_BY, COUNT_DISTINCT_ESTIMATE, LIST, UDAF @@ -1722,12 +1722,12 @@ TAggrFuncFactoryCallback BuildAggrFuncFactoryCallback( case TOPFREQ: factory = BuildTopFreqFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; - case TOP: - factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode); - break; - case TOP_BY: - factory = BuildTopFactoryAggregation<true>(pos, realFunctionName, factoryName, aggMode); - break; + case TOP: + factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode); + break; + case TOP_BY: + factory = BuildTopFactoryAggregation<true>(pos, realFunctionName, factoryName, aggMode); + break; case COUNT_DISTINCT_ESTIMATE: factory = BuildCountDistinctEstimateFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; @@ -2026,7 +2026,7 @@ struct TBuiltinFuncData { {"tablerecordindex", BuildNamedBuiltinFactoryCallback<TCallDirectRow>("TableRecord") }, {"weakfield", BuildSimpleBuiltinFactoryCallback<TWeakFieldOp>()}, {"tablerow", BuildSimpleBuiltinFactoryCallback<TTableRow>() }, - + // Hint builtins {"grouping", BuildSimpleBuiltinFactoryCallback<TGroupingNode>()}, @@ -2088,11 +2088,11 @@ struct TBuiltinFuncData { {"mode", BuildAggrFuncFactoryCallback("Mode", "topfreq_traits_factory", TOPFREQ) }, {"topfreq", BuildAggrFuncFactoryCallback("TopFreq", "topfreq_traits_factory", TOPFREQ) }, - {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)}, - {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)}, - {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)}, - {"bottomby", BuildAggrFuncFactoryCallback("BottomBy", "bottom_by_traits_factory", TOP_BY)}, - + {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)}, + {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)}, + {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)}, + {"bottomby", BuildAggrFuncFactoryCallback("BottomBy", "bottom_by_traits_factory", TOP_BY)}, + {"histogram", BuildAggrFuncFactoryCallback("AdaptiveWardHistogram", "histogram_adaptive_ward_traits_factory", HISTOGRAM, "Histogram")}, {"adaptivewardhistogram", BuildAggrFuncFactoryCallback("AdaptiveWardHistogram", "histogram_adaptive_ward_traits_factory", HISTOGRAM)}, {"adaptiveweighthistogram", BuildAggrFuncFactoryCallback("AdaptiveWeightHistogram", "histogram_adaptive_weight_traits_factory", HISTOGRAM)}, @@ -2334,9 +2334,9 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec return BuildUdf(ctx, pos, moduleName, name, newArgs); } - } else if (ns == "datetime2" && (name == "Format" || name == "Parse")) { - return BuildUdf(ctx, pos, nameSpace, name, args); - + } else if (ns == "datetime2" && (name == "Format" || name == "Parse")) { + return BuildUdf(ctx, pos, nameSpace, name, args); + } else if (scriptType != NKikimr::NMiniKQL::EScriptType::Unknown) { auto scriptName = NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType); return new TScriptUdf(pos, TString(scriptName), name, args); diff --git a/ydb/library/yql/sql/v0/node.cpp b/ydb/library/yql/sql/v0/node.cpp index 095e37fbf3..57f0c6a561 100644 --- a/ydb/library/yql/sql/v0/node.cpp +++ b/ydb/library/yql/sql/v0/node.cpp @@ -75,12 +75,12 @@ bool TTableRef::Check(TContext& ctx) const { return true; } -TColumnSchema::TColumnSchema(TPosition pos, const TString& name, const TString& type, bool nullable, bool isTypeString) +TColumnSchema::TColumnSchema(TPosition pos, const TString& name, const TString& type, bool nullable, bool isTypeString) : Pos(pos) , Name(name) , Type(type) , Nullable(nullable) - , IsTypeString(isTypeString) + , IsTypeString(isTypeString) { } @@ -794,9 +794,9 @@ TWindowSpecificationPtr TWindowSpecification::Clone() const { THoppingWindowSpecPtr THoppingWindowSpec::Clone() const { auto res = MakeIntrusive<THoppingWindowSpec>(); res->TimeExtractor = TimeExtractor->Clone(); - res->Hop = Hop->Clone(); - res->Interval = Interval->Clone(); - res->Delay = Delay->Clone(); + res->Hop = Hop->Clone(); + res->Interval = Interval->Clone(); + res->Delay = Delay->Clone(); return res; } @@ -1018,8 +1018,8 @@ void IAggregation::AddFactoryArguments(TNodePtr& apply) const { Y_UNUSED(apply); } -std::vector<ui32> IAggregation::GetFactoryColumnIndices() const { - return {0u}; +std::vector<ui32> IAggregation::GetFactoryColumnIndices() const { + return {0u}; } TNodePtr IAggregation::WindowTraits(const TNodePtr& type) const { @@ -1190,14 +1190,14 @@ const TVector<TString>& ISource::GetTmpWindowColumns() const { return TmpWindowColumns; } -void ISource::SetHoppingWindowSpec(THoppingWindowSpecPtr spec) { - HoppingWindowSpec = spec; -} - -THoppingWindowSpecPtr ISource::GetHoppingWindowSpec() const { - return HoppingWindowSpec; -} - +void ISource::SetHoppingWindowSpec(THoppingWindowSpecPtr spec) { + HoppingWindowSpec = spec; +} + +THoppingWindowSpecPtr ISource::GetHoppingWindowSpec() const { + return HoppingWindowSpec; +} + TWindowSpecificationPtr ISource::FindWindowSpecification(TContext& ctx, const TString& windowName) const { auto winIter = WinSpecs.find(windowName); if (winIter == WinSpecs.end()) { @@ -1321,10 +1321,10 @@ bool ISource::IsOverWindowSource() const { return !WinSpecs.empty(); } -bool ISource::IsStream() const { - return false; -} - +bool ISource::IsStream() const { + return false; +} + bool ISource::IsOrdered() const { return false; } @@ -1452,20 +1452,20 @@ TNodePtr ISource::BuildAggregation(const TString& label) { aggrArgs = L(aggrArgs, traits); } - if (HoppingWindowSpec) { - auto hoppingTraits = Y( - "HoppingTraits", - Y("ListItemType", listType), - BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor), - HoppingWindowSpec->Hop, - HoppingWindowSpec->Interval, + if (HoppingWindowSpec) { + auto hoppingTraits = Y( + "HoppingTraits", + Y("ListItemType", listType), + BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor), + HoppingWindowSpec->Hop, + HoppingWindowSpec->Interval, HoppingWindowSpec->Delay, Q("False")); - - return Y("Aggregate", label, Q(keysTuple), Q(aggrArgs), - Q(Y(Q(Y(BuildQuotedAtom(Pos, "hopping"), hoppingTraits))))); - } - + + return Y("Aggregate", label, Q(keysTuple), Q(aggrArgs), + Q(Y(Q(Y(BuildQuotedAtom(Pos, "hopping"), hoppingTraits))))); + } + return Y("Aggregate", label, Q(keysTuple), Q(aggrArgs)); } diff --git a/ydb/library/yql/sql/v0/node.h b/ydb/library/yql/sql/v0/node.h index ec88de65cd..4c7cfc273b 100644 --- a/ydb/library/yql/sql/v0/node.h +++ b/ydb/library/yql/sql/v0/node.h @@ -449,9 +449,9 @@ namespace NSQLTranslationV0 { TString Name; TString Type; bool Nullable; - bool IsTypeString; + bool IsTypeString; - TColumnSchema(TPosition pos, const TString& name, const TString& type, bool nullable, bool isTypeString); + TColumnSchema(TPosition pos, const TString& name, const TString& type, bool nullable, bool isTypeString); }; struct TColumns: public TSimpleRefCount<TColumns> { @@ -506,23 +506,23 @@ namespace NSQLTranslationV0 { ~TWindowSpecification() {} }; - struct THoppingWindowSpec: public TSimpleRefCount<THoppingWindowSpec> { - TNodePtr TimeExtractor; - TNodePtr Hop; - TNodePtr Interval; - TNodePtr Delay; + struct THoppingWindowSpec: public TSimpleRefCount<THoppingWindowSpec> { + TNodePtr TimeExtractor; + TNodePtr Hop; + TNodePtr Interval; + TNodePtr Delay; TIntrusivePtr<THoppingWindowSpec> Clone() const; ~THoppingWindowSpec() {} - }; - + }; + typedef TIntrusivePtr<TWindowSpecification> TWindowSpecificationPtr; typedef TMap<TString, TWindowSpecificationPtr> TWinSpecs; typedef TVector<TTableRef> TTableList; - typedef TIntrusivePtr<THoppingWindowSpec> THoppingWindowSpecPtr; - + typedef TIntrusivePtr<THoppingWindowSpec> THoppingWindowSpecPtr; + bool ValidateAllNodesForAggregation(TContext& ctx, const TVector<TNodePtr>& nodes); class TDeferredAtom { @@ -654,7 +654,7 @@ namespace NSQLTranslationV0 { virtual TNodePtr AggregationTraitsFactory() const = 0; - virtual std::vector<ui32> GetFactoryColumnIndices() const; + virtual std::vector<ui32> GetFactoryColumnIndices() const; virtual void AddFactoryArguments(TNodePtr& apply) const; @@ -718,14 +718,14 @@ namespace NSQLTranslationV0 { virtual void AddWindowSpecs(TWinSpecs winSpecs); virtual bool AddAggregationOverWindow(TContext& ctx, const TString& windowName, TAggregationPtr func); virtual bool AddFuncOverWindow(TContext& ctx, const TString& windowName, TNodePtr func); - virtual void SetHoppingWindowSpec(THoppingWindowSpecPtr spec); - virtual THoppingWindowSpecPtr GetHoppingWindowSpec() const; + virtual void SetHoppingWindowSpec(THoppingWindowSpecPtr spec); + virtual THoppingWindowSpecPtr GetHoppingWindowSpec() const; virtual bool IsCompositeSource() const; virtual bool IsGroupByColumn(const TString& column) const; virtual bool IsFlattenByColumns() const; virtual bool IsCalcOverWindow() const; virtual bool IsOverWindowSource() const; - virtual bool IsStream() const; + virtual bool IsStream() const; virtual bool IsOrdered() const; virtual TWriteSettings GetWriteSettings() const; virtual bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed); @@ -784,7 +784,7 @@ namespace NSQLTranslationV0 { TMultiMap<TString, TAggregationPtr> AggregationOverWindow; TMultiMap<TString, TNodePtr> FuncOverWindow; TWinSpecs WinSpecs; - THoppingWindowSpecPtr HoppingWindowSpec; + THoppingWindowSpecPtr HoppingWindowSpec; TVector<ISource*> UsedSources; TString FlattenMode; bool FlattenColumns = false; @@ -912,8 +912,8 @@ namespace NSQLTranslationV0 { TAggregationPtr BuildTwoArgsFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildLinearHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); - template <bool HasKey> - TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); + template <bool HasKey> + TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildCountDistinctEstimateFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildListFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); @@ -949,7 +949,7 @@ namespace NSQLTranslationV0 { TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources); TSourcePtr BuildFakeSource(TPosition pos); TSourcePtr BuildNodeSource(TPosition pos, const TNodePtr& node); - TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label = TString()); + TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label = TString()); TSourcePtr BuildInnerSource(TPosition pos, TNodePtr node, const TString& label = TString()); TSourcePtr BuildRefColumnSource(TPosition pos, const TString& partExpression); TSourcePtr BuildUnionAll(TPosition pos, TVector<TSourcePtr>&& sources); @@ -968,7 +968,7 @@ namespace NSQLTranslationV0 { const TVector<TSortSpecificationPtr>& orderBy, TNodePtr having, TWinSpecs&& windowSpec, - THoppingWindowSpecPtr hoppingWindowSpec, + THoppingWindowSpecPtr hoppingWindowSpec, TVector<TNodePtr>&& terms, bool distinct, TVector<TNodePtr>&& without, @@ -1006,8 +1006,8 @@ namespace NSQLTranslationV0 { TNodePtr BuildInputOptions(TPosition pos, const TVector<TString>& hints); TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubquery); TNodePtr BuildCreateTable(TPosition pos, const TTableRef& table, const TVector<TColumnSchema>& columns, - const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns, - const TVector<std::pair<TIdentifier, bool>>& orderByColumns); + const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns, + const TVector<std::pair<TIdentifier, bool>>& orderByColumns); TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TVector<TColumnSchema>& columns, EAlterTableIntentnt mode); TNodePtr BuildDropTable(TPosition pos, const TTableRef& table); TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options = nullptr); diff --git a/ydb/library/yql/sql/v0/query.cpp b/ydb/library/yql/sql/v0/query.cpp index fa169a2e48..7752b28b01 100644 --- a/ydb/library/yql/sql/v0/query.cpp +++ b/ydb/library/yql/sql/v0/query.cpp @@ -421,14 +421,14 @@ TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubque class TCreateTableNode final: public TAstListNode { public: TCreateTableNode(TPosition pos, const TTableRef& tr, const TVector<TColumnSchema>& columns, - const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns, - const TVector<std::pair<TIdentifier, bool>>& orderByColumns) + const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns, + const TVector<std::pair<TIdentifier, bool>>& orderByColumns) : TAstListNode(pos) , Table(tr) , Columns(columns) , PkColumns(pkColumns) - , PartitionByColumns(partitionByColumns) - , OrderByColumns(orderByColumns) + , PartitionByColumns(partitionByColumns) + , OrderByColumns(orderByColumns) {} bool DoInit(TContext& ctx, ISource* src) override { @@ -442,7 +442,7 @@ public: } keys = ctx.GroundBlockShortcutsForExpr(keys); - if (!PkColumns.empty() || !PartitionByColumns.empty() || !OrderByColumns.empty()) { + if (!PkColumns.empty() || !PartitionByColumns.empty() || !OrderByColumns.empty()) { THashSet<TString> columnsSet; for (auto& col : Columns) { columnsSet.insert(col.Name); @@ -454,23 +454,23 @@ public: return false; } } - for (auto& keyColumn : PartitionByColumns) { - if (!columnsSet.contains(keyColumn.Name)) { - ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name; - return false; - } - } - for (auto& keyColumn : OrderByColumns) { - if (!columnsSet.contains(keyColumn.first.Name)) { - ctx.Error(keyColumn.first.Pos) << "Undefined column: " << keyColumn.first.Name; - return false; - } - } + for (auto& keyColumn : PartitionByColumns) { + if (!columnsSet.contains(keyColumn.Name)) { + ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name; + return false; + } + } + for (auto& keyColumn : OrderByColumns) { + if (!columnsSet.contains(keyColumn.first.Name)) { + ctx.Error(keyColumn.first.Pos) << "Undefined column: " << keyColumn.first.Name; + return false; + } + } } auto columns = Y(); for (auto& col: Columns) { - auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos); + auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos); if (!type) { return false; } @@ -507,45 +507,45 @@ public: opts = L(opts, Q(Y(Q("mode"), Q("create")))); opts = L(opts, Q(Y(Q("columns"), Q(columns)))); - const auto serviceName = to_lower(Table.ServiceName(ctx)); - if (serviceName == RtmrProviderName) { - if (!PkColumns.empty() && !PartitionByColumns.empty()) { - ctx.Error() << "Only one of PRIMARY KEY or PARTITION BY constraints may be specified"; - return false; - } - } else { - if (!PartitionByColumns.empty() || !OrderByColumns.empty()) { - ctx.Error() << "PARTITION BY and ORDER BY are supported only for " << RtmrProviderName << " provider"; - return false; - } - } - + const auto serviceName = to_lower(Table.ServiceName(ctx)); + if (serviceName == RtmrProviderName) { + if (!PkColumns.empty() && !PartitionByColumns.empty()) { + ctx.Error() << "Only one of PRIMARY KEY or PARTITION BY constraints may be specified"; + return false; + } + } else { + if (!PartitionByColumns.empty() || !OrderByColumns.empty()) { + ctx.Error() << "PARTITION BY and ORDER BY are supported only for " << RtmrProviderName << " provider"; + return false; + } + } + if (!PkColumns.empty()) { - auto primaryKey = Y(); - for (auto& col : PkColumns) { - primaryKey = L(primaryKey, BuildQuotedAtom(col.Pos, col.Name)); - } + auto primaryKey = Y(); + for (auto& col : PkColumns) { + primaryKey = L(primaryKey, BuildQuotedAtom(col.Pos, col.Name)); + } opts = L(opts, Q(Y(Q("primarykey"), Q(primaryKey)))); - if (!OrderByColumns.empty()) { - ctx.Error() << "PRIMARY KEY cannot be used with ORDER BY, use PARTITION BY instead"; - return false; - } - } else { - if (!PartitionByColumns.empty()) { - auto partitionBy = Y(); - for (auto& col : PartitionByColumns) { - partitionBy = L(partitionBy, BuildQuotedAtom(col.Pos, col.Name)); - } - opts = L(opts, Q(Y(Q("partitionby"), Q(partitionBy)))); - } - if (!OrderByColumns.empty()) { - auto orderBy = Y(); - for (auto& col : OrderByColumns) { - orderBy = L(orderBy, Q(Y(BuildQuotedAtom(col.first.Pos, col.first.Name), col.second ? Q("1") : Q("0")))); - } - opts = L(opts, Q(Y(Q("orderby"), Q(orderBy)))); - } - } + if (!OrderByColumns.empty()) { + ctx.Error() << "PRIMARY KEY cannot be used with ORDER BY, use PARTITION BY instead"; + return false; + } + } else { + if (!PartitionByColumns.empty()) { + auto partitionBy = Y(); + for (auto& col : PartitionByColumns) { + partitionBy = L(partitionBy, BuildQuotedAtom(col.Pos, col.Name)); + } + opts = L(opts, Q(Y(Q("partitionby"), Q(partitionBy)))); + } + if (!OrderByColumns.empty()) { + auto orderBy = Y(); + for (auto& col : OrderByColumns) { + orderBy = L(orderBy, Q(Y(BuildQuotedAtom(col.first.Pos, col.first.Name), col.second ? Q("1") : Q("0")))); + } + opts = L(opts, Q(Y(Q("orderby"), Q(orderBy)))); + } + } Add("block", Q(Y( Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.ServiceName(ctx)), BuildQuotedAtom(Pos, Table.Cluster))), @@ -564,15 +564,15 @@ private: TTableRef Table; TVector<TColumnSchema> Columns; TVector<TIdentifier> PkColumns; - TVector<TIdentifier> PartitionByColumns; - TVector<std::pair<TIdentifier, bool>> OrderByColumns; // column, is desc? + TVector<TIdentifier> PartitionByColumns; + TVector<std::pair<TIdentifier, bool>> OrderByColumns; // column, is desc? }; TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TVector<TColumnSchema>& columns, - const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns, - const TVector<std::pair<TIdentifier, bool>>& orderByColumns) + const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns, + const TVector<std::pair<TIdentifier, bool>>& orderByColumns) { - return new TCreateTableNode(pos, tr, columns, pkColumns, partitionByColumns, orderByColumns); + return new TCreateTableNode(pos, tr, columns, pkColumns, partitionByColumns, orderByColumns); } class TAlterTableNode final: public TAstListNode { @@ -606,7 +606,7 @@ public: } else { auto columns = Y(); for (auto& col: Columns) { - auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos); + auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos); if (!type) { return false; } diff --git a/ydb/library/yql/sql/v0/select.cpp b/ydb/library/yql/sql/v0/select.cpp index 07937af1c2..ef47023be1 100644 --- a/ydb/library/yql/sql/v0/select.cpp +++ b/ydb/library/yql/sql/v0/select.cpp @@ -312,11 +312,11 @@ protected: return Source->ShouldUseSourceAsColumn(source); } - bool IsStream() const override { - Y_VERIFY_DEBUG(Source); - return Source->IsStream(); - } - + bool IsStream() const override { + Y_VERIFY_DEBUG(Source); + return Source->IsStream(); + } + bool IsOrdered() const override { Y_VERIFY_DEBUG(Source); return Source->IsOrdered(); @@ -535,9 +535,9 @@ public: return Node; } - bool IsStream() const override { - return Subquery->GetSource()->IsStream(); - } + bool IsStream() const override { + return Subquery->GetSource()->IsStream(); + } void DoUpdateState() const override { State.Set(ENodeState::Const, true); @@ -565,10 +565,10 @@ TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleInde class TTableSource: public IRealSource { public: - TTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label) + TTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label) : IRealSource(pos) , Table(table) - , Stream(stream) + , Stream(stream) { SetLabel(label.empty() ? Table.ShortName() : label); } @@ -637,10 +637,10 @@ public: return AstNode(Table.RefName); } - bool IsStream() const override { - return Stream; - } - + bool IsStream() const override { + return Stream; + } + TPtr DoClone() const final { return new TTableSource(Pos, Table, Stream, GetLabel()); } @@ -653,8 +653,8 @@ protected: const bool Stream; }; -TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label) { - return new TTableSource(pos, table, stream, label); +TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label) { + return new TTableSource(pos, table, stream, label); } class TInnerSource: public IProxySource { @@ -1049,7 +1049,7 @@ public: const TVector<TSortSpecificationPtr>& orderBy, TNodePtr having, TWinSpecs& winSpecs, - THoppingWindowSpecPtr hoppingWindowSpec, + THoppingWindowSpecPtr hoppingWindowSpec, const TVector<TNodePtr>& terms, bool distinct, const TVector<TNodePtr>& without, @@ -1067,7 +1067,7 @@ public: , Without(without) , Distinct(distinct) , HoppingWindowSpec(hoppingWindowSpec) - , Stream(stream) + , Stream(stream) , Settings(settings) { } @@ -1104,14 +1104,14 @@ public: if (!Source->Init(ctx, initSrc)) { return false; } - if (Stream && !Source->IsStream()) { - ctx.Error(Pos) << "SELECT STREAM is unsupported for non-streaming sources"; - return false; - } + if (Stream && !Source->IsStream()) { + ctx.Error(Pos) << "SELECT STREAM is unsupported for non-streaming sources"; + return false; + } if (!Stream && Source->IsStream() && !ctx.PragmaDirectRead) { - ctx.Error(Pos) << "SELECT STREAM must be used for streaming sources"; - return false; - } + ctx.Error(Pos) << "SELECT STREAM must be used for streaming sources"; + return false; + } ctx.PushBlockShortcuts(); auto src = Source.Get(); @@ -1649,7 +1649,7 @@ private: TVector<TNodePtr> Without; const bool Distinct; bool OrderByInit = false; - THoppingWindowSpecPtr HoppingWindowSpec; + THoppingWindowSpecPtr HoppingWindowSpec; const bool Stream; const TWriteSettings Settings; }; @@ -1960,7 +1960,7 @@ TSourcePtr BuildSelectCore( const TVector<TSortSpecificationPtr>& orderBy, TNodePtr having, TWinSpecs&& winSpecs, - THoppingWindowSpecPtr hoppingWindowSpec, + THoppingWindowSpecPtr hoppingWindowSpec, TVector<TNodePtr>&& terms, bool distinct, TVector<TNodePtr>&& without, diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp index 85a9b143d4..b028fc0439 100644 --- a/ydb/library/yql/sql/v0/sql.cpp +++ b/ydb/library/yql/sql/v0/sql.cpp @@ -300,74 +300,74 @@ static TColumnSchema ColumnSchemaImpl(const TRule_column_schema& node, TTranslat } static bool CreateTableEntry(const TRule_create_table_entry& node, TTranslation& ctx, - TVector<TColumnSchema>& columns, TVector<TIdentifier>& pkColumns, - TVector<TIdentifier>& partitionByColumns, TVector<std::pair<TIdentifier, bool>>& orderByColumns) -{ + TVector<TColumnSchema>& columns, TVector<TIdentifier>& pkColumns, + TVector<TIdentifier>& partitionByColumns, TVector<std::pair<TIdentifier, bool>>& orderByColumns) +{ switch (node.Alt_case()) { case TRule_create_table_entry::kAltCreateTableEntry1: columns.push_back(ColumnSchemaImpl(node.GetAlt_create_table_entry1().GetRule_column_schema1(), ctx)); break; - + case TRule_create_table_entry::kAltCreateTableEntry2: { auto& constraint = node.GetAlt_create_table_entry2().GetRule_table_constraint1(); - switch (constraint.Alt_case()) { - case TRule_table_constraint::kAltTableConstraint1: { - auto& pkConstraint = constraint.GetAlt_table_constraint1(); - pkColumns.push_back(IdEx(pkConstraint.GetRule_id4(), ctx)); - for (auto& block : pkConstraint.GetBlock5()) { - pkColumns.push_back(IdEx(block.GetRule_id2(), ctx)); - } - break; - } - case TRule_table_constraint::kAltTableConstraint2: { - auto& pbConstraint = constraint.GetAlt_table_constraint2(); - partitionByColumns.push_back(IdEx(pbConstraint.GetRule_id4(), ctx)); - for (auto& block : pbConstraint.GetBlock5()) { - partitionByColumns.push_back(IdEx(block.GetRule_id2(), ctx)); - } - break; - } - case TRule_table_constraint::kAltTableConstraint3: { - auto& obConstraint = constraint.GetAlt_table_constraint3(); - auto extractDirection = [&ctx] (const TRule_column_order_by_specification& spec, bool& desc) { - desc = false; - if (!spec.HasBlock2()) { - return true; - } - - auto& token = spec.GetBlock2().GetToken1(); - switch (token.GetId()) { - case SQLLexerTokens::TOKEN_ASC: - return true; - case SQLLexerTokens::TOKEN_DESC: - desc = true; - return true; - default: - ctx.Error() << "Unsupported direction token: " << token.GetId(); - return false; - } - }; - - bool desc = false; - auto& obSpec = obConstraint.GetRule_column_order_by_specification4(); - if (!extractDirection(obSpec, desc)) { - return false; - } - orderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_id1(), ctx), desc)); - - for (auto& block : obConstraint.GetBlock5()) { - auto& obSpec = block.GetRule_column_order_by_specification2(); - if (!extractDirection(obSpec, desc)) { - return false; - } - orderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_id1(), ctx), desc)); - } - break; - } - default: - ctx.AltNotImplemented("table_constraint", constraint); - return false; + switch (constraint.Alt_case()) { + case TRule_table_constraint::kAltTableConstraint1: { + auto& pkConstraint = constraint.GetAlt_table_constraint1(); + pkColumns.push_back(IdEx(pkConstraint.GetRule_id4(), ctx)); + for (auto& block : pkConstraint.GetBlock5()) { + pkColumns.push_back(IdEx(block.GetRule_id2(), ctx)); + } + break; + } + case TRule_table_constraint::kAltTableConstraint2: { + auto& pbConstraint = constraint.GetAlt_table_constraint2(); + partitionByColumns.push_back(IdEx(pbConstraint.GetRule_id4(), ctx)); + for (auto& block : pbConstraint.GetBlock5()) { + partitionByColumns.push_back(IdEx(block.GetRule_id2(), ctx)); + } + break; + } + case TRule_table_constraint::kAltTableConstraint3: { + auto& obConstraint = constraint.GetAlt_table_constraint3(); + auto extractDirection = [&ctx] (const TRule_column_order_by_specification& spec, bool& desc) { + desc = false; + if (!spec.HasBlock2()) { + return true; + } + + auto& token = spec.GetBlock2().GetToken1(); + switch (token.GetId()) { + case SQLLexerTokens::TOKEN_ASC: + return true; + case SQLLexerTokens::TOKEN_DESC: + desc = true; + return true; + default: + ctx.Error() << "Unsupported direction token: " << token.GetId(); + return false; + } + }; + + bool desc = false; + auto& obSpec = obConstraint.GetRule_column_order_by_specification4(); + if (!extractDirection(obSpec, desc)) { + return false; + } + orderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_id1(), ctx), desc)); + + for (auto& block : obConstraint.GetBlock5()) { + auto& obSpec = block.GetRule_column_order_by_specification2(); + if (!extractDirection(obSpec, desc)) { + return false; + } + orderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_id1(), ctx), desc)); + } + break; + } + default: + ctx.AltNotImplemented("table_constraint", constraint); + return false; } break; } @@ -1176,14 +1176,14 @@ public: , GroupSetContext(groupSetContext ? groupSetContext : TGroupByClauseCtx::TPtr(new TGroupByClauseCtx())) {} - bool Build(const TRule_group_by_clause& node, bool stream); + bool Build(const TRule_group_by_clause& node, bool stream); bool ParseList(const TRule_grouping_element_list& groupingListNode); void SetFeatures(const TString& field) const; TVector<TNodePtr>& Content(); TMap<TString, TNodePtr>& Aliases(); - THoppingWindowSpecPtr GetHoppingWindow(); - + THoppingWindowSpecPtr GetHoppingWindow(); + private: TVector<TNodePtr> MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const; void ResolveGroupByAndGrouping(); @@ -1191,7 +1191,7 @@ private: void FeedCollection(const TNodePtr& elem, TVector<TNodePtr>& collection, bool& hasEmpty) const; bool OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node); bool OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_list& node); - bool HoppingWindow(const TRule_hopping_window_specification& node); + bool HoppingWindow(const TRule_hopping_window_specification& node); bool IsNodeColumnsOrNamedExpression(const TVector<TNodePtr>& content, const TString& construction) const; @@ -1203,7 +1203,7 @@ private: TVector<TNodePtr> GroupBySet; TGroupByClauseCtx::TPtr GroupSetContext; - THoppingWindowSpecPtr HoppingWindowSpec; // stream queries + THoppingWindowSpecPtr HoppingWindowSpec; // stream queries static const TString AutogenerateNamePrefix; }; @@ -2654,7 +2654,7 @@ TSourcePtr TSqlSelect::SingleSource(const TRule_single_source& node) { const bool stream = serviceName == RtmrProviderName; return BuildTableSource(pos, table, stream); - } + } } case TRule_single_source::kAltSingleSource2: { const auto& alt = node.GetAlt_single_source2(); @@ -3084,14 +3084,14 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet selectPos = Ctx.Pos(); } - const bool stream = node.HasBlock3(); - const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier4()); + const bool stream = node.HasBlock3(); + const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier4()); if (distinct) { Ctx.IncrementMonCounter("sql_features", "DistinctInSelect"); } TSourcePtr source(BuildFakeSource(selectPos)); - if (node.HasBlock1() && node.HasBlock8()) { + if (node.HasBlock1() && node.HasBlock8()) { Token(node.GetBlock8().GetToken1()); Ctx.IncrementMonCounter("sql_errors", "DoubleFrom"); Ctx.Error() << "Only one FROM clause is allowed"; @@ -3100,21 +3100,21 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet if (node.HasBlock1()) { source = JoinSource(node.GetBlock1().GetRule_join_source2()); Ctx.IncrementMonCounter("sql_features", "FromInFront"); - } else if (node.HasBlock8()) { - source = JoinSource(node.GetBlock8().GetRule_join_source2()); + } else if (node.HasBlock8()) { + source = JoinSource(node.GetBlock8().GetRule_join_source2()); } if (!source) { return nullptr; } - + TVector<TNodePtr> without; - if (node.HasBlock7()) { - if (!ColumnList(without, node.GetBlock7().GetRule_column_list2())) { + if (node.HasBlock7()) { + if (!ColumnList(without, node.GetBlock7().GetRule_column_list2())) { return nullptr; } } - if (node.HasBlock9()) { - auto block = node.GetBlock9(); + if (node.HasBlock9()) { + auto block = node.GetBlock9(); Token(block.GetToken1()); TPosition pos(Ctx.Pos()); TSqlExpression expr(Ctx, Mode); @@ -3132,10 +3132,10 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet /// \todo merge gtoupByExpr and groupBy in one TVector<TNodePtr> groupByExpr, groupBy; - THoppingWindowSpecPtr hoppingWindowSpec; - if (node.HasBlock10()) { + THoppingWindowSpecPtr hoppingWindowSpec; + if (node.HasBlock10()) { TGroupByClause clause(Ctx, Mode); - if (!clause.Build(node.GetBlock10().GetRule_group_by_clause1(), stream)) { + if (!clause.Build(node.GetBlock10().GetRule_group_by_clause1(), stream)) { return nullptr; } for (const auto& exprAlias: clause.Aliases()) { @@ -3144,13 +3144,13 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet } groupBy = std::move(clause.Content()); clause.SetFeatures("sql_features"); - hoppingWindowSpec = clause.GetHoppingWindow(); + hoppingWindowSpec = clause.GetHoppingWindow(); } TNodePtr having; - if (node.HasBlock11()) { + if (node.HasBlock11()) { TSqlExpression expr(Ctx, Mode); - having = expr.Build(node.GetBlock11().GetRule_expr2()); + having = expr.Build(node.GetBlock11().GetRule_expr2()); if (!having) { return nullptr; } @@ -3158,33 +3158,33 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet } TWinSpecs windowSpec; - if (node.HasBlock12()) { - if (stream) { - Ctx.Error() << "WINDOW is not allowed in streaming queries"; + if (node.HasBlock12()) { + if (stream) { + Ctx.Error() << "WINDOW is not allowed in streaming queries"; + return nullptr; + } + if (!WindowClause(node.GetBlock12().GetRule_window_clause1(), windowSpec)) { return nullptr; } - if (!WindowClause(node.GetBlock12().GetRule_window_clause1(), windowSpec)) { - return nullptr; - } Ctx.IncrementMonCounter("sql_features", "WindowClause"); } TVector<TSortSpecificationPtr> orderBy; - if (node.HasBlock13()) { - if (stream) { - Ctx.Error() << "ORDER BY is not allowed in streaming queries"; + if (node.HasBlock13()) { + if (stream) { + Ctx.Error() << "ORDER BY is not allowed in streaming queries"; + return nullptr; + } + if (!OrderByClause(node.GetBlock13().GetRule_order_by_clause1(), orderBy)) { return nullptr; } - if (!OrderByClause(node.GetBlock13().GetRule_order_by_clause1(), orderBy)) { - return nullptr; - } Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(orderBy) ? "OrderBy" : "OrderByExpr"); } TVector<TNodePtr> terms; - if (!SelectTerm(terms, node.GetRule_result_column5())) { + if (!SelectTerm(terms, node.GetRule_result_column5())) { return nullptr; } - for (auto block: node.GetBlock6()) { + for (auto block: node.GetBlock6()) { if (!SelectTerm(terms, block.GetRule_result_column2())) { return nullptr; } @@ -3357,10 +3357,10 @@ bool TSqlSelect::OrderByClause(const TRule_order_by_clause& node, TVector<TSortS return SortSpecificationList(node.GetRule_sort_specification_list3(), orderBy); } -bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) { +bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) { const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier3()); if (distinct) { - Ctx.Error() << "DISTINCT is not supported in GROUP BY clause yet!"; + Ctx.Error() << "DISTINCT is not supported in GROUP BY clause yet!"; Ctx.IncrementMonCounter("sql_errors", "DistinctInGroupByNotSupported"); return false; } @@ -3368,14 +3368,14 @@ bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) { return false; } ResolveGroupByAndGrouping(); - if (stream && !HoppingWindowSpec) { - Ctx.Error() << "Streaming group by query must have a hopping window specification."; - return false; - } - if (!stream && HoppingWindowSpec) { - Ctx.Error() << "Hopping window specification is not supported in a non-streaming query."; - return false; - } + if (stream && !HoppingWindowSpec) { + Ctx.Error() << "Streaming group by query must have a hopping window specification."; + return false; + } + if (!stream && HoppingWindowSpec) { + Ctx.Error() << "Hopping window specification is not supported in a non-streaming query."; + return false; + } return true; } @@ -3422,10 +3422,10 @@ TMap<TString, TNodePtr>& TGroupByClause::Aliases() { return GroupSetContext->NodeAliases; } -THoppingWindowSpecPtr TGroupByClause::GetHoppingWindow() { - return HoppingWindowSpec; -} - +THoppingWindowSpecPtr TGroupByClause::GetHoppingWindow() { + return HoppingWindowSpec; +} + TVector<TNodePtr> TGroupByClause::MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const { TVector<TNodePtr> content; for (const auto& leftNode: lhs) { @@ -3559,12 +3559,12 @@ bool TGroupByClause::GroupingElement(const TRule_grouping_element& node) { Features().Set(EGroupByFeatures::GroupingSet); break; } - case TRule_grouping_element::kAltGroupingElement5: { - if (!HoppingWindow(node.GetAlt_grouping_element5().GetRule_hopping_window_specification1())) { - return false; - } - break; - } + case TRule_grouping_element::kAltGroupingElement5: { + if (!HoppingWindow(node.GetAlt_grouping_element5().GetRule_hopping_window_specification1())) { + return false; + } + break; + } default: Y_FAIL("You should change implementation according grammar changes"); } @@ -3641,69 +3641,69 @@ bool TGroupByClause::OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_l return true; } -bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& node) { - if (HoppingWindowSpec) { - Ctx.Error() << "Duplicate hopping window specification."; - return false; - } - HoppingWindowSpec = new THoppingWindowSpec; - { - TSqlExpression expr(Ctx, Mode); - HoppingWindowSpec->TimeExtractor = expr.Build(node.GetRule_expr3()); - if (!HoppingWindowSpec->TimeExtractor) { - return false; - } - } - auto processIntervalParam = [&] (const TRule_expr& rule) -> TNodePtr { - TSqlExpression expr(Ctx, Mode); - auto node = expr.Build(rule); - if (!node) { - return nullptr; - } - - auto literal = node->GetLiteral("String"); - if (!literal) { - return new TAstListNodeImpl(Ctx.Pos(), { - new TAstAtomNodeImpl(Ctx.Pos(), "EvaluateExpr", TNodeFlags::Default), - node - }); - } - +bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& node) { + if (HoppingWindowSpec) { + Ctx.Error() << "Duplicate hopping window specification."; + return false; + } + HoppingWindowSpec = new THoppingWindowSpec; + { + TSqlExpression expr(Ctx, Mode); + HoppingWindowSpec->TimeExtractor = expr.Build(node.GetRule_expr3()); + if (!HoppingWindowSpec->TimeExtractor) { + return false; + } + } + auto processIntervalParam = [&] (const TRule_expr& rule) -> TNodePtr { + TSqlExpression expr(Ctx, Mode); + auto node = expr.Build(rule); + if (!node) { + return nullptr; + } + + auto literal = node->GetLiteral("String"); + if (!literal) { + return new TAstListNodeImpl(Ctx.Pos(), { + new TAstAtomNodeImpl(Ctx.Pos(), "EvaluateExpr", TNodeFlags::Default), + node + }); + } + const auto out = NKikimr::NMiniKQL::ValueFromString(NKikimr::NUdf::EDataSlot::Interval, *literal); if (!out) { - Ctx.Error(node->GetPos()) << "Expected interval in ISO 8601 format"; - return nullptr; - } - + Ctx.Error(node->GetPos()) << "Expected interval in ISO 8601 format"; + return nullptr; + } + if ('T' == literal->back()) { Ctx.Warning(node->GetPos(), TIssuesIds::YQL_DEPRECATED_INTERVAL_CONSTANT) << "Time prefix 'T' at end of interval contant"; } - return new TAstListNodeImpl(Ctx.Pos(), { - new TAstAtomNodeImpl(Ctx.Pos(), "Interval", TNodeFlags::Default), - new TAstListNodeImpl(Ctx.Pos(), { - new TAstAtomNodeImpl(Ctx.Pos(), "quote", TNodeFlags::Default), + return new TAstListNodeImpl(Ctx.Pos(), { + new TAstAtomNodeImpl(Ctx.Pos(), "Interval", TNodeFlags::Default), + new TAstListNodeImpl(Ctx.Pos(), { + new TAstAtomNodeImpl(Ctx.Pos(), "quote", TNodeFlags::Default), new TAstAtomNodeImpl(Ctx.Pos(), ToString(out.Get<i64>()), TNodeFlags::Default) - }) - }); - }; - - HoppingWindowSpec->Hop = processIntervalParam(node.GetRule_expr5()); - if (!HoppingWindowSpec->Hop) { - return false; - } - HoppingWindowSpec->Interval = processIntervalParam(node.GetRule_expr7()); - if (!HoppingWindowSpec->Interval) { - return false; - } - HoppingWindowSpec->Delay = processIntervalParam(node.GetRule_expr9()); - if (!HoppingWindowSpec->Delay) { - return false; - } - - return true; -} - + }) + }); + }; + + HoppingWindowSpec->Hop = processIntervalParam(node.GetRule_expr5()); + if (!HoppingWindowSpec->Hop) { + return false; + } + HoppingWindowSpec->Interval = processIntervalParam(node.GetRule_expr7()); + if (!HoppingWindowSpec->Interval) { + return false; + } + HoppingWindowSpec->Delay = processIntervalParam(node.GetRule_expr9()); + if (!HoppingWindowSpec->Delay) { + return false; + } + + return true; +} + bool TGroupByClause::IsNodeColumnsOrNamedExpression(const TVector<TNodePtr>& content, const TString& construction) const { for (const auto& node: content) { if (IsAutogenerated(node->GetColumnName())) { @@ -4153,7 +4153,7 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const auto serviceName = to_lower(table.ServiceName(Ctx)); const bool isMapReduce = serviceName == YtProviderName; const bool isKikimr = serviceName == KikimrProviderName; - const bool isRtmr = serviceName == RtmrProviderName; + const bool isRtmr = serviceName == RtmrProviderName; const bool isStat = serviceName == StatProviderName; if (!isKikimr) { @@ -4180,12 +4180,12 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); return false; } - } else if (isRtmr) { - if (mode != ESQLWriteColumnMode::InsertInto) { - Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; - Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); - return false; - } + } else if (isRtmr) { + if (mode != ESQLWriteColumnMode::InsertInto) { + Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; + Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); + return false; + } } else if (isStat) { if (mode != ESQLWriteColumnMode::UpsertInto) { Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; @@ -4338,19 +4338,19 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& TVector<TColumnSchema> columns; TVector<TIdentifier> pkColumns; - TVector<TIdentifier> partitionByColumns; - TVector<std::pair<TIdentifier, bool>> orderByColumns; - - if (!CreateTableEntry(rule.GetRule_create_table_entry5(), *this, columns, pkColumns, partitionByColumns, orderByColumns)) { + TVector<TIdentifier> partitionByColumns; + TVector<std::pair<TIdentifier, bool>> orderByColumns; + + if (!CreateTableEntry(rule.GetRule_create_table_entry5(), *this, columns, pkColumns, partitionByColumns, orderByColumns)) { return false; } for (auto& block: rule.GetBlock6()) { - if (!CreateTableEntry(block.GetRule_create_table_entry2(), *this, columns, pkColumns, partitionByColumns, orderByColumns)) { + if (!CreateTableEntry(block.GetRule_create_table_entry2(), *this, columns, pkColumns, partitionByColumns, orderByColumns)) { return false; } } - AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, columns, pkColumns, partitionByColumns, orderByColumns)); + AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, columns, pkColumns, partitionByColumns, orderByColumns)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore5: { @@ -4561,7 +4561,7 @@ bool TSqlQuery::AlterTableAddColumns(TVector<TNodePtr>& blocks, const TRule_alte bool TSqlQuery::AlterTableDropColumn(TVector<TNodePtr>& blocks, const TRule_alter_table_drop_column& node, const TTableRef& tr) { TString name = Id(node.GetRule_id3(), *this); - TColumnSchema column(Ctx.Pos(), name, "", false, false); + TColumnSchema column(Ctx.Pos(), name, "", false, false); AddStatementToBlocks(blocks, BuildAlterTable(Ctx.Pos(), tr, TVector<TColumnSchema>{column}, EAlterTableIntentnt::DropColumn)); return true; } @@ -4885,17 +4885,17 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) { TTableRef table = SimpleTableRefImpl(stmt.GetRule_simple_table_ref3(), Mode, *this); - if (!table.Check(Ctx)) { - return nullptr; - } + if (!table.Check(Ctx)) { + return nullptr; + } auto serviceName = to_lower(table.ServiceName(Ctx)); const bool isKikimr = serviceName == KikimrProviderName; if (!isKikimr) { Ctx.Error(GetPos(stmt.GetToken1())) << "DELETE is unsupported for " << serviceName; - return nullptr; - } + return nullptr; + } TSourcePtr source = BuildTableSource(Ctx.Pos(), table, false); @@ -4934,17 +4934,17 @@ TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) { TNodePtr TSqlQuery::Build(const TRule_update_stmt& stmt) { TTableRef table = SimpleTableRefImpl(stmt.GetRule_simple_table_ref2(), Mode, *this); - if (!table.Check(Ctx)) { - return nullptr; - } + if (!table.Check(Ctx)) { + return nullptr; + } auto serviceName = to_lower(table.ServiceName(Ctx)); const bool isKikimr = serviceName == KikimrProviderName; if (!isKikimr) { Ctx.Error(GetPos(stmt.GetToken1())) << "UPDATE is unsupported for " << serviceName; - return nullptr; - } + return nullptr; + } switch (stmt.GetBlock3().Alt_case()) { case TRule_update_stmt_TBlock3::kAlt1: { diff --git a/ydb/library/yql/sql/v0/sql_ut.cpp b/ydb/library/yql/sql/v0/sql_ut.cpp index 64a372af7f..3a70aef93c 100644 --- a/ydb/library/yql/sql/v0/sql_ut.cpp +++ b/ydb/library/yql/sql/v0/sql_ut.cpp @@ -788,7 +788,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { VerifyProgram(res, elementStat, verifyLine); UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]); } - + Y_UNIT_TEST(ProcessUserTypeAuth) { NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData($ROWS, AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName)); UNIT_ASSERT(res.Root); @@ -808,19 +808,19 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { } Y_UNIT_TEST(SelectStreamRtmr) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - + NYql::TAstParseResult res = SqlToYql( + "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", + 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + Y_UNIT_TEST(GroupByHopRtmr) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; INSERT INTO Output SELECT STREAM key, SUM(value) AS value FROM Input - GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"); - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; INSERT INTO Output SELECT STREAM key, SUM(value) AS value FROM Input + GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"); + )", 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } Y_UNIT_TEST(KikimrInserts) { NYql::TAstParseResult res = SqlToYql(R"( diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 2a0d9b32e6..ef394c7a00 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -46,8 +46,8 @@ sql_stmt_core: | drop_role_stmt ; -expr: - or_subexpr (OR or_subexpr)* +expr: + or_subexpr (OR or_subexpr)* | type_name_composite; or_subexpr: and_subexpr (AND and_subexpr)*; @@ -70,15 +70,15 @@ match_op: LIKE | ILIKE | GLOB | REGEXP | RLIKE | MATCH; eq_subexpr: neq_subexpr ((LESS | LESS_OR_EQ | GREATER | GREATER_OR_EQ) neq_subexpr)*; -// workaround for recursive types, '??' and closing '>>' -shift_right: GREATER GREATER; -rot_right: GREATER GREATER PIPE; -double_question: QUESTION QUESTION; +// workaround for recursive types, '??' and closing '>>' +shift_right: GREATER GREATER; +rot_right: GREATER GREATER PIPE; +double_question: QUESTION QUESTION; neq_subexpr: bit_subexpr ((SHIFT_LEFT | shift_right | ROT_LEFT | rot_right | AMPERSAND | PIPE | CARET) bit_subexpr)* // trailing QUESTIONs are used in optional simple types (String?) and optional lambda args: ($x, $y?) -> ($x) ((double_question neq_subexpr) => double_question neq_subexpr | QUESTION+)?; - + bit_subexpr: add_subexpr ((PLUS | MINUS) add_subexpr)*; add_subexpr: mul_subexpr ((ASTERISK | SLASH | PERCENT) mul_subexpr)*; @@ -117,7 +117,7 @@ atom_expr: | exists_expr | case_expr | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) - | value_constructor + | value_constructor | bitcast_expr | list_literal | dict_literal @@ -134,7 +134,7 @@ in_atom_expr: | LPAREN select_stmt RPAREN // TODO: resolve ANTLR error: rule in_atom_expr has non-LL(*) decision due to recursive rule invocations reachable from alts 3,8 // | LPAREN values_stmt RPAREN - | value_constructor + | value_constructor | bitcast_expr | list_literal | dict_literal @@ -228,7 +228,7 @@ when_expr: WHEN expr THEN expr; literal_value: integer | real - | STRING_VALUE + | STRING_VALUE | BLOB // it's unused right now | NULL | CURRENT_TIME // it's unused right now @@ -249,61 +249,61 @@ signed_number: (PLUS | MINUS)? (integer | real); type_name_simple: an_id_pure; -integer_or_bind: integer | bind_parameter; -type_name_tag: id | STRING_VALUE | bind_parameter; - -struct_arg: type_name_tag COLON type_name_or_bind; +integer_or_bind: integer | bind_parameter; +type_name_tag: id | STRING_VALUE | bind_parameter; + +struct_arg: type_name_tag COLON type_name_or_bind; struct_arg_as: type_name_or_bind AS type_name_tag; -variant_arg: (type_name_tag COLON)? type_name_or_bind; -callable_arg: variant_arg (LBRACE_CURLY AUTOMAP RBRACE_CURLY)?; -callable_arg_list: callable_arg (COMMA callable_arg)*; - -type_name_decimal: DECIMAL LPAREN integer_or_bind COMMA integer_or_bind RPAREN; -type_name_optional: OPTIONAL LESS type_name_or_bind GREATER; +variant_arg: (type_name_tag COLON)? type_name_or_bind; +callable_arg: variant_arg (LBRACE_CURLY AUTOMAP RBRACE_CURLY)?; +callable_arg_list: callable_arg (COMMA callable_arg)*; + +type_name_decimal: DECIMAL LPAREN integer_or_bind COMMA integer_or_bind RPAREN; +type_name_optional: OPTIONAL LESS type_name_or_bind GREATER; type_name_tuple: TUPLE (LESS (type_name_or_bind (COMMA type_name_or_bind)* COMMA?)? GREATER | NOT_EQUALS2); type_name_struct: STRUCT (LESS (struct_arg (COMMA struct_arg)* COMMA?)? GREATER | NOT_EQUALS2); type_name_variant: VARIANT LESS variant_arg (COMMA variant_arg)* COMMA? GREATER; -type_name_list: LIST LESS type_name_or_bind GREATER; -type_name_stream: STREAM LESS type_name_or_bind GREATER; -type_name_flow: FLOW LESS type_name_or_bind GREATER; -type_name_dict: DICT LESS type_name_or_bind COMMA type_name_or_bind GREATER; -type_name_set: SET LESS type_name_or_bind GREATER; +type_name_list: LIST LESS type_name_or_bind GREATER; +type_name_stream: STREAM LESS type_name_or_bind GREATER; +type_name_flow: FLOW LESS type_name_or_bind GREATER; +type_name_dict: DICT LESS type_name_or_bind COMMA type_name_or_bind GREATER; +type_name_set: SET LESS type_name_or_bind GREATER; type_name_enum: ENUM LESS type_name_tag (COMMA type_name_tag)* COMMA? GREATER; -type_name_resource: RESOURCE LESS type_name_tag GREATER; -type_name_tagged: TAGGED LESS type_name_or_bind COMMA type_name_tag GREATER; -type_name_callable: CALLABLE LESS LPAREN callable_arg_list? COMMA? (LBRACE_SQUARE callable_arg_list RBRACE_SQUARE)? RPAREN ARROW type_name_or_bind GREATER; - +type_name_resource: RESOURCE LESS type_name_tag GREATER; +type_name_tagged: TAGGED LESS type_name_or_bind COMMA type_name_tag GREATER; +type_name_callable: CALLABLE LESS LPAREN callable_arg_list? COMMA? (LBRACE_SQUARE callable_arg_list RBRACE_SQUARE)? RPAREN ARROW type_name_or_bind GREATER; + type_name_composite: ( type_name_optional - | type_name_tuple - | type_name_struct - | type_name_variant - | type_name_list - | type_name_stream - | type_name_flow - | type_name_dict - | type_name_set - | type_name_enum - | type_name_resource - | type_name_tagged - | type_name_callable - ) QUESTION*; - + | type_name_tuple + | type_name_struct + | type_name_variant + | type_name_list + | type_name_stream + | type_name_flow + | type_name_dict + | type_name_set + | type_name_enum + | type_name_resource + | type_name_tagged + | type_name_callable + ) QUESTION*; + type_name: type_name_composite | (type_name_decimal | type_name_simple) QUESTION*; -type_name_or_bind: type_name | bind_parameter; - +type_name_or_bind: type_name | bind_parameter; + value_constructor_literal: STRING_VALUE; -value_constructor: +value_constructor: VARIANT LPAREN expr COMMA expr COMMA expr RPAREN | ENUM LPAREN expr COMMA expr RPAREN | CALLABLE LPAREN expr COMMA expr RPAREN ; - -declare_stmt: DECLARE bind_parameter AS type_name (EQUALS literal_value)?; - + +declare_stmt: DECLARE bind_parameter AS type_name (EQUALS literal_value)?; + module_path: DOT? an_id (DOT an_id)*; import_stmt: IMPORT module_path SYMBOLS named_bind_parameter_list; export_stmt: EXPORT bind_parameter_list; @@ -316,7 +316,7 @@ pragma_stmt: PRAGMA opt_id_prefix_or_type an_id (EQUALS pragma_value | LPAREN pr pragma_value: signed_number | id - | STRING_VALUE + | STRING_VALUE | bool_value | bind_parameter ; @@ -376,7 +376,7 @@ grouping_element: | cube_list | grouping_sets_specification //empty_grouping_set inside smart_parenthesis - | hopping_window_specification + | hopping_window_specification ; /// expect column (named column), or parenthesis list columns, or expression (named expression), or list expression @@ -389,8 +389,8 @@ cube_list: CUBE LPAREN ordinary_grouping_set_list RPAREN; /// SQL2003 grouping_set_list == grouping_element_list grouping_sets_specification: GROUPING SETS LPAREN grouping_element_list RPAREN; -hopping_window_specification: HOP LPAREN expr COMMA expr COMMA expr COMMA expr RPAREN; - +hopping_window_specification: HOP LPAREN expr COMMA expr COMMA expr COMMA expr RPAREN; + result_column: opt_id_prefix ASTERISK | expr ((AS an_id_or_type) | an_id_pure)? @@ -498,11 +498,11 @@ column_schema: an_id_schema type_name_or_bind family_relation? (NOT? NULL)?; family_relation: FAMILY an_id; column_order_by_specification: an_id (ASC | DESC)?; -table_constraint: +table_constraint: PRIMARY KEY LPAREN an_id (COMMA an_id)* RPAREN | PARTITION BY LPAREN an_id (COMMA an_id)* RPAREN - | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN -; + | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN +; table_index: INDEX an_id table_index_type (WITH LPAREN an_id EQUALS an_id COMMA (an_id EQUALS an_id)* RPAREN)? @@ -763,12 +763,12 @@ an_id_pure: identifier | STRING_VALUE; opt_id_prefix: (an_id DOT)?; cluster_expr: (an_id COLON)? (pure_column_or_named | ASTERISK); -id_or_type: id | type_id; +id_or_type: id | type_id; opt_id_prefix_or_type: (an_id_or_type DOT)?; id_or_at: AT? an_id_or_type; id_table_or_type: an_id_table | type_id; -id_table_or_at: AT? id_table_or_type; - +id_table_or_at: AT? id_table_or_type; + keyword: keyword_compat | keyword_expr_uncompat @@ -794,7 +794,7 @@ keyword_expr_uncompat: | EXISTS | FROM | FULL - | HOP + | HOP | JSON_EXISTS | JSON_VALUE | JSON_QUERY @@ -1016,22 +1016,22 @@ keyword_compat: ( | XOR ); -type_id: +type_id: OPTIONAL - | TUPLE - | STRUCT - | VARIANT - | LIST -// | STREAM - | FLOW - | DICT - | SET - | ENUM - | RESOURCE - | TAGGED - | CALLABLE -; - + | TUPLE + | STRUCT + | VARIANT + | LIST +// | STREAM + | FLOW + | DICT + | SET + | ENUM + | RESOURCE + | TAGGED + | CALLABLE +; + bool_value: (TRUE | FALSE); real: REAL; integer: DIGITS | INTEGER_VALUE; @@ -1128,14 +1128,14 @@ ASSUME: A S S U M E; ASYNC: A S Y N C; ATTACH: A T T A C H; AUTOINCREMENT: A U T O I N C R E M E N T; -AUTOMAP: A U T O M A P; +AUTOMAP: A U T O M A P; BEFORE: B E F O R E; BEGIN: B E G I N; BERNOULLI: B E R N O U L L I; BETWEEN: B E T W E E N; BITCAST: B I T C A S T; BY: B Y; -CALLABLE: C A L L A B L E; +CALLABLE: C A L L A B L E; CASCADE: C A S C A D E; CASE: C A S E; CAST: C A S T; @@ -1158,7 +1158,7 @@ CURRENT_TIME: C U R R E N T '_' T I M E; CURRENT_DATE: C U R R E N T '_' D A T E; CURRENT_TIMESTAMP: C U R R E N T '_' T I M E S T A M P; DATABASE: D A T A B A S E; -DECIMAL: D E C I M A L; +DECIMAL: D E C I M A L; DECLARE: D E C L A R E; DEFAULT: D E F A U L T; DEFERRABLE: D E F E R R A B L E; @@ -1180,7 +1180,7 @@ EMPTY: E M P T Y; EMPTY_ACTION: E M P T Y '_' A C T I O N; ENCRYPTED: E N C R Y P T E D; END: E N D; -ENUM: E N U M; +ENUM: E N U M; ERASE: E R A S E; ESCAPE: E S C A P E; EVALUATE: E V A L U A T E; @@ -1196,7 +1196,7 @@ FAIL: F A I L; FAMILY: F A M I L Y; FILTER: F I L T E R; FLATTEN: F L A T T E N; -FLOW: F L O W; +FLOW: F L O W; FOLLOWING: F O L L O W I N G; FOR: F O R; FOREIGN: F O R E I G N; @@ -1209,7 +1209,7 @@ GROUP: G R O U P; GROUPING: G R O U P I N G; GROUPS: G R O U P S; HAVING: H A V I N G; -HOP: H O P; +HOP: H O P; IF: I F; IGNORE: I G N O R E; ILIKE: I L I K E; @@ -1275,7 +1275,7 @@ RENAME: R E N A M E; REPEATABLE: R E P E A T A B L E; REPLACE: R E P L A C E; RESET: R E S E T; -RESOURCE: R E S O U R C E; +RESOURCE: R E S O U R C E; RESPECT: R E S P E C T; RESTRICT: R E S T R I C T; RESULT: R E S U L T; @@ -1296,15 +1296,15 @@ SEMI: S E M I; SET: S E T; SETS: S E T S; SORTED: S O R T E D; -STREAM: S T R E A M; -STRUCT: S T R U C T; +STREAM: S T R E A M; +STRUCT: S T R U C T; SUBQUERY: S U B Q U E R Y; SYMBOLS: S Y M B O L S; SYNC: S Y N C; SYSTEM: S Y S T E M; TABLE: T A B L E; TABLESAMPLE: T A B L E S A M P L E; -TAGGED: T A G G E D; +TAGGED: T A G G E D; TEMP: T E M P; TEMPORARY: T E M P O R A R Y; THEN: T H E N; @@ -1312,7 +1312,7 @@ TIES: T I E S; TO: T O; TRANSACTION: T R A N S A C T I O N; TRIGGER: T R I G G E R; -TUPLE: T U P L E; +TUPLE: T U P L E; UNBOUNDED: U N B O U N D E D; UNCONDITIONAL: U N C O N D I T I O N A L; UNION: U N I O N; @@ -1325,7 +1325,7 @@ USER: U S E R; USING: U S I N G; VACUUM: V A C U U M; VALUES: V A L U E S; -VARIANT: V A R I A N T; +VARIANT: V A R I A N T; VIEW: V I E W; VIRTUAL: V I R T U A L; WHEN: W H E N; @@ -1368,7 +1368,7 @@ fragment BINDIGITS: '0' B ('0' | '1')+; fragment DECDIGITS: DIGIT+; DIGITS: DECDIGITS | HEXDIGITS | OCTDIGITS | BINDIGITS; -INTEGER_VALUE: DIGITS (U? (L | S | T)?); +INTEGER_VALUE: DIGITS (U? (L | S | T)?); fragment FLOAT_EXP : E (PLUS | MINUS)? DECDIGITS ; REAL: diff --git a/ydb/library/yql/sql/v1/aggregation.cpp b/ydb/library/yql/sql/v1/aggregation.cpp index 99ddd78572..850e0e5056 100644 --- a/ydb/library/yql/sql/v1/aggregation.cpp +++ b/ydb/library/yql/sql/v1/aggregation.cpp @@ -248,8 +248,8 @@ private: } } - std::vector<ui32> GetFactoryColumnIndices() const final { - return {1u, 0u}; + std::vector<ui32> GetFactoryColumnIndices() const final { + return {1u, 0u}; } bool DoInit(TContext& ctx, ISource* src) final { @@ -330,8 +330,8 @@ private: return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload), BuildLambda(Pos, Y("row"), Predicate)); } - std::vector<ui32> GetFactoryColumnIndices() const final { - return {0u, 1u}; + std::vector<ui32> GetFactoryColumnIndices() const final { + return {0u, 1u}; } bool DoInit(TContext& ctx, ISource* src) final { @@ -507,8 +507,8 @@ private: apply = L(apply, Intervals); } - std::vector<ui32> GetFactoryColumnIndices() const final { - return {0u, 1u}; + std::vector<ui32> GetFactoryColumnIndices() const final { + return {0u, 1u}; } bool DoInit(TContext& ctx, ISource* src) final { @@ -849,119 +849,119 @@ TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& nam return new TTopFreqFactory(pos, name, factory, aggMode); } -template <bool HasKey> -class TTopAggregationFactory final : public TAggregationFactory { -public: - TTopAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) +template <bool HasKey> +class TTopAggregationFactory final : public TAggregationFactory { +public: + TTopAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) + : TAggregationFactory(pos, name, factory, aggMode) , FakeSource(BuildFakeSource(pos)) - {} - -private: - bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { - ui32 adjustArgsCount = isFactory ? 1 : (HasKey ? 3 : 2); - if (exprs.size() != adjustArgsCount) { - ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " - << adjustArgsCount << " arguments, given: " << exprs.size(); - return false; - } - + {} + +private: + bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { + ui32 adjustArgsCount = isFactory ? 1 : (HasKey ? 3 : 2); + if (exprs.size() != adjustArgsCount) { + ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " + << adjustArgsCount << " arguments, given: " << exprs.size(); + return false; + } + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } - if (!isFactory) { - Payload = exprs[0]; - if (HasKey) { - Key = exprs[1]; - } - } - - Count = exprs.back(); - - if (!isFactory) { + if (!isFactory) { + Payload = exprs[0]; + if (HasKey) { + Key = exprs[1]; + } + } + + Count = exprs.back(); + + if (!isFactory) { Name = src->MakeLocalName(Name); - } - - if (!Init(ctx, src)) { - return false; - } - - if (!isFactory) { - node.Add("Member", "row", Q(Name)); + } + + if (!Init(ctx, src)) { + return false; + } + + if (!isFactory) { + node.Add("Member", "row", Q(Name)); if (IsOverWindow()) { src->AddTmpWindowColumn(Name); } - } - - return true; - } - - TNodePtr DoClone() const final { - return new TTopAggregationFactory(Pos, Name, Func, AggMode); - } - - TNodePtr GetApply(const TNodePtr& type) const final { - TNodePtr apply; - if (HasKey) { - apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload)); - } else { - apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload)); - } - AddFactoryArguments(apply); - return apply; - } - - void AddFactoryArguments(TNodePtr& apply) const final { - apply = L(apply, Count); - } - - std::vector<ui32> GetFactoryColumnIndices() const final { - if (HasKey) { - return {1u, 0u}; - } else { - return {0u}; - } - } - - bool DoInit(TContext& ctx, ISource* src) final { + } + + return true; + } + + TNodePtr DoClone() const final { + return new TTopAggregationFactory(Pos, Name, Func, AggMode); + } + + TNodePtr GetApply(const TNodePtr& type) const final { + TNodePtr apply; + if (HasKey) { + apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload)); + } else { + apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload)); + } + AddFactoryArguments(apply); + return apply; + } + + void AddFactoryArguments(TNodePtr& apply) const final { + apply = L(apply, Count); + } + + std::vector<ui32> GetFactoryColumnIndices() const final { + if (HasKey) { + return {1u, 0u}; + } else { + return {0u}; + } + } + + bool DoInit(TContext& ctx, ISource* src) final { if (!Count->Init(ctx, FakeSource.Get())) { - return false; - } - - if (!Payload) { - return true; - } - - if (HasKey) { - if (!Key->Init(ctx, src)) { - return false; - } - } - - if (!Payload->Init(ctx, src)) { - return false; - } - - if ((HasKey && Key->IsAggregated()) || (!HasKey && Payload->IsAggregated())) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; - return false; - } - return true; - } - + return false; + } + + if (!Payload) { + return true; + } + + if (HasKey) { + if (!Key->Init(ctx, src)) { + return false; + } + } + + if (!Payload->Init(ctx, src)) { + return false; + } + + if ((HasKey && Key->IsAggregated()) || (!HasKey && Payload->IsAggregated())) { + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; + return false; + } + return true; + } + TSourcePtr FakeSource; - TNodePtr Key, Payload, Count; -}; - -template <bool HasKey> -TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TTopAggregationFactory<HasKey>(pos, name, factory, aggMode); -} - -template TAggregationPtr BuildTopFactoryAggregation<false>(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); -template TAggregationPtr BuildTopFactoryAggregation<true >(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); - + TNodePtr Key, Payload, Count; +}; + +template <bool HasKey> +TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { + return new TTopAggregationFactory<HasKey>(pos, name, factory, aggMode); +} + +template TAggregationPtr BuildTopFactoryAggregation<false>(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); +template TAggregationPtr BuildTopFactoryAggregation<true >(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); + class TCountDistinctEstimateAggregationFactory final : public TAggregationFactory { public: TCountDistinctEstimateAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) diff --git a/ydb/library/yql/sql/v1/builtin.cpp b/ydb/library/yql/sql/v1/builtin.cpp index 4e24ab7b28..43ef1243f1 100644 --- a/ydb/library/yql/sql/v1/builtin.cpp +++ b/ydb/library/yql/sql/v1/builtin.cpp @@ -215,14 +215,14 @@ public: auto factory = Aggr->AggregationTraitsFactory(); auto apply = Y("Apply", factory, Y("ListType", "type")); - - auto columnIndices = Aggr->GetFactoryColumnIndices(); - if (columnIndices.size() == 1) { + + auto columnIndices = Aggr->GetFactoryColumnIndices(); + if (columnIndices.size() == 1) { apply = L(apply, "extractor"); } else { // make several extractors from main that returns a tuple - for (ui32 arg = 0; arg < columnIndices.size(); ++arg) { - auto partial = BuildLambda(Pos, Y("row"), Y("Nth", Y("Apply", "extractor", "row"), Q(ToString(columnIndices[arg])))); + for (ui32 arg = 0; arg < columnIndices.size(); ++arg) { + auto partial = BuildLambda(Pos, Y("row"), Y("Nth", Y("Apply", "extractor", "row"), Q(ToString(columnIndices[arg])))); apply = L(apply, partial); } } @@ -1037,12 +1037,12 @@ public: } auto aliasNode = BuildFileNameArgument(Args[1]->GetPos(), Args[1]); OpName = "Apply"; - Args[0] = Y("Udf", Q("File.ByLines"), Y("Void"), - Y("TupleType", - Y("TupleType", Y("DataType", dataTypeStringNode)), - Y("StructType"), - Y("TupleType"))); - + Args[0] = Y("Udf", Q("File.ByLines"), Y("Void"), + Y("TupleType", + Y("TupleType", Y("DataType", dataTypeStringNode)), + Y("StructType"), + Y("TupleType"))); + Args[1] = Y("FilePath", aliasNode); return TCallNode::DoInit(ctx, src); } @@ -2218,47 +2218,47 @@ private: TString Mode; }; -template <bool IsStart> -class THoppingTime final: public TAstListNode { -public: - THoppingTime(TPosition pos, const TVector<TNodePtr>& args = {}) - : TAstListNode(pos) +template <bool IsStart> +class THoppingTime final: public TAstListNode { +public: + THoppingTime(TPosition pos, const TVector<TNodePtr>& args = {}) + : TAstListNode(pos) { Y_UNUSED(args); } - -private: - TNodePtr DoClone() const override { - return new THoppingTime(GetPos()); - } - - bool DoInit(TContext& ctx, ISource* src) override { - Y_UNUSED(ctx); - - auto window = src->GetHoppingWindowSpec(); - if (!window) { - ctx.Error(Pos) << "No hopping window parameters in aggregation"; - return false; - } - - Nodes.clear(); - - if (!IsStart) { - Add("Member", "row", Q("_yql_time")); - return true; - } - - Add("Sub", - Y("Member", "row", Q("_yql_time")), - window->Interval); - return true; - } - - void DoUpdateState() const override { - State.Set(ENodeState::Aggregated, true); - } -}; - + +private: + TNodePtr DoClone() const override { + return new THoppingTime(GetPos()); + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(ctx); + + auto window = src->GetHoppingWindowSpec(); + if (!window) { + ctx.Error(Pos) << "No hopping window parameters in aggregation"; + return false; + } + + Nodes.clear(); + + if (!IsStart) { + Add("Member", "row", Q("_yql_time")); + return true; + } + + Add("Sub", + Y("Member", "row", Q("_yql_time")), + window->Interval); + return true; + } + + void DoUpdateState() const override { + State.Set(ENodeState::Aggregated, true); + } +}; + class TInvalidBuiltin final: public INode { public: TInvalidBuiltin(TPosition pos, const TString& info) @@ -2293,8 +2293,8 @@ enum EAggrFuncTypeCallback { LINEAR_HISTOGRAM, PERCENTILE, TOPFREQ, - TOP, - TOP_BY, + TOP, + TOP_BY, COUNT_DISTINCT_ESTIMATE, LIST, UDAF @@ -2363,12 +2363,12 @@ TAggrFuncFactoryCallback BuildAggrFuncFactoryCallback( case TOPFREQ: factory = BuildTopFreqFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; - case TOP: - factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode); - break; - case TOP_BY: - factory = BuildTopFactoryAggregation<true>(pos, realFunctionName, factoryName, aggMode); - break; + case TOP: + factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode); + break; + case TOP_BY: + factory = BuildTopFactoryAggregation<true>(pos, realFunctionName, factoryName, aggMode); + break; case COUNT_DISTINCT_ESTIMATE: factory = BuildCountDistinctEstimateFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; @@ -2712,7 +2712,7 @@ struct TBuiltinFuncData { {"jointablerow", BuildSimpleBuiltinFactoryCallback<TTableRow<true>>() }, {"tablerows", BuildSimpleBuiltinFactoryCallback<TTableRows>() }, {"weakfield", BuildSimpleBuiltinFactoryCallback<TWeakFieldOp>()}, - + // Hint builtins {"grouping", BuildSimpleBuiltinFactoryCallback<TGroupingNode>()}, @@ -2779,11 +2779,11 @@ struct TBuiltinFuncData { {"mode", BuildAggrFuncFactoryCallback("Mode", "topfreq_traits_factory", TOPFREQ) }, {"topfreq", BuildAggrFuncFactoryCallback("TopFreq", "topfreq_traits_factory", TOPFREQ) }, - {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)}, - {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)}, - {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)}, - {"bottomby", BuildAggrFuncFactoryCallback("BottomBy", "bottom_by_traits_factory", TOP_BY)}, - + {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)}, + {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)}, + {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)}, + {"bottomby", BuildAggrFuncFactoryCallback("BottomBy", "bottom_by_traits_factory", TOP_BY)}, + {"histogram", BuildAggrFuncFactoryCallback("AdaptiveWardHistogram", "histogram_adaptive_ward_traits_factory", HISTOGRAM, "Histogram")}, {"adaptivewardhistogram", BuildAggrFuncFactoryCallback("AdaptiveWardHistogram", "histogram_adaptive_ward_traits_factory", HISTOGRAM)}, {"adaptiveweighthistogram", BuildAggrFuncFactoryCallback("AdaptiveWeightHistogram", "histogram_adaptive_weight_traits_factory", HISTOGRAM)}, @@ -2990,13 +2990,13 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec return BuildUdf(ctx, pos, moduleName, name, newArgs); } - } else if (ns == "datetime2" && (name == "Format" || name == "Parse")) { - return BuildUdf(ctx, pos, nameSpace, name, args); + } else if (ns == "datetime2" && (name == "Format" || name == "Parse")) { + return BuildUdf(ctx, pos, nameSpace, name, args); } else if (name == "MakeLibraPreprocessor") { if (args.size() != 1) { return new TInvalidBuiltin(pos, TStringBuilder() << name << " requires exactly one argument"); } - + auto settings = NYT::TNode::CreateMap(); auto makeUdfArgs = [&args, &pos, &settings]() { diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp index 5a3622bd46..870bbd9cc7 100644 --- a/ydb/library/yql/sql/v1/node.cpp +++ b/ydb/library/yql/sql/v1/node.cpp @@ -996,9 +996,9 @@ TWindowSpecificationPtr TWindowSpecification::Clone() const { THoppingWindowSpecPtr THoppingWindowSpec::Clone() const { auto res = MakeIntrusive<THoppingWindowSpec>(); res->TimeExtractor = TimeExtractor->Clone(); - res->Hop = Hop->Clone(); - res->Interval = Interval->Clone(); - res->Delay = Delay->Clone(); + res->Hop = Hop->Clone(); + res->Interval = Interval->Clone(); + res->Delay = Delay->Clone(); res->DataWatermarks = DataWatermarks; return res; } @@ -1247,8 +1247,8 @@ void IAggregation::AddFactoryArguments(TNodePtr& apply) const { Y_UNUSED(apply); } -std::vector<ui32> IAggregation::GetFactoryColumnIndices() const { - return {0u}; +std::vector<ui32> IAggregation::GetFactoryColumnIndices() const { + return {0u}; } TNodePtr IAggregation::WindowTraits(const TNodePtr& type) const { @@ -1443,14 +1443,14 @@ const TVector<TString>& ISource::GetTmpWindowColumns() const { return TmpWindowColumns; } -void ISource::SetHoppingWindowSpec(THoppingWindowSpecPtr spec) { - HoppingWindowSpec = spec; -} - -THoppingWindowSpecPtr ISource::GetHoppingWindowSpec() const { - return HoppingWindowSpec; -} - +void ISource::SetHoppingWindowSpec(THoppingWindowSpecPtr spec) { + HoppingWindowSpec = spec; +} + +THoppingWindowSpecPtr ISource::GetHoppingWindowSpec() const { + return HoppingWindowSpec; +} + TNodePtr ISource::GetSessionWindowSpec() const { return SessionWindow; } @@ -1581,10 +1581,10 @@ bool ISource::IsOverWindowSource() const { return !WinSpecs.empty(); } -bool ISource::IsStream() const { - return false; -} - +bool ISource::IsStream() const { + return false; +} + EOrderKind ISource::GetOrderKind() const { return EOrderKind::None; } @@ -1773,19 +1773,19 @@ TNodePtr ISource::BuildAggregation(const TString& label) { options = L(options, Q(Y(Q("compact")))); } - if (HoppingWindowSpec) { - auto hoppingTraits = Y( - "HoppingTraits", - Y("ListItemType", listType), - BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor), - HoppingWindowSpec->Hop, - HoppingWindowSpec->Interval, + if (HoppingWindowSpec) { + auto hoppingTraits = Y( + "HoppingTraits", + Y("ListItemType", listType), + BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor), + HoppingWindowSpec->Hop, + HoppingWindowSpec->Interval, HoppingWindowSpec->Delay, HoppingWindowSpec->DataWatermarks ? Q("true") : Q("false")); - + options = L(options, Q(Y(Q("hopping"), hoppingTraits))); - } - + } + if (SessionWindow) { YQL_ENSURE(SessionWindow->GetLabel()); auto sessionWindow = dynamic_cast<TSessionWindow*>(SessionWindow.Get()); diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index e912e20c18..de82b45a23 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -531,7 +531,7 @@ namespace NSQLTranslationV1 { struct TColumnSchema { TPosition Pos; TString Name; - TNodePtr Type; + TNodePtr Type; bool Nullable; TVector<TIdentifier> Families; @@ -604,18 +604,18 @@ namespace NSQLTranslationV1 { }; typedef TIntrusivePtr<TFrameSpecification> TFrameSpecificationPtr; - struct THoppingWindowSpec: public TSimpleRefCount<THoppingWindowSpec> { - TNodePtr TimeExtractor; - TNodePtr Hop; - TNodePtr Interval; - TNodePtr Delay; + struct THoppingWindowSpec: public TSimpleRefCount<THoppingWindowSpec> { + TNodePtr TimeExtractor; + TNodePtr Hop; + TNodePtr Interval; + TNodePtr Delay; bool DataWatermarks; TIntrusivePtr<THoppingWindowSpec> Clone() const; ~THoppingWindowSpec() {} - }; + }; typedef TIntrusivePtr<THoppingWindowSpec> THoppingWindowSpecPtr; - + struct TWindowSpecification: public TSimpleRefCount<TWindowSpecification> { TMaybe<TString> ExistingWindowName; TVector<TNodePtr> Partitions; @@ -756,7 +756,7 @@ namespace NSQLTranslationV1 { virtual TNodePtr AggregationTraitsFactory() const = 0; - virtual std::vector<ui32> GetFactoryColumnIndices() const; + virtual std::vector<ui32> GetFactoryColumnIndices() const; virtual void AddFactoryArguments(TNodePtr& apply) const; @@ -833,8 +833,8 @@ namespace NSQLTranslationV1 { virtual void AddWindowSpecs(TWinSpecs winSpecs); virtual bool AddAggregationOverWindow(TContext& ctx, const TString& windowName, TAggregationPtr func); virtual bool AddFuncOverWindow(TContext& ctx, const TString& windowName, TNodePtr func); - virtual void SetHoppingWindowSpec(THoppingWindowSpecPtr spec); - virtual THoppingWindowSpecPtr GetHoppingWindowSpec() const; + virtual void SetHoppingWindowSpec(THoppingWindowSpecPtr spec); + virtual THoppingWindowSpecPtr GetHoppingWindowSpec() const; virtual TNodePtr GetSessionWindowSpec() const; virtual bool IsCompositeSource() const; virtual bool IsGroupByColumn(const TString& column) const; @@ -842,7 +842,7 @@ namespace NSQLTranslationV1 { virtual bool IsFlattenByExprs() const; virtual bool IsCalcOverWindow() const; virtual bool IsOverWindowSource() const; - virtual bool IsStream() const; + virtual bool IsStream() const; virtual EOrderKind GetOrderKind() const; virtual TWriteSettings GetWriteSettings() const; virtual bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed); @@ -910,7 +910,7 @@ namespace NSQLTranslationV1 { TMap<TString, TVector<TAggregationPtr>> AggregationOverWindow; TMap<TString, TVector<TNodePtr>> FuncOverWindow; TWinSpecs WinSpecs; - THoppingWindowSpecPtr HoppingWindowSpec; + THoppingWindowSpecPtr HoppingWindowSpec; TNodePtr SessionWindow; TVector<ISource*> UsedSources; TString FlattenMode; @@ -1227,8 +1227,8 @@ namespace NSQLTranslationV1 { TAggregationPtr BuildTwoArgsFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildLinearHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); - template <bool HasKey> - TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); + template <bool HasKey> + TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildCountDistinctEstimateFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); TAggregationPtr BuildListFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode); @@ -1283,7 +1283,7 @@ namespace NSQLTranslationV1 { const TVector<TSortSpecificationPtr>& orderBy, TNodePtr having, TWinSpecs&& windowSpec, - THoppingWindowSpecPtr hoppingWindowSpec, + THoppingWindowSpecPtr hoppingWindowSpec, TVector<TNodePtr>&& terms, bool distinct, TVector<TNodePtr>&& without, diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 2fe459d1e7..725356b9f5 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -595,17 +595,17 @@ public: } } for (auto& keyColumn : Params.PartitionByColumns) { - if (!columnsSet.contains(keyColumn.Name)) { - ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name; - return false; - } - } + if (!columnsSet.contains(keyColumn.Name)) { + ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name; + return false; + } + } for (auto& keyColumn : Params.OrderByColumns) { - if (!columnsSet.contains(keyColumn.first.Name)) { - ctx.Error(keyColumn.first.Pos) << "Undefined column: " << keyColumn.first.Name; - return false; - } - } + if (!columnsSet.contains(keyColumn.first.Name)) { + ctx.Error(keyColumn.first.Pos) << "Undefined column: " << keyColumn.first.Name; + return false; + } + } THashSet<TString> indexNames; for (const auto& index : Params.Indexes) { @@ -641,9 +641,9 @@ public: for (auto& col : Params.Columns) { auto columnDesc = Y(); columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); - auto type = col.Type; + auto type = col.Type; if (col.Nullable) { - type = Y("OptionalType", type); + type = Y("OptionalType", type); } columnDesc = L(columnDesc, type); if (col.Families) { @@ -668,33 +668,33 @@ public: if (Table.Service == RtmrProviderName) { if (!Params.PkColumns.empty() && !Params.PartitionByColumns.empty()) { - ctx.Error() << "Only one of PRIMARY KEY or PARTITION BY constraints may be specified"; - return false; - } - } else { + ctx.Error() << "Only one of PRIMARY KEY or PARTITION BY constraints may be specified"; + return false; + } + } else { if (!Params.OrderByColumns.empty()) { ctx.Error() << "ORDER BY is supported only for " << RtmrProviderName << " provider"; - return false; - } - } - + return false; + } + } + if (!Params.PkColumns.empty()) { - auto primaryKey = Y(); + auto primaryKey = Y(); for (auto& col : Params.PkColumns) { - primaryKey = L(primaryKey, BuildQuotedAtom(col.Pos, col.Name)); - } + primaryKey = L(primaryKey, BuildQuotedAtom(col.Pos, col.Name)); + } opts = L(opts, Q(Y(Q("primarykey"), Q(primaryKey)))); if (!Params.OrderByColumns.empty()) { - ctx.Error() << "PRIMARY KEY cannot be used with ORDER BY, use PARTITION BY instead"; - return false; - } + ctx.Error() << "PRIMARY KEY cannot be used with ORDER BY, use PARTITION BY instead"; + return false; + } } if (!Params.PartitionByColumns.empty()) { auto partitionBy = Y(); for (auto& col : Params.PartitionByColumns) { partitionBy = L(partitionBy, BuildQuotedAtom(col.Pos, col.Name)); - } + } opts = L(opts, Q(Y(Q("partitionby"), Q(partitionBy)))); } @@ -702,9 +702,9 @@ public: auto orderBy = Y(); for (auto& col : Params.OrderByColumns) { orderBy = L(orderBy, Q(Y(BuildQuotedAtom(col.first.Pos, col.first.Name), col.second ? Q("1") : Q("0")))); - } + } opts = L(opts, Q(Y(Q("orderby"), Q(orderBy)))); - } + } for (const auto& index : Params.Indexes) { const auto& desc = CreateIndexDesc(index, *this); @@ -840,9 +840,9 @@ public: for (auto& col : Params.AddColumns) { auto columnDesc = Y(); columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); - auto type = col.Type; + auto type = col.Type; if (col.Nullable) { - type = Y("OptionalType", type); + type = Y("OptionalType", type); } columnDesc = L(columnDesc, type); if (col.Families) { diff --git a/ydb/library/yql/sql/v1/select.cpp b/ydb/library/yql/sql/v1/select.cpp index f1f6f2d3ff..2841f05a5b 100644 --- a/ydb/library/yql/sql/v1/select.cpp +++ b/ydb/library/yql/sql/v1/select.cpp @@ -366,11 +366,11 @@ protected: return Source->ShouldUseSourceAsColumn(source); } - bool IsStream() const override { - Y_VERIFY_DEBUG(Source); - return Source->IsStream(); - } - + bool IsStream() const override { + Y_VERIFY_DEBUG(Source); + return Source->IsStream(); + } + EOrderKind GetOrderKind() const override { Y_VERIFY_DEBUG(Source); return Source->GetOrderKind(); @@ -614,9 +614,9 @@ public: return SetSamplingRate(ctx, samplingRate); } - bool IsStream() const override { - return Subquery->GetSource()->IsStream(); - } + bool IsStream() const override { + return Subquery->GetSource()->IsStream(); + } void DoUpdateState() const override { State.Set(ENodeState::Const, true); @@ -739,10 +739,10 @@ public: return AstNode(Table.RefName); } - bool IsStream() const override { + bool IsStream() const override { return IsStreamingService(Table.Service); - } - + } + TPtr DoClone() const final { return new TTableSource(Pos, Table, GetLabel()); } @@ -1400,7 +1400,7 @@ public: const TVector<TSortSpecificationPtr>& orderBy, TNodePtr having, TWinSpecs& winSpecs, - THoppingWindowSpecPtr hoppingWindowSpec, + THoppingWindowSpecPtr hoppingWindowSpec, const TVector<TNodePtr>& terms, bool distinct, const TVector<TNodePtr>& without, @@ -1445,9 +1445,9 @@ public: return false; } if (SelectStream && !Source->IsStream()) { - ctx.Error(Pos) << "SELECT STREAM is unsupported for non-streaming sources"; - return false; - } + ctx.Error(Pos) << "SELECT STREAM is unsupported for non-streaming sources"; + return false; + } auto src = Source.Get(); bool hasError = false; @@ -2091,7 +2091,7 @@ private: TVector<TNodePtr> Without; const bool Distinct; bool OrderByInit = false; - THoppingWindowSpecPtr HoppingWindowSpec; + THoppingWindowSpecPtr HoppingWindowSpec; const bool SelectStream; const TWriteSettings Settings; }; @@ -2461,7 +2461,7 @@ TSourcePtr DoBuildSelectCore( const TVector<TSortSpecificationPtr>& orderBy, TNodePtr having, TWinSpecs&& winSpecs, - THoppingWindowSpecPtr hoppingWindowSpec, + THoppingWindowSpecPtr hoppingWindowSpec, TVector<TNodePtr>&& terms, bool distinct, TVector<TNodePtr>&& without, diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index 624c31c5d0..ab5a864788 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -92,7 +92,7 @@ inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword& node) { case TRule_keyword::kAltKeyword9: return GetIdentifier(ctx, node.GetAlt_keyword9().GetRule_keyword_schema_uncompat1()); default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); } } @@ -118,21 +118,21 @@ static TString Id(const TRule_id& node, TTranslation& ctx) { case TRule_id::kAltId2: return GetKeyword(ctx, node.GetAlt_id2().GetRule_keyword1()); default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_id_or_type& node, TTranslation& ctx) { + switch (node.Alt_case()) { + case TRule_id_or_type::kAltIdOrType1: + return Id(node.GetAlt_id_or_type1().GetRule_id1(), ctx); + case TRule_id_or_type::kAltIdOrType2: + return ctx.Identifier(node.GetAlt_id_or_type2().GetRule_type_id1().GetToken1()); + default: + Y_FAIL("You should change implementation according to grammar changes"); } } -static TString Id(const TRule_id_or_type& node, TTranslation& ctx) { - switch (node.Alt_case()) { - case TRule_id_or_type::kAltIdOrType1: - return Id(node.GetAlt_id_or_type1().GetRule_id1(), ctx); - case TRule_id_or_type::kAltIdOrType2: - return ctx.Identifier(node.GetAlt_id_or_type2().GetRule_type_id1().GetToken1()); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - static TString Id(const TRule_id_schema& node, TTranslation& ctx) { //id_schema: // identifier @@ -162,7 +162,7 @@ static TString Id(const TRule_id_schema& node, TTranslation& ctx) { case TRule_id_schema::kAltIdSchema7: return GetKeyword(ctx, node.GetAlt_id_schema7().GetRule_keyword_hint_uncompat1()); default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); } } @@ -230,17 +230,17 @@ static TString Id(const TRule_an_id_table& node, TTranslation& ctx) { } } -static TString Id(const TRule_id_table_or_type& node, TTranslation& ctx) { - switch (node.Alt_case()) { - case TRule_id_table_or_type::kAltIdTableOrType1: +static TString Id(const TRule_id_table_or_type& node, TTranslation& ctx) { + switch (node.Alt_case()) { + case TRule_id_table_or_type::kAltIdTableOrType1: return Id(node.GetAlt_id_table_or_type1().GetRule_an_id_table1(), ctx); - case TRule_id_table_or_type::kAltIdTableOrType2: - return ctx.Identifier(node.GetAlt_id_table_or_type2().GetRule_type_id1().GetToken1()); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - + case TRule_id_table_or_type::kAltIdTableOrType2: + return ctx.Identifier(node.GetAlt_id_table_or_type2().GetRule_type_id1().GetToken1()); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + static TString Id(const TRule_id_expr& node, TTranslation& ctx) { //id_expr: // identifier @@ -270,7 +270,7 @@ static TString Id(const TRule_id_expr& node, TTranslation& ctx) { case TRule_id_expr::kAltIdExpr7: return GetKeyword(ctx, node.GetAlt_id_expr7().GetRule_keyword_schema_uncompat1()); default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); } } @@ -310,7 +310,7 @@ static TString Id(const TRule_id_expr_in& node, TTranslation& ctx) { case TRule_id_expr_in::kAltIdExprIn6: return GetKeyword(ctx, node.GetAlt_id_expr_in6().GetRule_keyword_schema_uncompat1()); default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); } } @@ -431,7 +431,7 @@ static TString Id(const TRule_an_id& node, TTranslation& ctx) { case TRule_an_id::kAltAnId2: return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id2().GetToken1())); default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); } } @@ -521,13 +521,13 @@ static TString OptIdPrefixAsStr(const TRule_opt_id_prefix& node, TTranslation& c return Id(node.GetBlock1().GetRule_an_id1(), ctx); } -static TString OptIdPrefixAsStr(const TRule_opt_id_prefix_or_type& node, TTranslation& ctx, const TString& defaultStr = {}) { - if (!node.HasBlock1()) { - return defaultStr; - } +static TString OptIdPrefixAsStr(const TRule_opt_id_prefix_or_type& node, TTranslation& ctx, const TString& defaultStr = {}) { + if (!node.HasBlock1()) { + return defaultStr; + } return Id(node.GetBlock1().GetRule_an_id_or_type1(), ctx); -} - +} + static void PureColumnListStr(const TRule_pure_column_list& node, TTranslation& ctx, TVector<TString>& outList) { outList.push_back(Id(node.GetRule_an_id2(), ctx)); for (auto& block: node.GetBlock3()) { @@ -576,7 +576,7 @@ static TDeferredAtom PureColumnOrNamed(const TRule_pure_column_or_named& node, T case TRule_pure_column_or_named::kAltPureColumnOrNamed2: return TDeferredAtom(ctx.Context().Pos(), Id(node.GetAlt_pure_column_or_named2().GetRule_an_id1(), ctx)); default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); } } @@ -623,7 +623,7 @@ static bool CreateTableIndex(const TRule_table_index& node, TTranslation& ctx, T ctx.AltNotImplemented("local", indexType); return false; default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); } if (node.HasBlock4()) { @@ -839,9 +839,9 @@ protected: bool ClusterExpr(const TRule_cluster_expr& node, bool allowWildcard, TString& service, TDeferredAtom& cluster); bool ClusterExprOrBinding(const TRule_cluster_expr& node, TString& service, TDeferredAtom& cluster, bool& isBinding); - TMaybe<TColumnSchema> ColumnSchemaImpl(const TRule_column_schema& node); + TMaybe<TColumnSchema> ColumnSchemaImpl(const TRule_column_schema& node); bool CreateTableEntry(const TRule_create_table_entry& node, TCreateTableParameters& params); - + bool FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family); bool FillFamilySettings(const TRule_family_settings& settingsNode, TFamilyEntry& family); bool CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings); @@ -854,15 +854,15 @@ protected: TNodePtr TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed); TNodePtr TypeDecimal(const TRule_type_name_decimal& node); TNodePtr AddOptionals(const TNodePtr& node, size_t optionalCount); - TMaybe<std::pair<TVector<TNodePtr>, bool>> CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted); - - TNodePtr IntegerOrBind(const TRule_integer_or_bind& node); - TNodePtr TypeNameTag(const TRule_type_name_tag& node); - TNodePtr TypeNodeOrBind(const TRule_type_name_or_bind& node); + TMaybe<std::pair<TVector<TNodePtr>, bool>> CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted); + + TNodePtr IntegerOrBind(const TRule_integer_or_bind& node); + TNodePtr TypeNameTag(const TRule_type_name_tag& node); + TNodePtr TypeNodeOrBind(const TRule_type_name_or_bind& node); TNodePtr TypeNode(const TRule_type_name& node); TNodePtr TypeNode(const TRule_type_name_composite& node); - TNodePtr ValueConstructorLiteral(const TRule_value_constructor_literal& node); - TNodePtr ValueConstructor(const TRule_value_constructor& node); + TNodePtr ValueConstructorLiteral(const TRule_value_constructor_literal& node); + TNodePtr ValueConstructor(const TRule_value_constructor& node); TNodePtr ListLiteral(const TRule_list_literal& node); TNodePtr DictLiteral(const TRule_dict_literal& node); TNodePtr StructLiteral(const TRule_struct_literal& node); @@ -872,7 +872,7 @@ protected: TWindowSpecificationPtr WindowSpecification(const TRule_window_specification_details& rule); bool OrderByClause(const TRule_order_by_clause& node, TVector<TSortSpecificationPtr>& orderBy); bool SortSpecificationList(const TRule_sort_specification_list& node, TVector<TSortSpecificationPtr>& sortSpecs); - + bool IsDistinctOptSet(const TRule_opt_set_quantifier& node) const; bool IsDistinctOptSet(const TRule_opt_set_quantifier& node, TPosition& distinctPos) const; @@ -908,21 +908,21 @@ public: } TNodePtr Build(const TRule_expr& node) { - // expr: - // or_subexpr (OR or_subexpr)* + // expr: + // or_subexpr (OR or_subexpr)* // | type_name_composite - switch (node.Alt_case()) { - case TRule_expr::kAltExpr1: { - auto getNode = [](const TRule_expr_TAlt1_TBlock2& b) -> const TRule_or_subexpr& { return b.GetRule_or_subexpr2(); }; - return BinOper("Or", node.GetAlt_expr1().GetRule_or_subexpr1(), getNode, + switch (node.Alt_case()) { + case TRule_expr::kAltExpr1: { + auto getNode = [](const TRule_expr_TAlt1_TBlock2& b) -> const TRule_or_subexpr& { return b.GetRule_or_subexpr2(); }; + return BinOper("Or", node.GetAlt_expr1().GetRule_or_subexpr1(), getNode, node.GetAlt_expr1().GetBlock2().begin(), node.GetAlt_expr1().GetBlock2().end(), {}); - } - case TRule_expr::kAltExpr2: { + } + case TRule_expr::kAltExpr2: { return TypeNode(node.GetAlt_expr2().GetRule_type_name_composite1()); - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } } void SetSmartParenthesisMode(ESmartParenthesis mode) { @@ -1015,7 +1015,7 @@ private: auto getNode = [](const TRule_neq_subexpr::TBlock2& b) -> const TRule_bit_subexpr& { return b.GetRule_bit_subexpr2(); }; auto result = BinOpList(node.GetRule_bit_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); if (!result) { - return {}; + return {}; } if (node.HasBlock3()) { auto& block = node.GetBlock3(); @@ -1054,9 +1054,9 @@ private: template <typename TNode, typename TGetNode, typename TIter> TNodePtr BinOpList(const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); - template <typename TGetNode, typename TIter> + template <typename TGetNode, typename TIter> TNodePtr BinOpList(const TRule_bit_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); - + template <typename TGetNode, typename TIter> TNodePtr BinOpList(const TRule_eq_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); @@ -1115,7 +1115,7 @@ public: void InitExpr(const TNodePtr& expr); bool Init(const TRule_using_call_expr& node); - bool Init(const TRule_value_constructor& node); + bool Init(const TRule_value_constructor& node); bool Init(const TRule_invoke_expr& node); bool ConfigureExternalCall(const TRule_external_call_settings& node); void IncCounters(); @@ -1789,22 +1789,22 @@ TMaybe<TSourcePtr> TSqlTranslation::AsTableImpl(const TRule_table_ref& node) { return Nothing(); } -TMaybe<TColumnSchema> TSqlTranslation::ColumnSchemaImpl(const TRule_column_schema& node) { +TMaybe<TColumnSchema> TSqlTranslation::ColumnSchemaImpl(const TRule_column_schema& node) { const bool nullable = !node.HasBlock4() || !node.GetBlock4().HasBlock1(); const TString name(Id(node.GetRule_an_id_schema1(), *this)); - const TPosition pos(Context().Pos()); - const auto type = TypeNodeOrBind(node.GetRule_type_name_or_bind2()); - if (!type) { - return {}; - } + const TPosition pos(Context().Pos()); + const auto type = TypeNodeOrBind(node.GetRule_type_name_or_bind2()); + if (!type) { + return {}; + } TVector<TIdentifier> families; if (node.HasBlock3()) { const auto& familyRelation = node.GetBlock3().GetRule_family_relation1(); families.push_back(IdEx(familyRelation.GetRule_an_id2(), *this)); } return TColumnSchema(pos, name, type, nullable, families); -} - +} + bool TSqlTranslation::FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family) { TIdentifier id = IdEx(settingNode.GetRule_an_id1(), *this); const TRule_family_setting_value& value = settingNode.GetRule_family_setting_value3(); @@ -1834,107 +1834,107 @@ bool TSqlTranslation::FillFamilySettings(const TRule_family_settings& settingsNo } bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCreateTableParameters& params) -{ - switch (node.Alt_case()) { - case TRule_create_table_entry::kAltCreateTableEntry1: - { +{ + switch (node.Alt_case()) { + case TRule_create_table_entry::kAltCreateTableEntry1: + { // column_schema - auto columnSchema = ColumnSchemaImpl(node.GetAlt_create_table_entry1().GetRule_column_schema1()); - if (!columnSchema) { - return false; - } + auto columnSchema = ColumnSchemaImpl(node.GetAlt_create_table_entry1().GetRule_column_schema1()); + if (!columnSchema) { + return false; + } if (columnSchema->Families.size() > 1) { Ctx.Error() << "Several column families for a single column are not yet supported"; return false; } params.Columns.push_back(*columnSchema); - break; - } - case TRule_create_table_entry::kAltCreateTableEntry2: - { + break; + } + case TRule_create_table_entry::kAltCreateTableEntry2: + { // table_constraint - auto& constraint = node.GetAlt_create_table_entry2().GetRule_table_constraint1(); - switch (constraint.Alt_case()) { - case TRule_table_constraint::kAltTableConstraint1: { + auto& constraint = node.GetAlt_create_table_entry2().GetRule_table_constraint1(); + switch (constraint.Alt_case()) { + case TRule_table_constraint::kAltTableConstraint1: { if (!params.PkColumns.empty()) { Ctx.Error() << "PRIMARY KEY statement must be specified only once"; return false; } - auto& pkConstraint = constraint.GetAlt_table_constraint1(); + auto& pkConstraint = constraint.GetAlt_table_constraint1(); params.PkColumns.push_back(IdEx(pkConstraint.GetRule_an_id4(), *this)); - for (auto& block : pkConstraint.GetBlock5()) { + for (auto& block : pkConstraint.GetBlock5()) { params.PkColumns.push_back(IdEx(block.GetRule_an_id2(), *this)); - } - break; - } - case TRule_table_constraint::kAltTableConstraint2: { + } + break; + } + case TRule_table_constraint::kAltTableConstraint2: { if (!params.PartitionByColumns.empty()) { Ctx.Error() << "PARTITION BY statement must be specified only once"; return false; } - auto& pbConstraint = constraint.GetAlt_table_constraint2(); + auto& pbConstraint = constraint.GetAlt_table_constraint2(); params.PartitionByColumns.push_back(IdEx(pbConstraint.GetRule_an_id4(), *this)); - for (auto& block : pbConstraint.GetBlock5()) { + for (auto& block : pbConstraint.GetBlock5()) { params.PartitionByColumns.push_back(IdEx(block.GetRule_an_id2(), *this)); - } - break; - } - case TRule_table_constraint::kAltTableConstraint3: { + } + break; + } + case TRule_table_constraint::kAltTableConstraint3: { if (!params.OrderByColumns.empty()) { Ctx.Error() << "ORDER BY statement must be specified only once"; return false; } - auto& obConstraint = constraint.GetAlt_table_constraint3(); - auto extractDirection = [this] (const TRule_column_order_by_specification& spec, bool& desc) { - desc = false; - if (!spec.HasBlock2()) { - return true; - } - - auto& token = spec.GetBlock2().GetToken1(); - switch (token.GetId()) { - case SQLv1LexerTokens::TOKEN_ASC: - return true; - case SQLv1LexerTokens::TOKEN_DESC: - desc = true; - return true; - default: - Ctx.Error() << "Unsupported direction token: " << token.GetId(); - return false; - } - }; - - bool desc = false; - auto& obSpec = obConstraint.GetRule_column_order_by_specification4(); - if (!extractDirection(obSpec, desc)) { - return false; - } + auto& obConstraint = constraint.GetAlt_table_constraint3(); + auto extractDirection = [this] (const TRule_column_order_by_specification& spec, bool& desc) { + desc = false; + if (!spec.HasBlock2()) { + return true; + } + + auto& token = spec.GetBlock2().GetToken1(); + switch (token.GetId()) { + case SQLv1LexerTokens::TOKEN_ASC: + return true; + case SQLv1LexerTokens::TOKEN_DESC: + desc = true; + return true; + default: + Ctx.Error() << "Unsupported direction token: " << token.GetId(); + return false; + } + }; + + bool desc = false; + auto& obSpec = obConstraint.GetRule_column_order_by_specification4(); + if (!extractDirection(obSpec, desc)) { + return false; + } params.OrderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_an_id1(), *this), desc)); - - for (auto& block : obConstraint.GetBlock5()) { - auto& obSpec = block.GetRule_column_order_by_specification2(); - if (!extractDirection(obSpec, desc)) { - return false; - } + + for (auto& block : obConstraint.GetBlock5()) { + auto& obSpec = block.GetRule_column_order_by_specification2(); + if (!extractDirection(obSpec, desc)) { + return false; + } params.OrderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_an_id1(), *this), desc)); - } - break; - } - default: - AltNotImplemented("table_constraint", constraint); - return false; - } - break; - } - case TRule_create_table_entry::kAltCreateTableEntry3: - { + } + break; + } + default: + AltNotImplemented("table_constraint", constraint); + return false; + } + break; + } + case TRule_create_table_entry::kAltCreateTableEntry3: + { // table_index - auto& table_index = node.GetAlt_create_table_entry3().GetRule_table_index1(); + auto& table_index = node.GetAlt_create_table_entry3().GetRule_table_index1(); if (!CreateTableIndex(table_index, *this, params.Indexes)) { - return false; - } - break; - } + return false; + } + break; + } case TRule_create_table_entry::kAltCreateTableEntry4: { // family_entry @@ -1957,13 +1957,13 @@ bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCr } break; } - default: - AltNotImplemented("create_table_entry", node); - return false; - } - return true; -} - + default: + AltNotImplemented("create_table_entry", node); + return false; + } + return true; +} + TNodePtr LiteralNumber(TContext& ctx, const TRule_integer& node); namespace { @@ -2308,70 +2308,70 @@ bool TSqlTranslation::CreateTableSettings(const TRule_with_table_settings& setti return true; } -bool ParseNumbers(TContext& ctx, const TString& strOrig, ui64& value, TString& suffix); - -TNodePtr TSqlTranslation::IntegerOrBind(const TRule_integer_or_bind& node) { - switch (node.Alt_case()) { - case TRule_integer_or_bind::kAltIntegerOrBind1: { - const TString intString = Ctx.Token(node.GetAlt_integer_or_bind1().GetRule_integer1().GetToken1()); - ui64 value; - TString suffix; - if (!ParseNumbers(Ctx, intString, value, suffix)) { - return {}; - } - return BuildQuotedAtom(Ctx.Pos(), ToString(value), TNodeFlags::ArbitraryContent); - } - case TRule_integer_or_bind::kAltIntegerOrBind2: { +bool ParseNumbers(TContext& ctx, const TString& strOrig, ui64& value, TString& suffix); + +TNodePtr TSqlTranslation::IntegerOrBind(const TRule_integer_or_bind& node) { + switch (node.Alt_case()) { + case TRule_integer_or_bind::kAltIntegerOrBind1: { + const TString intString = Ctx.Token(node.GetAlt_integer_or_bind1().GetRule_integer1().GetToken1()); + ui64 value; + TString suffix; + if (!ParseNumbers(Ctx, intString, value, suffix)) { + return {}; + } + return BuildQuotedAtom(Ctx.Pos(), ToString(value), TNodeFlags::ArbitraryContent); + } + case TRule_integer_or_bind::kAltIntegerOrBind2: { TString bindName; if (!NamedNodeImpl(node.GetAlt_integer_or_bind2().GetRule_bind_parameter1(), bindName, *this)) { return {}; } - auto namedNode = GetNamedNode(bindName); - if (!namedNode) { - return {}; - } - auto atom = MakeAtomFromExpression(Ctx, namedNode); - return atom.Build(); - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -TNodePtr TSqlTranslation::TypeNameTag(const TRule_type_name_tag& node) { - switch (node.Alt_case()) { - case TRule_type_name_tag::kAltTypeNameTag1: { - auto content = Id(node.GetAlt_type_name_tag1().GetRule_id1(), *this); - auto atom = TDeferredAtom(Ctx.Pos(), content); - return atom.Build(); - } - case TRule_type_name_tag::kAltTypeNameTag2: { + auto namedNode = GetNamedNode(bindName); + if (!namedNode) { + return {}; + } + auto atom = MakeAtomFromExpression(Ctx, namedNode); + return atom.Build(); + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +TNodePtr TSqlTranslation::TypeNameTag(const TRule_type_name_tag& node) { + switch (node.Alt_case()) { + case TRule_type_name_tag::kAltTypeNameTag1: { + auto content = Id(node.GetAlt_type_name_tag1().GetRule_id1(), *this); + auto atom = TDeferredAtom(Ctx.Pos(), content); + return atom.Build(); + } + case TRule_type_name_tag::kAltTypeNameTag2: { auto value = Token(node.GetAlt_type_name_tag2().GetToken1()); auto parsed = StringContentOrIdContent(Ctx, Ctx.Pos(), value); if (!parsed) { - return {}; - } + return {}; + } auto atom = TDeferredAtom(Ctx.Pos(), parsed->Content); - return atom.Build(); - } - case TRule_type_name_tag::kAltTypeNameTag3: { + return atom.Build(); + } + case TRule_type_name_tag::kAltTypeNameTag3: { TString bindName; if (!NamedNodeImpl(node.GetAlt_type_name_tag3().GetRule_bind_parameter1(), bindName, *this)) { return {}; } - auto namedNode = GetNamedNode(bindName); - if (!namedNode) { - return {}; - } - TDeferredAtom atom; - MakeTableFromExpression(Ctx, namedNode, atom); - return atom.Build(); - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - + auto namedNode = GetNamedNode(bindName); + if (!namedNode) { + return {}; + } + TDeferredAtom atom; + MakeTableFromExpression(Ctx, namedNode, atom); + return atom.Build(); + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + TNodePtr TSqlTranslation::TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed) { const TString origName = Id(node.GetRule_an_id_pure1(), *this); if (origName.empty()) { @@ -2382,19 +2382,19 @@ TNodePtr TSqlTranslation::TypeSimple(const TRule_type_name_simple& node, bool on TNodePtr TSqlTranslation::TypeDecimal(const TRule_type_name_decimal& node) { auto pos = Ctx.Pos(); - auto flags = TNodeFlags::Default; - - auto paramOne = IntegerOrBind(node.GetRule_integer_or_bind3()); - if (!paramOne) { - return {}; - } - auto paramTwo = IntegerOrBind(node.GetRule_integer_or_bind5()); - if (!paramTwo) { - return {}; - } + auto flags = TNodeFlags::Default; + + auto paramOne = IntegerOrBind(node.GetRule_integer_or_bind3()); + if (!paramOne) { + return {}; + } + auto paramTwo = IntegerOrBind(node.GetRule_integer_or_bind5()); + if (!paramTwo) { + return {}; + } return new TCallNodeImpl(pos, "DataType", { BuildQuotedAtom(pos, "Decimal", flags), paramOne, paramTwo }); -} - +} + TNodePtr TSqlTranslation::AddOptionals(const TNodePtr& node, size_t optionalCount) { TNodePtr result = node; if (node) { @@ -2407,83 +2407,83 @@ TNodePtr TSqlTranslation::AddOptionals(const TNodePtr& node, size_t optionalCoun } -TMaybe<std::pair<TVector<TNodePtr>, bool>> TSqlTranslation::CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted) { - auto pos = Ctx.Pos(); - auto flags = TNodeFlags::Default; - auto& arg1 = argList.GetRule_callable_arg1(); - auto& varArg = arg1.GetRule_variant_arg1(); - TVector<TNodePtr> result; - TVector<TNodePtr> items; - auto typeNode = TypeNodeOrBind(varArg.GetRule_type_name_or_bind2()); - if (!typeNode) { - return {}; - } - items.push_back(typeNode); - if (varArg.HasBlock1()) { - namedArgsStarted = true; - auto tag = TypeNameTag(varArg.GetBlock1().GetRule_type_name_tag1()); - if (!tag) { - return {}; - } - items.push_back(tag); - } - if (arg1.HasBlock2()) { - if (!varArg.HasBlock1()) { - items.push_back(BuildQuotedAtom(pos, "", flags)); - } - items.push_back(BuildQuotedAtom(pos, "1", flags)); - } - result.push_back(new TAstListNodeImpl(pos, items)); - - for (auto& arg : argList.GetBlock2()) { - auto& varArg = arg.GetRule_callable_arg2().GetRule_variant_arg1(); - TVector<TNodePtr> items; - auto typeNode = TypeNodeOrBind(varArg.GetRule_type_name_or_bind2()); - if (!typeNode) { - return {}; - } - items.push_back(typeNode); - if (varArg.HasBlock1()) { - auto tag = TypeNameTag(varArg.GetBlock1().GetRule_type_name_tag1()); - if (!tag) { - return {}; - } - items.push_back(tag); - } else { - if (namedArgsStarted) { - Ctx.Error() << "Expected named argument, previous argument was named"; - return {}; - } - items.push_back(BuildQuotedAtom(pos, "", flags)); - } - if (arg.GetRule_callable_arg2().HasBlock2()) { - if (!varArg.HasBlock1()) { - items.push_back(BuildQuotedAtom(pos, "", flags)); - } - items.push_back(BuildQuotedAtom(pos, "1", flags)); - } - result.push_back(new TAstListNodeImpl(pos, items)); - } - return std::make_pair(result, namedArgsStarted); -} - -TNodePtr TSqlTranslation::TypeNodeOrBind(const TRule_type_name_or_bind& node) { - switch (node.Alt_case()) { - case TRule_type_name_or_bind::kAltTypeNameOrBind1: { - return TypeNode(node.GetAlt_type_name_or_bind1().GetRule_type_name1()); - } - case TRule_type_name_or_bind::kAltTypeNameOrBind2: { +TMaybe<std::pair<TVector<TNodePtr>, bool>> TSqlTranslation::CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted) { + auto pos = Ctx.Pos(); + auto flags = TNodeFlags::Default; + auto& arg1 = argList.GetRule_callable_arg1(); + auto& varArg = arg1.GetRule_variant_arg1(); + TVector<TNodePtr> result; + TVector<TNodePtr> items; + auto typeNode = TypeNodeOrBind(varArg.GetRule_type_name_or_bind2()); + if (!typeNode) { + return {}; + } + items.push_back(typeNode); + if (varArg.HasBlock1()) { + namedArgsStarted = true; + auto tag = TypeNameTag(varArg.GetBlock1().GetRule_type_name_tag1()); + if (!tag) { + return {}; + } + items.push_back(tag); + } + if (arg1.HasBlock2()) { + if (!varArg.HasBlock1()) { + items.push_back(BuildQuotedAtom(pos, "", flags)); + } + items.push_back(BuildQuotedAtom(pos, "1", flags)); + } + result.push_back(new TAstListNodeImpl(pos, items)); + + for (auto& arg : argList.GetBlock2()) { + auto& varArg = arg.GetRule_callable_arg2().GetRule_variant_arg1(); + TVector<TNodePtr> items; + auto typeNode = TypeNodeOrBind(varArg.GetRule_type_name_or_bind2()); + if (!typeNode) { + return {}; + } + items.push_back(typeNode); + if (varArg.HasBlock1()) { + auto tag = TypeNameTag(varArg.GetBlock1().GetRule_type_name_tag1()); + if (!tag) { + return {}; + } + items.push_back(tag); + } else { + if (namedArgsStarted) { + Ctx.Error() << "Expected named argument, previous argument was named"; + return {}; + } + items.push_back(BuildQuotedAtom(pos, "", flags)); + } + if (arg.GetRule_callable_arg2().HasBlock2()) { + if (!varArg.HasBlock1()) { + items.push_back(BuildQuotedAtom(pos, "", flags)); + } + items.push_back(BuildQuotedAtom(pos, "1", flags)); + } + result.push_back(new TAstListNodeImpl(pos, items)); + } + return std::make_pair(result, namedArgsStarted); +} + +TNodePtr TSqlTranslation::TypeNodeOrBind(const TRule_type_name_or_bind& node) { + switch (node.Alt_case()) { + case TRule_type_name_or_bind::kAltTypeNameOrBind1: { + return TypeNode(node.GetAlt_type_name_or_bind1().GetRule_type_name1()); + } + case TRule_type_name_or_bind::kAltTypeNameOrBind2: { TString bindName; if (!NamedNodeImpl(node.GetAlt_type_name_or_bind2().GetRule_bind_parameter1(), bindName, *this)) { return {}; } return GetNamedNode(bindName); - } + } default: Y_FAIL("You should change implementation according to grammar changes"); - } -} - + } +} + TNodePtr TSqlTranslation::TypeNode(const TRule_type_name& node) { //type_name: // type_name_composite @@ -2492,8 +2492,8 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name& node) { return TypeNode(node.GetAlt_type_name1().GetRule_type_name_composite1()); } - TNodePtr result; - TPosition pos = Ctx.Pos(); + TNodePtr result; + TPosition pos = Ctx.Pos(); auto& alt = node.GetAlt_type_name2(); auto& block = alt.GetBlock1(); @@ -2533,39 +2533,39 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { // ) QUESTION*; TNodePtr result; TPosition pos = Ctx.Pos(); - auto flags = TNodeFlags::Default; - - auto wrapOneParamType = [&] (const TRule_type_name_or_bind& param, const char* type) -> TNodePtr { - auto node = TypeNodeOrBind(param); - return node ? new TAstListNodeImpl(pos, { BuildAtom(pos, type, flags), node }) : nullptr; - }; - auto makeVoid = [&] () -> TNodePtr { - return new TAstListNodeImpl(pos, { BuildAtom(pos, "VoidType", flags) }); - }; - auto makeQuote = [&] (const TNodePtr& node) -> TNodePtr { - return new TAstListNodeImpl(pos, { new TAstAtomNodeImpl(pos, "quote", 0), node }); - }; - - auto& block = node.GetBlock1(); - switch (block.Alt_case()) { + auto flags = TNodeFlags::Default; + + auto wrapOneParamType = [&] (const TRule_type_name_or_bind& param, const char* type) -> TNodePtr { + auto node = TypeNodeOrBind(param); + return node ? new TAstListNodeImpl(pos, { BuildAtom(pos, type, flags), node }) : nullptr; + }; + auto makeVoid = [&] () -> TNodePtr { + return new TAstListNodeImpl(pos, { BuildAtom(pos, "VoidType", flags) }); + }; + auto makeQuote = [&] (const TNodePtr& node) -> TNodePtr { + return new TAstListNodeImpl(pos, { new TAstAtomNodeImpl(pos, "quote", 0), node }); + }; + + auto& block = node.GetBlock1(); + switch (block.Alt_case()) { case TRule_type_name_composite_TBlock1::kAlt1: { auto& optionalType = block.GetAlt1().GetRule_type_name_optional1(); - result = wrapOneParamType(optionalType.GetRule_type_name_or_bind3(), "OptionalType"); - break; - } + result = wrapOneParamType(optionalType.GetRule_type_name_or_bind3(), "OptionalType"); + break; + } case TRule_type_name_composite_TBlock1::kAlt2: { auto& tupleType = block.GetAlt2().GetRule_type_name_tuple1(); - TVector<TNodePtr> items; - items.push_back(BuildAtom(pos, "TupleType", flags)); + TVector<TNodePtr> items; + items.push_back(BuildAtom(pos, "TupleType", flags)); switch (tupleType.GetBlock2().Alt_case()) { case TRule_type_name_tuple::TBlock2::kAlt1: { if (tupleType.GetBlock2().GetAlt1().HasBlock2()) { auto typeNode = TypeNodeOrBind(tupleType.GetBlock2().GetAlt1().GetBlock2().GetRule_type_name_or_bind1()); - if (!typeNode) { - return {}; - } - items.push_back(typeNode); + if (!typeNode) { + return {}; + } + items.push_back(typeNode); for (auto& arg : tupleType.GetBlock2().GetAlt1().GetBlock2().GetBlock2()) { auto typeNode = TypeNodeOrBind(arg.GetRule_type_name_or_bind2()); if (!typeNode) { @@ -2573,22 +2573,22 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { } items.push_back(typeNode); } - } + } [[fallthrough]]; // AUTOGENERATED_FALLTHROUGH_FIXME - } + } case TRule_type_name_tuple::TBlock2::kAlt2: break; default: Y_FAIL("You should change implementation according to grammar changes"); } - result = new TAstListNodeImpl(pos, items); - break; - } + result = new TAstListNodeImpl(pos, items); + break; + } case TRule_type_name_composite_TBlock1::kAlt3: { auto& structType = block.GetAlt3().GetRule_type_name_struct1(); - TVector<TNodePtr> items; - items.push_back(BuildAtom(pos, "StructType", flags)); + TVector<TNodePtr> items; + items.push_back(BuildAtom(pos, "StructType", flags)); switch (structType.GetBlock2().Alt_case()) { case TRule_type_name_struct::TBlock2::kAlt1: { @@ -2617,110 +2617,110 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { } } [[fallthrough]]; // AUTOGENERATED_FALLTHROUGH_FIXME - } + } case TRule_type_name_struct::TBlock2::kAlt2: break; default: Y_FAIL("You should change implementation according to grammar changes"); - } + } - result = new TAstListNodeImpl(pos, items); - break; - } + result = new TAstListNodeImpl(pos, items); + break; + } case TRule_type_name_composite_TBlock1::kAlt4: { auto& variantType = block.GetAlt4().GetRule_type_name_variant1(); - TVector<TNodePtr> items; - bool overStruct = false; - auto& variantArg = variantType.GetRule_variant_arg3(); - auto typeNode = TypeNodeOrBind(variantArg.GetRule_type_name_or_bind2()); - if (!typeNode) { - return {}; - } - if (variantArg.HasBlock1()) { - items.push_back(BuildAtom(pos, "StructType", flags)); - overStruct = true; - auto tag = TypeNameTag(variantArg.GetBlock1().GetRule_type_name_tag1()); - if (!tag) { - return {}; - } - items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode }))); - } else { - items.push_back(BuildAtom(pos, "TupleType", flags)); - items.push_back(typeNode); - } - - for (auto& arg : variantType.GetBlock4()) { - auto typeNode = TypeNodeOrBind(arg.GetRule_variant_arg2().GetRule_type_name_or_bind2()); - if (!typeNode) { - return {}; - } - if (overStruct) { - if (!arg.GetRule_variant_arg2().HasBlock1()) { - Ctx.Error() << "Variant over struct and tuple mixture"; - return {}; - } - auto tag = TypeNameTag(arg.GetRule_variant_arg2().GetBlock1().GetRule_type_name_tag1()); - if (!tag) { - return {}; - } - items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode }))); - } else { - if (arg.GetRule_variant_arg2().HasBlock1()) { - Ctx.Error() << "Variant over struct and tuple mixture"; - return {}; - } - items.push_back(typeNode); - } - } - typeNode = new TAstListNodeImpl(pos, items); - result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode }); - break; - } + TVector<TNodePtr> items; + bool overStruct = false; + auto& variantArg = variantType.GetRule_variant_arg3(); + auto typeNode = TypeNodeOrBind(variantArg.GetRule_type_name_or_bind2()); + if (!typeNode) { + return {}; + } + if (variantArg.HasBlock1()) { + items.push_back(BuildAtom(pos, "StructType", flags)); + overStruct = true; + auto tag = TypeNameTag(variantArg.GetBlock1().GetRule_type_name_tag1()); + if (!tag) { + return {}; + } + items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode }))); + } else { + items.push_back(BuildAtom(pos, "TupleType", flags)); + items.push_back(typeNode); + } + + for (auto& arg : variantType.GetBlock4()) { + auto typeNode = TypeNodeOrBind(arg.GetRule_variant_arg2().GetRule_type_name_or_bind2()); + if (!typeNode) { + return {}; + } + if (overStruct) { + if (!arg.GetRule_variant_arg2().HasBlock1()) { + Ctx.Error() << "Variant over struct and tuple mixture"; + return {}; + } + auto tag = TypeNameTag(arg.GetRule_variant_arg2().GetBlock1().GetRule_type_name_tag1()); + if (!tag) { + return {}; + } + items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode }))); + } else { + if (arg.GetRule_variant_arg2().HasBlock1()) { + Ctx.Error() << "Variant over struct and tuple mixture"; + return {}; + } + items.push_back(typeNode); + } + } + typeNode = new TAstListNodeImpl(pos, items); + result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode }); + break; + } case TRule_type_name_composite_TBlock1::kAlt5: { auto& listType = block.GetAlt5().GetRule_type_name_list1(); - result = wrapOneParamType(listType.GetRule_type_name_or_bind3(), "ListType"); - break; - } + result = wrapOneParamType(listType.GetRule_type_name_or_bind3(), "ListType"); + break; + } case TRule_type_name_composite_TBlock1::kAlt6: { auto& streamType = block.GetAlt6().GetRule_type_name_stream1(); - result = wrapOneParamType(streamType.GetRule_type_name_or_bind3(), "StreamType"); - break; - } + result = wrapOneParamType(streamType.GetRule_type_name_or_bind3(), "StreamType"); + break; + } case TRule_type_name_composite_TBlock1::kAlt7: { auto& flowType = block.GetAlt7().GetRule_type_name_flow1(); - result = wrapOneParamType(flowType.GetRule_type_name_or_bind3(), "FlowType"); - break; - } + result = wrapOneParamType(flowType.GetRule_type_name_or_bind3(), "FlowType"); + break; + } case TRule_type_name_composite_TBlock1::kAlt8: { auto& dictType = block.GetAlt8().GetRule_type_name_dict1(); - TVector<TNodePtr> items; - items.push_back(BuildAtom(pos, "DictType", flags)); - auto typeNode = TypeNodeOrBind(dictType.GetRule_type_name_or_bind3()); - if (!typeNode) { - return {}; - } - items.push_back(typeNode); - typeNode = TypeNodeOrBind(dictType.GetRule_type_name_or_bind5()); - if (!typeNode) { - return {}; - } - items.push_back(typeNode); - result = new TAstListNodeImpl(pos, items); - break; - } + TVector<TNodePtr> items; + items.push_back(BuildAtom(pos, "DictType", flags)); + auto typeNode = TypeNodeOrBind(dictType.GetRule_type_name_or_bind3()); + if (!typeNode) { + return {}; + } + items.push_back(typeNode); + typeNode = TypeNodeOrBind(dictType.GetRule_type_name_or_bind5()); + if (!typeNode) { + return {}; + } + items.push_back(typeNode); + result = new TAstListNodeImpl(pos, items); + break; + } case TRule_type_name_composite_TBlock1::kAlt9: { auto& setType = block.GetAlt9().GetRule_type_name_set1(); - auto typeNode = TypeNodeOrBind(setType.GetRule_type_name_or_bind3()); - if (!typeNode) { - return {}; - } - result = new TAstListNodeImpl(pos, { BuildAtom(pos, "DictType", flags), typeNode, makeVoid() }); - break; - } + auto typeNode = TypeNodeOrBind(setType.GetRule_type_name_or_bind3()); + if (!typeNode) { + return {}; + } + result = new TAstListNodeImpl(pos, { BuildAtom(pos, "DictType", flags), typeNode, makeVoid() }); + break; + } case TRule_type_name_composite_TBlock1::kAlt10: { auto& enumType = block.GetAlt10().GetRule_type_name_enum1(); - TVector<TNodePtr> items; - items.push_back(BuildAtom(pos, "StructType", flags)); + TVector<TNodePtr> items; + items.push_back(BuildAtom(pos, "StructType", flags)); auto tag = TypeNameTag(enumType.GetRule_type_name_tag3()); if (!tag) { return {}; @@ -2728,91 +2728,91 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, makeVoid() }))); for (auto& arg : enumType.GetBlock4()) { auto tag = TypeNameTag(arg.GetRule_type_name_tag2()); - if (!tag) { - return {}; - } - items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, makeVoid() }))); - } - auto typeNode = new TAstListNodeImpl(pos, items); - result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode }); - break; - } + if (!tag) { + return {}; + } + items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, makeVoid() }))); + } + auto typeNode = new TAstListNodeImpl(pos, items); + result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode }); + break; + } case TRule_type_name_composite_TBlock1::kAlt11: { auto& resourceType = block.GetAlt11().GetRule_type_name_resource1(); - auto tag = TypeNameTag(resourceType.GetRule_type_name_tag3()); - if (!tag) { - return {}; - } - result = new TAstListNodeImpl(pos, { BuildAtom(pos, "ResourceType", flags), tag }); - break; - } + auto tag = TypeNameTag(resourceType.GetRule_type_name_tag3()); + if (!tag) { + return {}; + } + result = new TAstListNodeImpl(pos, { BuildAtom(pos, "ResourceType", flags), tag }); + break; + } case TRule_type_name_composite_TBlock1::kAlt12: { auto& taggedType = block.GetAlt12().GetRule_type_name_tagged1(); - auto typeNode = TypeNodeOrBind(taggedType.GetRule_type_name_or_bind3()); - if (!typeNode) { - return {}; - } - auto tag = TypeNameTag(taggedType.GetRule_type_name_tag5()); - if (!tag) { - return {}; - } - result = new TAstListNodeImpl(pos, { BuildAtom(pos, "TaggedType", flags), typeNode, tag }); - break; - } + auto typeNode = TypeNodeOrBind(taggedType.GetRule_type_name_or_bind3()); + if (!typeNode) { + return {}; + } + auto tag = TypeNameTag(taggedType.GetRule_type_name_tag5()); + if (!tag) { + return {}; + } + result = new TAstListNodeImpl(pos, { BuildAtom(pos, "TaggedType", flags), typeNode, tag }); + break; + } case TRule_type_name_composite_TBlock1::kAlt13: { auto& callableType = block.GetAlt13().GetRule_type_name_callable1(); - TMaybe<std::pair<TVector<TNodePtr>, bool>> requiredArgs, optionalArgs; - bool namedArgsStarted = false; - size_t optionalArgsCount = 0; - if (callableType.HasBlock4()) { - auto& argList = callableType.GetBlock4().GetRule_callable_arg_list1(); - requiredArgs = CallableArgList(argList, namedArgsStarted); - if (!requiredArgs) { - return {}; - } - namedArgsStarted = requiredArgs->second; - } - if (callableType.HasBlock6()) { - auto& argList = callableType.GetBlock6().GetRule_callable_arg_list2(); - optionalArgs = CallableArgList(argList, namedArgsStarted); - if (!optionalArgs) { - return {}; - } - optionalArgsCount = optionalArgs->first.size(); - } - auto returnType = TypeNodeOrBind(callableType.GetRule_type_name_or_bind9()); - if (!returnType) { - return {}; - } - TVector<TNodePtr> items; - items.push_back(BuildAtom(pos, "CallableType", flags)); - if (optionalArgsCount) { - items.push_back(makeQuote(new TAstListNodeImpl(pos, - { BuildQuotedAtom(pos, ToString(optionalArgsCount), flags) }))); - } else { - items.push_back(makeQuote(new TAstListNodeImpl(pos, {}))); - } - items.push_back(makeQuote(new TAstListNodeImpl(pos, { returnType }))); - if (requiredArgs) { - for (auto& arg: requiredArgs->first) { - items.push_back(makeQuote(arg)); - } - } - if (optionalArgs) { - for (auto& arg: optionalArgs->first) { - items.push_back(makeQuote(arg)); - } - } - result = new TAstListNodeImpl(pos, items); - break; - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - + TMaybe<std::pair<TVector<TNodePtr>, bool>> requiredArgs, optionalArgs; + bool namedArgsStarted = false; + size_t optionalArgsCount = 0; + if (callableType.HasBlock4()) { + auto& argList = callableType.GetBlock4().GetRule_callable_arg_list1(); + requiredArgs = CallableArgList(argList, namedArgsStarted); + if (!requiredArgs) { + return {}; + } + namedArgsStarted = requiredArgs->second; + } + if (callableType.HasBlock6()) { + auto& argList = callableType.GetBlock6().GetRule_callable_arg_list2(); + optionalArgs = CallableArgList(argList, namedArgsStarted); + if (!optionalArgs) { + return {}; + } + optionalArgsCount = optionalArgs->first.size(); + } + auto returnType = TypeNodeOrBind(callableType.GetRule_type_name_or_bind9()); + if (!returnType) { + return {}; + } + TVector<TNodePtr> items; + items.push_back(BuildAtom(pos, "CallableType", flags)); + if (optionalArgsCount) { + items.push_back(makeQuote(new TAstListNodeImpl(pos, + { BuildQuotedAtom(pos, ToString(optionalArgsCount), flags) }))); + } else { + items.push_back(makeQuote(new TAstListNodeImpl(pos, {}))); + } + items.push_back(makeQuote(new TAstListNodeImpl(pos, { returnType }))); + if (requiredArgs) { + for (auto& arg: requiredArgs->first) { + items.push_back(makeQuote(arg)); + } + } + if (optionalArgs) { + for (auto& arg: optionalArgs->first) { + items.push_back(makeQuote(arg)); + } + } + result = new TAstListNodeImpl(pos, items); + break; + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + return AddOptionals(result, node.GetBlock2().size()); -} - +} + bool Expr(TSqlExpression& sqlExpr, TVector<TNodePtr>& exprNodes, const TRule_expr& node) { TNodePtr exprNode = sqlExpr.Build(node); if (!exprNode) { @@ -2835,18 +2835,18 @@ bool ExprList(TSqlExpression& sqlExpr, TVector<TNodePtr>& exprNodes, const TRule return true; } -TNodePtr TSqlTranslation::ValueConstructorLiteral(const TRule_value_constructor_literal& node) { +TNodePtr TSqlTranslation::ValueConstructorLiteral(const TRule_value_constructor_literal& node) { return BuildLiteralSmartString(Ctx, Token(node.GetToken1())); -} - -TNodePtr TSqlTranslation::ValueConstructor(const TRule_value_constructor& node) { - TSqlCallExpr call(Ctx, Mode); - if (!call.Init(node)) { - return {}; - } - return call.BuildCall(); -} - +} + +TNodePtr TSqlTranslation::ValueConstructor(const TRule_value_constructor& node) { + TSqlCallExpr call(Ctx, Mode); + if (!call.Init(node)) { + return {}; + } + return call.BuildCall(); +} + TNodePtr TSqlTranslation::ListLiteral(const TRule_list_literal& node) { TVector<TNodePtr> values; values.push_back(new TAstAtomNodeImpl(Ctx.Pos(), "AsList", TNodeFlags::Default)); @@ -3143,23 +3143,23 @@ bool TSqlTranslation::SimpleTableRefImpl(const TRule_simple_table_ref& node, TTa return true; } -bool TSqlCallExpr::Init(const TRule_value_constructor& node) { - switch (node.Alt_case()) { - case TRule_value_constructor::kAltValueConstructor1: { - auto& ctor = node.GetAlt_value_constructor1(); - Func = "Variant"; - TSqlExpression expr(Ctx, Mode); - if (!Expr(expr, Args, ctor.GetRule_expr3())) { - return false; - } - if (!Expr(expr, Args, ctor.GetRule_expr5())) { - return false; - } +bool TSqlCallExpr::Init(const TRule_value_constructor& node) { + switch (node.Alt_case()) { + case TRule_value_constructor::kAltValueConstructor1: { + auto& ctor = node.GetAlt_value_constructor1(); + Func = "Variant"; + TSqlExpression expr(Ctx, Mode); + if (!Expr(expr, Args, ctor.GetRule_expr3())) { + return false; + } + if (!Expr(expr, Args, ctor.GetRule_expr5())) { + return false; + } if (!Expr(expr, Args, ctor.GetRule_expr7())) { - return false; - } - break; - } + return false; + } + break; + } case TRule_value_constructor::kAltValueConstructor2: { auto& ctor = node.GetAlt_value_constructor2(); Func = "Enum"; @@ -3184,13 +3184,13 @@ bool TSqlCallExpr::Init(const TRule_value_constructor& node) { } break; } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - PositionalArgs = Args; - return true; -} - + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + PositionalArgs = Args; + return true; +} + bool TSqlCallExpr::ExtractCallParam(const TRule_external_call_param& node) { TString paramName = Id(node.GetRule_an_id1(), *this); paramName = to_lower(paramName); @@ -3264,7 +3264,7 @@ bool TSqlCallExpr::Init(const TRule_using_call_expr& node) { break; } default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); } YQL_ENSURE(!DistinctAllowed); UsingCallExpr = true; @@ -3581,7 +3581,7 @@ public: , CompactGroupBy(false) {} - bool Build(const TRule_group_by_clause& node, bool stream); + bool Build(const TRule_group_by_clause& node, bool stream); bool ParseList(const TRule_grouping_element_list& groupingListNode, EGroupByFeatures featureContext); void SetFeatures(const TString& field) const; @@ -3589,7 +3589,7 @@ public: TMap<TString, TNodePtr>& Aliases(); THoppingWindowSpecPtr GetHoppingWindow() const; bool IsCompactGroupBy() const; - + private: TMaybe<TVector<TNodePtr>> MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const; bool ResolveGroupByAndGrouping(); @@ -3597,7 +3597,7 @@ private: void FeedCollection(const TNodePtr& elem, TVector<TNodePtr>& collection, bool& hasEmpty) const; bool OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node, EGroupByFeatures featureContext); bool OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_list& node, EGroupByFeatures featureContext); - bool HoppingWindow(const TRule_hopping_window_specification& node); + bool HoppingWindow(const TRule_hopping_window_specification& node); bool AllowUnnamed(TPosition pos, EGroupByFeatures featureContext); @@ -3609,7 +3609,7 @@ private: TVector<TNodePtr> GroupBySet; TGroupByClauseCtx::TPtr GroupSetContext; - THoppingWindowSpecPtr HoppingWindowSpec; // stream queries + THoppingWindowSpecPtr HoppingWindowSpec; // stream queries static const TString AutogenerateNamePrefix; bool CompactGroupBy; }; @@ -4589,7 +4589,7 @@ TMaybe<TExprOrIdent> TSqlExpression::AtomExpr(const TRule_atom_expr& node, const // | exists_expr // | case_expr // | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) - // | value_constructor + // | value_constructor // | bitcast_expr // | list_literal // | dict_literal @@ -4647,7 +4647,7 @@ TMaybe<TExprOrIdent> TSqlExpression::AtomExpr(const TRule_atom_expr& node, const case TRule_atom_expr::kAltAtomExpr8: { result.Expr = ValueConstructor(node.GetAlt_atom_expr8().GetRule_value_constructor1()); break; - } + } case TRule_atom_expr::kAltAtomExpr9: result.Expr = BitCastRule(node.GetAlt_atom_expr9().GetRule_bitcast_expr1()); break; @@ -4678,7 +4678,7 @@ TMaybe<TExprOrIdent> TSqlExpression::InAtomExpr(const TRule_in_atom_expr& node, // | case_expr // | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) // | LPAREN select_stmt RPAREN - // | value_constructor + // | value_constructor // | bitcast_expr // | list_literal // | dict_literal @@ -4745,10 +4745,10 @@ TMaybe<TExprOrIdent> TSqlExpression::InAtomExpr(const TRule_in_atom_expr& node, result.Expr = BuildSelectResult(pos, std::move(source), false, Mode == NSQLTranslation::ESqlMode::SUBQUERY, Ctx.Scoped); break; } - case TRule_in_atom_expr::kAltInAtomExpr8: { + case TRule_in_atom_expr::kAltInAtomExpr8: { result.Expr = ValueConstructor(node.GetAlt_in_atom_expr8().GetRule_value_constructor1()); break; - } + } case TRule_in_atom_expr::kAltInAtomExpr9: result.Expr = BitCastRule(node.GetAlt_in_atom_expr9().GetRule_bitcast_expr1()); break; @@ -5238,91 +5238,91 @@ TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter be opName = "%"; Ctx.IncrementMonCounter("sql_binary_operations", "Mod"); break; - default: - Ctx.IncrementMonCounter("sql_errors", "UnsupportedBinaryOperation"); - Error() << "Unsupported binary operation token: " << tokenId; - return nullptr; - } - + default: + Ctx.IncrementMonCounter("sql_errors", "UnsupportedBinaryOperation"); + Error() << "Unsupported binary operation token: " << tokenId; + return nullptr; + } + partialResult = BuildBinaryOp(Ctx, pos, opName, partialResult, SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{})); - ++begin; - } - - return partialResult; -} - -template <typename TGetNode, typename TIter> + ++begin; + } + + return partialResult; +} + +template <typename TGetNode, typename TIter> TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) { TNodePtr partialResult = SubExpr(node, (begin == end) ? tail : TTrailingQuestions{}); - while (begin != end) { - Ctx.IncrementMonCounter("sql_features", "BinaryOperation"); - TString opName; - switch (begin->GetBlock1().Alt_case()) { - case TRule_neq_subexpr_TBlock2_TBlock1::kAlt1: { - Token(begin->GetBlock1().GetAlt1().GetToken1()); - auto tokenId = begin->GetBlock1().GetAlt1().GetToken1().GetId(); - if (tokenId != SQLv1LexerTokens::TOKEN_SHIFT_LEFT) { - Error() << "Unsupported binary operation token: " << tokenId; - return {}; - } + while (begin != end) { + Ctx.IncrementMonCounter("sql_features", "BinaryOperation"); + TString opName; + switch (begin->GetBlock1().Alt_case()) { + case TRule_neq_subexpr_TBlock2_TBlock1::kAlt1: { + Token(begin->GetBlock1().GetAlt1().GetToken1()); + auto tokenId = begin->GetBlock1().GetAlt1().GetToken1().GetId(); + if (tokenId != SQLv1LexerTokens::TOKEN_SHIFT_LEFT) { + Error() << "Unsupported binary operation token: " << tokenId; + return {}; + } opName = "ShiftLeft"; Ctx.IncrementMonCounter("sql_binary_operations", "ShiftLeft"); break; - } - case TRule_neq_subexpr_TBlock2_TBlock1::kAlt2: { + } + case TRule_neq_subexpr_TBlock2_TBlock1::kAlt2: { opName = "ShiftRight"; Ctx.IncrementMonCounter("sql_binary_operations", "ShiftRight"); break; - } - case TRule_neq_subexpr_TBlock2_TBlock1::kAlt3: { - Token(begin->GetBlock1().GetAlt3().GetToken1()); - auto tokenId = begin->GetBlock1().GetAlt3().GetToken1().GetId(); - if (tokenId != SQLv1LexerTokens::TOKEN_ROT_LEFT) { - Error() << "Unsupported binary operation token: " << tokenId; - return {}; - } + } + case TRule_neq_subexpr_TBlock2_TBlock1::kAlt3: { + Token(begin->GetBlock1().GetAlt3().GetToken1()); + auto tokenId = begin->GetBlock1().GetAlt3().GetToken1().GetId(); + if (tokenId != SQLv1LexerTokens::TOKEN_ROT_LEFT) { + Error() << "Unsupported binary operation token: " << tokenId; + return {}; + } opName = "RotLeft"; Ctx.IncrementMonCounter("sql_binary_operations", "RotLeft"); break; - } - case TRule_neq_subexpr_TBlock2_TBlock1::kAlt4: { + } + case TRule_neq_subexpr_TBlock2_TBlock1::kAlt4: { opName = "RotRight"; Ctx.IncrementMonCounter("sql_binary_operations", "RotRight"); break; - } - case TRule_neq_subexpr_TBlock2_TBlock1::kAlt5: { - Token(begin->GetBlock1().GetAlt5().GetToken1()); - auto tokenId = begin->GetBlock1().GetAlt5().GetToken1().GetId(); - if (tokenId != SQLv1LexerTokens::TOKEN_AMPERSAND) { - Error() << "Unsupported binary operation token: " << tokenId; - return {}; - } - opName = "BitAnd"; - Ctx.IncrementMonCounter("sql_binary_operations", "BitAnd"); - break; - } - case TRule_neq_subexpr_TBlock2_TBlock1::kAlt6: { - Token(begin->GetBlock1().GetAlt6().GetToken1()); - auto tokenId = begin->GetBlock1().GetAlt6().GetToken1().GetId(); - if (tokenId != SQLv1LexerTokens::TOKEN_PIPE) { - Error() << "Unsupported binary operation token: " << tokenId; - return {}; - } - opName = "BitOr"; - Ctx.IncrementMonCounter("sql_binary_operations", "BitOr"); - break; - } - case TRule_neq_subexpr_TBlock2_TBlock1::kAlt7: { - Token(begin->GetBlock1().GetAlt7().GetToken1()); - auto tokenId = begin->GetBlock1().GetAlt7().GetToken1().GetId(); - if (tokenId != SQLv1LexerTokens::TOKEN_CARET) { - Error() << "Unsupported binary operation token: " << tokenId; - return {}; - } - opName = "BitXor"; - Ctx.IncrementMonCounter("sql_binary_operations", "BitXor"); - break; - } + } + case TRule_neq_subexpr_TBlock2_TBlock1::kAlt5: { + Token(begin->GetBlock1().GetAlt5().GetToken1()); + auto tokenId = begin->GetBlock1().GetAlt5().GetToken1().GetId(); + if (tokenId != SQLv1LexerTokens::TOKEN_AMPERSAND) { + Error() << "Unsupported binary operation token: " << tokenId; + return {}; + } + opName = "BitAnd"; + Ctx.IncrementMonCounter("sql_binary_operations", "BitAnd"); + break; + } + case TRule_neq_subexpr_TBlock2_TBlock1::kAlt6: { + Token(begin->GetBlock1().GetAlt6().GetToken1()); + auto tokenId = begin->GetBlock1().GetAlt6().GetToken1().GetId(); + if (tokenId != SQLv1LexerTokens::TOKEN_PIPE) { + Error() << "Unsupported binary operation token: " << tokenId; + return {}; + } + opName = "BitOr"; + Ctx.IncrementMonCounter("sql_binary_operations", "BitOr"); + break; + } + case TRule_neq_subexpr_TBlock2_TBlock1::kAlt7: { + Token(begin->GetBlock1().GetAlt7().GetToken1()); + auto tokenId = begin->GetBlock1().GetAlt7().GetToken1().GetId(); + if (tokenId != SQLv1LexerTokens::TOKEN_CARET) { + Error() << "Unsupported binary operation token: " << tokenId; + return {}; + } + opName = "BitXor"; + Ctx.IncrementMonCounter("sql_binary_operations", "BitXor"); + break; + } default: Y_FAIL("You should change implementation according to grammar changes"); } @@ -6001,7 +6001,7 @@ TSourcePtr TSqlSelect::SingleSource(const TRule_single_source& node, const TVect TPosition pos(Ctx.Pos()); Ctx.IncrementMonCounter("sql_select_clusters", table.Cluster.GetLiteral() ? *table.Cluster.GetLiteral() : "unknown"); return BuildTableSource(pos, table); - } + } } case TRule_single_source::kAltSingleSource2: { const auto& alt = node.GetAlt_single_source2(); @@ -6561,7 +6561,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet selectPos = Ctx.Pos(); } - const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier4()); + const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier4()); if (distinct) { Ctx.IncrementMonCounter("sql_features", "DistinctInSelect"); } @@ -6582,7 +6582,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet if (!source) { return nullptr; } - + const bool selectStream = node.HasBlock3(); TVector<TNodePtr> without; if (node.HasBlock8()) { @@ -6613,7 +6613,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet /// \todo merge gtoupByExpr and groupBy in one TVector<TNodePtr> groupByExpr, groupBy; - THoppingWindowSpecPtr hoppingWindowSpec; + THoppingWindowSpecPtr hoppingWindowSpec; bool compactGroupBy = false; if (node.HasBlock11()) { TGroupByClause clause(Ctx, Mode); @@ -6626,7 +6626,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet } groupBy = std::move(clause.Content()); clause.SetFeatures("sql_features"); - hoppingWindowSpec = clause.GetHoppingWindow(); + hoppingWindowSpec = clause.GetHoppingWindow(); compactGroupBy = clause.IsCompactGroupBy(); } @@ -6644,12 +6644,12 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet TWinSpecs windowSpec; if (node.HasBlock13()) { if (source->IsStream()) { - Ctx.Error() << "WINDOW is not allowed in streaming queries"; + Ctx.Error() << "WINDOW is not allowed in streaming queries"; return nullptr; } if (!WindowClause(node.GetBlock13().GetRule_window_clause1(), windowSpec)) { - return nullptr; - } + return nullptr; + } Ctx.IncrementMonCounter("sql_features", "WindowClause"); } @@ -6662,7 +6662,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet Token(orderBlock.GetRule_order_by_clause2().GetToken1()); if (source->IsStream()) { - Ctx.Error() << "ORDER BY is not allowed in streaming queries"; + Ctx.Error() << "ORDER BY is not allowed in streaming queries"; return nullptr; } @@ -6671,8 +6671,8 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet } if (!OrderByClause(orderBlock.GetRule_order_by_clause2(), orderBy)) { - return nullptr; - } + return nullptr; + } Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(orderBy) ? (assumeSorted ? "AssumeOrderBy" : "OrderBy") : (assumeSorted ? "AssumeOrderByExpr" : "OrderByExpr") @@ -7030,7 +7030,7 @@ bool TSqlTranslation::OrderByClause(const TRule_order_by_clause& node, TVector<T return SortSpecificationList(node.GetRule_sort_specification_list3(), orderBy); } -bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) { +bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) { CompactGroupBy = node.HasBlock2(); TPosition distinctPos; if (IsDistinctOptSet(node.GetRule_opt_set_quantifier4(), distinctPos)) { @@ -7049,7 +7049,7 @@ bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) { Ctx.Error() << "Streaming group by query must have a hopping window specification."; return false; } - } + } return true; } @@ -7097,9 +7097,9 @@ TMap<TString, TNodePtr>& TGroupByClause::Aliases() { } THoppingWindowSpecPtr TGroupByClause::GetHoppingWindow() const { - return HoppingWindowSpec; -} - + return HoppingWindowSpec; +} + bool TGroupByClause::IsCompactGroupBy() const { return CompactGroupBy; } @@ -7253,12 +7253,12 @@ bool TGroupByClause::GroupingElement(const TRule_grouping_element& node, EGroupB Features().Set(EGroupByFeatures::GroupingSet); break; } - case TRule_grouping_element::kAltGroupingElement5: { - if (!HoppingWindow(node.GetAlt_grouping_element5().GetRule_hopping_window_specification1())) { - return false; - } - break; - } + case TRule_grouping_element::kAltGroupingElement5: { + if (!HoppingWindow(node.GetAlt_grouping_element5().GetRule_hopping_window_specification1())) { + return false; + } + break; + } default: Y_FAIL("You should change implementation according to grammar changes"); } @@ -7346,72 +7346,72 @@ bool TGroupByClause::OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_l return true; } -bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& node) { - if (HoppingWindowSpec) { - Ctx.Error() << "Duplicate hopping window specification."; - return false; - } - HoppingWindowSpec = new THoppingWindowSpec; - { +bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& node) { + if (HoppingWindowSpec) { + Ctx.Error() << "Duplicate hopping window specification."; + return false; + } + HoppingWindowSpec = new THoppingWindowSpec; + { TColumnRefScope scope(Ctx, EColumnRefState::Allow); - TSqlExpression expr(Ctx, Mode); - HoppingWindowSpec->TimeExtractor = expr.Build(node.GetRule_expr3()); - if (!HoppingWindowSpec->TimeExtractor) { - return false; - } - } - auto processIntervalParam = [&] (const TRule_expr& rule) -> TNodePtr { - TSqlExpression expr(Ctx, Mode); - auto node = expr.Build(rule); - if (!node) { - return nullptr; - } - - auto literal = node->GetLiteral("String"); - if (!literal) { - return new TAstListNodeImpl(Ctx.Pos(), { - new TAstAtomNodeImpl(Ctx.Pos(), "EvaluateExpr", TNodeFlags::Default), - node - }); - } - + TSqlExpression expr(Ctx, Mode); + HoppingWindowSpec->TimeExtractor = expr.Build(node.GetRule_expr3()); + if (!HoppingWindowSpec->TimeExtractor) { + return false; + } + } + auto processIntervalParam = [&] (const TRule_expr& rule) -> TNodePtr { + TSqlExpression expr(Ctx, Mode); + auto node = expr.Build(rule); + if (!node) { + return nullptr; + } + + auto literal = node->GetLiteral("String"); + if (!literal) { + return new TAstListNodeImpl(Ctx.Pos(), { + new TAstAtomNodeImpl(Ctx.Pos(), "EvaluateExpr", TNodeFlags::Default), + node + }); + } + const auto out = NKikimr::NMiniKQL::ValueFromString(NKikimr::NUdf::EDataSlot::Interval, *literal); if (!out) { - Ctx.Error(node->GetPos()) << "Expected interval in ISO 8601 format"; - return nullptr; - } - + Ctx.Error(node->GetPos()) << "Expected interval in ISO 8601 format"; + return nullptr; + } + if ('T' == literal->back()) { Ctx.Error(node->GetPos()) << "Time prefix 'T' at end of interval constant. The designator 'T' shall be absent if all of the time components are absent."; return nullptr; } - return new TAstListNodeImpl(Ctx.Pos(), { - new TAstAtomNodeImpl(Ctx.Pos(), "Interval", TNodeFlags::Default), - new TAstListNodeImpl(Ctx.Pos(), { - new TAstAtomNodeImpl(Ctx.Pos(), "quote", TNodeFlags::Default), + return new TAstListNodeImpl(Ctx.Pos(), { + new TAstAtomNodeImpl(Ctx.Pos(), "Interval", TNodeFlags::Default), + new TAstListNodeImpl(Ctx.Pos(), { + new TAstAtomNodeImpl(Ctx.Pos(), "quote", TNodeFlags::Default), new TAstAtomNodeImpl(Ctx.Pos(), ToString(out.Get<i64>()), TNodeFlags::Default) - }) - }); - }; - - HoppingWindowSpec->Hop = processIntervalParam(node.GetRule_expr5()); - if (!HoppingWindowSpec->Hop) { - return false; - } - HoppingWindowSpec->Interval = processIntervalParam(node.GetRule_expr7()); - if (!HoppingWindowSpec->Interval) { - return false; - } - HoppingWindowSpec->Delay = processIntervalParam(node.GetRule_expr9()); - if (!HoppingWindowSpec->Delay) { - return false; - } + }) + }); + }; + + HoppingWindowSpec->Hop = processIntervalParam(node.GetRule_expr5()); + if (!HoppingWindowSpec->Hop) { + return false; + } + HoppingWindowSpec->Interval = processIntervalParam(node.GetRule_expr7()); + if (!HoppingWindowSpec->Interval) { + return false; + } + HoppingWindowSpec->Delay = processIntervalParam(node.GetRule_expr9()); + if (!HoppingWindowSpec->Delay) { + return false; + } HoppingWindowSpec->DataWatermarks = Ctx.PragmaDataWatermarks; - - return true; -} - + + return true; +} + bool TGroupByClause::AllowUnnamed(TPosition pos, EGroupByFeatures featureContext) { TStringBuf feature; switch (featureContext) { @@ -8015,7 +8015,7 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { break; } default: - Y_FAIL("You should change implementation according to grammar changes"); + Y_FAIL("You should change implementation according to grammar changes"); } } @@ -8090,7 +8090,7 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const auto serviceName = table.Service; const bool isMapReduce = serviceName == YtProviderName; const bool isKikimr = serviceName == KikimrProviderName || serviceName == YdbProviderName; - const bool isRtmr = serviceName == RtmrProviderName; + const bool isRtmr = serviceName == RtmrProviderName; const bool isStat = serviceName == StatProviderName; if (!isKikimr) { @@ -8117,12 +8117,12 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); return false; } - } else if (isRtmr) { - if (mode != ESQLWriteColumnMode::InsertInto) { - Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; - Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); - return false; - } + } else if (isRtmr) { + if (mode != ESQLWriteColumnMode::InsertInto) { + Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; + Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); + return false; + } } else if (isStat) { if (mode != ESQLWriteColumnMode::UpsertInto) { Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; @@ -8382,7 +8382,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } TCreateTableParameters params; - + if (!CreateTableEntry(rule.GetRule_create_table_entry5(), params)) { return false; } @@ -8795,7 +8795,7 @@ bool TSqlQuery::DeclareStatement(const TRule_declare_stmt& stmt) { return false; } const auto varPos = Ctx.Pos(); - const auto typeNode = TypeNode(stmt.GetRule_type_name4()); + const auto typeNode = TypeNode(stmt.GetRule_type_name4()); if (!typeNode) { return false; } @@ -8958,9 +8958,9 @@ bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTab bool TSqlQuery::AlterTableAddColumn(const TRule_alter_table_add_column& node, TAlterTableParameters& params) { auto columnSchema = ColumnSchemaImpl(node.GetRule_column_schema3()); - if (!columnSchema) { - return false; - } + if (!columnSchema) { + return false; + } if (columnSchema->Families.size() > 1) { Ctx.Error() << "Several column families for a single column are not yet supported"; return false; @@ -8968,7 +8968,7 @@ bool TSqlQuery::AlterTableAddColumn(const TRule_alter_table_add_column& node, TA params.AddColumns.push_back(*columnSchema); return true; } - + bool TSqlQuery::AlterTableDropColumn(const TRule_alter_table_drop_column& node, TAlterTableParameters& params) { TString name = Id(node.GetRule_an_id3(), *this); params.DropColumns.push_back(name); @@ -8991,8 +8991,8 @@ bool TSqlQuery::AlterTableAddFamily(const TRule_family_entry& node, TAlterTableP TFamilyEntry family(IdEx(node.GetRule_an_id2(), *this)); if (node.HasBlock3()) { if (!FillFamilySettings(node.GetBlock3().GetRule_family_settings1(), family)) { - return false; - } + return false; + } } params.AddColumnFamilies.push_back(family); return true; @@ -9133,7 +9133,7 @@ void TSqlQuery::AlterTableDropChangefeed(const TRule_alter_table_drop_changefeed TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success) { success = false; - const TString& prefix = OptIdPrefixAsStr(stmt.GetRule_opt_id_prefix_or_type2(), *this); + const TString& prefix = OptIdPrefixAsStr(stmt.GetRule_opt_id_prefix_or_type2(), *this); const TString& lowerPrefix = to_lower(prefix); const TString pragma(Id(stmt.GetRule_an_id3(), *this)); TString normalizedPragma(pragma); @@ -9683,14 +9683,14 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) { TTableRef table; if (!SimpleTableRefImpl(stmt.GetRule_simple_table_ref3(), table)) { - return nullptr; - } + return nullptr; + } const bool isKikimr = table.Service == KikimrProviderName; if (!isKikimr) { Ctx.Error(GetPos(stmt.GetToken1())) << "DELETE is unsupported for " << table.Service; - return nullptr; - } + return nullptr; + } TSourcePtr source = BuildTableSource(Ctx.Pos(), table); @@ -9731,15 +9731,15 @@ TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) { TNodePtr TSqlQuery::Build(const TRule_update_stmt& stmt) { TTableRef table; if (!SimpleTableRefImpl(stmt.GetRule_simple_table_ref2(), table)) { - return nullptr; - } + return nullptr; + } const bool isKikimr = table.Service == KikimrProviderName; if (!isKikimr) { Ctx.Error(GetPos(stmt.GetToken1())) << "UPDATE is unsupported for " << table.Service; - return nullptr; - } + return nullptr; + } switch (stmt.GetBlock3().Alt_case()) { case TRule_update_stmt_TBlock3::kAlt1: { diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 553f92c7ac..85b6e0dfcd 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -860,7 +860,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { VerifyProgram(res, elementStat, verifyLine); UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]); } - + Y_UNIT_TEST(ProcessUserTypeAuth) { NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData(TableRows(), AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName)); UNIT_ASSERT(res.Root); @@ -879,17 +879,17 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { } Y_UNIT_TEST(SelectStreamRtmr) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); + NYql::TAstParseResult res = SqlToYql( + "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", + 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); res = SqlToYql( "USE plato; INSERT INTO Output SELECT key FROM Input;", 10, TString(NYql::RtmrProviderName)); UNIT_ASSERT(res.Root); - } - + } + Y_UNIT_TEST(SelectStreamRtmrJoinWithYt) { NYql::TAstParseResult res = SqlToYql( "USE plato; INSERT INTO Output SELECT STREAM key FROM Input LEFT JOIN hahn.ttt as t ON Input.key = t.Name;", @@ -906,12 +906,12 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { } Y_UNIT_TEST(GroupByHopRtmr) { - NYql::TAstParseResult res = SqlToYql(R"( + NYql::TAstParseResult res = SqlToYql(R"( USE plato; INSERT INTO Output SELECT key, SUM(value) AS value FROM Input - GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"); - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } + GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"); + )", 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } Y_UNIT_TEST(GroupByHopRtmrSubquery) { // 'use plato' intentially avoided diff --git a/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp b/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp index eef6927185..763fea8024 100644 --- a/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp +++ b/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp @@ -1,225 +1,225 @@ #include <ydb/library/yql/public/udf/udf_helpers.h> #include <ydb/library/yql/public/udf/tz/udf_tz.h> #include <ydb/library/yql/minikql/mkql_type_ops.h> - -#include <util/datetime/base.h> - -using namespace NKikimr; -using namespace NUdf; - -extern const char SplitName[] = "Split"; -extern const char ToSecondsName[] = "ToSeconds"; -extern const char ToMillisecondsName[] = "ToMilliseconds"; -extern const char ToMicrosecondsName[] = "ToMicroseconds"; - -extern const char TMResourceName[] = "DateTime2.TM"; - -namespace { - - constexpr size_t MAX_TIMEZONE_NAME_LEN = 64; - - struct TTMStorage { - unsigned int Year : 12; - unsigned int DayOfYear : 9; - unsigned int WeekOfYear : 6; + +#include <util/datetime/base.h> + +using namespace NKikimr; +using namespace NUdf; + +extern const char SplitName[] = "Split"; +extern const char ToSecondsName[] = "ToSeconds"; +extern const char ToMillisecondsName[] = "ToMilliseconds"; +extern const char ToMicrosecondsName[] = "ToMicroseconds"; + +extern const char TMResourceName[] = "DateTime2.TM"; + +namespace { + + constexpr size_t MAX_TIMEZONE_NAME_LEN = 64; + + struct TTMStorage { + unsigned int Year : 12; + unsigned int DayOfYear : 9; + unsigned int WeekOfYear : 6; unsigned int WeekOfYearIso8601 : 6; - unsigned int DayOfWeek : 3; - unsigned int Month : 4; - unsigned int Day : 5; - unsigned int Hour : 5; - unsigned int Minute : 6; - unsigned int Second : 6; - unsigned int Microsecond : 20; - unsigned int TimezoneId : 16; - - inline static bool IsUniversal(ui16 timezoneId) { - return timezoneId == 0; - } - - inline void MakeDefault() { - Year = 1970; - Month = 1; - Day = 1; - Hour = 0; - Minute = 0; - Second = 0; - Microsecond = 0; - TimezoneId = 0; - } - - inline void FromDate(const IDateBuilder& builder, ui16 value, ui16 timezoneId = 0) { + unsigned int DayOfWeek : 3; + unsigned int Month : 4; + unsigned int Day : 5; + unsigned int Hour : 5; + unsigned int Minute : 6; + unsigned int Second : 6; + unsigned int Microsecond : 20; + unsigned int TimezoneId : 16; + + inline static bool IsUniversal(ui16 timezoneId) { + return timezoneId == 0; + } + + inline void MakeDefault() { + Year = 1970; + Month = 1; + Day = 1; + Hour = 0; + Minute = 0; + Second = 0; + Microsecond = 0; + TimezoneId = 0; + } + + inline void FromDate(const IDateBuilder& builder, ui16 value, ui16 timezoneId = 0) { ui32 year, month, day, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek; - + if (!builder.FullSplitDate2(value, year, month, day, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, timezoneId)) { ythrow yexception() << "Error in FullSplitDate"; - } - + } + TimezoneId = timezoneId; - Year = year; - Month = month; - Day = day; - - DayOfYear = dayOfYear; - WeekOfYear = weekOfYear; + Year = year; + Month = month; + Day = day; + + DayOfYear = dayOfYear; + WeekOfYear = weekOfYear; WeekOfYearIso8601 = weekOfYearIso8601; - DayOfWeek = dayOfWeek; - } - + DayOfWeek = dayOfWeek; + } + inline ui16 ToDate(const IDateBuilder& builder, bool local) const { - if (!IsUniversal(TimezoneId)) { - ui32 datetime; + if (!IsUniversal(TimezoneId)) { + ui32 datetime; if (!builder.MakeDatetime(Year, Month, Day, local ? 0 : Hour, local ? 0 : Minute, local ? 0 : Second, datetime, TimezoneId)) { ythrow yexception() << "Error in MakeDatetime"; } - return datetime / 86400u; - } else { - ui16 date; + return datetime / 86400u; + } else { + ui16 date; if (!builder.MakeDate(Year, Month, Day, date)) { ythrow yexception() << "Error in MakeDate"; } - return date; - } - } - - inline void FromDatetime(const IDateBuilder& builder, ui32 value, ui16 timezoneId = 0) { + return date; + } + } + + inline void FromDatetime(const IDateBuilder& builder, ui32 value, ui16 timezoneId = 0) { ui32 year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek; - + if (!builder.FullSplitDatetime2(value, year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, timezoneId)) { ythrow yexception() << "Error in FullSplitDatetime"; } - + TimezoneId = timezoneId; - Year = year; - Month = month; - Day = day; - Hour = hour; - Minute = minute; - Second = second; - - DayOfYear = dayOfYear; - WeekOfYear = weekOfYear; + Year = year; + Month = month; + Day = day; + Hour = hour; + Minute = minute; + Second = second; + + DayOfYear = dayOfYear; + WeekOfYear = weekOfYear; WeekOfYearIso8601 = weekOfYearIso8601; - DayOfWeek = dayOfWeek; - } - - inline ui32 ToDatetime(const IDateBuilder& builder) const { + DayOfWeek = dayOfWeek; + } + + inline ui32 ToDatetime(const IDateBuilder& builder) const { ui32 datetime = 0; if (!builder.MakeDatetime(Year, Month, Day, Hour, Minute, Second, datetime, TimezoneId)) { ythrow yexception() << "Error in MakeDatetime"; } - return datetime; - } - - inline void FromTimestamp(const IDateBuilder& builder, ui64 value, ui16 timezoneId = 0) { + return datetime; + } + + inline void FromTimestamp(const IDateBuilder& builder, ui64 value, ui16 timezoneId = 0) { const ui32 seconds = value / 1000000ull; - FromDatetime(builder, seconds, timezoneId); - Microsecond = value - seconds * 1000000ull; - } - - inline ui64 ToTimestamp(const IDateBuilder& builder) const { - return ToDatetime(builder) * 1000000ull + Microsecond; - } - - inline bool Validate(const IDateBuilder& builder) { - ui32 datetime; - if (!builder.MakeDatetime(Year, Month, Day, Hour, Minute, Second, datetime, TimezoneId)) { - return false; - } - + FromDatetime(builder, seconds, timezoneId); + Microsecond = value - seconds * 1000000ull; + } + + inline ui64 ToTimestamp(const IDateBuilder& builder) const { + return ToDatetime(builder) * 1000000ull + Microsecond; + } + + inline bool Validate(const IDateBuilder& builder) { + ui32 datetime; + if (!builder.MakeDatetime(Year, Month, Day, Hour, Minute, Second, datetime, TimezoneId)) { + return false; + } + ui32 year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek; if (!builder.FullSplitDatetime2(datetime, year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, TimezoneId)) { ythrow yexception() << "Error in FullSplitDatetime."; } - - DayOfYear = dayOfYear; - WeekOfYear = weekOfYear; + + DayOfYear = dayOfYear; + WeekOfYear = weekOfYear; WeekOfYearIso8601 = weekOfYearIso8601; - DayOfWeek = dayOfWeek; - - return true; - } - - inline void FromTimeOfDay(ui64 value) { - Hour = value / 3600000000ull; - value -= Hour * 3600000000ull; - Minute = value / 60000000ull; - value -= Minute * 60000000ull; - Second = value / 1000000ull; - Microsecond = value - Second * 1000000ull; - } - - inline ui64 ToTimeOfDay() const { - return ((Hour * 60ull + Minute) * 60ull + Second) * 1000000ull + Microsecond; - } - - static const TTMStorage& Reference(const TUnboxedValuePod& value) { - return *reinterpret_cast<const TTMStorage*>(value.GetRawPtr()); - } - - static TTMStorage& Reference(TUnboxedValuePod& value) { - return *reinterpret_cast<TTMStorage*>(value.GetRawPtr()); - } - }; - - static_assert(sizeof(TTMStorage) == 16, "TTMStorage size must be equal to TUnboxedValuePod size"); - -#define ACCESSORS(field, type) \ - inline type Get##field(const TUnboxedValuePod& tm) { \ - return (type)TTMStorage::Reference(tm).field; \ - } \ - Y_DECLARE_UNUSED inline void Set##field(TUnboxedValuePod& tm, type value) { \ - TTMStorage::Reference(tm).field = value; \ - } - - ACCESSORS(Year, ui16) - ACCESSORS(DayOfYear, ui16) - ACCESSORS(WeekOfYear, ui8) + DayOfWeek = dayOfWeek; + + return true; + } + + inline void FromTimeOfDay(ui64 value) { + Hour = value / 3600000000ull; + value -= Hour * 3600000000ull; + Minute = value / 60000000ull; + value -= Minute * 60000000ull; + Second = value / 1000000ull; + Microsecond = value - Second * 1000000ull; + } + + inline ui64 ToTimeOfDay() const { + return ((Hour * 60ull + Minute) * 60ull + Second) * 1000000ull + Microsecond; + } + + static const TTMStorage& Reference(const TUnboxedValuePod& value) { + return *reinterpret_cast<const TTMStorage*>(value.GetRawPtr()); + } + + static TTMStorage& Reference(TUnboxedValuePod& value) { + return *reinterpret_cast<TTMStorage*>(value.GetRawPtr()); + } + }; + + static_assert(sizeof(TTMStorage) == 16, "TTMStorage size must be equal to TUnboxedValuePod size"); + +#define ACCESSORS(field, type) \ + inline type Get##field(const TUnboxedValuePod& tm) { \ + return (type)TTMStorage::Reference(tm).field; \ + } \ + Y_DECLARE_UNUSED inline void Set##field(TUnboxedValuePod& tm, type value) { \ + TTMStorage::Reference(tm).field = value; \ + } + + ACCESSORS(Year, ui16) + ACCESSORS(DayOfYear, ui16) + ACCESSORS(WeekOfYear, ui8) ACCESSORS(WeekOfYearIso8601, ui8) - ACCESSORS(DayOfWeek, ui8) - ACCESSORS(Month, ui8) - ACCESSORS(Day, ui8) - ACCESSORS(Hour, ui8) - ACCESSORS(Minute, ui8) - ACCESSORS(Second, ui8) - ACCESSORS(Microsecond, ui32) - ACCESSORS(TimezoneId, ui16) - -#undef ACCESSORS - - inline bool ValidateYear(ui16 year) { - return year >= NUdf::MIN_YEAR - 1 || year <= NUdf::MAX_YEAR + 1; - } - - inline bool ValidateMonth(ui8 month) { - return month >= 1 && month <= 12; - } - - inline bool ValidateDay(ui8 day) { - return day >= 1 && day <= 31; - } - - inline bool ValidateHour(ui8 hour) { - return hour < 24; - } - - inline bool ValidateMinute(ui8 minute) { - return minute < 60; - } - - inline bool ValidateSecond(ui8 second) { - return second < 60; - } - - inline bool ValidateMicrosecond(ui32 microsecond) { - return microsecond < 1000000; - } - - inline bool ValidateTimezoneId(ui16 timezoneId) { + ACCESSORS(DayOfWeek, ui8) + ACCESSORS(Month, ui8) + ACCESSORS(Day, ui8) + ACCESSORS(Hour, ui8) + ACCESSORS(Minute, ui8) + ACCESSORS(Second, ui8) + ACCESSORS(Microsecond, ui32) + ACCESSORS(TimezoneId, ui16) + +#undef ACCESSORS + + inline bool ValidateYear(ui16 year) { + return year >= NUdf::MIN_YEAR - 1 || year <= NUdf::MAX_YEAR + 1; + } + + inline bool ValidateMonth(ui8 month) { + return month >= 1 && month <= 12; + } + + inline bool ValidateDay(ui8 day) { + return day >= 1 && day <= 31; + } + + inline bool ValidateHour(ui8 hour) { + return hour < 24; + } + + inline bool ValidateMinute(ui8 minute) { + return minute < 60; + } + + inline bool ValidateSecond(ui8 second) { + return second < 60; + } + + inline bool ValidateMicrosecond(ui32 microsecond) { + return microsecond < 1000000; + } + + inline bool ValidateTimezoneId(ui16 timezoneId) { const auto& zones = NUdf::GetTimezones(); return timezoneId < zones.size() && !zones[timezoneId].empty(); - } - + } + inline bool ValidateMonthShortName(const std::string_view& monthName, ui8& month) { static constexpr auto cmp = [](const std::string_view& a, const std::string_view& b) { int cmp = strnicmp(a.data(), b.data(), std::min(a.size(), b.size())); @@ -290,44 +290,44 @@ namespace { return interval > -i64(MAX_TIMESTAMP) && interval < i64(MAX_TIMESTAMP); } - // Split - - template <typename TUserDataType> - class TSplit : public TBoxedValue { + // Split + + template <typename TUserDataType> + class TSplit : public TBoxedValue { const TSourcePosition Pos_; - public: + public: explicit TSplit(TSourcePosition pos) : Pos_(pos) {} - TUnboxedValue Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const override; - - static void DeclareSignature( - TType* userType, - IFunctionTypeInfoBuilder& builder, - bool typesOnly) - { - builder.UserType(userType); + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override; + + static void DeclareSignature( + TType* userType, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) + { + builder.UserType(userType); builder.Args()->Add<TUserDataType>().Flags(ICallablePayload::TArgumentFlags::AutoMap); - builder.Returns(builder.Resource(TMResourceName)); - - if (!typesOnly) { + builder.Returns(builder.Resource(TMResourceName)); + + if (!typesOnly) { builder.Implementation(new TSplit<TUserDataType>(builder.GetSourcePosition())); - } - } - }; - - template <> - TUnboxedValue TSplit<TDate>::Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const - { + } + } + }; + + template <> + TUnboxedValue TSplit<TDate>::Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const + { try { EMPTY_RESULT_ON_EMPTY_ARG(0); - + auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); auto& storage = TTMStorage::Reference(result); @@ -336,16 +336,16 @@ namespace { } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } - } - - template <> - TUnboxedValue TSplit<TDatetime>::Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const - { + } + + template <> + TUnboxedValue TSplit<TDatetime>::Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const + { try { EMPTY_RESULT_ON_EMPTY_ARG(0); - + auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); auto& storage = TTMStorage::Reference(result); @@ -354,16 +354,16 @@ namespace { } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } - } - - template <> - TUnboxedValue TSplit<TTimestamp>::Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const - { + } + + template <> + TUnboxedValue TSplit<TTimestamp>::Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const + { try { EMPTY_RESULT_ON_EMPTY_ARG(0); - + auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); auto& storage = TTMStorage::Reference(result); @@ -372,16 +372,16 @@ namespace { } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } - } - - template <> - TUnboxedValue TSplit<TTzDate>::Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const - { + } + + template <> + TUnboxedValue TSplit<TTzDate>::Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const + { try { EMPTY_RESULT_ON_EMPTY_ARG(0); - + auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); auto& storage = TTMStorage::Reference(result); @@ -390,16 +390,16 @@ namespace { } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } - } - - template <> - TUnboxedValue TSplit<TTzDatetime>::Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const - { + } + + template <> + TUnboxedValue TSplit<TTzDatetime>::Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const + { try { EMPTY_RESULT_ON_EMPTY_ARG(0); - + auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); auto& storage = TTMStorage::Reference(result); @@ -408,16 +408,16 @@ namespace { } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } - } - - template <> - TUnboxedValue TSplit<TTzTimestamp>::Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const - { + } + + template <> + TUnboxedValue TSplit<TTzTimestamp>::Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const + { try { EMPTY_RESULT_ON_EMPTY_ARG(0); - + auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); auto& storage = TTMStorage::Reference(result); @@ -426,65 +426,65 @@ namespace { } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } - } - - // Make* - - SIMPLE_UDF(TMakeDate, TDate(TAutoMap<TResource<TMResourceName>>)) { - auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = TTMStorage::Reference(args[0]); + } + + // Make* + + SIMPLE_UDF(TMakeDate, TDate(TAutoMap<TResource<TMResourceName>>)) { + auto& builder = valueBuilder->GetDateBuilder(); + auto& storage = TTMStorage::Reference(args[0]); return TUnboxedValuePod(storage.ToDate(builder, false)); - } - - SIMPLE_UDF(TMakeDatetime, TDatetime(TAutoMap<TResource<TMResourceName>>)) { - auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = TTMStorage::Reference(args[0]); - return TUnboxedValuePod(storage.ToDatetime(builder)); - } - - SIMPLE_UDF(TMakeTimestamp, TTimestamp(TAutoMap<TResource<TMResourceName>>)) { - auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = TTMStorage::Reference(args[0]); - return TUnboxedValuePod(storage.ToTimestamp(builder)); - } - - SIMPLE_UDF(TMakeTzDate, TTzDate(TAutoMap<TResource<TMResourceName>>)) { - auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = TTMStorage::Reference(args[0]); + } + + SIMPLE_UDF(TMakeDatetime, TDatetime(TAutoMap<TResource<TMResourceName>>)) { + auto& builder = valueBuilder->GetDateBuilder(); + auto& storage = TTMStorage::Reference(args[0]); + return TUnboxedValuePod(storage.ToDatetime(builder)); + } + + SIMPLE_UDF(TMakeTimestamp, TTimestamp(TAutoMap<TResource<TMResourceName>>)) { + auto& builder = valueBuilder->GetDateBuilder(); + auto& storage = TTMStorage::Reference(args[0]); + return TUnboxedValuePod(storage.ToTimestamp(builder)); + } + + SIMPLE_UDF(TMakeTzDate, TTzDate(TAutoMap<TResource<TMResourceName>>)) { + auto& builder = valueBuilder->GetDateBuilder(); + auto& storage = TTMStorage::Reference(args[0]); TUnboxedValuePod result(storage.ToDate(builder, true)); - result.SetTimezoneId(storage.TimezoneId); - return result; - } - - SIMPLE_UDF(TMakeTzDatetime, TTzDatetime(TAutoMap<TResource<TMResourceName>>)) { - auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = TTMStorage::Reference(args[0]); - TUnboxedValuePod result(storage.ToDatetime(builder)); - result.SetTimezoneId(storage.TimezoneId); - return result; - } - - SIMPLE_UDF(TMakeTzTimestamp, TTzTimestamp(TAutoMap<TResource<TMResourceName>>)) { - auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = TTMStorage::Reference(args[0]); - TUnboxedValuePod result(storage.ToTimestamp(builder)); - result.SetTimezoneId(storage.TimezoneId); - return result; - } - - // Get* - -#define GET_METHOD(field, type) \ - SIMPLE_UDF(TGet##field, type(TAutoMap<TResource<TMResourceName>>)) { \ - Y_UNUSED(valueBuilder); \ - return TUnboxedValuePod(Get##field(args[0])); \ - } - - GET_METHOD(Year, ui16) - GET_METHOD(DayOfYear, ui16) - GET_METHOD(Month, ui8) - - SIMPLE_UDF(TGetMonthName, char*(TAutoMap<TResource<TMResourceName>>)) { + result.SetTimezoneId(storage.TimezoneId); + return result; + } + + SIMPLE_UDF(TMakeTzDatetime, TTzDatetime(TAutoMap<TResource<TMResourceName>>)) { + auto& builder = valueBuilder->GetDateBuilder(); + auto& storage = TTMStorage::Reference(args[0]); + TUnboxedValuePod result(storage.ToDatetime(builder)); + result.SetTimezoneId(storage.TimezoneId); + return result; + } + + SIMPLE_UDF(TMakeTzTimestamp, TTzTimestamp(TAutoMap<TResource<TMResourceName>>)) { + auto& builder = valueBuilder->GetDateBuilder(); + auto& storage = TTMStorage::Reference(args[0]); + TUnboxedValuePod result(storage.ToTimestamp(builder)); + result.SetTimezoneId(storage.TimezoneId); + return result; + } + + // Get* + +#define GET_METHOD(field, type) \ + SIMPLE_UDF(TGet##field, type(TAutoMap<TResource<TMResourceName>>)) { \ + Y_UNUSED(valueBuilder); \ + return TUnboxedValuePod(Get##field(args[0])); \ + } + + GET_METHOD(Year, ui16) + GET_METHOD(DayOfYear, ui16) + GET_METHOD(Month, ui8) + + SIMPLE_UDF(TGetMonthName, char*(TAutoMap<TResource<TMResourceName>>)) { Y_UNUSED(valueBuilder); static const std::array<TUnboxedValue, 12U> monthNames = {{ TUnboxedValuePod::Embedded(TStringRef::Of("January")), @@ -501,19 +501,19 @@ namespace { TUnboxedValuePod::Embedded(TStringRef::Of("December")) }}; return monthNames.at(GetMonth(*args) - 1U); - } - - GET_METHOD(WeekOfYear, ui8) + } + + GET_METHOD(WeekOfYear, ui8) GET_METHOD(WeekOfYearIso8601, ui8) - - SIMPLE_UDF(TGetDayOfMonth, ui8(TAutoMap<TResource<TMResourceName>>)) { - Y_UNUSED(valueBuilder); - return TUnboxedValuePod(GetDay(args[0])); - } - - GET_METHOD(DayOfWeek, ui8) - - SIMPLE_UDF(TGetDayOfWeekName, char*(TAutoMap<TResource<TMResourceName>>)) { + + SIMPLE_UDF(TGetDayOfMonth, ui8(TAutoMap<TResource<TMResourceName>>)) { + Y_UNUSED(valueBuilder); + return TUnboxedValuePod(GetDay(args[0])); + } + + GET_METHOD(DayOfWeek, ui8) + + SIMPLE_UDF(TGetDayOfWeekName, char*(TAutoMap<TResource<TMResourceName>>)) { Y_UNUSED(valueBuilder); static const std::array<TUnboxedValue, 7U> dayNames = {{ TUnboxedValuePod::Embedded(TStringRef::Of("Monday")), @@ -525,98 +525,98 @@ namespace { TUnboxedValuePod::Embedded(TStringRef::Of("Sunday")) }}; return dayNames.at(GetDayOfWeek(*args) - 1U); - } - - GET_METHOD(Hour, ui8) - GET_METHOD(Minute, ui8) - GET_METHOD(Second, ui8) - - SIMPLE_UDF(TGetMillisecondOfSecond, ui32(TAutoMap<TResource<TMResourceName>>)) { - Y_UNUSED(valueBuilder); - return TUnboxedValuePod(GetMicrosecond(args[0]) / 1000u); - } - - SIMPLE_UDF(TGetMicrosecondOfSecond, ui32(TAutoMap<TResource<TMResourceName>>)) { - Y_UNUSED(valueBuilder); - return TUnboxedValuePod(GetMicrosecond(args[0])); - } - - GET_METHOD(TimezoneId, ui16) - - SIMPLE_UDF(TGetTimezoneName, char*(TAutoMap<TResource<TMResourceName>>)) { - auto timezoneId = GetTimezoneId(args[0]); + } + + GET_METHOD(Hour, ui8) + GET_METHOD(Minute, ui8) + GET_METHOD(Second, ui8) + + SIMPLE_UDF(TGetMillisecondOfSecond, ui32(TAutoMap<TResource<TMResourceName>>)) { + Y_UNUSED(valueBuilder); + return TUnboxedValuePod(GetMicrosecond(args[0]) / 1000u); + } + + SIMPLE_UDF(TGetMicrosecondOfSecond, ui32(TAutoMap<TResource<TMResourceName>>)) { + Y_UNUSED(valueBuilder); + return TUnboxedValuePod(GetMicrosecond(args[0])); + } + + GET_METHOD(TimezoneId, ui16) + + SIMPLE_UDF(TGetTimezoneName, char*(TAutoMap<TResource<TMResourceName>>)) { + auto timezoneId = GetTimezoneId(args[0]); if (timezoneId >= NUdf::GetTimezones().size()) { - return TUnboxedValuePod(); - } - return valueBuilder->NewString(NUdf::GetTimezones()[timezoneId]); - } - - // Update - - class TUpdate : public TBoxedValue { + return TUnboxedValuePod(); + } + return valueBuilder->NewString(NUdf::GetTimezones()[timezoneId]); + } + + // Update + + class TUpdate : public TBoxedValue { const TSourcePosition Pos_; public: explicit TUpdate(TSourcePosition pos) : Pos_(pos) {} - TUnboxedValue Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const override - { + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override + { try { EMPTY_RESULT_ON_EMPTY_ARG(0); auto result = args[0]; - + if (args[1]) { auto year = args[1].Get<ui16>(); if (!ValidateYear(year)) { return TUnboxedValuePod(); } SetYear(result, year); - } + } if (args[2]) { auto month = args[2].Get<ui8>(); if (!ValidateMonth(month)) { return TUnboxedValuePod(); } SetMonth(result, month); - } + } if (args[3]) { auto day = args[3].Get<ui8>(); if (!ValidateDay(day)) { return TUnboxedValuePod(); } SetDay(result, day); - } + } if (args[4]) { auto hour = args[4].Get<ui8>(); if (!ValidateHour(hour)) { return TUnboxedValuePod(); } SetHour(result, hour); - } + } if (args[5]) { auto minute = args[5].Get<ui8>(); if (!ValidateMinute(minute)) { return TUnboxedValuePod(); } SetMinute(result, minute); - } + } if (args[6]) { auto second = args[6].Get<ui8>(); if (!ValidateSecond(second)) { return TUnboxedValuePod(); } SetSecond(result, second); - } + } if (args[7]) { auto microsecond = args[7].Get<ui32>(); if (!ValidateMicrosecond(microsecond)) { return TUnboxedValuePod(); } SetMicrosecond(result, microsecond); - } + } if (args[8]) { auto timezoneId = args[8].Get<ui16>(); if (!ValidateTimezoneId(timezoneId)) { @@ -628,320 +628,320 @@ namespace { auto& builder = valueBuilder->GetDateBuilder(); auto& storage = TTMStorage::Reference(result); if (!storage.Validate(builder)) { - return TUnboxedValuePod(); - } + return TUnboxedValuePod(); + } return result; } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } - } - - static const TStringRef& Name() { - static auto name = TStringRef::Of("Update"); - return name; - } - - static bool DeclareSignature( - const TStringRef& name, - TType*, - IFunctionTypeInfoBuilder& builder, - bool typesOnly) - { - if (Name() != name) { - return false; - } - - auto resourceType = builder.Resource(TMResourceName); - auto optionalResourceType = builder.Optional()->Item(resourceType).Build(); - - builder.OptionalArgs(8).Args()->Add(resourceType).Flags(ICallablePayload::TArgumentFlags::AutoMap) - .Add(builder.Optional()->Item<ui16>().Build()).Name("Year") - .Add(builder.Optional()->Item<ui8>().Build()).Name("Month") - .Add(builder.Optional()->Item<ui8>().Build()).Name("Day") - .Add(builder.Optional()->Item<ui8>().Build()).Name("Hour") - .Add(builder.Optional()->Item<ui8>().Build()).Name("Minute") - .Add(builder.Optional()->Item<ui8>().Build()).Name("Second") - .Add(builder.Optional()->Item<ui32>().Build()).Name("Microsecond") + } + } + + static const TStringRef& Name() { + static auto name = TStringRef::Of("Update"); + return name; + } + + static bool DeclareSignature( + const TStringRef& name, + TType*, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) + { + if (Name() != name) { + return false; + } + + auto resourceType = builder.Resource(TMResourceName); + auto optionalResourceType = builder.Optional()->Item(resourceType).Build(); + + builder.OptionalArgs(8).Args()->Add(resourceType).Flags(ICallablePayload::TArgumentFlags::AutoMap) + .Add(builder.Optional()->Item<ui16>().Build()).Name("Year") + .Add(builder.Optional()->Item<ui8>().Build()).Name("Month") + .Add(builder.Optional()->Item<ui8>().Build()).Name("Day") + .Add(builder.Optional()->Item<ui8>().Build()).Name("Hour") + .Add(builder.Optional()->Item<ui8>().Build()).Name("Minute") + .Add(builder.Optional()->Item<ui8>().Build()).Name("Second") + .Add(builder.Optional()->Item<ui32>().Build()).Name("Microsecond") .Add(builder.Optional()->Item<ui16>().Build()).Name("TimezoneId"); - - builder.Returns(optionalResourceType); - - if (!typesOnly) { + + builder.Returns(optionalResourceType); + + if (!typesOnly) { builder.Implementation(new TUpdate(builder.GetSourcePosition())); - } - - return true; - } - }; - - // From* - + } + + return true; + } + }; + + // From* + SIMPLE_UDF(TFromSeconds, TOptional<TTimestamp>(TAutoMap<ui32>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); auto res = args[0].Get<ui32>(); if (!ValidateDatetime(res)) { return TUnboxedValuePod(); } return TUnboxedValuePod((ui64)(res * 1000000ull)); - } - + } + SIMPLE_UDF(TFromMilliseconds, TOptional<TTimestamp>(TAutoMap<ui64>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); auto res = args[0].Get<ui64>(); if (res >= MAX_TIMESTAMP / 1000u) { return TUnboxedValuePod(); } return TUnboxedValuePod(res * 1000u); - } - + } + SIMPLE_UDF(TFromMicroseconds, TOptional<TTimestamp>(TAutoMap<ui64>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); auto res = args[0].Get<ui64>(); if (!ValidateTimestamp(res)) { return TUnboxedValuePod(); } return TUnboxedValuePod(res); - } - + } + SIMPLE_UDF(TIntervalFromDays, TOptional<TInterval>(TAutoMap<i32>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); const i64 res = i64(args[0].Get<i32>()) * 86400000000ll; return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod(); - } - + } + SIMPLE_UDF(TIntervalFromHours, TOptional<TInterval>(TAutoMap<i32>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); const i64 res = i64(args[0].Get<i32>()) * 3600000000ll; return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod(); - } - + } + SIMPLE_UDF(TIntervalFromMinutes, TOptional<TInterval>(TAutoMap<i32>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); const i64 res = i64(args[0].Get<i32>()) * 60000000ll; return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod(); - } - + } + SIMPLE_UDF(TIntervalFromSeconds, TOptional<TInterval>(TAutoMap<i32>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); const i64 res = i64(args[0].Get<i32>()) * 1000000ll; return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod(); - } - + } + SIMPLE_UDF(TIntervalFromMilliseconds, TOptional<TInterval>(TAutoMap<i64>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); const i64 res = i64(args[0].Get<i64>()) * 1000ll; return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod(); - } - + } + SIMPLE_UDF(TIntervalFromMicroseconds, TOptional<TInterval>(TAutoMap<i64>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); const i64 res = args[0].Get<i64>(); return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod(); - } - - // To* - + } + + // To* + SIMPLE_UDF(TToDays, i32(TAutoMap<TInterval>)) { - Y_UNUSED(valueBuilder); + Y_UNUSED(valueBuilder); return TUnboxedValuePod(i32(args[0].Get<i64>() / 86400000000ll)); - } - - SIMPLE_UDF(TToHours, i32(TAutoMap<TInterval>)) { - Y_UNUSED(valueBuilder); + } + + SIMPLE_UDF(TToHours, i32(TAutoMap<TInterval>)) { + Y_UNUSED(valueBuilder); return TUnboxedValuePod(i32(args[0].Get<i64>() / 3600000000ll)); - } - - SIMPLE_UDF(TToMinutes, i32(TAutoMap<TInterval>)) { - Y_UNUSED(valueBuilder); + } + + SIMPLE_UDF(TToMinutes, i32(TAutoMap<TInterval>)) { + Y_UNUSED(valueBuilder); return TUnboxedValuePod(i32(args[0].Get<i64>() / 60000000ll)); - } - -#define DECLARE_TO_VALUE(units, TSigned, TUnsigned) \ - template <typename TUserDataType> \ - class TTo##units : public TBoxedValue { \ - private: \ - TUnboxedValue Run( \ - const IValueBuilder* valueBuilder, \ - const TUnboxedValuePod* args) const override; \ - public: \ - static void DeclareSignature( \ - TType* userType, \ - IFunctionTypeInfoBuilder& builder, \ - bool typesOnly) \ - { \ - builder.UserType(userType); \ - builder.Args()->Add<TUserDataType>() \ - .Flags(ICallablePayload::TArgumentFlags::AutoMap).Done(); \ - \ - if (TDataType<TUserDataType>::Id == TDataType<TInterval>::Id) { \ - builder.Returns<TSigned>(); \ - } else { \ - builder.Returns<TUnsigned>(); \ - } \ - \ - if (!typesOnly) { \ - builder.Implementation(new TTo##units<TUserDataType>); \ - } \ - } \ - }; - - DECLARE_TO_VALUE(Seconds, i32, ui32); - DECLARE_TO_VALUE(Milliseconds, i64, ui64); - DECLARE_TO_VALUE(Microseconds, i64, ui64); - -#define TO_METHOD(units, type, expr) \ - template <> \ - TUnboxedValue TTo##units<type>::Run( \ - const IValueBuilder* valueBuilder, \ - const TUnboxedValuePod* args) const \ - { \ - Y_UNUSED(valueBuilder); \ - EMPTY_RESULT_ON_EMPTY_ARG(0); \ - return TUnboxedValuePod(expr); \ - } - - TO_METHOD(Seconds, TDate, (ui32)(args[0].Get<ui16>() * 86400u)); - TO_METHOD(Seconds, TDatetime, args[0].Get<ui32>()); - TO_METHOD(Seconds, TTimestamp, (ui32)(args[0].Get<ui64>() / 1000000ull)); - TO_METHOD(Seconds, TInterval, (i64)(args[0].Get<i64>() / 1000000ll)); - TO_METHOD(Seconds, TTzDate, (ui32)(args[0].Get<ui16>() * 86400u)); - TO_METHOD(Seconds, TTzDatetime, args[0].Get<ui32>()); - TO_METHOD(Seconds, TTzTimestamp, (ui32)(args[0].Get<ui64>() / 1000000u)); - - TO_METHOD(Milliseconds, TDate, (ui64)(args[0].Get<ui16>() * 86400000ull)); - TO_METHOD(Milliseconds, TDatetime, (ui64)(args[0].Get<ui32>() * 1000ull)); - TO_METHOD(Milliseconds, TTimestamp, (ui64)(args[0].Get<ui64>() / 1000ull)); - TO_METHOD(Milliseconds, TInterval, (i64)(args[0].Get<i64>() / 1000ll)); - TO_METHOD(Milliseconds, TTzDate, (ui64)(args[0].Get<ui16>() * 86400000ull)); - TO_METHOD(Milliseconds, TTzDatetime, (ui64)(args[0].Get<ui32>() * 1000ull)); - TO_METHOD(Milliseconds, TTzTimestamp, (ui64)(args[0].Get<ui64>() / 1000ull)); - - TO_METHOD(Microseconds, TDate, (ui64)(args[0].Get<ui16>() * 86400000000ull)); - TO_METHOD(Microseconds, TDatetime, (ui64)(args[0].Get<ui32>() * 1000000ull)); - TO_METHOD(Microseconds, TTimestamp, args[0].Get<ui64>()); - TO_METHOD(Microseconds, TInterval, args[0].Get<i64>()); - TO_METHOD(Microseconds, TTzDate, (ui64)(args[0].Get<ui16>() * 86400000000ull)); - TO_METHOD(Microseconds, TTzDatetime, (ui64)(args[0].Get<ui32>() * 1000000ull)); - TO_METHOD(Microseconds, TTzTimestamp, args[0].Get<ui64>()); - - // StartOf* - - SIMPLE_UDF(TStartOfYear, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = TTMStorage::Reference(result); - storage.Month = 1; - storage.Day = 1; - storage.Hour = 0; - storage.Minute = 0; - storage.Second = 0; - storage.Microsecond = 0; - - auto& builder = valueBuilder->GetDateBuilder(); - if (!storage.Validate(builder)) { - return TUnboxedValuePod(); - } - return result; - } - - SIMPLE_UDF(TStartOfQuarter, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = TTMStorage::Reference(result); - storage.Month = (storage.Month - 1) / 3 * 3 + 1; - storage.Day = 1; - storage.Hour = 0; - storage.Minute = 0; - storage.Second = 0; - storage.Microsecond = 0; - - auto& builder = valueBuilder->GetDateBuilder(); - if (!storage.Validate(builder)) { - return TUnboxedValuePod(); - } - return result; - } - - SIMPLE_UDF(TStartOfMonth, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = TTMStorage::Reference(result); - storage.Day = 1; - storage.Hour = 0; - storage.Minute = 0; - storage.Second = 0; - storage.Microsecond = 0; - - auto& builder = valueBuilder->GetDateBuilder(); - if (!storage.Validate(builder)) { - return TUnboxedValuePod(); - } - return result; - } - - SIMPLE_UDF(TStartOfWeek, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = TTMStorage::Reference(result); - auto& builder = valueBuilder->GetDateBuilder(); - + } + +#define DECLARE_TO_VALUE(units, TSigned, TUnsigned) \ + template <typename TUserDataType> \ + class TTo##units : public TBoxedValue { \ + private: \ + TUnboxedValue Run( \ + const IValueBuilder* valueBuilder, \ + const TUnboxedValuePod* args) const override; \ + public: \ + static void DeclareSignature( \ + TType* userType, \ + IFunctionTypeInfoBuilder& builder, \ + bool typesOnly) \ + { \ + builder.UserType(userType); \ + builder.Args()->Add<TUserDataType>() \ + .Flags(ICallablePayload::TArgumentFlags::AutoMap).Done(); \ + \ + if (TDataType<TUserDataType>::Id == TDataType<TInterval>::Id) { \ + builder.Returns<TSigned>(); \ + } else { \ + builder.Returns<TUnsigned>(); \ + } \ + \ + if (!typesOnly) { \ + builder.Implementation(new TTo##units<TUserDataType>); \ + } \ + } \ + }; + + DECLARE_TO_VALUE(Seconds, i32, ui32); + DECLARE_TO_VALUE(Milliseconds, i64, ui64); + DECLARE_TO_VALUE(Microseconds, i64, ui64); + +#define TO_METHOD(units, type, expr) \ + template <> \ + TUnboxedValue TTo##units<type>::Run( \ + const IValueBuilder* valueBuilder, \ + const TUnboxedValuePod* args) const \ + { \ + Y_UNUSED(valueBuilder); \ + EMPTY_RESULT_ON_EMPTY_ARG(0); \ + return TUnboxedValuePod(expr); \ + } + + TO_METHOD(Seconds, TDate, (ui32)(args[0].Get<ui16>() * 86400u)); + TO_METHOD(Seconds, TDatetime, args[0].Get<ui32>()); + TO_METHOD(Seconds, TTimestamp, (ui32)(args[0].Get<ui64>() / 1000000ull)); + TO_METHOD(Seconds, TInterval, (i64)(args[0].Get<i64>() / 1000000ll)); + TO_METHOD(Seconds, TTzDate, (ui32)(args[0].Get<ui16>() * 86400u)); + TO_METHOD(Seconds, TTzDatetime, args[0].Get<ui32>()); + TO_METHOD(Seconds, TTzTimestamp, (ui32)(args[0].Get<ui64>() / 1000000u)); + + TO_METHOD(Milliseconds, TDate, (ui64)(args[0].Get<ui16>() * 86400000ull)); + TO_METHOD(Milliseconds, TDatetime, (ui64)(args[0].Get<ui32>() * 1000ull)); + TO_METHOD(Milliseconds, TTimestamp, (ui64)(args[0].Get<ui64>() / 1000ull)); + TO_METHOD(Milliseconds, TInterval, (i64)(args[0].Get<i64>() / 1000ll)); + TO_METHOD(Milliseconds, TTzDate, (ui64)(args[0].Get<ui16>() * 86400000ull)); + TO_METHOD(Milliseconds, TTzDatetime, (ui64)(args[0].Get<ui32>() * 1000ull)); + TO_METHOD(Milliseconds, TTzTimestamp, (ui64)(args[0].Get<ui64>() / 1000ull)); + + TO_METHOD(Microseconds, TDate, (ui64)(args[0].Get<ui16>() * 86400000000ull)); + TO_METHOD(Microseconds, TDatetime, (ui64)(args[0].Get<ui32>() * 1000000ull)); + TO_METHOD(Microseconds, TTimestamp, args[0].Get<ui64>()); + TO_METHOD(Microseconds, TInterval, args[0].Get<i64>()); + TO_METHOD(Microseconds, TTzDate, (ui64)(args[0].Get<ui16>() * 86400000000ull)); + TO_METHOD(Microseconds, TTzDatetime, (ui64)(args[0].Get<ui32>() * 1000000ull)); + TO_METHOD(Microseconds, TTzTimestamp, args[0].Get<ui64>()); + + // StartOf* + + SIMPLE_UDF(TStartOfYear, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { + auto result = args[0]; + auto& storage = TTMStorage::Reference(result); + storage.Month = 1; + storage.Day = 1; + storage.Hour = 0; + storage.Minute = 0; + storage.Second = 0; + storage.Microsecond = 0; + + auto& builder = valueBuilder->GetDateBuilder(); + if (!storage.Validate(builder)) { + return TUnboxedValuePod(); + } + return result; + } + + SIMPLE_UDF(TStartOfQuarter, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { + auto result = args[0]; + auto& storage = TTMStorage::Reference(result); + storage.Month = (storage.Month - 1) / 3 * 3 + 1; + storage.Day = 1; + storage.Hour = 0; + storage.Minute = 0; + storage.Second = 0; + storage.Microsecond = 0; + + auto& builder = valueBuilder->GetDateBuilder(); + if (!storage.Validate(builder)) { + return TUnboxedValuePod(); + } + return result; + } + + SIMPLE_UDF(TStartOfMonth, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { + auto result = args[0]; + auto& storage = TTMStorage::Reference(result); + storage.Day = 1; + storage.Hour = 0; + storage.Minute = 0; + storage.Second = 0; + storage.Microsecond = 0; + + auto& builder = valueBuilder->GetDateBuilder(); + if (!storage.Validate(builder)) { + return TUnboxedValuePod(); + } + return result; + } + + SIMPLE_UDF(TStartOfWeek, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { + auto result = args[0]; + auto& storage = TTMStorage::Reference(result); + auto& builder = valueBuilder->GetDateBuilder(); + const auto date = storage.ToDatetime(builder); const ui32 shift = 86400u * (storage.DayOfWeek - 1u); - if (shift > date) { - return TUnboxedValuePod(); - } + if (shift > date) { + return TUnboxedValuePod(); + } storage.FromDatetime(builder, date - shift, storage.TimezoneId); - - storage.Hour = 0; - storage.Minute = 0; - storage.Second = 0; - storage.Microsecond = 0; - return result; - } - - SIMPLE_UDF(TStartOfDay, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = TTMStorage::Reference(result); - storage.Hour = 0; - storage.Minute = 0; - storage.Second = 0; - storage.Microsecond = 0; - - auto& builder = valueBuilder->GetDateBuilder(); - if (!storage.Validate(builder)) { - return TUnboxedValuePod(); - } - return result; - } - + + storage.Hour = 0; + storage.Minute = 0; + storage.Second = 0; + storage.Microsecond = 0; + return result; + } + + SIMPLE_UDF(TStartOfDay, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { + auto result = args[0]; + auto& storage = TTMStorage::Reference(result); + storage.Hour = 0; + storage.Minute = 0; + storage.Second = 0; + storage.Microsecond = 0; + + auto& builder = valueBuilder->GetDateBuilder(); + if (!storage.Validate(builder)) { + return TUnboxedValuePod(); + } + return result; + } + SIMPLE_UDF(TStartOf, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>, TAutoMap<TInterval>)) { - auto result = args[0]; - ui64 interval = std::abs(args[1].Get<i64>()); - if (interval == 0) { - return result; - } - auto& storage = TTMStorage::Reference(result); - if (interval >= 86400000000ull) { - // treat as StartOfDay - storage.Hour = 0; - storage.Minute = 0; - storage.Second = 0; - storage.Microsecond = 0; - } else { - auto current = storage.ToTimeOfDay(); - auto rounded = current / interval * interval; - storage.FromTimeOfDay(rounded); - } - - auto& builder = valueBuilder->GetDateBuilder(); - if (!storage.Validate(builder)) { - return TUnboxedValuePod(); - } - return result; - } - - SIMPLE_UDF(TTimeOfDay, TInterval(TAutoMap<TResource<TMResourceName>>)) { - Y_UNUSED(valueBuilder); - auto& storage = TTMStorage::Reference(args[0]); - return TUnboxedValuePod((i64)storage.ToTimeOfDay()); - } - + auto result = args[0]; + ui64 interval = std::abs(args[1].Get<i64>()); + if (interval == 0) { + return result; + } + auto& storage = TTMStorage::Reference(result); + if (interval >= 86400000000ull) { + // treat as StartOfDay + storage.Hour = 0; + storage.Minute = 0; + storage.Second = 0; + storage.Microsecond = 0; + } else { + auto current = storage.ToTimeOfDay(); + auto rounded = current / interval * interval; + storage.FromTimeOfDay(rounded); + } + + auto& builder = valueBuilder->GetDateBuilder(); + if (!storage.Validate(builder)) { + return TUnboxedValuePod(); + } + return result; + } + + SIMPLE_UDF(TTimeOfDay, TInterval(TAutoMap<TResource<TMResourceName>>)) { + Y_UNUSED(valueBuilder); + auto& storage = TTMStorage::Reference(args[0]); + return TUnboxedValuePod((i64)storage.ToTimeOfDay()); + } + // Add ... SIMPLE_UDF(TShiftYears, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>, i32)) { @@ -1016,79 +1016,79 @@ namespace { } }; - // Format - - class TFormat : public TBoxedValue { - public: - class TFactory : public TBoxedValue { - public: - explicit TFactory(TSourcePosition pos) - : Pos_(pos) - {} - - private: + // Format + + class TFormat : public TBoxedValue { + public: + class TFactory : public TBoxedValue { + public: + explicit TFactory(TSourcePosition pos) + : Pos_(pos) + {} + + private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const final try { return TUnboxedValuePod(new TFormat(args[0], Pos_)); } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } + } const TSourcePosition Pos_; - }; - - static const TStringRef& Name() { - static auto name = TStringRef::Of("Format"); - return name; - } - - static bool DeclareSignature( - const TStringRef& name, - TType*, - IFunctionTypeInfoBuilder& builder, - bool typesOnly) - { - if (Name() != name) { - return false; - } - - auto resourceType = builder.Resource(TMResourceName); - + }; + + static const TStringRef& Name() { + static auto name = TStringRef::Of("Format"); + return name; + } + + static bool DeclareSignature( + const TStringRef& name, + TType*, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) + { + if (Name() != name) { + return false; + } + + auto resourceType = builder.Resource(TMResourceName); + builder.Args()->Add(resourceType).Flags(ICallablePayload::TArgumentFlags::AutoMap); - builder.RunConfig<char*>().Returns<char*>(); - - if (!typesOnly) { - builder.Implementation(new TFormat::TFactory(builder.GetSourcePosition())); - } - - return true; - } - - private: + builder.RunConfig<char*>().Returns<char*>(); + + if (!typesOnly) { + builder.Implementation(new TFormat::TFactory(builder.GetSourcePosition())); + } + + return true; + } + + private: const TSourcePosition Pos_; const TUnboxedValue Format_; std::vector<std::function<size_t(char*, const TUnboxedValuePod&, const IDateBuilder&)>> Printers_; - - size_t ReservedSize_; - - struct TDataPrinter { + + size_t ReservedSize_; + + struct TDataPrinter { const std::string_view Data; - - size_t operator()(char* out, const TUnboxedValuePod&, const IDateBuilder&) const { + + size_t operator()(char* out, const TUnboxedValuePod&, const IDateBuilder&) const { std::memcpy(out, Data.data(), Data.size()); return Data.size(); - } - }; - - private: - TUnboxedValue Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const override - { + } + }; + + private: + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override + { try { EMPTY_RESULT_ON_EMPTY_ARG(0); const auto value = args[0]; - + auto& builder = valueBuilder->GetDateBuilder(); - + auto result = valueBuilder->NewStringNotFilled(ReservedSize_); auto pos = result.AsStringRef().Data(); ui32 size = 0U; @@ -1099,7 +1099,7 @@ namespace { pos += plus; } } - + if (size < ReservedSize_) { result = valueBuilder->SubString(result.Release(), 0U, size); } @@ -1108,84 +1108,84 @@ namespace { } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } - } - + } + TFormat(const TUnboxedValuePod& runConfig, TSourcePosition pos) : Pos_(pos) , Format_(runConfig) - { + { const std::string_view formatView(Format_.AsStringRef()); auto dataStart = formatView.begin(); size_t dataSize = 0U; ReservedSize_ = 0U; - + for (auto ptr = formatView.begin(); formatView.end() != ptr; ++ptr) { - if (*ptr != '%') { + if (*ptr != '%') { ++dataSize; - continue; - } - + continue; + } + if (dataSize) { Printers_.emplace_back(TDataPrinter{std::string_view(&*dataStart, dataSize)}); ReservedSize_ += dataSize; dataSize = 0U; - } - + } + if (formatView.end() == ++ptr) { ythrow yexception() << "format string ends with single %%"; - } - - switch (*ptr) { - case '%': { + } + + switch (*ptr) { + case '%': { static constexpr size_t size = 1; Printers_.emplace_back([](char* out, const TUnboxedValuePod&, const IDateBuilder&) { - *out = '%'; - return size; - }); - ReservedSize_ += size; - break; - } - case 'Y': { + *out = '%'; + return size; + }); + ReservedSize_ += size; + break; + } + case 'Y': { static constexpr size_t size = 4; Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) { return PrintNDigits<size>::Do(GetYear(value), out); - }); - ReservedSize_ += size; - break; - } - case 'm': { + }); + ReservedSize_ += size; + break; + } + case 'm': { static constexpr size_t size = 2; Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) { return PrintNDigits<size>::Do(GetMonth(value), out); - }); - ReservedSize_ += size; - break; - } - case 'd': { + }); + ReservedSize_ += size; + break; + } + case 'd': { static constexpr size_t size = 2; Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) { return PrintNDigits<size>::Do(GetDay(value), out); - }); - ReservedSize_ += size; - break; - } - case 'H': { + }); + ReservedSize_ += size; + break; + } + case 'H': { static constexpr size_t size = 2; Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) { return PrintNDigits<size>::Do(GetHour(value), out); - }); - ReservedSize_ += size; - break; - } - case 'M': { + }); + ReservedSize_ += size; + break; + } + case 'M': { static constexpr size_t size = 2; Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) { return PrintNDigits<size>::Do(GetMinute(value), out); - }); - ReservedSize_ += size; - break; - } - case 'S': + }); + ReservedSize_ += size; + break; + } + case 'S': Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) { constexpr size_t size = 2; if (const auto microsecond = GetMicrosecond(value)) { @@ -1193,46 +1193,46 @@ namespace { *out++ = '.'; constexpr size_t msize = 6; return size + 1U + PrintNDigits<msize, false>::Do(microsecond, out); - } + } return PrintNDigits<size>::Do(GetSecond(value), out); - }); - ReservedSize_ += 9; - break; - - case 'z': { + }); + ReservedSize_ += 9; + break; + + case 'z': { static constexpr size_t size = 5; Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder& builder) { - auto timezoneId = GetTimezoneId(value); - if (TTMStorage::IsUniversal(timezoneId)) { + auto timezoneId = GetTimezoneId(value); + if (TTMStorage::IsUniversal(timezoneId)) { std::memcpy(out, "+0000", size); - return size; - } - i32 shift; - if (!builder.GetTimezoneShift(GetYear(value), GetMonth(value), GetDay(value), - GetHour(value), GetMinute(value), GetSecond(value), timezoneId, shift)) - { + return size; + } + i32 shift; + if (!builder.GetTimezoneShift(GetYear(value), GetMonth(value), GetDay(value), + GetHour(value), GetMinute(value), GetSecond(value), timezoneId, shift)) + { std::memcpy(out, "+0000", size); - return size; - } + return size; + } *out++ = shift > 0 ? '+' : '-'; shift = std::abs(shift); out += PrintNDigits<2U>::Do(shift / 60U, out); out += PrintNDigits<2U>::Do(shift % 60U, out); - return size; - }); - ReservedSize_ += size; - break; - } - case 'Z': + return size; + }); + ReservedSize_ += size; + break; + } + case 'Z': Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) { const auto timezoneId = GetTimezoneId(value); const auto tzName = NUdf::GetTimezones()[timezoneId]; std::memcpy(out, tzName.data(), std::min(tzName.size(), MAX_TIMEZONE_NAME_LEN)); return tzName.size(); - }); - ReservedSize_ += MAX_TIMEZONE_NAME_LEN; - break; + }); + ReservedSize_ += MAX_TIMEZONE_NAME_LEN; + break; case 'b': { static constexpr size_t size = 3; Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) { @@ -1283,20 +1283,20 @@ namespace { ReservedSize_ += 9U; // MAX_MONTH_FULL_NAME_LEN break; } - default: - ythrow yexception() << "invalid format character: " << *ptr; - } - + default: + ythrow yexception() << "invalid format character: " << *ptr; + } + dataStart = ptr + 1U; - } - + } + if (dataSize) { Printers_.emplace_back(TDataPrinter{std::string_view(dataStart, dataSize)}); ReservedSize_ += dataSize; - } - } - }; - + } + } + }; + template<size_t Digits> struct ParseExaclyNDigits; @@ -1322,90 +1322,90 @@ namespace { } }; - // Parse - - class TParse : public TBoxedValue { - public: - class TFactory : public TBoxedValue { - public: - explicit TFactory(TSourcePosition pos) - : Pos_(pos) - {} - - private: + // Parse + + class TParse : public TBoxedValue { + public: + class TFactory : public TBoxedValue { + public: + explicit TFactory(TSourcePosition pos) + : Pos_(pos) + {} + + private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const final try { return TUnboxedValuePod(new TParse(args[0], Pos_)); } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } - + } + const TSourcePosition Pos_; - }; - - static const TStringRef& Name() { - static auto name = TStringRef::Of("Parse"); - return name; - } - - static bool DeclareSignature( - const TStringRef& name, - TType*, - IFunctionTypeInfoBuilder& builder, - bool typesOnly) - { - if (Name() != name) { - return false; - } - - auto resourceType = builder.Resource(TMResourceName); - auto optionalResourceType = builder.Optional()->Item(resourceType).Build(); - - builder.Args()->Add<char*>().Flags(ICallablePayload::TArgumentFlags::AutoMap) - .Add(builder.Optional()->Item<ui16>()) - .Done() - .OptionalArgs(1); - builder.RunConfig<char*>().Returns(optionalResourceType); - - if (!typesOnly) { - builder.Implementation(new TParse::TFactory(builder.GetSourcePosition())); - } - - return true; - } - - private: + }; + + static const TStringRef& Name() { + static auto name = TStringRef::Of("Parse"); + return name; + } + + static bool DeclareSignature( + const TStringRef& name, + TType*, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) + { + if (Name() != name) { + return false; + } + + auto resourceType = builder.Resource(TMResourceName); + auto optionalResourceType = builder.Optional()->Item(resourceType).Build(); + + builder.Args()->Add<char*>().Flags(ICallablePayload::TArgumentFlags::AutoMap) + .Add(builder.Optional()->Item<ui16>()) + .Done() + .OptionalArgs(1); + builder.RunConfig<char*>().Returns(optionalResourceType); + + if (!typesOnly) { + builder.Implementation(new TParse::TFactory(builder.GetSourcePosition())); + } + + return true; + } + + private: const TSourcePosition Pos_; const TUnboxedValue Format_; - + std::vector<std::function<bool(std::string_view::const_iterator& it, size_t, TUnboxedValuePod&, const IDateBuilder&)>> Scanners_; - - struct TDataScanner { + + struct TDataScanner { const std::string_view Data_; - + bool operator()(std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod&, const IDateBuilder&) const { if (limit < Data_.size() || !std::equal(Data_.begin(), Data_.end(), it)) { return false; - } + } std::advance(it, Data_.size()); return true; - } - }; - - TUnboxedValue Run( - const IValueBuilder* valueBuilder, - const TUnboxedValuePod* args) const override - { + } + }; + + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override + { try { EMPTY_RESULT_ON_EMPTY_ARG(0); - + const std::string_view buffer = args[0].AsStringRef(); - + TUnboxedValuePod result(0); auto& storage = TTMStorage::Reference(result); storage.MakeDefault(); - + auto& builder = valueBuilder->GetDateBuilder(); - + auto it = buffer.begin(); for (const auto& scanner : Scanners_) { if (!scanner(it, std::distance(it, buffer.end()), result, builder)) { @@ -1414,159 +1414,159 @@ namespace { } if (buffer.end() != it || !storage.Validate(builder)) { - return TUnboxedValuePod(); - } + return TUnboxedValuePod(); + } return result; } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } - } - + } + } + TParse(const TUnboxedValuePod& runConfig, TSourcePosition pos) : Pos_(pos) , Format_(runConfig) - { + { const std::string_view formatView(Format_.AsStringRef()); auto dataStart = formatView.begin(); size_t dataSize = 0U; - + for (auto ptr = formatView.begin(); formatView.end() != ptr; ++ptr) { - if (*ptr != '%') { + if (*ptr != '%') { ++dataSize; - continue; - } - + continue; + } + if (dataSize) { Scanners_.emplace_back(TDataScanner{std::string_view(&*dataStart, dataSize)}); dataSize = 0; - } - + } + if (++ptr == formatView.end()) { ythrow yexception() << "format string ends with single %%"; - } - - switch (*ptr) { - case '%': + } + + switch (*ptr) { + case '%': Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod&, const IDateBuilder&) { return limit > 0U && *it++ == '%'; - }); - break; - - case 'Y': { + }); + break; + + case 'Y': { static constexpr size_t size = 4; Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) { ui32 year = 0U; if (limit < size || !ParseExaclyNDigits<size>::Do(it, year) || !ValidateYear(year)) { return false; - } - SetYear(result, year); + } + SetYear(result, year); return true; - }); - break; - } - case 'm': { + }); + break; + } + case 'm': { static constexpr size_t size = 2; Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) { ui32 month = 0U; if (limit < size || !ParseExaclyNDigits<size>::Do(it, month) || !ValidateMonth(month)) { return false; - } - SetMonth(result, month); + } + SetMonth(result, month); return true; - }); - break; - } - case 'd': { + }); + break; + } + case 'd': { static constexpr size_t size = 2; Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) { ui32 day = 0U; if (limit < size || !ParseExaclyNDigits<size>::Do(it, day) || !ValidateDay(day)) { return false; - } - SetDay(result, day); + } + SetDay(result, day); return true; - }); - break; - } - case 'H': { + }); + break; + } + case 'H': { static constexpr size_t size = 2; Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) { ui32 hour = 0U; if (limit < size || !ParseExaclyNDigits<size>::Do(it, hour) || !ValidateHour(hour)) { return false; - } - SetHour(result, hour); + } + SetHour(result, hour); return true; - }); - break; - } - case 'M': { + }); + break; + } + case 'M': { static constexpr size_t size = 2; Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) { ui32 minute = 0U; if (limit < size || !ParseExaclyNDigits<size>::Do(it, minute) || !ValidateMinute(minute)) { return false; - } - SetMinute(result, minute); + } + SetMinute(result, minute); return true; - }); - break; - } - case 'S': { + }); + break; + } + case 'S': { static constexpr size_t size = 2; Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) { ui32 second = 0U; if (limit < size || !ParseExaclyNDigits<size>::Do(it, second) || !ValidateSecond(second)) { return false; - } - SetSecond(result, second); + } + SetSecond(result, second); limit -= size; if (!limit || *it != '.') { return true; - } - + } + ++it; --limit; ui32 usec = 0U; - + size_t digits = 6U; for (; limit; --limit) { const auto c = *it; if (!digits || !std::isdigit(c)) { - break; - } + break; + } usec *= 10U; usec += c - '0'; ++it; --digits; - } + } while (digits--) { usec *= 10U; - } - SetMicrosecond(result, usec); + } + SetMicrosecond(result, usec); return true; - }); - break; - } - case 'Z': + }); + break; + } + case 'Z': Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder& builder) { const auto start = it; while (limit > 0 && (std::isalnum(*it) || *it == '/' || *it == '_' || *it == '-' || *it == '+')) { ++it; - --limit; - } + --limit; + } const auto size = std::distance(start, it); - - ui32 timezoneId; + + ui32 timezoneId; if (!builder.FindTimezoneId(TStringRef(&*start, size), timezoneId)) { return false; - } - SetTimezoneId(result, timezoneId); + } + SetTimezoneId(result, timezoneId); return true; - }); - break; + }); + break; case 'b': { static constexpr size_t size = 3; Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) { @@ -1607,135 +1607,135 @@ namespace { }); break; } - default: - ythrow yexception() << "invalid format character: " << *ptr; - } - + default: + ythrow yexception() << "invalid format character: " << *ptr; + } + dataStart = ptr + 1U; - } - + } + if (dataSize) { Scanners_.emplace_back(TDataScanner{std::string_view(&*dataStart, dataSize)}); - } - } - }; - -#define PARSE_SPECIFIC_FORMAT(format) \ - SIMPLE_UDF(TParse##format, TOptional<TResource<TMResourceName>>(TAutoMap<char*>)) { \ - auto str = args[0].AsStringRef(); \ - TInstant instant; \ - if (!TInstant::TryParse##format(TStringBuf(str.Data(), str.Size()), instant)) { \ - return TUnboxedValuePod(); \ - } \ - auto& builder = valueBuilder->GetDateBuilder(); \ - TUnboxedValuePod result(0); \ - auto& storage = TTMStorage::Reference(result); \ - storage.FromTimestamp(builder, instant.MicroSeconds()); \ - return result; \ - } - - PARSE_SPECIFIC_FORMAT(Rfc822); - PARSE_SPECIFIC_FORMAT(Iso8601); - PARSE_SPECIFIC_FORMAT(Http); - PARSE_SPECIFIC_FORMAT(X509); - - SIMPLE_MODULE(TDateTime2Module, - TUserDataTypeFuncFactory<true, SplitName, TSplit, - TDate, - TDatetime, - TTimestamp, - TTzDate, - TTzDatetime, - TTzTimestamp>, - - TMakeDate, - TMakeDatetime, - TMakeTimestamp, - TMakeTzDate, - TMakeTzDatetime, - TMakeTzTimestamp, - - TGetYear, - TGetDayOfYear, - TGetMonth, - TGetMonthName, - TGetWeekOfYear, + } + } + }; + +#define PARSE_SPECIFIC_FORMAT(format) \ + SIMPLE_UDF(TParse##format, TOptional<TResource<TMResourceName>>(TAutoMap<char*>)) { \ + auto str = args[0].AsStringRef(); \ + TInstant instant; \ + if (!TInstant::TryParse##format(TStringBuf(str.Data(), str.Size()), instant)) { \ + return TUnboxedValuePod(); \ + } \ + auto& builder = valueBuilder->GetDateBuilder(); \ + TUnboxedValuePod result(0); \ + auto& storage = TTMStorage::Reference(result); \ + storage.FromTimestamp(builder, instant.MicroSeconds()); \ + return result; \ + } + + PARSE_SPECIFIC_FORMAT(Rfc822); + PARSE_SPECIFIC_FORMAT(Iso8601); + PARSE_SPECIFIC_FORMAT(Http); + PARSE_SPECIFIC_FORMAT(X509); + + SIMPLE_MODULE(TDateTime2Module, + TUserDataTypeFuncFactory<true, SplitName, TSplit, + TDate, + TDatetime, + TTimestamp, + TTzDate, + TTzDatetime, + TTzTimestamp>, + + TMakeDate, + TMakeDatetime, + TMakeTimestamp, + TMakeTzDate, + TMakeTzDatetime, + TMakeTzTimestamp, + + TGetYear, + TGetDayOfYear, + TGetMonth, + TGetMonthName, + TGetWeekOfYear, TGetWeekOfYearIso8601, - TGetDayOfMonth, - TGetDayOfWeek, - TGetDayOfWeekName, - TGetHour, - TGetMinute, - TGetSecond, - TGetMillisecondOfSecond, - TGetMicrosecondOfSecond, - TGetTimezoneId, - TGetTimezoneName, - - TUpdate, - - TFromSeconds, - TFromMilliseconds, - TFromMicroseconds, - - TIntervalFromDays, - TIntervalFromHours, - TIntervalFromMinutes, - TIntervalFromSeconds, - TIntervalFromMilliseconds, - TIntervalFromMicroseconds, - - TToDays, - TToHours, - TToMinutes, - - TStartOfYear, - TStartOfQuarter, - TStartOfMonth, - TStartOfWeek, - TStartOfDay, - TStartOf, - TTimeOfDay, - + TGetDayOfMonth, + TGetDayOfWeek, + TGetDayOfWeekName, + TGetHour, + TGetMinute, + TGetSecond, + TGetMillisecondOfSecond, + TGetMicrosecondOfSecond, + TGetTimezoneId, + TGetTimezoneName, + + TUpdate, + + TFromSeconds, + TFromMilliseconds, + TFromMicroseconds, + + TIntervalFromDays, + TIntervalFromHours, + TIntervalFromMinutes, + TIntervalFromSeconds, + TIntervalFromMilliseconds, + TIntervalFromMicroseconds, + + TToDays, + TToHours, + TToMinutes, + + TStartOfYear, + TStartOfQuarter, + TStartOfMonth, + TStartOfWeek, + TStartOfDay, + TStartOf, + TTimeOfDay, + TShiftYears, TShiftQuarters, TShiftMonths, - TUserDataTypeFuncFactory<true, ToSecondsName, TToSeconds, - TDate, - TDatetime, - TTimestamp, - TInterval, - TTzDate, - TTzDatetime, - TTzTimestamp>, - - TUserDataTypeFuncFactory<true, ToMillisecondsName, TToMilliseconds, - TDate, - TDatetime, - TTimestamp, - TInterval, - TTzDate, - TTzDatetime, - TTzTimestamp>, - - TUserDataTypeFuncFactory<true, ToMicrosecondsName, TToMicroseconds, - TDate, - TDatetime, - TTimestamp, - TInterval, - TTzDate, - TTzDatetime, - TTzTimestamp>, - - TFormat, - TParse, - - TParseRfc822, - TParseIso8601, - TParseHttp, - TParseX509 - ) -} - -REGISTER_MODULES(TDateTime2Module) + TUserDataTypeFuncFactory<true, ToSecondsName, TToSeconds, + TDate, + TDatetime, + TTimestamp, + TInterval, + TTzDate, + TTzDatetime, + TTzTimestamp>, + + TUserDataTypeFuncFactory<true, ToMillisecondsName, TToMilliseconds, + TDate, + TDatetime, + TTimestamp, + TInterval, + TTzDate, + TTzDatetime, + TTzTimestamp>, + + TUserDataTypeFuncFactory<true, ToMicrosecondsName, TToMicroseconds, + TDate, + TDatetime, + TTimestamp, + TInterval, + TTzDate, + TTzDatetime, + TTzTimestamp>, + + TFormat, + TParse, + + TParseRfc822, + TParseIso8601, + TParseHttp, + TParseX509 + ) +} + +REGISTER_MODULES(TDateTime2Module) diff --git a/ydb/library/yql/udfs/common/datetime2/ya.make b/ydb/library/yql/udfs/common/datetime2/ya.make index faa95811b4..3d0db781ed 100644 --- a/ydb/library/yql/udfs/common/datetime2/ya.make +++ b/ydb/library/yql/udfs/common/datetime2/ya.make @@ -1,24 +1,24 @@ -YQL_UDF(datetime2_udf) - -YQL_ABI_VERSION( - 2 +YQL_UDF(datetime2_udf) + +YQL_ABI_VERSION( + 2 24 - 0 -) - + 0 +) + OWNER( g:yql g:yql_ydb_core ) - -SRCS( - datetime_udf.cpp -) - -PEERDIR( - util/draft + +SRCS( + datetime_udf.cpp +) + +PEERDIR( + util/draft ydb/library/yql/minikql ydb/library/yql/public/udf/tz -) - -END() +) + +END() diff --git a/ydb/library/yql/udfs/common/set/set_udf.cpp b/ydb/library/yql/udfs/common/set/set_udf.cpp index ac009ed71d..f282c22842 100644 --- a/ydb/library/yql/udfs/common/set/set_udf.cpp +++ b/ydb/library/yql/udfs/common/set/set_udf.cpp @@ -1,52 +1,52 @@ - + #include <ydb/library/yql/public/udf/udf_type_ops.h> #include <ydb/library/yql/public/udf/udf_helpers.h> - -#include <unordered_set> - -using namespace NKikimr; -using namespace NUdf; - -namespace { - -template <typename THash, typename TEquals> -class TSetBase { -private: + +#include <unordered_set> + +using namespace NKikimr; +using namespace NUdf; + +namespace { + +template <typename THash, typename TEquals> +class TSetBase { +private: std::unordered_set<TUnboxedValue, THash, TEquals, TUnboxedValue::TAllocator> Set; - ui32 MaxSize = 0; + ui32 MaxSize = 0; bool WasChanged = false; - -protected: - TSetBase(THash hash, TEquals equals) - : Set(1, hash, equals) - {} - - void Init(const TUnboxedValuePod& value, ui32 maxSize) { - MaxSize = maxSize ? maxSize : std::numeric_limits<ui32>::max(); - AddValue(value); - } - - void Merge(const TSetBase& left, const TSetBase& right) { - MaxSize = std::max(left.MaxSize, right.MaxSize); - for (const auto& item : left.Set) { - AddValue(item); - } - for (const auto& item : right.Set) { - AddValue(item); - } - } - - void Deserialize(const TUnboxedValuePod& serialized) { - MaxSize = serialized.GetElement(0).Get<ui32>(); - auto list = serialized.GetElement(1); - - const auto listIter = list.GetListIterator(); - for (TUnboxedValue current; listIter.Next(current);) { - AddValue(current); - } - } - -public: + +protected: + TSetBase(THash hash, TEquals equals) + : Set(1, hash, equals) + {} + + void Init(const TUnboxedValuePod& value, ui32 maxSize) { + MaxSize = maxSize ? maxSize : std::numeric_limits<ui32>::max(); + AddValue(value); + } + + void Merge(const TSetBase& left, const TSetBase& right) { + MaxSize = std::max(left.MaxSize, right.MaxSize); + for (const auto& item : left.Set) { + AddValue(item); + } + for (const auto& item : right.Set) { + AddValue(item); + } + } + + void Deserialize(const TUnboxedValuePod& serialized) { + MaxSize = serialized.GetElement(0).Get<ui32>(); + auto list = serialized.GetElement(1); + + const auto listIter = list.GetListIterator(); + for (TUnboxedValue current; listIter.Next(current);) { + AddValue(current); + } + } + +public: void ResetChanged() { WasChanged = false; } @@ -55,182 +55,182 @@ public: return WasChanged; } - TUnboxedValue Serialize(const IValueBuilder* builder) { - TUnboxedValue* values = nullptr; - auto list = builder->NewArray(Set.size(), values); - - for (const auto& item : Set) { - *values++ = item; - } - - TUnboxedValue* items = nullptr; - auto result = builder->NewArray(2U, items); - items[0] = TUnboxedValuePod(MaxSize); - items[1] = list; - - return result; - } - - TUnboxedValue GetResult(const IValueBuilder* builder) { - TUnboxedValue* values = nullptr; - auto result = builder->NewArray(Set.size(), values); - - for (const auto& item : Set) { - *values++ = item; - } - return result; - } - - void AddValue(const TUnboxedValuePod& value) { - if (Set.size() < MaxSize) { + TUnboxedValue Serialize(const IValueBuilder* builder) { + TUnboxedValue* values = nullptr; + auto list = builder->NewArray(Set.size(), values); + + for (const auto& item : Set) { + *values++ = item; + } + + TUnboxedValue* items = nullptr; + auto result = builder->NewArray(2U, items); + items[0] = TUnboxedValuePod(MaxSize); + items[1] = list; + + return result; + } + + TUnboxedValue GetResult(const IValueBuilder* builder) { + TUnboxedValue* values = nullptr; + auto result = builder->NewArray(Set.size(), values); + + for (const auto& item : Set) { + *values++ = item; + } + return result; + } + + void AddValue(const TUnboxedValuePod& value) { + if (Set.size() < MaxSize) { WasChanged = Set.insert(TUnboxedValuePod(value)).second; - } - } -}; - -template <EDataSlot Slot> -class TSetData - : public TSetBase<TUnboxedValueHash<Slot>, TUnboxedValueEquals<Slot>> -{ -public: - using TBase = TSetBase<TUnboxedValueHash<Slot>, TUnboxedValueEquals<Slot>>; - - TSetData(const TUnboxedValuePod& value, ui32 maxSize) - : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) - { - TBase::Init(value, maxSize); - } - - TSetData(const TSetData& left, const TSetData& right) - : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) - { - TBase::Merge(left, right); - } - - explicit TSetData(const TUnboxedValuePod& serialized) - : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) - { - TBase::Deserialize(serialized); - } -}; - -struct TGenericHash { - IHash::TPtr Hash; - - std::size_t operator()(const TUnboxedValuePod& value) const { - return Hash->Hash(value); - } -}; - -struct TGenericEquals { - IEquate::TPtr Equate; - - bool operator()(const TUnboxedValuePod& left, const TUnboxedValuePod& right) const { - return Equate->Equals(left, right); - } -}; - -class TSetGeneric - : public TSetBase<TGenericHash, TGenericEquals> -{ -public: - using TBase = TSetBase<TGenericHash, TGenericEquals>; - - TSetGeneric(const TUnboxedValuePod& value, ui32 maxSize, - IHash::TPtr hash, IEquate::TPtr equate) - : TBase(TGenericHash{hash}, TGenericEquals{equate}) - { - TBase::Init(value, maxSize); - } - - TSetGeneric(const TSetGeneric& left, const TSetGeneric& right, - IHash::TPtr hash, IEquate::TPtr equate) - : TBase(TGenericHash{hash}, TGenericEquals{equate}) - { - TBase::Merge(left, right); - } - - TSetGeneric(const TUnboxedValuePod& serialized, - IHash::TPtr hash, IEquate::TPtr equate) - : TBase(TGenericHash{hash}, TGenericEquals{equate}) - { - TBase::Deserialize(serialized); - } -}; - -extern const char SetResourceNameGeneric[] = "Set.SetResource.Generic"; -class TSetResource: - public TBoxedResource<TSetGeneric, SetResourceNameGeneric> -{ -public: - template <typename... Args> - inline TSetResource(Args&&... args) - : TBoxedResource(std::forward<Args>(args)...) - {} -}; - -template <EDataSlot Slot> -class TSetResourceData; - -template <EDataSlot Slot> -TSetResourceData<Slot>* GetSetResourceData(const TUnboxedValuePod& arg) { - TSetResourceData<Slot>::Validate(arg); - return static_cast<TSetResourceData<Slot>*>(arg.AsBoxed().Get()); -} - -TSetResource* GetSetResource(const TUnboxedValuePod& arg) { - TSetResource::Validate(arg); - return static_cast<TSetResource*>(arg.AsBoxed().Get()); -} - - -template <EDataSlot Slot> -class TSetCreateData: public TBoxedValue { -private: + } + } +}; + +template <EDataSlot Slot> +class TSetData + : public TSetBase<TUnboxedValueHash<Slot>, TUnboxedValueEquals<Slot>> +{ +public: + using TBase = TSetBase<TUnboxedValueHash<Slot>, TUnboxedValueEquals<Slot>>; + + TSetData(const TUnboxedValuePod& value, ui32 maxSize) + : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) + { + TBase::Init(value, maxSize); + } + + TSetData(const TSetData& left, const TSetData& right) + : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) + { + TBase::Merge(left, right); + } + + explicit TSetData(const TUnboxedValuePod& serialized) + : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) + { + TBase::Deserialize(serialized); + } +}; + +struct TGenericHash { + IHash::TPtr Hash; + + std::size_t operator()(const TUnboxedValuePod& value) const { + return Hash->Hash(value); + } +}; + +struct TGenericEquals { + IEquate::TPtr Equate; + + bool operator()(const TUnboxedValuePod& left, const TUnboxedValuePod& right) const { + return Equate->Equals(left, right); + } +}; + +class TSetGeneric + : public TSetBase<TGenericHash, TGenericEquals> +{ +public: + using TBase = TSetBase<TGenericHash, TGenericEquals>; + + TSetGeneric(const TUnboxedValuePod& value, ui32 maxSize, + IHash::TPtr hash, IEquate::TPtr equate) + : TBase(TGenericHash{hash}, TGenericEquals{equate}) + { + TBase::Init(value, maxSize); + } + + TSetGeneric(const TSetGeneric& left, const TSetGeneric& right, + IHash::TPtr hash, IEquate::TPtr equate) + : TBase(TGenericHash{hash}, TGenericEquals{equate}) + { + TBase::Merge(left, right); + } + + TSetGeneric(const TUnboxedValuePod& serialized, + IHash::TPtr hash, IEquate::TPtr equate) + : TBase(TGenericHash{hash}, TGenericEquals{equate}) + { + TBase::Deserialize(serialized); + } +}; + +extern const char SetResourceNameGeneric[] = "Set.SetResource.Generic"; +class TSetResource: + public TBoxedResource<TSetGeneric, SetResourceNameGeneric> +{ +public: + template <typename... Args> + inline TSetResource(Args&&... args) + : TBoxedResource(std::forward<Args>(args)...) + {} +}; + +template <EDataSlot Slot> +class TSetResourceData; + +template <EDataSlot Slot> +TSetResourceData<Slot>* GetSetResourceData(const TUnboxedValuePod& arg) { + TSetResourceData<Slot>::Validate(arg); + return static_cast<TSetResourceData<Slot>*>(arg.AsBoxed().Get()); +} + +TSetResource* GetSetResource(const TUnboxedValuePod& arg) { + TSetResource::Validate(arg); + return static_cast<TSetResource*>(arg.AsBoxed().Get()); +} + + +template <EDataSlot Slot> +class TSetCreateData: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return TUnboxedValuePod(new TSetResourceData<Slot>(args[0], args[1].Get<ui32>())); - } -}; - -class TSetCreate: public TBoxedValue { -private: + return TUnboxedValuePod(new TSetResourceData<Slot>(args[0], args[1].Get<ui32>())); + } +}; + +class TSetCreate: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return TUnboxedValuePod(new TSetResource(args[0], args[1].Get<ui32>(), Hash_, Equate_)); - } - -public: - TSetCreate(IHash::TPtr hash, IEquate::TPtr equate) - : Hash_(hash) - , Equate_(equate) - {} - -private: - IHash::TPtr Hash_; - IEquate::TPtr Equate_; -}; - -template <EDataSlot Slot> -class TSetAddValueData: public TBoxedValue { -private: + return TUnboxedValuePod(new TSetResource(args[0], args[1].Get<ui32>(), Hash_, Equate_)); + } + +public: + TSetCreate(IHash::TPtr hash, IEquate::TPtr equate) + : Hash_(hash) + , Equate_(equate) + {} + +private: + IHash::TPtr Hash_; + IEquate::TPtr Equate_; +}; + +template <EDataSlot Slot> +class TSetAddValueData: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - auto resource = GetSetResourceData<Slot>(args[0]); + auto resource = GetSetResourceData<Slot>(args[0]); resource->Get()->ResetChanged(); - resource->Get()->AddValue(args[1]); - return TUnboxedValuePod(resource); - } -}; - -class TSetAddValue: public TBoxedValue { -private: + resource->Get()->AddValue(args[1]); + return TUnboxedValuePod(resource); + } +}; + +class TSetAddValue: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - auto resource = GetSetResource(args[0]); + auto resource = GetSetResource(args[0]); resource->Get()->ResetChanged(); - resource->Get()->AddValue(args[1]); - return TUnboxedValuePod(resource); - } -}; - -template <EDataSlot Slot> + resource->Get()->AddValue(args[1]); + return TUnboxedValuePod(resource); + } +}; + +template <EDataSlot Slot> class TSetWasChangedData: public TBoxedValue { private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { @@ -248,239 +248,239 @@ private: }; template <EDataSlot Slot> -class TSetSerializeData: public TBoxedValue { -private: +class TSetSerializeData: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - return GetSetResourceData<Slot>(args[0])->Get()->Serialize(valueBuilder); - } -}; - -class TSetSerialize: public TBoxedValue { -private: + return GetSetResourceData<Slot>(args[0])->Get()->Serialize(valueBuilder); + } +}; + +class TSetSerialize: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - return GetSetResource(args[0])->Get()->Serialize(valueBuilder); - } -}; - -template <EDataSlot Slot> -class TSetDeserializeData: public TBoxedValue { -private: + return GetSetResource(args[0])->Get()->Serialize(valueBuilder); + } +}; + +template <EDataSlot Slot> +class TSetDeserializeData: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return TUnboxedValuePod(new TSetResourceData<Slot>(args[0])); - } -}; - -class TSetDeserialize: public TBoxedValue { -private: + return TUnboxedValuePod(new TSetResourceData<Slot>(args[0])); + } +}; + +class TSetDeserialize: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return TUnboxedValuePod(new TSetResource(args[0], Hash_, Equate_)); - } - -public: - TSetDeserialize(IHash::TPtr hash, IEquate::TPtr equate) - : Hash_(hash) - , Equate_(equate) - {} - -private: - IHash::TPtr Hash_; - IEquate::TPtr Equate_; -}; - -template <EDataSlot Slot> -class TSetMergeData: public TBoxedValue { -private: + return TUnboxedValuePod(new TSetResource(args[0], Hash_, Equate_)); + } + +public: + TSetDeserialize(IHash::TPtr hash, IEquate::TPtr equate) + : Hash_(hash) + , Equate_(equate) + {} + +private: + IHash::TPtr Hash_; + IEquate::TPtr Equate_; +}; + +template <EDataSlot Slot> +class TSetMergeData: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - auto left = GetSetResourceData<Slot>(args[0]); - auto right = GetSetResourceData<Slot>(args[1]); - return TUnboxedValuePod(new TSetResourceData<Slot>(*left->Get(), *right->Get())); - } -}; - -class TSetMerge: public TBoxedValue { -private: + auto left = GetSetResourceData<Slot>(args[0]); + auto right = GetSetResourceData<Slot>(args[1]); + return TUnboxedValuePod(new TSetResourceData<Slot>(*left->Get(), *right->Get())); + } +}; + +class TSetMerge: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - auto left = GetSetResource(args[0]); - auto right = GetSetResource(args[1]); - return TUnboxedValuePod(new TSetResource(*left->Get(), *right->Get(), Hash_, Equate_)); - } - -public: - TSetMerge(IHash::TPtr hash, IEquate::TPtr equate) - : Hash_(hash) - , Equate_(equate) - {} - -private: - IHash::TPtr Hash_; - IEquate::TPtr Equate_; -}; - -template <EDataSlot Slot> -class TSetGetResultData: public TBoxedValue { -private: + auto left = GetSetResource(args[0]); + auto right = GetSetResource(args[1]); + return TUnboxedValuePod(new TSetResource(*left->Get(), *right->Get(), Hash_, Equate_)); + } + +public: + TSetMerge(IHash::TPtr hash, IEquate::TPtr equate) + : Hash_(hash) + , Equate_(equate) + {} + +private: + IHash::TPtr Hash_; + IEquate::TPtr Equate_; +}; + +template <EDataSlot Slot> +class TSetGetResultData: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - return GetSetResourceData<Slot>(args[0])->Get()->GetResult(valueBuilder); - } -}; - -class TSetGetResult: public TBoxedValue { -private: + return GetSetResourceData<Slot>(args[0])->Get()->GetResult(valueBuilder); + } +}; + +class TSetGetResult: public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - return GetSetResource(args[0])->Get()->GetResult(valueBuilder); - } -}; - - -#define MAKE_RESOURCE(slot, ...) \ -extern const char SetResourceName##slot[] = "Set.SetResource."#slot; \ -template <> \ -class TSetResourceData<EDataSlot::slot>: \ - public TBoxedResource<TSetData<EDataSlot::slot>, SetResourceName##slot> \ -{ \ -public: \ - template <typename... Args> \ - inline TSetResourceData(Args&&... args) \ - : TBoxedResource(std::forward<Args>(args)...) \ - {} \ -}; - -UDF_TYPE_ID_MAP(MAKE_RESOURCE) - -#define MAKE_IMPL(operation, slot) \ -case EDataSlot::slot: \ - builder.Implementation(new operation<EDataSlot::slot>); \ - break; - -#define MAKE_CREATE(slot, ...) MAKE_IMPL(TSetCreateData, slot) -#define MAKE_ADD_VALUE(slot, ...) MAKE_IMPL(TSetAddValueData, slot) + return GetSetResource(args[0])->Get()->GetResult(valueBuilder); + } +}; + + +#define MAKE_RESOURCE(slot, ...) \ +extern const char SetResourceName##slot[] = "Set.SetResource."#slot; \ +template <> \ +class TSetResourceData<EDataSlot::slot>: \ + public TBoxedResource<TSetData<EDataSlot::slot>, SetResourceName##slot> \ +{ \ +public: \ + template <typename... Args> \ + inline TSetResourceData(Args&&... args) \ + : TBoxedResource(std::forward<Args>(args)...) \ + {} \ +}; + +UDF_TYPE_ID_MAP(MAKE_RESOURCE) + +#define MAKE_IMPL(operation, slot) \ +case EDataSlot::slot: \ + builder.Implementation(new operation<EDataSlot::slot>); \ + break; + +#define MAKE_CREATE(slot, ...) MAKE_IMPL(TSetCreateData, slot) +#define MAKE_ADD_VALUE(slot, ...) MAKE_IMPL(TSetAddValueData, slot) #define MAKE_WAS_CHANGED(slot, ...) MAKE_IMPL(TSetWasChangedData, slot) -#define MAKE_SERIALIZE(slot, ...) MAKE_IMPL(TSetSerializeData, slot) -#define MAKE_DESERIALIZE(slot, ...) MAKE_IMPL(TSetDeserializeData, slot) -#define MAKE_MERGE(slot, ...) MAKE_IMPL(TSetMergeData, slot) -#define MAKE_GET_RESULT(slot, ...) MAKE_IMPL(TSetGetResultData, slot) - -#define MAKE_TYPE(slot, ...) \ -case EDataSlot::slot: \ - setType = builder.Resource(SetResourceName##slot); \ - break; - - -static const auto CreateName = TStringRef::Of("Create"); -static const auto AddValueName = TStringRef::Of("AddValue"); +#define MAKE_SERIALIZE(slot, ...) MAKE_IMPL(TSetSerializeData, slot) +#define MAKE_DESERIALIZE(slot, ...) MAKE_IMPL(TSetDeserializeData, slot) +#define MAKE_MERGE(slot, ...) MAKE_IMPL(TSetMergeData, slot) +#define MAKE_GET_RESULT(slot, ...) MAKE_IMPL(TSetGetResultData, slot) + +#define MAKE_TYPE(slot, ...) \ +case EDataSlot::slot: \ + setType = builder.Resource(SetResourceName##slot); \ + break; + + +static const auto CreateName = TStringRef::Of("Create"); +static const auto AddValueName = TStringRef::Of("AddValue"); static const auto WasChangedName = TStringRef::Of("WasChanged"); // must be used right after AddValue -static const auto SerializeName = TStringRef::Of("Serialize"); -static const auto DeserializeName = TStringRef::Of("Deserialize"); -static const auto MergeName = TStringRef::Of("Merge"); -static const auto GetResultName = TStringRef::Of("GetResult"); - -class TSetModule: public IUdfModule { -public: - TStringRef Name() const { - return TStringRef::Of("Set"); - } - - void CleanupOnTerminate() const final { - } - - void GetAllFunctions(IFunctionsSink& sink) const final { - sink.Add(CreateName)->SetTypeAwareness(); - sink.Add(AddValueName)->SetTypeAwareness(); +static const auto SerializeName = TStringRef::Of("Serialize"); +static const auto DeserializeName = TStringRef::Of("Deserialize"); +static const auto MergeName = TStringRef::Of("Merge"); +static const auto GetResultName = TStringRef::Of("GetResult"); + +class TSetModule: public IUdfModule { +public: + TStringRef Name() const { + return TStringRef::Of("Set"); + } + + void CleanupOnTerminate() const final { + } + + void GetAllFunctions(IFunctionsSink& sink) const final { + sink.Add(CreateName)->SetTypeAwareness(); + sink.Add(AddValueName)->SetTypeAwareness(); sink.Add(WasChangedName)->SetTypeAwareness(); - sink.Add(SerializeName)->SetTypeAwareness(); - sink.Add(DeserializeName)->SetTypeAwareness(); - sink.Add(MergeName)->SetTypeAwareness(); - sink.Add(GetResultName)->SetTypeAwareness(); - } - - void BuildFunctionTypeInfo( - const TStringRef& name, - TType* userType, - const TStringRef& typeConfig, - ui32 flags, - IFunctionTypeInfoBuilder& builder) const final - { - Y_UNUSED(typeConfig); - - try { - const bool typesOnly = (flags & TFlags::TypesOnly); - builder.UserType(userType); - - auto typeHelper = builder.TypeInfoHelper(); - - auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); - if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) { - builder.SetError("User type is not a 3-tuple"); - return; - } - - bool isGeneric = false; - IHash::TPtr hash; - IEquate::TPtr equate; - TMaybe<EDataSlot> slot; - - auto valueType = userTypeInspector.GetElementType(2); - auto valueTypeInspector = TDataTypeInspector(*typeHelper, valueType); - if (!valueTypeInspector) { - isGeneric = true; - hash = builder.MakeHash(valueType); - equate = builder.MakeEquate(valueType); - if (!hash || !equate) { - return; - } - } else { - slot = FindDataSlot(valueTypeInspector.GetTypeId()); - if (!slot) { - builder.SetError("Unknown data type"); - return; - } - const auto& features = NUdf::GetDataTypeInfo(*slot).Features; - if (!(features & NUdf::CanHash) || !(features & NUdf::CanEquate)) { - builder.SetError("Data type is not hashable or equatable"); - return; - } - } - - auto serializedListType = builder.List()->Item(valueType).Build(); - auto serializedType = builder.Tuple()->Add<ui32>().Add(serializedListType).Build(); - - TType* setType = nullptr; - if (isGeneric) { - setType = builder.Resource(SetResourceNameGeneric); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_TYPE) - } - } - - if (name == CreateName) { - builder.Args()->Add(valueType).Add<ui32>().Done().Returns(setType); - - if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TSetCreate(hash, equate)); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_CREATE) - } - } - } - } - - if (name == AddValueName) { - builder.Args()->Add(setType).Add(valueType).Done().Returns(setType); - - if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TSetAddValue); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_ADD_VALUE) - } - } - } - } - + sink.Add(SerializeName)->SetTypeAwareness(); + sink.Add(DeserializeName)->SetTypeAwareness(); + sink.Add(MergeName)->SetTypeAwareness(); + sink.Add(GetResultName)->SetTypeAwareness(); + } + + void BuildFunctionTypeInfo( + const TStringRef& name, + TType* userType, + const TStringRef& typeConfig, + ui32 flags, + IFunctionTypeInfoBuilder& builder) const final + { + Y_UNUSED(typeConfig); + + try { + const bool typesOnly = (flags & TFlags::TypesOnly); + builder.UserType(userType); + + auto typeHelper = builder.TypeInfoHelper(); + + auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); + if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) { + builder.SetError("User type is not a 3-tuple"); + return; + } + + bool isGeneric = false; + IHash::TPtr hash; + IEquate::TPtr equate; + TMaybe<EDataSlot> slot; + + auto valueType = userTypeInspector.GetElementType(2); + auto valueTypeInspector = TDataTypeInspector(*typeHelper, valueType); + if (!valueTypeInspector) { + isGeneric = true; + hash = builder.MakeHash(valueType); + equate = builder.MakeEquate(valueType); + if (!hash || !equate) { + return; + } + } else { + slot = FindDataSlot(valueTypeInspector.GetTypeId()); + if (!slot) { + builder.SetError("Unknown data type"); + return; + } + const auto& features = NUdf::GetDataTypeInfo(*slot).Features; + if (!(features & NUdf::CanHash) || !(features & NUdf::CanEquate)) { + builder.SetError("Data type is not hashable or equatable"); + return; + } + } + + auto serializedListType = builder.List()->Item(valueType).Build(); + auto serializedType = builder.Tuple()->Add<ui32>().Add(serializedListType).Build(); + + TType* setType = nullptr; + if (isGeneric) { + setType = builder.Resource(SetResourceNameGeneric); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_TYPE) + } + } + + if (name == CreateName) { + builder.Args()->Add(valueType).Add<ui32>().Done().Returns(setType); + + if (!typesOnly) { + if (isGeneric) { + builder.Implementation(new TSetCreate(hash, equate)); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_CREATE) + } + } + } + } + + if (name == AddValueName) { + builder.Args()->Add(setType).Add(valueType).Done().Returns(setType); + + if (!typesOnly) { + if (isGeneric) { + builder.Implementation(new TSetAddValue); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_ADD_VALUE) + } + } + } + } + if (name == WasChangedName) { builder.Args()->Add(setType).Done().Returns<bool>(); @@ -495,70 +495,70 @@ public: } } - if (name == MergeName) { - builder.Args()->Add(setType).Add(setType).Done().Returns(setType); - - if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TSetMerge(hash, equate)); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_MERGE) - } - } - } - } - - if (name == SerializeName) { - builder.Args()->Add(setType).Done().Returns(serializedType); - - if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TSetSerialize); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_SERIALIZE) - } - } - } - } - - if (name == DeserializeName) { - builder.Args()->Add(serializedType).Done().Returns(setType); - - if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TSetDeserialize(hash, equate)); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_DESERIALIZE) - } - } - } - } - - if (name == GetResultName) { - auto resultType = builder.List()->Item(valueType).Build(); - - builder.Args()->Add(setType).Done().Returns(resultType); - - if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TSetGetResult); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_GET_RESULT) - } - } - } - } - - } catch (const std::exception& e) { - builder.SetError(CurrentExceptionMessage()); - } - } -}; - -} // namespace - -REGISTER_MODULES(TSetModule) + if (name == MergeName) { + builder.Args()->Add(setType).Add(setType).Done().Returns(setType); + + if (!typesOnly) { + if (isGeneric) { + builder.Implementation(new TSetMerge(hash, equate)); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_MERGE) + } + } + } + } + + if (name == SerializeName) { + builder.Args()->Add(setType).Done().Returns(serializedType); + + if (!typesOnly) { + if (isGeneric) { + builder.Implementation(new TSetSerialize); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_SERIALIZE) + } + } + } + } + + if (name == DeserializeName) { + builder.Args()->Add(serializedType).Done().Returns(setType); + + if (!typesOnly) { + if (isGeneric) { + builder.Implementation(new TSetDeserialize(hash, equate)); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_DESERIALIZE) + } + } + } + } + + if (name == GetResultName) { + auto resultType = builder.List()->Item(valueType).Build(); + + builder.Args()->Add(setType).Done().Returns(resultType); + + if (!typesOnly) { + if (isGeneric) { + builder.Implementation(new TSetGetResult); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_GET_RESULT) + } + } + } + } + + } catch (const std::exception& e) { + builder.SetError(CurrentExceptionMessage()); + } + } +}; + +} // namespace + +REGISTER_MODULES(TSetModule) diff --git a/ydb/library/yql/udfs/common/set/ya.make b/ydb/library/yql/udfs/common/set/ya.make index 1c047bb108..bfc20204e9 100644 --- a/ydb/library/yql/udfs/common/set/ya.make +++ b/ydb/library/yql/udfs/common/set/ya.make @@ -1,15 +1,15 @@ -YQL_UDF(set_udf) - -YQL_ABI_VERSION( - 2 +YQL_UDF(set_udf) + +YQL_ABI_VERSION( + 2 19 - 0 -) - + 0 +) + OWNER(g:yql g:yql_ydb_core) - -SRCS( - set_udf.cpp -) - -END() + +SRCS( + set_udf.cpp +) + +END() diff --git a/ydb/library/yql/udfs/common/top/top_udf.cpp b/ydb/library/yql/udfs/common/top/top_udf.cpp index c6f78b444a..748ffe489b 100644 --- a/ydb/library/yql/udfs/common/top/top_udf.cpp +++ b/ydb/library/yql/udfs/common/top/top_udf.cpp @@ -1,68 +1,68 @@ #include <ydb/library/yql/public/udf/udf_helpers.h> #include <ydb/library/yql/public/udf/udf_type_ops.h> - + #include <library/cpp/containers/top_keeper/top_keeper.h> - + #include <util/generic/set.h> #include <algorithm> #include <iterator> -using namespace NKikimr; -using namespace NUdf; - -namespace { - -using TUnboxedValuePair = std::pair<TUnboxedValue, TUnboxedValue>; - -template <EDataSlot Slot, bool IsTop> -struct TDataCompare { +using namespace NKikimr; +using namespace NUdf; + +namespace { + +using TUnboxedValuePair = std::pair<TUnboxedValue, TUnboxedValue>; + +template <EDataSlot Slot, bool IsTop> +struct TDataCompare { bool operator()(const TUnboxedValue& left, const TUnboxedValue& right) const { - if (IsTop) { - return CompareValues<Slot>(left, right) > 0; - } else { - return CompareValues<Slot>(left, right) < 0; - } - } -}; - -template <EDataSlot Slot, bool IsTop> -struct TDataPairCompare { + if (IsTop) { + return CompareValues<Slot>(left, right) > 0; + } else { + return CompareValues<Slot>(left, right) < 0; + } + } +}; + +template <EDataSlot Slot, bool IsTop> +struct TDataPairCompare { bool operator()(const TUnboxedValuePair& left, const TUnboxedValuePair& right) const { - if (IsTop) { + if (IsTop) { return CompareValues<Slot>(left.first, right.first) > 0; - } else { + } else { return CompareValues<Slot>(left.first, right.first) < 0; - } - } -}; - -template <bool IsTop> -struct TGenericCompare { - ICompare::TPtr Compare; - + } + } +}; + +template <bool IsTop> +struct TGenericCompare { + ICompare::TPtr Compare; + bool operator()(const TUnboxedValue& left, const TUnboxedValue& right) const { - if (IsTop) { - return Compare->Less(right, left); - } else { - return Compare->Less(left, right); - } - } -}; - -template <bool IsTop> -struct TGenericPairCompare { - ICompare::TPtr Compare; - + if (IsTop) { + return Compare->Less(right, left); + } else { + return Compare->Less(left, right); + } + } +}; + +template <bool IsTop> +struct TGenericPairCompare { + ICompare::TPtr Compare; + bool operator()(const TUnboxedValuePair& left, const TUnboxedValuePair& right) const { - if (IsTop) { + if (IsTop) { return Compare->Less(right.first, left.first); - } else { + } else { return Compare->Less(left.first, right.first); - } - } -}; - + } + } +}; + template <typename TValue, typename TCompare, typename TAllocator> class TTopKeeperContainer { TTopKeeper<TValue, TCompare, true, TAllocator> Keeper; @@ -144,806 +144,806 @@ public: } }; -template <typename TCompare> -class TTopKeeperWrapperBase { -protected: +template <typename TCompare> +class TTopKeeperWrapperBase { +protected: TTopKeeperContainer<TUnboxedValue, TCompare, TUnboxedValue::TAllocator> Keeper; - -protected: - explicit TTopKeeperWrapperBase(TCompare compare) + +protected: + explicit TTopKeeperWrapperBase(TCompare compare) : Keeper(compare) - {} - - void Init(const TUnboxedValuePod& value, ui32 maxSize) { - Keeper.SetMaxSize(maxSize); - AddValue(value); - } - - void Merge(TTopKeeperWrapperBase& left, TTopKeeperWrapperBase& right) { - Keeper.SetMaxSize(left.Keeper.GetMaxSize()); - for (const auto& item : left.Keeper.GetInternal()) { - AddValue(item); - } - for (const auto& item : right.Keeper.GetInternal()) { - AddValue(item); - } - } - - void Deserialize(const TUnboxedValuePod& serialized) { - auto maxSize = serialized.GetElement(0).Get<ui32>(); - auto list = serialized.GetElement(1); - - Keeper.SetMaxSize(maxSize); - const auto listIter = list.GetListIterator(); - for (TUnboxedValue current; listIter.Next(current);) { - AddValue(current); - } - } - -public: - void AddValue(const TUnboxedValuePod& value) { - Keeper.Insert(TUnboxedValuePod(value)); - } - - TUnboxedValue Serialize(const IValueBuilder* builder) { - TUnboxedValue* values = nullptr; - auto list = builder->NewArray(Keeper.GetSize(), values); - - for (const auto& item : Keeper.GetInternal()) { - *values++ = item; - } - - TUnboxedValue* items = nullptr; - auto result = builder->NewArray(2U, items); - items[0] = TUnboxedValuePod((ui32)Keeper.GetMaxSize()); - items[1] = list; - - return result; - } - - TUnboxedValue GetResult(const IValueBuilder* builder) { - TUnboxedValue* values = nullptr; - auto list = builder->NewArray(Keeper.GetSize(), values); - - for (const auto& item : Keeper.GetInternal()) { - *values++ = item; - } - return list; - } -}; - -template <typename TCompare> -class TTopKeeperPairWrapperBase { -protected: + {} + + void Init(const TUnboxedValuePod& value, ui32 maxSize) { + Keeper.SetMaxSize(maxSize); + AddValue(value); + } + + void Merge(TTopKeeperWrapperBase& left, TTopKeeperWrapperBase& right) { + Keeper.SetMaxSize(left.Keeper.GetMaxSize()); + for (const auto& item : left.Keeper.GetInternal()) { + AddValue(item); + } + for (const auto& item : right.Keeper.GetInternal()) { + AddValue(item); + } + } + + void Deserialize(const TUnboxedValuePod& serialized) { + auto maxSize = serialized.GetElement(0).Get<ui32>(); + auto list = serialized.GetElement(1); + + Keeper.SetMaxSize(maxSize); + const auto listIter = list.GetListIterator(); + for (TUnboxedValue current; listIter.Next(current);) { + AddValue(current); + } + } + +public: + void AddValue(const TUnboxedValuePod& value) { + Keeper.Insert(TUnboxedValuePod(value)); + } + + TUnboxedValue Serialize(const IValueBuilder* builder) { + TUnboxedValue* values = nullptr; + auto list = builder->NewArray(Keeper.GetSize(), values); + + for (const auto& item : Keeper.GetInternal()) { + *values++ = item; + } + + TUnboxedValue* items = nullptr; + auto result = builder->NewArray(2U, items); + items[0] = TUnboxedValuePod((ui32)Keeper.GetMaxSize()); + items[1] = list; + + return result; + } + + TUnboxedValue GetResult(const IValueBuilder* builder) { + TUnboxedValue* values = nullptr; + auto list = builder->NewArray(Keeper.GetSize(), values); + + for (const auto& item : Keeper.GetInternal()) { + *values++ = item; + } + return list; + } +}; + +template <typename TCompare> +class TTopKeeperPairWrapperBase { +protected: TTopKeeperContainer<TUnboxedValuePair, TCompare, TStdAllocatorForUdf<TUnboxedValuePair>> Keeper; - -protected: - explicit TTopKeeperPairWrapperBase(TCompare compare) + +protected: + explicit TTopKeeperPairWrapperBase(TCompare compare) : Keeper(compare) - {} - - void Init(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize) { - Keeper.SetMaxSize(maxSize); - AddValue(key, payload); - } - - void Merge(TTopKeeperPairWrapperBase& left, TTopKeeperPairWrapperBase& right) { - Keeper.SetMaxSize(left.Keeper.GetMaxSize()); - for (const auto& item : left.Keeper.GetInternal()) { - AddValue(item.first, item.second); - } - for (const auto& item : right.Keeper.GetInternal()) { - AddValue(item.first, item.second); - } - } - - void Deserialize(const TUnboxedValuePod& serialized) { - auto maxSize = serialized.GetElement(0).Get<ui32>(); - auto list = serialized.GetElement(1); - - Keeper.SetMaxSize(maxSize); - const auto listIter = list.GetListIterator(); - for (TUnboxedValue current; listIter.Next(current);) { - AddValue(current.GetElement(0), current.GetElement(1)); - } - } - -public: - void AddValue(const TUnboxedValuePod& key, const TUnboxedValuePod& payload) { - Keeper.Insert(std::make_pair(TUnboxedValuePod(key), TUnboxedValuePod(payload))); - } - - TUnboxedValue Serialize(const IValueBuilder* builder) { - TUnboxedValue* values = nullptr; - auto list = builder->NewArray(Keeper.GetSize(), values); - - for (const auto& item : Keeper.GetInternal()) { - TUnboxedValue* items = nullptr; - auto pair = builder->NewArray(2U, items); - items[0] = item.first; - items[1] = item.second; - *values++ = pair; - } - - TUnboxedValue* items = nullptr; - auto result = builder->NewArray(2U, items); - items[0] = TUnboxedValuePod((ui32)Keeper.GetMaxSize()); - items[1] = list; - - return result; - } - - TUnboxedValue GetResult(const IValueBuilder* builder) { - TUnboxedValue* values = nullptr; - auto list = builder->NewArray(Keeper.GetSize(), values); - - for (const auto& item : Keeper.GetInternal()) { - *values++ = item.second; - } - return list; - } -}; - - -template <EDataSlot Slot, bool HasKey, bool IsTop> -class TTopKeeperDataWrapper; - -template <EDataSlot Slot, bool IsTop> -class TTopKeeperDataWrapper<Slot, false, IsTop> - : public TTopKeeperWrapperBase<TDataCompare<Slot, IsTop>> -{ -public: - using TBase = TTopKeeperWrapperBase<TDataCompare<Slot, IsTop>>; - - TTopKeeperDataWrapper(const TUnboxedValuePod& value, ui32 maxSize) - : TBase(TDataCompare<Slot, IsTop>()) - { - TBase::Init(value, maxSize); - } - - TTopKeeperDataWrapper(TTopKeeperDataWrapper& left, TTopKeeperDataWrapper& right) - : TBase(TDataCompare<Slot, IsTop>()) - { - TBase::Merge(left, right); - } - - explicit TTopKeeperDataWrapper(const TUnboxedValuePod& serialized) - : TBase(TDataCompare<Slot, IsTop>()) - { - TBase::Deserialize(serialized); - } -}; - -template <EDataSlot Slot, bool IsTop> -class TTopKeeperDataWrapper<Slot, true, IsTop> - : public TTopKeeperPairWrapperBase<TDataPairCompare<Slot, IsTop>> -{ -public: - using TBase = TTopKeeperPairWrapperBase<TDataPairCompare<Slot, IsTop>>; - - TTopKeeperDataWrapper(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize) - : TBase(TDataPairCompare<Slot, IsTop>()) - { - TBase::Init(key, payload, maxSize); - } - - TTopKeeperDataWrapper(TTopKeeperDataWrapper& left, TTopKeeperDataWrapper& right) - : TBase(TDataPairCompare<Slot, IsTop>()) - { - TBase::Merge(left, right); - } - - explicit TTopKeeperDataWrapper(const TUnboxedValuePod& serialized) - : TBase(TDataPairCompare<Slot, IsTop>()) - { - TBase::Deserialize(serialized); - } -}; - -template <bool HasKey, bool IsTop> -class TTopKeeperWrapper; - -template <bool IsTop> -class TTopKeeperWrapper<false, IsTop> - : public TTopKeeperWrapperBase<TGenericCompare<IsTop>> -{ -public: - using TBase = TTopKeeperWrapperBase<TGenericCompare<IsTop>>; - - TTopKeeperWrapper(const TUnboxedValuePod& value, ui32 maxSize, ICompare::TPtr compare) - : TBase(TGenericCompare<IsTop>{compare}) - { - TBase::Init(value, maxSize); - } - - TTopKeeperWrapper(TTopKeeperWrapper& left, TTopKeeperWrapper& right, ICompare::TPtr compare) - : TBase(TGenericCompare<IsTop>{compare}) - { - TBase::Merge(left, right); - } - - TTopKeeperWrapper(const TUnboxedValuePod& serialized, ICompare::TPtr compare) - : TBase(TGenericCompare<IsTop>{compare}) - { - TBase::Deserialize(serialized); - } -}; - -template <bool IsTop> -class TTopKeeperWrapper<true, IsTop> - : public TTopKeeperPairWrapperBase<TGenericPairCompare<IsTop>> -{ -public: - using TBase = TTopKeeperPairWrapperBase<TGenericPairCompare<IsTop>>; - - TTopKeeperWrapper(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize, ICompare::TPtr compare) - : TBase(TGenericPairCompare<IsTop>{compare}) - { - TBase::Init(key, payload, maxSize); - } - - TTopKeeperWrapper(TTopKeeperWrapper& left, TTopKeeperWrapper& right, ICompare::TPtr compare) - : TBase(TGenericPairCompare<IsTop>{compare}) - { - TBase::Merge(left, right); - } - - TTopKeeperWrapper(const TUnboxedValuePod& serialized, ICompare::TPtr compare) - : TBase(TGenericPairCompare<IsTop>{compare}) - { - TBase::Deserialize(serialized); - } -}; - - -template <EDataSlot Slot, bool HasKey, bool IsTop> -class TTopResourceData; - -template <EDataSlot Slot, bool HasKey, bool IsTop> -TTopResourceData<Slot, HasKey, IsTop>* GetTopResourceData(const TUnboxedValuePod& arg) { - TTopResourceData<Slot, HasKey, IsTop>::Validate(arg); - return static_cast<TTopResourceData<Slot, HasKey, IsTop>*>(arg.AsBoxed().Get()); -} - -template <bool HasKey, bool IsTop> -class TTopResource; - -template <bool HasKey, bool IsTop> -TTopResource<HasKey, IsTop>* GetTopResource(const TUnboxedValuePod& arg) { - TTopResource<HasKey, IsTop>::Validate(arg); - return static_cast<TTopResource<HasKey, IsTop>*>(arg.AsBoxed().Get()); -} - - -template <EDataSlot Slot, bool HasKey, bool IsTop> -class TTopCreateData : public TBoxedValue { -private: - template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr> - TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { - return TUnboxedValuePod( - new TTopResourceData<Slot, HasKey, IsTop>(args[0], args[1].Get<ui32>())); - } - - template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr> - TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { - return TUnboxedValuePod( - new TTopResourceData<Slot, HasKey, IsTop>(args[0], args[1], args[2].Get<ui32>())); - } - - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return RunImpl(args); - } -}; - -template <bool HasKey, bool IsTop> -class TTopCreate : public TBoxedValue { -private: - template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr> - TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { - return TUnboxedValuePod( - new TTopResource<HasKey, IsTop>(args[0], args[1].Get<ui32>(), Compare_)); - } - - template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr> - TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { - return TUnboxedValuePod( - new TTopResource<HasKey, IsTop>(args[0], args[1], args[2].Get<ui32>(), Compare_)); - } - - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return RunImpl(args); - } - -public: - explicit TTopCreate(ICompare::TPtr compare) - : Compare_(compare) - {} - -private: - ICompare::TPtr Compare_; -}; - -template <EDataSlot Slot, bool HasKey, bool IsTop> -class TTopAddValueData : public TBoxedValue { -private: - template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr> - TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { - auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); - resource->Get()->AddValue(args[1]); - return TUnboxedValuePod(resource); - } - - template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr> - TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { - auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); - resource->Get()->AddValue(args[1], args[2]); - return TUnboxedValuePod(resource); - } - - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return RunImpl(args); - } -}; - -template <bool HasKey, bool IsTop> -class TTopAddValue : public TBoxedValue { -private: - template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr> - TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { - auto resource = GetTopResource<HasKey, IsTop>(args[0]); - resource->Get()->AddValue(args[1]); - return TUnboxedValuePod(resource); - } - - template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr> - TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { - auto resource = GetTopResource<HasKey, IsTop>(args[0]); - resource->Get()->AddValue(args[1], args[2]); - return TUnboxedValuePod(resource); - } - - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return RunImpl(args); - } - -public: - explicit TTopAddValue(ICompare::TPtr) - {} -}; - -template <EDataSlot Slot, bool HasKey, bool IsTop> -class TTopSerializeData : public TBoxedValue { -private: + {} + + void Init(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize) { + Keeper.SetMaxSize(maxSize); + AddValue(key, payload); + } + + void Merge(TTopKeeperPairWrapperBase& left, TTopKeeperPairWrapperBase& right) { + Keeper.SetMaxSize(left.Keeper.GetMaxSize()); + for (const auto& item : left.Keeper.GetInternal()) { + AddValue(item.first, item.second); + } + for (const auto& item : right.Keeper.GetInternal()) { + AddValue(item.first, item.second); + } + } + + void Deserialize(const TUnboxedValuePod& serialized) { + auto maxSize = serialized.GetElement(0).Get<ui32>(); + auto list = serialized.GetElement(1); + + Keeper.SetMaxSize(maxSize); + const auto listIter = list.GetListIterator(); + for (TUnboxedValue current; listIter.Next(current);) { + AddValue(current.GetElement(0), current.GetElement(1)); + } + } + +public: + void AddValue(const TUnboxedValuePod& key, const TUnboxedValuePod& payload) { + Keeper.Insert(std::make_pair(TUnboxedValuePod(key), TUnboxedValuePod(payload))); + } + + TUnboxedValue Serialize(const IValueBuilder* builder) { + TUnboxedValue* values = nullptr; + auto list = builder->NewArray(Keeper.GetSize(), values); + + for (const auto& item : Keeper.GetInternal()) { + TUnboxedValue* items = nullptr; + auto pair = builder->NewArray(2U, items); + items[0] = item.first; + items[1] = item.second; + *values++ = pair; + } + + TUnboxedValue* items = nullptr; + auto result = builder->NewArray(2U, items); + items[0] = TUnboxedValuePod((ui32)Keeper.GetMaxSize()); + items[1] = list; + + return result; + } + + TUnboxedValue GetResult(const IValueBuilder* builder) { + TUnboxedValue* values = nullptr; + auto list = builder->NewArray(Keeper.GetSize(), values); + + for (const auto& item : Keeper.GetInternal()) { + *values++ = item.second; + } + return list; + } +}; + + +template <EDataSlot Slot, bool HasKey, bool IsTop> +class TTopKeeperDataWrapper; + +template <EDataSlot Slot, bool IsTop> +class TTopKeeperDataWrapper<Slot, false, IsTop> + : public TTopKeeperWrapperBase<TDataCompare<Slot, IsTop>> +{ +public: + using TBase = TTopKeeperWrapperBase<TDataCompare<Slot, IsTop>>; + + TTopKeeperDataWrapper(const TUnboxedValuePod& value, ui32 maxSize) + : TBase(TDataCompare<Slot, IsTop>()) + { + TBase::Init(value, maxSize); + } + + TTopKeeperDataWrapper(TTopKeeperDataWrapper& left, TTopKeeperDataWrapper& right) + : TBase(TDataCompare<Slot, IsTop>()) + { + TBase::Merge(left, right); + } + + explicit TTopKeeperDataWrapper(const TUnboxedValuePod& serialized) + : TBase(TDataCompare<Slot, IsTop>()) + { + TBase::Deserialize(serialized); + } +}; + +template <EDataSlot Slot, bool IsTop> +class TTopKeeperDataWrapper<Slot, true, IsTop> + : public TTopKeeperPairWrapperBase<TDataPairCompare<Slot, IsTop>> +{ +public: + using TBase = TTopKeeperPairWrapperBase<TDataPairCompare<Slot, IsTop>>; + + TTopKeeperDataWrapper(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize) + : TBase(TDataPairCompare<Slot, IsTop>()) + { + TBase::Init(key, payload, maxSize); + } + + TTopKeeperDataWrapper(TTopKeeperDataWrapper& left, TTopKeeperDataWrapper& right) + : TBase(TDataPairCompare<Slot, IsTop>()) + { + TBase::Merge(left, right); + } + + explicit TTopKeeperDataWrapper(const TUnboxedValuePod& serialized) + : TBase(TDataPairCompare<Slot, IsTop>()) + { + TBase::Deserialize(serialized); + } +}; + +template <bool HasKey, bool IsTop> +class TTopKeeperWrapper; + +template <bool IsTop> +class TTopKeeperWrapper<false, IsTop> + : public TTopKeeperWrapperBase<TGenericCompare<IsTop>> +{ +public: + using TBase = TTopKeeperWrapperBase<TGenericCompare<IsTop>>; + + TTopKeeperWrapper(const TUnboxedValuePod& value, ui32 maxSize, ICompare::TPtr compare) + : TBase(TGenericCompare<IsTop>{compare}) + { + TBase::Init(value, maxSize); + } + + TTopKeeperWrapper(TTopKeeperWrapper& left, TTopKeeperWrapper& right, ICompare::TPtr compare) + : TBase(TGenericCompare<IsTop>{compare}) + { + TBase::Merge(left, right); + } + + TTopKeeperWrapper(const TUnboxedValuePod& serialized, ICompare::TPtr compare) + : TBase(TGenericCompare<IsTop>{compare}) + { + TBase::Deserialize(serialized); + } +}; + +template <bool IsTop> +class TTopKeeperWrapper<true, IsTop> + : public TTopKeeperPairWrapperBase<TGenericPairCompare<IsTop>> +{ +public: + using TBase = TTopKeeperPairWrapperBase<TGenericPairCompare<IsTop>>; + + TTopKeeperWrapper(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize, ICompare::TPtr compare) + : TBase(TGenericPairCompare<IsTop>{compare}) + { + TBase::Init(key, payload, maxSize); + } + + TTopKeeperWrapper(TTopKeeperWrapper& left, TTopKeeperWrapper& right, ICompare::TPtr compare) + : TBase(TGenericPairCompare<IsTop>{compare}) + { + TBase::Merge(left, right); + } + + TTopKeeperWrapper(const TUnboxedValuePod& serialized, ICompare::TPtr compare) + : TBase(TGenericPairCompare<IsTop>{compare}) + { + TBase::Deserialize(serialized); + } +}; + + +template <EDataSlot Slot, bool HasKey, bool IsTop> +class TTopResourceData; + +template <EDataSlot Slot, bool HasKey, bool IsTop> +TTopResourceData<Slot, HasKey, IsTop>* GetTopResourceData(const TUnboxedValuePod& arg) { + TTopResourceData<Slot, HasKey, IsTop>::Validate(arg); + return static_cast<TTopResourceData<Slot, HasKey, IsTop>*>(arg.AsBoxed().Get()); +} + +template <bool HasKey, bool IsTop> +class TTopResource; + +template <bool HasKey, bool IsTop> +TTopResource<HasKey, IsTop>* GetTopResource(const TUnboxedValuePod& arg) { + TTopResource<HasKey, IsTop>::Validate(arg); + return static_cast<TTopResource<HasKey, IsTop>*>(arg.AsBoxed().Get()); +} + + +template <EDataSlot Slot, bool HasKey, bool IsTop> +class TTopCreateData : public TBoxedValue { +private: + template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr> + TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { + return TUnboxedValuePod( + new TTopResourceData<Slot, HasKey, IsTop>(args[0], args[1].Get<ui32>())); + } + + template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr> + TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { + return TUnboxedValuePod( + new TTopResourceData<Slot, HasKey, IsTop>(args[0], args[1], args[2].Get<ui32>())); + } + + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { + return RunImpl(args); + } +}; + +template <bool HasKey, bool IsTop> +class TTopCreate : public TBoxedValue { +private: + template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr> + TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { + return TUnboxedValuePod( + new TTopResource<HasKey, IsTop>(args[0], args[1].Get<ui32>(), Compare_)); + } + + template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr> + TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { + return TUnboxedValuePod( + new TTopResource<HasKey, IsTop>(args[0], args[1], args[2].Get<ui32>(), Compare_)); + } + + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { + return RunImpl(args); + } + +public: + explicit TTopCreate(ICompare::TPtr compare) + : Compare_(compare) + {} + +private: + ICompare::TPtr Compare_; +}; + +template <EDataSlot Slot, bool HasKey, bool IsTop> +class TTopAddValueData : public TBoxedValue { +private: + template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr> + TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { + auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); + resource->Get()->AddValue(args[1]); + return TUnboxedValuePod(resource); + } + + template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr> + TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { + auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); + resource->Get()->AddValue(args[1], args[2]); + return TUnboxedValuePod(resource); + } + + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { + return RunImpl(args); + } +}; + +template <bool HasKey, bool IsTop> +class TTopAddValue : public TBoxedValue { +private: + template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr> + TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { + auto resource = GetTopResource<HasKey, IsTop>(args[0]); + resource->Get()->AddValue(args[1]); + return TUnboxedValuePod(resource); + } + + template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr> + TUnboxedValue RunImpl(const TUnboxedValuePod* args) const { + auto resource = GetTopResource<HasKey, IsTop>(args[0]); + resource->Get()->AddValue(args[1], args[2]); + return TUnboxedValuePod(resource); + } + + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { + return RunImpl(args); + } + +public: + explicit TTopAddValue(ICompare::TPtr) + {} +}; + +template <EDataSlot Slot, bool HasKey, bool IsTop> +class TTopSerializeData : public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); - return resource->Get()->Serialize(valueBuilder); - } -}; - -template <bool HasKey, bool IsTop> -class TTopSerialize : public TBoxedValue { -private: + auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); + return resource->Get()->Serialize(valueBuilder); + } +}; + +template <bool HasKey, bool IsTop> +class TTopSerialize : public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - auto resource = GetTopResource<HasKey, IsTop>(args[0]); - return resource->Get()->Serialize(valueBuilder); - } - -public: - explicit TTopSerialize(ICompare::TPtr) - {} -}; - -template <EDataSlot Slot, bool HasKey, bool IsTop> -class TTopDeserializeData : public TBoxedValue { -private: + auto resource = GetTopResource<HasKey, IsTop>(args[0]); + return resource->Get()->Serialize(valueBuilder); + } + +public: + explicit TTopSerialize(ICompare::TPtr) + {} +}; + +template <EDataSlot Slot, bool HasKey, bool IsTop> +class TTopDeserializeData : public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return TUnboxedValuePod(new TTopResourceData<Slot, HasKey, IsTop>(args[0])); - } -}; - -template <bool HasKey, bool IsTop> -class TTopDeserialize : public TBoxedValue { -private: + return TUnboxedValuePod(new TTopResourceData<Slot, HasKey, IsTop>(args[0])); + } +}; + +template <bool HasKey, bool IsTop> +class TTopDeserialize : public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - return TUnboxedValuePod(new TTopResource<HasKey, IsTop>(args[0], Compare_)); - } - -public: - explicit TTopDeserialize(ICompare::TPtr compare) - : Compare_(compare) - {} - -private: - ICompare::TPtr Compare_; -}; - -template <EDataSlot Slot, bool HasKey, bool IsTop> -class TTopMergeData : public TBoxedValue { -private: + return TUnboxedValuePod(new TTopResource<HasKey, IsTop>(args[0], Compare_)); + } + +public: + explicit TTopDeserialize(ICompare::TPtr compare) + : Compare_(compare) + {} + +private: + ICompare::TPtr Compare_; +}; + +template <EDataSlot Slot, bool HasKey, bool IsTop> +class TTopMergeData : public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - auto left = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); - auto right = GetTopResourceData<Slot, HasKey, IsTop>(args[1]); - return TUnboxedValuePod(new TTopResourceData<Slot, HasKey, IsTop>(*left->Get(), *right->Get())); - } -}; - -template <bool HasKey, bool IsTop> -class TTopMerge : public TBoxedValue { -private: + auto left = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); + auto right = GetTopResourceData<Slot, HasKey, IsTop>(args[1]); + return TUnboxedValuePod(new TTopResourceData<Slot, HasKey, IsTop>(*left->Get(), *right->Get())); + } +}; + +template <bool HasKey, bool IsTop> +class TTopMerge : public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override { - auto left = GetTopResource<HasKey, IsTop>(args[0]); - auto right = GetTopResource<HasKey, IsTop>(args[1]); - return TUnboxedValuePod(new TTopResource<HasKey, IsTop>(*left->Get(), *right->Get(), Compare_)); - } - -public: - explicit TTopMerge(ICompare::TPtr compare) - : Compare_(compare) - {} - -private: - ICompare::TPtr Compare_; -}; - -template <EDataSlot Slot, bool HasKey, bool IsTop> -class TTopGetResultData : public TBoxedValue { -private: + auto left = GetTopResource<HasKey, IsTop>(args[0]); + auto right = GetTopResource<HasKey, IsTop>(args[1]); + return TUnboxedValuePod(new TTopResource<HasKey, IsTop>(*left->Get(), *right->Get(), Compare_)); + } + +public: + explicit TTopMerge(ICompare::TPtr compare) + : Compare_(compare) + {} + +private: + ICompare::TPtr Compare_; +}; + +template <EDataSlot Slot, bool HasKey, bool IsTop> +class TTopGetResultData : public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); - return resource->Get()->GetResult(valueBuilder); - } -}; - -template <bool HasKey, bool IsTop> -class TTopGetResult : public TBoxedValue { -private: + auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]); + return resource->Get()->GetResult(valueBuilder); + } +}; + +template <bool HasKey, bool IsTop> +class TTopGetResult : public TBoxedValue { +private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - auto resource = GetTopResource<HasKey, IsTop>(args[0]); - return resource->Get()->GetResult(valueBuilder); - } - -public: - explicit TTopGetResult(ICompare::TPtr) - {} -}; - - -#define RESOURCE(slot, hasKey, isTop) \ -extern const char TopResourceName_##slot##_##hasKey##_##isTop[] = \ - "Top.TopResource."#slot"."#hasKey"."#isTop; \ -template <> \ -class TTopResourceData<EDataSlot::slot, hasKey, isTop>: \ - public TBoxedResource< \ - TTopKeeperDataWrapper<EDataSlot::slot, hasKey, isTop>, \ - TopResourceName_##slot##_##hasKey##_##isTop> \ -{ \ -public: \ - template <typename... Args> \ - inline TTopResourceData(Args&&... args) \ - : TBoxedResource(std::forward<Args>(args)...) \ - {} \ -}; - -#define RESOURCE_00(slot, ...) RESOURCE(slot, false, false) -#define RESOURCE_01(slot, ...) RESOURCE(slot, false, true) -#define RESOURCE_10(slot, ...) RESOURCE(slot, true, false) -#define RESOURCE_11(slot, ...) RESOURCE(slot, true, true) - -UDF_TYPE_ID_MAP(RESOURCE_00) -UDF_TYPE_ID_MAP(RESOURCE_01) -UDF_TYPE_ID_MAP(RESOURCE_10) -UDF_TYPE_ID_MAP(RESOURCE_11) - -#define MAKE_IMPL(operation, slot, hasKey, isTop) \ - case EDataSlot::slot: \ - builder.Implementation(new operation<EDataSlot::slot, hasKey, isTop>); \ - break; - -#define CREATE_00(slot, ...) MAKE_IMPL(TTopCreateData, slot, false, false) -#define CREATE_01(slot, ...) MAKE_IMPL(TTopCreateData, slot, false, true) -#define CREATE_10(slot, ...) MAKE_IMPL(TTopCreateData, slot, true, false) -#define CREATE_11(slot, ...) MAKE_IMPL(TTopCreateData, slot, true, true) - -#define ADD_VALUE_00(slot, ...) MAKE_IMPL(TTopAddValueData, slot, false, false) -#define ADD_VALUE_01(slot, ...) MAKE_IMPL(TTopAddValueData, slot, false, true) -#define ADD_VALUE_10(slot, ...) MAKE_IMPL(TTopAddValueData, slot, true, false) -#define ADD_VALUE_11(slot, ...) MAKE_IMPL(TTopAddValueData, slot, true, true) - -#define MERGE_00(slot, ...) MAKE_IMPL(TTopMergeData, slot, false, false) -#define MERGE_01(slot, ...) MAKE_IMPL(TTopMergeData, slot, false, true) -#define MERGE_10(slot, ...) MAKE_IMPL(TTopMergeData, slot, true, false) -#define MERGE_11(slot, ...) MAKE_IMPL(TTopMergeData, slot, true, true) - -#define SERIALIZE_00(slot, ...) MAKE_IMPL(TTopSerializeData, slot, false, false) -#define SERIALIZE_01(slot, ...) MAKE_IMPL(TTopSerializeData, slot, false, true) -#define SERIALIZE_10(slot, ...) MAKE_IMPL(TTopSerializeData, slot, true, false) -#define SERIALIZE_11(slot, ...) MAKE_IMPL(TTopSerializeData, slot, true, true) - -#define DESERIALIZE_00(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, false, false) -#define DESERIALIZE_01(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, false, true) -#define DESERIALIZE_10(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, true, false) -#define DESERIALIZE_11(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, true, true) - -#define GET_RESULT_00(slot, ...) MAKE_IMPL(TTopGetResultData, slot, false, false) -#define GET_RESULT_01(slot, ...) MAKE_IMPL(TTopGetResultData, slot, false, true) -#define GET_RESULT_10(slot, ...) MAKE_IMPL(TTopGetResultData, slot, true, false) -#define GET_RESULT_11(slot, ...) MAKE_IMPL(TTopGetResultData, slot, true, true) - -#define MAKE_TYPE(slot, hasKey, isTop) \ - case EDataSlot::slot: \ - topType = builder.Resource(TopResourceName_##slot##_##hasKey##_##isTop); \ - break; - -#define TYPE_00(slot, ...) MAKE_TYPE(slot, false, false) -#define TYPE_01(slot, ...) MAKE_TYPE(slot, false, true) -#define TYPE_10(slot, ...) MAKE_TYPE(slot, true, false) -#define TYPE_11(slot, ...) MAKE_TYPE(slot, true, true) - -#define PARAMETRIZE(action) \ - if (hasKey) { \ - if (isTop) { \ - switch (*slot) { \ - UDF_TYPE_ID_MAP(action##_11) \ - } \ - } else { \ - switch (*slot) { \ - UDF_TYPE_ID_MAP(action##_10) \ - } \ - } \ - } else { \ - if (isTop) { \ - switch (*slot) { \ - UDF_TYPE_ID_MAP(action##_01) \ - } \ - } else { \ - switch (*slot) { \ - UDF_TYPE_ID_MAP(action##_00) \ - } \ - } \ - } - - -#define RESOURCE_GENERIC(hasKey, isTop) \ -extern const char TopResourceName_Generic_##hasKey##_##isTop[] = \ - "Top.TopResource.Generic."#hasKey"."#isTop; \ -template <> \ -class TTopResource<hasKey, isTop>: \ - public TBoxedResource< \ - TTopKeeperWrapper<hasKey, isTop>, \ - TopResourceName_Generic_##hasKey##_##isTop> \ -{ \ -public: \ - template <typename... Args> \ - inline TTopResource(Args&&... args) \ - : TBoxedResource(std::forward<Args>(args)...) \ - {} \ -}; - -RESOURCE_GENERIC(false, false) -RESOURCE_GENERIC(false, true) -RESOURCE_GENERIC(true, false) -RESOURCE_GENERIC(true, true) - -#define MAKE_IMPL_GENERIC(operation, hasKey, isTop) \ - builder.Implementation(new operation<hasKey, isTop>(compare)); - -#define CREATE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopCreate, hasKey, isTop) -#define ADD_VALUE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopAddValue, hasKey, isTop) -#define MERGE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopMerge, hasKey, isTop) -#define SERIALIZE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopSerialize, hasKey, isTop) -#define DESERIALIZE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopDeserialize, hasKey, isTop) -#define GET_RESULT_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopGetResult, hasKey, isTop) - -#define TYPE_GENERIC(hasKey, isTop) \ - topType = builder.Resource(TopResourceName_Generic_##hasKey##_##isTop); - -#define PARAMETRIZE_GENERIC(action) \ - if (hasKey) { \ - if (isTop) { \ - action(true, true) \ - } else { \ - action(true, false) \ - } \ - } else { \ - if (isTop) { \ - action(false, true) \ - } else { \ - action(false, false) \ - } \ - } - - -static const auto CreateName = TStringRef::Of("Create"); -static const auto AddValueName = TStringRef::Of("AddValue"); -static const auto SerializeName = TStringRef::Of("Serialize"); -static const auto DeserializeName = TStringRef::Of("Deserialize"); -static const auto MergeName = TStringRef::Of("Merge"); -static const auto GetResultName = TStringRef::Of("GetResult"); - -class TTopModule : public IUdfModule { -public: - TStringRef Name() const { - return TStringRef::Of("Top"); - } - - void CleanupOnTerminate() const final { - } - - void GetAllFunctions(IFunctionsSink& sink) const final { - sink.Add(CreateName)->SetTypeAwareness(); - sink.Add(AddValueName)->SetTypeAwareness(); - sink.Add(SerializeName)->SetTypeAwareness(); - sink.Add(DeserializeName)->SetTypeAwareness(); - sink.Add(MergeName)->SetTypeAwareness(); - sink.Add(GetResultName)->SetTypeAwareness(); - } - - void BuildFunctionTypeInfo( - const TStringRef& name, - TType* userType, - const TStringRef& typeConfig, - ui32 flags, - IFunctionTypeInfoBuilder& builder) const final - { - Y_UNUSED(typeConfig); - - try { - bool typesOnly = (flags & TFlags::TypesOnly); - builder.UserType(userType); - - if (typeConfig.Size() != 2) { - builder.SetError(TStringBuilder() << "Invalid type config: " << typeConfig.Data()); - return; - } - - bool hasKey = (typeConfig.Data()[0] == '1'); - bool isTop = (typeConfig.Data()[1] == '1'); - - auto typeHelper = builder.TypeInfoHelper(); - - auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); - if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) { - builder.SetError("User type is not a 3-tuple"); - return; - } - - auto valueType = userTypeInspector.GetElementType(2); - auto keyType = valueType; - auto payloadType = valueType; - - if (hasKey) { - auto keyPayloadTypeInspector = TTupleTypeInspector(*typeHelper, valueType); - if (!keyPayloadTypeInspector || keyPayloadTypeInspector.GetElementsCount() != 2) { - builder.SetError("Key/payload type is not a 2-tuple"); - return; - } - - keyType = keyPayloadTypeInspector.GetElementType(0); - payloadType = keyPayloadTypeInspector.GetElementType(1); - } - - bool isGeneric = false; - ICompare::TPtr compare; - TMaybe<EDataSlot> slot; - - auto keyTypeInspector = TDataTypeInspector(*typeHelper, keyType); - if (!keyTypeInspector) { - isGeneric = true; - compare = builder.MakeCompare(keyType); - if (!compare) { - return; - } - } else { - slot = FindDataSlot(keyTypeInspector.GetTypeId()); - if (!slot) { - builder.SetError("Unknown data type"); - return; - } - if (!(GetDataTypeInfo(*slot).Features & NUdf::CanCompare)) { - builder.SetError("Data type is not comparable"); - return; - } - } - - auto serializedListType = builder.List()->Item(valueType).Build(); - auto serializedType = builder.Tuple()->Add<ui32>().Add(serializedListType).Build(); - - TType* topType = nullptr; - if (isGeneric) { - PARAMETRIZE_GENERIC(TYPE_GENERIC) - } else { - PARAMETRIZE(TYPE) - } - - if (name == CreateName) { - if (hasKey) { - builder.Args()->Add(keyType).Add(payloadType).Add<ui32>().Done().Returns(topType); - } else { - builder.Args()->Add(valueType).Add<ui32>().Done().Returns(topType); - } - - if (!typesOnly) { - if (isGeneric) { - PARAMETRIZE_GENERIC(CREATE_GENERIC) - } else { - PARAMETRIZE(CREATE) - } - } - } - - if (name == AddValueName) { - if (hasKey) { - builder.Args()->Add(topType).Add(keyType).Add(payloadType).Done().Returns(topType); - } else { - builder.Args()->Add(topType).Add(valueType).Done().Returns(topType); - } - - if (!typesOnly) { - if (isGeneric) { - PARAMETRIZE_GENERIC(ADD_VALUE_GENERIC) - } else { - PARAMETRIZE(ADD_VALUE) - } - } - } - - if (name == SerializeName) { - builder.Args()->Add(topType).Done().Returns(serializedType); - - if (!typesOnly) { - if (isGeneric) { - PARAMETRIZE_GENERIC(SERIALIZE_GENERIC) - } else { - PARAMETRIZE(SERIALIZE) - } - } - } - - if (name == DeserializeName) { - builder.Args()->Add(serializedType).Done().Returns(topType); - - if (!typesOnly) { - if (isGeneric) { - PARAMETRIZE_GENERIC(DESERIALIZE_GENERIC) - } else { - PARAMETRIZE(DESERIALIZE) - } - } - } - - if (name == MergeName) { - builder.Args()->Add(topType).Add(topType).Done().Returns(topType); - - if (!typesOnly) { - if (isGeneric) { - PARAMETRIZE_GENERIC(MERGE_GENERIC) - } else { - PARAMETRIZE(MERGE) - } - } - } - - if (name == GetResultName) { - auto listType = builder.List()->Item(payloadType).Build(); - - builder.Args()->Add(topType).Done().Returns(listType); - - if (!typesOnly) { - if (isGeneric) { - PARAMETRIZE_GENERIC(GET_RESULT_GENERIC) - } else { - PARAMETRIZE(GET_RESULT) - } - } - } - - } catch (const std::exception& e) { - builder.SetError(CurrentExceptionMessage()); - } - } -}; - -} // namespace - -REGISTER_MODULES(TTopModule) - + auto resource = GetTopResource<HasKey, IsTop>(args[0]); + return resource->Get()->GetResult(valueBuilder); + } + +public: + explicit TTopGetResult(ICompare::TPtr) + {} +}; + + +#define RESOURCE(slot, hasKey, isTop) \ +extern const char TopResourceName_##slot##_##hasKey##_##isTop[] = \ + "Top.TopResource."#slot"."#hasKey"."#isTop; \ +template <> \ +class TTopResourceData<EDataSlot::slot, hasKey, isTop>: \ + public TBoxedResource< \ + TTopKeeperDataWrapper<EDataSlot::slot, hasKey, isTop>, \ + TopResourceName_##slot##_##hasKey##_##isTop> \ +{ \ +public: \ + template <typename... Args> \ + inline TTopResourceData(Args&&... args) \ + : TBoxedResource(std::forward<Args>(args)...) \ + {} \ +}; + +#define RESOURCE_00(slot, ...) RESOURCE(slot, false, false) +#define RESOURCE_01(slot, ...) RESOURCE(slot, false, true) +#define RESOURCE_10(slot, ...) RESOURCE(slot, true, false) +#define RESOURCE_11(slot, ...) RESOURCE(slot, true, true) + +UDF_TYPE_ID_MAP(RESOURCE_00) +UDF_TYPE_ID_MAP(RESOURCE_01) +UDF_TYPE_ID_MAP(RESOURCE_10) +UDF_TYPE_ID_MAP(RESOURCE_11) + +#define MAKE_IMPL(operation, slot, hasKey, isTop) \ + case EDataSlot::slot: \ + builder.Implementation(new operation<EDataSlot::slot, hasKey, isTop>); \ + break; + +#define CREATE_00(slot, ...) MAKE_IMPL(TTopCreateData, slot, false, false) +#define CREATE_01(slot, ...) MAKE_IMPL(TTopCreateData, slot, false, true) +#define CREATE_10(slot, ...) MAKE_IMPL(TTopCreateData, slot, true, false) +#define CREATE_11(slot, ...) MAKE_IMPL(TTopCreateData, slot, true, true) + +#define ADD_VALUE_00(slot, ...) MAKE_IMPL(TTopAddValueData, slot, false, false) +#define ADD_VALUE_01(slot, ...) MAKE_IMPL(TTopAddValueData, slot, false, true) +#define ADD_VALUE_10(slot, ...) MAKE_IMPL(TTopAddValueData, slot, true, false) +#define ADD_VALUE_11(slot, ...) MAKE_IMPL(TTopAddValueData, slot, true, true) + +#define MERGE_00(slot, ...) MAKE_IMPL(TTopMergeData, slot, false, false) +#define MERGE_01(slot, ...) MAKE_IMPL(TTopMergeData, slot, false, true) +#define MERGE_10(slot, ...) MAKE_IMPL(TTopMergeData, slot, true, false) +#define MERGE_11(slot, ...) MAKE_IMPL(TTopMergeData, slot, true, true) + +#define SERIALIZE_00(slot, ...) MAKE_IMPL(TTopSerializeData, slot, false, false) +#define SERIALIZE_01(slot, ...) MAKE_IMPL(TTopSerializeData, slot, false, true) +#define SERIALIZE_10(slot, ...) MAKE_IMPL(TTopSerializeData, slot, true, false) +#define SERIALIZE_11(slot, ...) MAKE_IMPL(TTopSerializeData, slot, true, true) + +#define DESERIALIZE_00(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, false, false) +#define DESERIALIZE_01(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, false, true) +#define DESERIALIZE_10(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, true, false) +#define DESERIALIZE_11(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, true, true) + +#define GET_RESULT_00(slot, ...) MAKE_IMPL(TTopGetResultData, slot, false, false) +#define GET_RESULT_01(slot, ...) MAKE_IMPL(TTopGetResultData, slot, false, true) +#define GET_RESULT_10(slot, ...) MAKE_IMPL(TTopGetResultData, slot, true, false) +#define GET_RESULT_11(slot, ...) MAKE_IMPL(TTopGetResultData, slot, true, true) + +#define MAKE_TYPE(slot, hasKey, isTop) \ + case EDataSlot::slot: \ + topType = builder.Resource(TopResourceName_##slot##_##hasKey##_##isTop); \ + break; + +#define TYPE_00(slot, ...) MAKE_TYPE(slot, false, false) +#define TYPE_01(slot, ...) MAKE_TYPE(slot, false, true) +#define TYPE_10(slot, ...) MAKE_TYPE(slot, true, false) +#define TYPE_11(slot, ...) MAKE_TYPE(slot, true, true) + +#define PARAMETRIZE(action) \ + if (hasKey) { \ + if (isTop) { \ + switch (*slot) { \ + UDF_TYPE_ID_MAP(action##_11) \ + } \ + } else { \ + switch (*slot) { \ + UDF_TYPE_ID_MAP(action##_10) \ + } \ + } \ + } else { \ + if (isTop) { \ + switch (*slot) { \ + UDF_TYPE_ID_MAP(action##_01) \ + } \ + } else { \ + switch (*slot) { \ + UDF_TYPE_ID_MAP(action##_00) \ + } \ + } \ + } + + +#define RESOURCE_GENERIC(hasKey, isTop) \ +extern const char TopResourceName_Generic_##hasKey##_##isTop[] = \ + "Top.TopResource.Generic."#hasKey"."#isTop; \ +template <> \ +class TTopResource<hasKey, isTop>: \ + public TBoxedResource< \ + TTopKeeperWrapper<hasKey, isTop>, \ + TopResourceName_Generic_##hasKey##_##isTop> \ +{ \ +public: \ + template <typename... Args> \ + inline TTopResource(Args&&... args) \ + : TBoxedResource(std::forward<Args>(args)...) \ + {} \ +}; + +RESOURCE_GENERIC(false, false) +RESOURCE_GENERIC(false, true) +RESOURCE_GENERIC(true, false) +RESOURCE_GENERIC(true, true) + +#define MAKE_IMPL_GENERIC(operation, hasKey, isTop) \ + builder.Implementation(new operation<hasKey, isTop>(compare)); + +#define CREATE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopCreate, hasKey, isTop) +#define ADD_VALUE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopAddValue, hasKey, isTop) +#define MERGE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopMerge, hasKey, isTop) +#define SERIALIZE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopSerialize, hasKey, isTop) +#define DESERIALIZE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopDeserialize, hasKey, isTop) +#define GET_RESULT_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopGetResult, hasKey, isTop) + +#define TYPE_GENERIC(hasKey, isTop) \ + topType = builder.Resource(TopResourceName_Generic_##hasKey##_##isTop); + +#define PARAMETRIZE_GENERIC(action) \ + if (hasKey) { \ + if (isTop) { \ + action(true, true) \ + } else { \ + action(true, false) \ + } \ + } else { \ + if (isTop) { \ + action(false, true) \ + } else { \ + action(false, false) \ + } \ + } + + +static const auto CreateName = TStringRef::Of("Create"); +static const auto AddValueName = TStringRef::Of("AddValue"); +static const auto SerializeName = TStringRef::Of("Serialize"); +static const auto DeserializeName = TStringRef::Of("Deserialize"); +static const auto MergeName = TStringRef::Of("Merge"); +static const auto GetResultName = TStringRef::Of("GetResult"); + +class TTopModule : public IUdfModule { +public: + TStringRef Name() const { + return TStringRef::Of("Top"); + } + + void CleanupOnTerminate() const final { + } + + void GetAllFunctions(IFunctionsSink& sink) const final { + sink.Add(CreateName)->SetTypeAwareness(); + sink.Add(AddValueName)->SetTypeAwareness(); + sink.Add(SerializeName)->SetTypeAwareness(); + sink.Add(DeserializeName)->SetTypeAwareness(); + sink.Add(MergeName)->SetTypeAwareness(); + sink.Add(GetResultName)->SetTypeAwareness(); + } + + void BuildFunctionTypeInfo( + const TStringRef& name, + TType* userType, + const TStringRef& typeConfig, + ui32 flags, + IFunctionTypeInfoBuilder& builder) const final + { + Y_UNUSED(typeConfig); + + try { + bool typesOnly = (flags & TFlags::TypesOnly); + builder.UserType(userType); + + if (typeConfig.Size() != 2) { + builder.SetError(TStringBuilder() << "Invalid type config: " << typeConfig.Data()); + return; + } + + bool hasKey = (typeConfig.Data()[0] == '1'); + bool isTop = (typeConfig.Data()[1] == '1'); + + auto typeHelper = builder.TypeInfoHelper(); + + auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); + if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) { + builder.SetError("User type is not a 3-tuple"); + return; + } + + auto valueType = userTypeInspector.GetElementType(2); + auto keyType = valueType; + auto payloadType = valueType; + + if (hasKey) { + auto keyPayloadTypeInspector = TTupleTypeInspector(*typeHelper, valueType); + if (!keyPayloadTypeInspector || keyPayloadTypeInspector.GetElementsCount() != 2) { + builder.SetError("Key/payload type is not a 2-tuple"); + return; + } + + keyType = keyPayloadTypeInspector.GetElementType(0); + payloadType = keyPayloadTypeInspector.GetElementType(1); + } + + bool isGeneric = false; + ICompare::TPtr compare; + TMaybe<EDataSlot> slot; + + auto keyTypeInspector = TDataTypeInspector(*typeHelper, keyType); + if (!keyTypeInspector) { + isGeneric = true; + compare = builder.MakeCompare(keyType); + if (!compare) { + return; + } + } else { + slot = FindDataSlot(keyTypeInspector.GetTypeId()); + if (!slot) { + builder.SetError("Unknown data type"); + return; + } + if (!(GetDataTypeInfo(*slot).Features & NUdf::CanCompare)) { + builder.SetError("Data type is not comparable"); + return; + } + } + + auto serializedListType = builder.List()->Item(valueType).Build(); + auto serializedType = builder.Tuple()->Add<ui32>().Add(serializedListType).Build(); + + TType* topType = nullptr; + if (isGeneric) { + PARAMETRIZE_GENERIC(TYPE_GENERIC) + } else { + PARAMETRIZE(TYPE) + } + + if (name == CreateName) { + if (hasKey) { + builder.Args()->Add(keyType).Add(payloadType).Add<ui32>().Done().Returns(topType); + } else { + builder.Args()->Add(valueType).Add<ui32>().Done().Returns(topType); + } + + if (!typesOnly) { + if (isGeneric) { + PARAMETRIZE_GENERIC(CREATE_GENERIC) + } else { + PARAMETRIZE(CREATE) + } + } + } + + if (name == AddValueName) { + if (hasKey) { + builder.Args()->Add(topType).Add(keyType).Add(payloadType).Done().Returns(topType); + } else { + builder.Args()->Add(topType).Add(valueType).Done().Returns(topType); + } + + if (!typesOnly) { + if (isGeneric) { + PARAMETRIZE_GENERIC(ADD_VALUE_GENERIC) + } else { + PARAMETRIZE(ADD_VALUE) + } + } + } + + if (name == SerializeName) { + builder.Args()->Add(topType).Done().Returns(serializedType); + + if (!typesOnly) { + if (isGeneric) { + PARAMETRIZE_GENERIC(SERIALIZE_GENERIC) + } else { + PARAMETRIZE(SERIALIZE) + } + } + } + + if (name == DeserializeName) { + builder.Args()->Add(serializedType).Done().Returns(topType); + + if (!typesOnly) { + if (isGeneric) { + PARAMETRIZE_GENERIC(DESERIALIZE_GENERIC) + } else { + PARAMETRIZE(DESERIALIZE) + } + } + } + + if (name == MergeName) { + builder.Args()->Add(topType).Add(topType).Done().Returns(topType); + + if (!typesOnly) { + if (isGeneric) { + PARAMETRIZE_GENERIC(MERGE_GENERIC) + } else { + PARAMETRIZE(MERGE) + } + } + } + + if (name == GetResultName) { + auto listType = builder.List()->Item(payloadType).Build(); + + builder.Args()->Add(topType).Done().Returns(listType); + + if (!typesOnly) { + if (isGeneric) { + PARAMETRIZE_GENERIC(GET_RESULT_GENERIC) + } else { + PARAMETRIZE(GET_RESULT) + } + } + } + + } catch (const std::exception& e) { + builder.SetError(CurrentExceptionMessage()); + } + } +}; + +} // namespace + +REGISTER_MODULES(TTopModule) + diff --git a/ydb/library/yql/udfs/common/top/ya.make b/ydb/library/yql/udfs/common/top/ya.make index 1daeb15eba..e99fe77c36 100644 --- a/ydb/library/yql/udfs/common/top/ya.make +++ b/ydb/library/yql/udfs/common/top/ya.make @@ -1,19 +1,19 @@ -YQL_UDF(top_udf) - -YQL_ABI_VERSION( - 2 - 10 - 0 -) - +YQL_UDF(top_udf) + +YQL_ABI_VERSION( + 2 + 10 + 0 +) + OWNER(g:yql g:yql_ydb_core) - -SRCS( - top_udf.cpp -) - -PEERDIR( + +SRCS( + top_udf.cpp +) + +PEERDIR( library/cpp/containers/top_keeper -) - -END() +) + +END() diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp b/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp index 1621bf5162..31fcd88b54 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp +++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp @@ -5,50 +5,50 @@ using namespace NKikimr; using namespace NUdf; -template <typename THash, typename TEquals> -TTopFreqBase<THash, TEquals>::TTopFreqBase(THash hash, TEquals equals) - : Indices_(0, hash, equals) -{} - -template <typename THash, typename TEquals> -void TTopFreqBase<THash, TEquals>::Init(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize) { - MinSize_ = minSize; - MaxSize_ = maxSize; - +template <typename THash, typename TEquals> +TTopFreqBase<THash, TEquals>::TTopFreqBase(THash hash, TEquals equals) + : Indices_(0, hash, equals) +{} + +template <typename THash, typename TEquals> +void TTopFreqBase<THash, TEquals>::Init(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize) { + MinSize_ = minSize; + MaxSize_ = maxSize; + Freqs_.reserve(MaxSize_ + 1); Indices_.reserve(MaxSize_ + 1); - - AddValue(value); -} - -template <typename THash, typename TEquals> -void TTopFreqBase<THash, TEquals>::Merge(const TTopFreqBase& topFreq1, const TTopFreqBase& topFreq2) { - MinSize_ = std::max(topFreq1.MinSize_, topFreq2.MinSize_); - MaxSize_ = std::max(topFreq1.MaxSize_, topFreq2.MaxSize_); - - Freqs_.reserve(std::max(MaxSize_ + 1, ui32(topFreq1.Freqs_.size() + topFreq2.Freqs_.size()))); - Indices_.reserve(MaxSize_ + 1); - - Add(topFreq1); - Add(topFreq2); -} - -template <typename THash, typename TEquals> -void TTopFreqBase<THash, TEquals>::Deserialize(const TUnboxedValuePod& serialized) { - MinSize_ = serialized.GetElement(0).Get<ui32>(); - MaxSize_ = serialized.GetElement(1).Get<ui32>(); - + + AddValue(value); +} + +template <typename THash, typename TEquals> +void TTopFreqBase<THash, TEquals>::Merge(const TTopFreqBase& topFreq1, const TTopFreqBase& topFreq2) { + MinSize_ = std::max(topFreq1.MinSize_, topFreq2.MinSize_); + MaxSize_ = std::max(topFreq1.MaxSize_, topFreq2.MaxSize_); + + Freqs_.reserve(std::max(MaxSize_ + 1, ui32(topFreq1.Freqs_.size() + topFreq2.Freqs_.size()))); + Indices_.reserve(MaxSize_ + 1); + + Add(topFreq1); + Add(topFreq2); +} + +template <typename THash, typename TEquals> +void TTopFreqBase<THash, TEquals>::Deserialize(const TUnboxedValuePod& serialized) { + MinSize_ = serialized.GetElement(0).Get<ui32>(); + MaxSize_ = serialized.GetElement(1).Get<ui32>(); + Freqs_.reserve(MaxSize_ + 1); Indices_.reserve(MaxSize_ + 1); - - const auto listIter = serialized.GetElement(2).GetListIterator(); + + const auto listIter = serialized.GetElement(2).GetListIterator(); for (TUnboxedValue current; listIter.Next(current);) { Update(current.GetElement(1), current.GetElement(0).Get<ui64>()); } } -template <typename THash, typename TEquals> -TUnboxedValue TTopFreqBase<THash, TEquals>::Convert(const IValueBuilder* valueBuilder) const { +template <typename THash, typename TEquals> +TUnboxedValue TTopFreqBase<THash, TEquals>::Convert(const IValueBuilder* valueBuilder) const { TUnboxedValue* values = nullptr; const auto list = valueBuilder->NewArray(Freqs_.size(), values); for (const auto& item : Freqs_) { @@ -60,8 +60,8 @@ TUnboxedValue TTopFreqBase<THash, TEquals>::Convert(const IValueBuilder* valueBu return list; } -template <typename THash, typename TEquals> -void TTopFreqBase<THash, TEquals>::Add(const TTopFreqBase& otherModeCalc) { +template <typename THash, typename TEquals> +void TTopFreqBase<THash, TEquals>::Add(const TTopFreqBase& otherModeCalc) { for (auto& it : otherModeCalc.Freqs_) { Update(it.first, it.second); } @@ -69,23 +69,23 @@ void TTopFreqBase<THash, TEquals>::Add(const TTopFreqBase& otherModeCalc) { TryCompress(); } -template <typename THash, typename TEquals> -TUnboxedValue TTopFreqBase<THash, TEquals>::Get(const IValueBuilder* builder, ui32 resultSize) { +template <typename THash, typename TEquals> +TUnboxedValue TTopFreqBase<THash, TEquals>::Get(const IValueBuilder* builder, ui32 resultSize) { resultSize = std::min(resultSize, ui32(Freqs_.size())); Compress(resultSize, true); return Convert(builder); } -template <typename THash, typename TEquals> -void TTopFreqBase<THash, TEquals>::AddValue(const TUnboxedValuePod& value) { +template <typename THash, typename TEquals> +void TTopFreqBase<THash, TEquals>::AddValue(const TUnboxedValuePod& value) { Update(value, 1); TryCompress(); } -template <typename THash, typename TEquals> -void TTopFreqBase<THash, TEquals>::Update(const TUnboxedValuePod& value, ui64 freq) { +template <typename THash, typename TEquals> +void TTopFreqBase<THash, TEquals>::Update(const TUnboxedValuePod& value, ui64 freq) { Freqs_.emplace_back(TUnboxedValuePod(value), freq); - auto mapInsertResult = Indices_.emplace(TUnboxedValuePod(value), Freqs_.size() - 1); + auto mapInsertResult = Indices_.emplace(TUnboxedValuePod(value), Freqs_.size() - 1); if (!mapInsertResult.second) { Freqs_[mapInsertResult.first->second].second += freq; @@ -93,16 +93,16 @@ void TTopFreqBase<THash, TEquals>::Update(const TUnboxedValuePod& value, ui64 fr } } -template <typename THash, typename TEquals> -void TTopFreqBase<THash, TEquals>::TryCompress() { +template <typename THash, typename TEquals> +void TTopFreqBase<THash, TEquals>::TryCompress() { ui32 freqSize = Freqs_.size(); if (freqSize > MaxSize_) { Compress(MinSize_); } } -template <typename THash, typename TEquals> -void TTopFreqBase<THash, TEquals>::Compress(ui32 newSize, bool sort) { +template <typename THash, typename TEquals> +void TTopFreqBase<THash, TEquals>::Compress(ui32 newSize, bool sort) { auto compare = [](const TVectorElement& v1, const TVectorElement& v2) { return v1.second > v2.second; }; @@ -117,12 +117,12 @@ void TTopFreqBase<THash, TEquals>::Compress(ui32 newSize, bool sort) { Freqs_.resize(newSize); for (ui32 i = 0; i < newSize; i++) { - Indices_[Freqs_[i].first] = i; + Indices_[Freqs_[i].first] = i; } } -template <typename THash, typename TEquals> -TUnboxedValue TTopFreqBase<THash, TEquals>::Serialize(const IValueBuilder* builder) { +template <typename THash, typename TEquals> +TUnboxedValue TTopFreqBase<THash, TEquals>::Serialize(const IValueBuilder* builder) { if (ui32(Freqs_.size()) > MinSize_) { Compress(MinSize_); } @@ -135,79 +135,79 @@ TUnboxedValue TTopFreqBase<THash, TEquals>::Serialize(const IValueBuilder* build return tuple; } -template <EDataSlot Slot> -TTopFreqData<Slot>::TTopFreqData(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize) - : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) -{ - TBase::Init(value, minSize, maxSize); -} - -template <EDataSlot Slot> -TTopFreqData<Slot>::TTopFreqData(const TTopFreqData& topFreq1, const TTopFreqData& topFreq2) - : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) -{ - TBase::Merge(topFreq1, topFreq2); -} - -template <EDataSlot Slot> -TTopFreqData<Slot>::TTopFreqData(const TUnboxedValuePod& serialized) - : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) -{ - TBase::Deserialize(serialized); -} - -template <EDataSlot Slot> -TUnboxedValue TTopFreqData<Slot>::Serialize(const IValueBuilder* builder) { - return TBase::Serialize(builder); -} - -template <EDataSlot Slot> -TUnboxedValue TTopFreqData<Slot>::Get(const IValueBuilder* builder, ui32 resultSize) { - return TBase::Get(builder, resultSize); -} - -template <EDataSlot Slot> -void TTopFreqData<Slot>::AddValue(const TUnboxedValuePod& value) { - TBase::AddValue(value); -} - -#define INSTANCE_FOR(slot, ...) \ - template class TTopFreqData<EDataSlot::slot>; - -UDF_TYPE_ID_MAP(INSTANCE_FOR) - +template <EDataSlot Slot> +TTopFreqData<Slot>::TTopFreqData(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize) + : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) +{ + TBase::Init(value, minSize, maxSize); +} + +template <EDataSlot Slot> +TTopFreqData<Slot>::TTopFreqData(const TTopFreqData& topFreq1, const TTopFreqData& topFreq2) + : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) +{ + TBase::Merge(topFreq1, topFreq2); +} + +template <EDataSlot Slot> +TTopFreqData<Slot>::TTopFreqData(const TUnboxedValuePod& serialized) + : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>()) +{ + TBase::Deserialize(serialized); +} + +template <EDataSlot Slot> +TUnboxedValue TTopFreqData<Slot>::Serialize(const IValueBuilder* builder) { + return TBase::Serialize(builder); +} + +template <EDataSlot Slot> +TUnboxedValue TTopFreqData<Slot>::Get(const IValueBuilder* builder, ui32 resultSize) { + return TBase::Get(builder, resultSize); +} + +template <EDataSlot Slot> +void TTopFreqData<Slot>::AddValue(const TUnboxedValuePod& value) { + TBase::AddValue(value); +} + +#define INSTANCE_FOR(slot, ...) \ + template class TTopFreqData<EDataSlot::slot>; + +UDF_TYPE_ID_MAP(INSTANCE_FOR) + #undef INSTANCE_FOR - -TTopFreqGeneric::TTopFreqGeneric(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize, - IHash::TPtr hash, IEquate::TPtr equate) - : TBase(TGenericHash{hash}, TGenericEquals{equate}) -{ - TBase::Init(value, minSize, maxSize); -} - -TTopFreqGeneric::TTopFreqGeneric(const TTopFreqGeneric& topFreq1, const TTopFreqGeneric& topFreq2, - IHash::TPtr hash, IEquate::TPtr equate) - : TBase(TGenericHash{hash}, TGenericEquals{equate}) -{ - TBase::Merge(topFreq1, topFreq2); -} - -TTopFreqGeneric::TTopFreqGeneric(const TUnboxedValuePod& serialized, - IHash::TPtr hash, IEquate::TPtr equate) - : TBase(TGenericHash{hash}, TGenericEquals{equate}) -{ - TBase::Deserialize(serialized); -} - -TUnboxedValue TTopFreqGeneric::Serialize(const IValueBuilder* builder) { - return TBase::Serialize(builder); -} - -TUnboxedValue TTopFreqGeneric::Get(const IValueBuilder* builder, ui32 resultSize) { - return TBase::Get(builder, resultSize); -} - -void TTopFreqGeneric::AddValue(const TUnboxedValuePod& value) { - TBase::AddValue(value); -} - + +TTopFreqGeneric::TTopFreqGeneric(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize, + IHash::TPtr hash, IEquate::TPtr equate) + : TBase(TGenericHash{hash}, TGenericEquals{equate}) +{ + TBase::Init(value, minSize, maxSize); +} + +TTopFreqGeneric::TTopFreqGeneric(const TTopFreqGeneric& topFreq1, const TTopFreqGeneric& topFreq2, + IHash::TPtr hash, IEquate::TPtr equate) + : TBase(TGenericHash{hash}, TGenericEquals{equate}) +{ + TBase::Merge(topFreq1, topFreq2); +} + +TTopFreqGeneric::TTopFreqGeneric(const TUnboxedValuePod& serialized, + IHash::TPtr hash, IEquate::TPtr equate) + : TBase(TGenericHash{hash}, TGenericEquals{equate}) +{ + TBase::Deserialize(serialized); +} + +TUnboxedValue TTopFreqGeneric::Serialize(const IValueBuilder* builder) { + return TBase::Serialize(builder); +} + +TUnboxedValue TTopFreqGeneric::Get(const IValueBuilder* builder, ui32 resultSize) { + return TBase::Get(builder, resultSize); +} + +void TTopFreqGeneric::AddValue(const TUnboxedValuePod& value) { + TBase::AddValue(value); +} + diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq.h b/ydb/library/yql/udfs/common/topfreq/static/topfreq.h index f329322615..411190d257 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/topfreq.h +++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq.h @@ -6,9 +6,9 @@ #include <unordered_map> -template <typename THash, typename TEquals> -class TTopFreqBase { -protected: +template <typename THash, typename TEquals> +class TTopFreqBase { +protected: using TUnboxedValuePod = NKikimr::NUdf::TUnboxedValuePod; using TUnboxedValue = NKikimr::NUdf::TUnboxedValue; using IValueBuilder = NKikimr::NUdf::IValueBuilder; @@ -21,77 +21,77 @@ protected: ui32 MinSize_ = 0; ui32 MaxSize_ = 0; - void Add(const TTopFreqBase& otherCalc); + void Add(const TTopFreqBase& otherCalc); void Update(const TUnboxedValuePod& key, const ui64 value); void TryCompress(); void Compress(ui32 newSize, bool sort = false); TUnboxedValue Convert(const IValueBuilder* valueBuilder) const; -protected: - TTopFreqBase(THash hash, TEquals equals); +protected: + TTopFreqBase(THash hash, TEquals equals); + + void Init(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize); + void Merge(const TTopFreqBase& TopFreq1, const TTopFreqBase& TopFreq2); + void Deserialize(const TUnboxedValuePod& serialized); - void Init(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize); - void Merge(const TTopFreqBase& TopFreq1, const TTopFreqBase& TopFreq2); - void Deserialize(const TUnboxedValuePod& serialized); - TUnboxedValue Serialize(const IValueBuilder* builder); TUnboxedValue Get(const IValueBuilder* builder, ui32 resultSize); - void AddValue(const TUnboxedValuePod& value); + void AddValue(const TUnboxedValuePod& value); +}; + +template <NKikimr::NUdf::EDataSlot Slot> +class TTopFreqData + : public TTopFreqBase< + NKikimr::NUdf::TUnboxedValueHash<Slot>, + NKikimr::NUdf::TUnboxedValueEquals<Slot>> +{ +public: + using TBase = TTopFreqBase< + NKikimr::NUdf::TUnboxedValueHash<Slot>, + NKikimr::NUdf::TUnboxedValueEquals<Slot>>; + + TTopFreqData(const NKikimr::NUdf::TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize); + TTopFreqData(const TTopFreqData& topFreq1, const TTopFreqData& topFreq2); + TTopFreqData(const NKikimr::NUdf::TUnboxedValuePod& serialized); + + NKikimr::NUdf::TUnboxedValue Serialize(const NKikimr::NUdf::IValueBuilder* builder); + NKikimr::NUdf::TUnboxedValue Get(const NKikimr::NUdf::IValueBuilder* builder, ui32 resultSize); + void AddValue(const NKikimr::NUdf::TUnboxedValuePod& value); +}; + +struct TGenericHash { + NKikimr::NUdf::IHash::TPtr Hash; + + std::size_t operator()(const NKikimr::NUdf::TUnboxedValuePod& value) const { + return Hash->Hash(value); + } +}; + +struct TGenericEquals { + NKikimr::NUdf::IEquate::TPtr Equate; + + bool operator()( + const NKikimr::NUdf::TUnboxedValuePod& left, + const NKikimr::NUdf::TUnboxedValuePod& right) const + { + return Equate->Equals(left, right); + } +}; + +class TTopFreqGeneric + : public TTopFreqBase<TGenericHash, TGenericEquals> +{ +public: + using TBase = TTopFreqBase<TGenericHash, TGenericEquals>; + + TTopFreqGeneric(const NKikimr::NUdf::TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize, + NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate); + TTopFreqGeneric(const TTopFreqGeneric& topFreq1, const TTopFreqGeneric& topFreq2, + NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate); + TTopFreqGeneric(const NKikimr::NUdf::TUnboxedValuePod& serialized, + NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate); + + NKikimr::NUdf::TUnboxedValue Serialize(const NKikimr::NUdf::IValueBuilder* builder); + NKikimr::NUdf::TUnboxedValue Get(const NKikimr::NUdf::IValueBuilder* builder, ui32 resultSize); + void AddValue(const NKikimr::NUdf::TUnboxedValuePod& value); }; - -template <NKikimr::NUdf::EDataSlot Slot> -class TTopFreqData - : public TTopFreqBase< - NKikimr::NUdf::TUnboxedValueHash<Slot>, - NKikimr::NUdf::TUnboxedValueEquals<Slot>> -{ -public: - using TBase = TTopFreqBase< - NKikimr::NUdf::TUnboxedValueHash<Slot>, - NKikimr::NUdf::TUnboxedValueEquals<Slot>>; - - TTopFreqData(const NKikimr::NUdf::TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize); - TTopFreqData(const TTopFreqData& topFreq1, const TTopFreqData& topFreq2); - TTopFreqData(const NKikimr::NUdf::TUnboxedValuePod& serialized); - - NKikimr::NUdf::TUnboxedValue Serialize(const NKikimr::NUdf::IValueBuilder* builder); - NKikimr::NUdf::TUnboxedValue Get(const NKikimr::NUdf::IValueBuilder* builder, ui32 resultSize); - void AddValue(const NKikimr::NUdf::TUnboxedValuePod& value); -}; - -struct TGenericHash { - NKikimr::NUdf::IHash::TPtr Hash; - - std::size_t operator()(const NKikimr::NUdf::TUnboxedValuePod& value) const { - return Hash->Hash(value); - } -}; - -struct TGenericEquals { - NKikimr::NUdf::IEquate::TPtr Equate; - - bool operator()( - const NKikimr::NUdf::TUnboxedValuePod& left, - const NKikimr::NUdf::TUnboxedValuePod& right) const - { - return Equate->Equals(left, right); - } -}; - -class TTopFreqGeneric - : public TTopFreqBase<TGenericHash, TGenericEquals> -{ -public: - using TBase = TTopFreqBase<TGenericHash, TGenericEquals>; - - TTopFreqGeneric(const NKikimr::NUdf::TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize, - NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate); - TTopFreqGeneric(const TTopFreqGeneric& topFreq1, const TTopFreqGeneric& topFreq2, - NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate); - TTopFreqGeneric(const NKikimr::NUdf::TUnboxedValuePod& serialized, - NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate); - - NKikimr::NUdf::TUnboxedValue Serialize(const NKikimr::NUdf::IValueBuilder* builder); - NKikimr::NUdf::TUnboxedValue Get(const NKikimr::NUdf::IValueBuilder* builder, ui32 resultSize); - void AddValue(const NKikimr::NUdf::TUnboxedValuePod& value); -}; diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h b/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h index ba483f2f4a..5ea0a5f729 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h +++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h @@ -15,203 +15,203 @@ using namespace NYql; using namespace NUdf; namespace { - extern const char TopFreqResourceNameGeneric[] = "TopFreq.TopFreqResource.Generic"; - class TTopFreqResource: - public TBoxedResource<TTopFreqGeneric, TopFreqResourceNameGeneric> - { - public: - template <typename... Args> - inline TTopFreqResource(Args&&... args) - : TBoxedResource(std::forward<Args>(args)...) - {} - }; - - template <EDataSlot Slot> - class TTopFreqResourceData; - - template <EDataSlot Slot> - TTopFreqResourceData<Slot>* GetTopFreqResourceData(const TUnboxedValuePod& arg) { - TTopFreqResourceData<Slot>::Validate(arg); - return static_cast<TTopFreqResourceData<Slot>*>(arg.AsBoxed().Get()); + extern const char TopFreqResourceNameGeneric[] = "TopFreq.TopFreqResource.Generic"; + class TTopFreqResource: + public TBoxedResource<TTopFreqGeneric, TopFreqResourceNameGeneric> + { + public: + template <typename... Args> + inline TTopFreqResource(Args&&... args) + : TBoxedResource(std::forward<Args>(args)...) + {} + }; + + template <EDataSlot Slot> + class TTopFreqResourceData; + + template <EDataSlot Slot> + TTopFreqResourceData<Slot>* GetTopFreqResourceData(const TUnboxedValuePod& arg) { + TTopFreqResourceData<Slot>::Validate(arg); + return static_cast<TTopFreqResourceData<Slot>*>(arg.AsBoxed().Get()); } - TTopFreqResource* GetTopFreqResource(const TUnboxedValuePod& arg) { - TTopFreqResource::Validate(arg); - return static_cast<TTopFreqResource*>(arg.AsBoxed().Get()); - } - - - template <EDataSlot Slot> - class TTopFreq_CreateData: public TBoxedValue { - private: - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { - ui32 minSize = args[1].Get<ui32>(); - return TUnboxedValuePod(new TTopFreqResourceData<Slot>(args[0], minSize, minSize * 2)); - } - }; - + TTopFreqResource* GetTopFreqResource(const TUnboxedValuePod& arg) { + TTopFreqResource::Validate(arg); + return static_cast<TTopFreqResource*>(arg.AsBoxed().Get()); + } + + + template <EDataSlot Slot> + class TTopFreq_CreateData: public TBoxedValue { + private: + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { + ui32 minSize = args[1].Get<ui32>(); + return TUnboxedValuePod(new TTopFreqResourceData<Slot>(args[0], minSize, minSize * 2)); + } + }; + class TTopFreq_Create: public TBoxedValue { private: - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { ui32 minSize = args[1].Get<ui32>(); - return TUnboxedValuePod(new TTopFreqResource(args[0], minSize, minSize * 2, Hash_, Equate_)); + return TUnboxedValuePod(new TTopFreqResource(args[0], minSize, minSize * 2, Hash_, Equate_)); } public: - TTopFreq_Create(IHash::TPtr hash, IEquate::TPtr equate) - : Hash_(hash) - , Equate_(equate) - {} - - private: - IHash::TPtr Hash_; - IEquate::TPtr Equate_; - }; - - template <EDataSlot Slot> - class TTopFreq_AddValueData: public TBoxedValue { - private: - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { - const auto topFreq = GetTopFreqResourceData<Slot>(args[0]); - topFreq->Get()->AddValue(args[1]); - return TUnboxedValuePod(topFreq); + TTopFreq_Create(IHash::TPtr hash, IEquate::TPtr equate) + : Hash_(hash) + , Equate_(equate) + {} + + private: + IHash::TPtr Hash_; + IEquate::TPtr Equate_; + }; + + template <EDataSlot Slot> + class TTopFreq_AddValueData: public TBoxedValue { + private: + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { + const auto topFreq = GetTopFreqResourceData<Slot>(args[0]); + topFreq->Get()->AddValue(args[1]); + return TUnboxedValuePod(topFreq); } }; class TTopFreq_AddValue: public TBoxedValue { private: - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { - const auto topFreq = GetTopFreqResource(args[0]); + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { + const auto topFreq = GetTopFreqResource(args[0]); topFreq->Get()->AddValue(args[1]); return TUnboxedValuePod(topFreq); } - }; + }; - template <EDataSlot Slot> - class TTopFreq_SerializeData: public TBoxedValue { - private: - TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - return GetTopFreqResourceData<Slot>(args[0])->Get()->Serialize(valueBuilder); + template <EDataSlot Slot> + class TTopFreq_SerializeData: public TBoxedValue { + private: + TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { + return GetTopFreqResourceData<Slot>(args[0])->Get()->Serialize(valueBuilder); } }; class TTopFreq_Serialize: public TBoxedValue { private: - TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - return GetTopFreqResource(args[0])->Get()->Serialize(valueBuilder); + TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { + return GetTopFreqResource(args[0])->Get()->Serialize(valueBuilder); } - }; + }; - template <EDataSlot Slot> - class TTopFreq_DeserializeData: public TBoxedValue { - private: - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { - return TUnboxedValuePod(new TTopFreqResourceData<Slot>(args[0])); + template <EDataSlot Slot> + class TTopFreq_DeserializeData: public TBoxedValue { + private: + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { + return TUnboxedValuePod(new TTopFreqResourceData<Slot>(args[0])); } }; class TTopFreq_Deserialize: public TBoxedValue { private: - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { - return TUnboxedValuePod(new TTopFreqResource(args[0], Hash_, Equate_)); + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { + return TUnboxedValuePod(new TTopFreqResource(args[0], Hash_, Equate_)); } public: - TTopFreq_Deserialize(IHash::TPtr hash, IEquate::TPtr equate) - : Hash_(hash) - , Equate_(equate) - {} - - private: - IHash::TPtr Hash_; - IEquate::TPtr Equate_; - }; - - template <EDataSlot Slot> - class TTopFreq_MergeData: public TBoxedValue { - private: - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { - const auto topFreq0 = GetTopFreqResourceData<Slot>(args[0]); - const auto topFreq1 = GetTopFreqResourceData<Slot>(args[1]); - return TUnboxedValuePod(new TTopFreqResourceData<Slot>(*topFreq0->Get(), *topFreq1->Get())); + TTopFreq_Deserialize(IHash::TPtr hash, IEquate::TPtr equate) + : Hash_(hash) + , Equate_(equate) + {} + + private: + IHash::TPtr Hash_; + IEquate::TPtr Equate_; + }; + + template <EDataSlot Slot> + class TTopFreq_MergeData: public TBoxedValue { + private: + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { + const auto topFreq0 = GetTopFreqResourceData<Slot>(args[0]); + const auto topFreq1 = GetTopFreqResourceData<Slot>(args[1]); + return TUnboxedValuePod(new TTopFreqResourceData<Slot>(*topFreq0->Get(), *topFreq1->Get())); } }; class TTopFreq_Merge: public TBoxedValue { private: - TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { - const auto topFreq0 = GetTopFreqResource(args[0]); - const auto topFreq1 = GetTopFreqResource(args[1]); - return TUnboxedValuePod(new TTopFreqResource(*topFreq0->Get(), *topFreq1->Get(), Hash_, Equate_)); + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { + const auto topFreq0 = GetTopFreqResource(args[0]); + const auto topFreq1 = GetTopFreqResource(args[1]); + return TUnboxedValuePod(new TTopFreqResource(*topFreq0->Get(), *topFreq1->Get(), Hash_, Equate_)); } public: - TTopFreq_Merge(IHash::TPtr hash, IEquate::TPtr equate) - : Hash_(hash) - , Equate_(equate) - {} - - private: - IHash::TPtr Hash_; - IEquate::TPtr Equate_; + TTopFreq_Merge(IHash::TPtr hash, IEquate::TPtr equate) + : Hash_(hash) + , Equate_(equate) + {} + + private: + IHash::TPtr Hash_; + IEquate::TPtr Equate_; }; - template <EDataSlot Slot> - class TTopFreq_GetData: public TBoxedValue { + template <EDataSlot Slot> + class TTopFreq_GetData: public TBoxedValue { private: - TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - return GetTopFreqResourceData<Slot>(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>()); + TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { + return GetTopFreqResourceData<Slot>(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>()); } - }; + }; - class TTopFreq_Get: public TBoxedValue { - private: - TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - return GetTopFreqResource(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>()); + class TTopFreq_Get: public TBoxedValue { + private: + TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { + return GetTopFreqResource(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>()); } }; - -#define MAKE_RESOURCE(slot, ...) \ - extern const char TopFreqResourceName##slot[] = "TopFreq.TopFreqResource."#slot; \ - template <> \ - class TTopFreqResourceData<EDataSlot::slot>: \ - public TBoxedResource<TTopFreqData<EDataSlot::slot>, TopFreqResourceName##slot> \ - { \ - public: \ - template <typename... Args> \ - inline TTopFreqResourceData(Args&&... args) \ - : TBoxedResource(std::forward<Args>(args)...) \ - {} \ - }; - - UDF_TYPE_ID_MAP(MAKE_RESOURCE) - -#define MAKE_IMPL(operation, slot) \ - case EDataSlot::slot: \ - builder.Implementation(new operation<EDataSlot::slot>); \ - break; - -#define MAKE_CREATE(slot, ...) MAKE_IMPL(TTopFreq_CreateData, slot) -#define MAKE_ADD_VALUE(slot, ...) MAKE_IMPL(TTopFreq_AddValueData, slot) -#define MAKE_SERIALIZE(slot, ...) MAKE_IMPL(TTopFreq_SerializeData, slot) -#define MAKE_DESERIALIZE(slot, ...) MAKE_IMPL(TTopFreq_DeserializeData, slot) -#define MAKE_MERGE(slot, ...) MAKE_IMPL(TTopFreq_MergeData, slot) -#define MAKE_GET(slot, ...) MAKE_IMPL(TTopFreq_GetData, slot) - -#define MAKE_TYPE(slot, ...) \ - case EDataSlot::slot: \ - topFreqType = builder.Resource(TopFreqResourceName##slot); \ - break; - - - static const auto CreateName = TStringRef::Of("TopFreq_Create"); - static const auto AddValueName = TStringRef::Of("TopFreq_AddValue"); - static const auto SerializeName = TStringRef::Of("TopFreq_Serialize"); - static const auto DeserializeName = TStringRef::Of("TopFreq_Deserialize"); - static const auto MergeName = TStringRef::Of("TopFreq_Merge"); - static const auto GetName = TStringRef::Of("TopFreq_Get"); - + +#define MAKE_RESOURCE(slot, ...) \ + extern const char TopFreqResourceName##slot[] = "TopFreq.TopFreqResource."#slot; \ + template <> \ + class TTopFreqResourceData<EDataSlot::slot>: \ + public TBoxedResource<TTopFreqData<EDataSlot::slot>, TopFreqResourceName##slot> \ + { \ + public: \ + template <typename... Args> \ + inline TTopFreqResourceData(Args&&... args) \ + : TBoxedResource(std::forward<Args>(args)...) \ + {} \ + }; + + UDF_TYPE_ID_MAP(MAKE_RESOURCE) + +#define MAKE_IMPL(operation, slot) \ + case EDataSlot::slot: \ + builder.Implementation(new operation<EDataSlot::slot>); \ + break; + +#define MAKE_CREATE(slot, ...) MAKE_IMPL(TTopFreq_CreateData, slot) +#define MAKE_ADD_VALUE(slot, ...) MAKE_IMPL(TTopFreq_AddValueData, slot) +#define MAKE_SERIALIZE(slot, ...) MAKE_IMPL(TTopFreq_SerializeData, slot) +#define MAKE_DESERIALIZE(slot, ...) MAKE_IMPL(TTopFreq_DeserializeData, slot) +#define MAKE_MERGE(slot, ...) MAKE_IMPL(TTopFreq_MergeData, slot) +#define MAKE_GET(slot, ...) MAKE_IMPL(TTopFreq_GetData, slot) + +#define MAKE_TYPE(slot, ...) \ + case EDataSlot::slot: \ + topFreqType = builder.Resource(TopFreqResourceName##slot); \ + break; + + + static const auto CreateName = TStringRef::Of("TopFreq_Create"); + static const auto AddValueName = TStringRef::Of("TopFreq_AddValue"); + static const auto SerializeName = TStringRef::Of("TopFreq_Serialize"); + static const auto DeserializeName = TStringRef::Of("TopFreq_Deserialize"); + static const auto MergeName = TStringRef::Of("TopFreq_Merge"); + static const auto GetName = TStringRef::Of("TopFreq_Get"); + class TTopFreqModule: public IUdfModule { public: TStringRef Name() const { @@ -222,167 +222,167 @@ namespace { } void GetAllFunctions(IFunctionsSink& sink) const final { - sink.Add(CreateName)->SetTypeAwareness(); - sink.Add(AddValueName)->SetTypeAwareness(); - sink.Add(SerializeName)->SetTypeAwareness(); - sink.Add(DeserializeName)->SetTypeAwareness(); - sink.Add(MergeName)->SetTypeAwareness(); - sink.Add(GetName)->SetTypeAwareness(); + sink.Add(CreateName)->SetTypeAwareness(); + sink.Add(AddValueName)->SetTypeAwareness(); + sink.Add(SerializeName)->SetTypeAwareness(); + sink.Add(DeserializeName)->SetTypeAwareness(); + sink.Add(MergeName)->SetTypeAwareness(); + sink.Add(GetName)->SetTypeAwareness(); } void BuildFunctionTypeInfo( const TStringRef& name, - TType* userType, + TType* userType, const TStringRef& typeConfig, ui32 flags, - IFunctionTypeInfoBuilder& builder) const final - { + IFunctionTypeInfoBuilder& builder) const final + { Y_UNUSED(typeConfig); try { const bool typesOnly = (flags & TFlags::TypesOnly); builder.UserType(userType); - auto typeHelper = builder.TypeInfoHelper(); - - auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); - if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) { - builder.SetError("User type is not a 3-tuple"); - return; - } - - bool isGeneric = false; - IHash::TPtr hash; - IEquate::TPtr equate; - TMaybe<EDataSlot> slot; - - auto valueType = userTypeInspector.GetElementType(2); - auto valueTypeInspector = TDataTypeInspector(*typeHelper, valueType); - if (!valueTypeInspector) { - isGeneric = true; - hash = builder.MakeHash(valueType); - equate = builder.MakeEquate(valueType); - if (!hash || !equate) { - return; - } - } else { - slot = FindDataSlot(valueTypeInspector.GetTypeId()); - if (!slot) { - builder.SetError("Unknown data type"); - return; - } - const auto& features = NUdf::GetDataTypeInfo(*slot).Features; - if (!(features & NUdf::CanHash) || !(features & NUdf::CanEquate)) { - builder.SetError("Data type is not hashable or equatable"); - return; - } - } - - auto serializedItemType = builder.Tuple()->Add<ui64>().Add(valueType).Build(); - auto serializedListType = builder.List()->Item(serializedItemType).Build(); - auto serializedType = builder.Tuple()->Add<ui32>().Add<ui32>().Add(serializedListType).Build(); - - TType* topFreqType = nullptr; - if (isGeneric) { - topFreqType = builder.Resource(TopFreqResourceNameGeneric); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_TYPE) - } - } - - if (name == CreateName) { - builder.Args()->Add(valueType).Add<ui32>().Done().Returns(topFreqType); + auto typeHelper = builder.TypeInfoHelper(); + + auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); + if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) { + builder.SetError("User type is not a 3-tuple"); + return; + } + + bool isGeneric = false; + IHash::TPtr hash; + IEquate::TPtr equate; + TMaybe<EDataSlot> slot; + + auto valueType = userTypeInspector.GetElementType(2); + auto valueTypeInspector = TDataTypeInspector(*typeHelper, valueType); + if (!valueTypeInspector) { + isGeneric = true; + hash = builder.MakeHash(valueType); + equate = builder.MakeEquate(valueType); + if (!hash || !equate) { + return; + } + } else { + slot = FindDataSlot(valueTypeInspector.GetTypeId()); + if (!slot) { + builder.SetError("Unknown data type"); + return; + } + const auto& features = NUdf::GetDataTypeInfo(*slot).Features; + if (!(features & NUdf::CanHash) || !(features & NUdf::CanEquate)) { + builder.SetError("Data type is not hashable or equatable"); + return; + } + } + + auto serializedItemType = builder.Tuple()->Add<ui64>().Add(valueType).Build(); + auto serializedListType = builder.List()->Item(serializedItemType).Build(); + auto serializedType = builder.Tuple()->Add<ui32>().Add<ui32>().Add(serializedListType).Build(); + + TType* topFreqType = nullptr; + if (isGeneric) { + topFreqType = builder.Resource(TopFreqResourceNameGeneric); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_TYPE) + } + } + + if (name == CreateName) { + builder.Args()->Add(valueType).Add<ui32>().Done().Returns(topFreqType); if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TTopFreq_Create(hash, equate)); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_CREATE) - } + if (isGeneric) { + builder.Implementation(new TTopFreq_Create(hash, equate)); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_CREATE) + } } } } - if (name == AddValueName) { - builder.Args()->Add(topFreqType).Add(valueType).Done().Returns(topFreqType); + if (name == AddValueName) { + builder.Args()->Add(topFreqType).Add(valueType).Done().Returns(topFreqType); if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TTopFreq_AddValue); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_ADD_VALUE) - } + if (isGeneric) { + builder.Implementation(new TTopFreq_AddValue); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_ADD_VALUE) + } } } } - if (name == MergeName) { + if (name == MergeName) { builder.Args()->Add(topFreqType).Add(topFreqType).Done().Returns(topFreqType); if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TTopFreq_Merge(hash, equate)); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_MERGE) - } + if (isGeneric) { + builder.Implementation(new TTopFreq_Merge(hash, equate)); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_MERGE) + } } } } - if (name == SerializeName) { - builder.Args()->Add(topFreqType).Done().Returns(serializedType); + if (name == SerializeName) { + builder.Args()->Add(topFreqType).Done().Returns(serializedType); if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TTopFreq_Serialize); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_SERIALIZE) - } + if (isGeneric) { + builder.Implementation(new TTopFreq_Serialize); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_SERIALIZE) + } } } } - if (name == DeserializeName) { - builder.Args()->Add(serializedType).Done().Returns(topFreqType); + if (name == DeserializeName) { + builder.Args()->Add(serializedType).Done().Returns(topFreqType); if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TTopFreq_Deserialize(hash, equate)); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_DESERIALIZE) - } + if (isGeneric) { + builder.Implementation(new TTopFreq_Deserialize(hash, equate)); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_DESERIALIZE) + } } } } - if (name == GetName) { + if (name == GetName) { ui32 indexF, indexV; - auto itemType = builder.Struct()->AddField<ui64>("Frequency", &indexF).AddField("Value", valueType, &indexV).Build(); - auto resultType = builder.List()->Item(itemType).Build(); + auto itemType = builder.Struct()->AddField<ui64>("Frequency", &indexF).AddField("Value", valueType, &indexV).Build(); + auto resultType = builder.List()->Item(itemType).Build(); - builder.Args()->Add(topFreqType).Add<ui32>().Done().Returns(resultType); + builder.Args()->Add(topFreqType).Add<ui32>().Done().Returns(resultType); if (!typesOnly) { - if (isGeneric) { - builder.Implementation(new TTopFreq_Get); - } else { - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_GET) - } + if (isGeneric) { + builder.Implementation(new TTopFreq_Get); + } else { + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_GET) + } } } } - + } catch (const std::exception& e) { builder.SetError(CurrentExceptionMessage()); } } }; - -} // namespace + +} // namespace diff --git a/ydb/library/yql/udfs/common/topfreq/static/ya.make b/ydb/library/yql/udfs/common/topfreq/static/ya.make index 64f5124e41..ce5d98cb6b 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/ya.make +++ b/ydb/library/yql/udfs/common/topfreq/static/ya.make @@ -2,7 +2,7 @@ LIBRARY() YQL_ABI_VERSION( 2 - 10 + 10 0 ) diff --git a/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp b/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp index 12597a7f92..c6f6ecab72 100644 --- a/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp +++ b/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp @@ -155,7 +155,7 @@ namespace NYql { auto getArgsType = pgmBuilder.NewTupleType({resourceType, ui32Type}); auto getUserType = pgmBuilder.NewTupleType({getArgsType, emptyStructType, valueType}); auto udfTopFreq_Get = pgmBuilder.Udf("TopFreq.TopFreq_Get", TRuntimeNode(), getUserType); - + TRuntimeNode pgmTopFreq; { auto value = pgmBuilder.NewDataLiteral<ui64>(1); @@ -245,22 +245,22 @@ namespace NYql { auto addValueArgsType = pgmBuilder.NewTupleType({resourceType, valueType}); auto addValueUserType = pgmBuilder.NewTupleType({addValueArgsType, emptyStructType, valueType}); auto udfTopFreq_AddValue = pgmBuilder.Udf("TopFreq.TopFreq_AddValue", TRuntimeNode(), addValueUserType); - + auto getArgsType = pgmBuilder.NewTupleType({resourceType, ui32Type}); auto getUserType = pgmBuilder.NewTupleType({getArgsType, emptyStructType, valueType}); auto udfTopFreq_Get = pgmBuilder.Udf("TopFreq.TopFreq_Get", TRuntimeNode(), getUserType); - + auto serializeArgsType = pgmBuilder.NewTupleType({resourceType}); auto serializeUserType = pgmBuilder.NewTupleType({serializeArgsType, emptyStructType, valueType}); auto udfTopFreq_Serialize = pgmBuilder.Udf("TopFreq.TopFreq_Serialize", TRuntimeNode(), serializeUserType); - + auto serializedType = pgmBuilder.NewTupleType({ui32Type, ui32Type, pgmBuilder.NewListType(pgmBuilder.NewTupleType({ui64Type, valueType}))}); - + auto deserializeArgsType = pgmBuilder.NewTupleType({serializedType}); auto deserializeUserType = pgmBuilder.NewTupleType({deserializeArgsType, emptyStructType, valueType}); auto udfTopFreq_Deserialize = pgmBuilder.Udf("TopFreq.TopFreq_Deserialize", TRuntimeNode(), deserializeUserType); - + TRuntimeNode pgmTopFreq; { auto value = pgmBuilder.NewDataLiteral<bool>(true); @@ -338,22 +338,22 @@ namespace NYql { auto mergeArgsType = pgmBuilder.NewTupleType({resourceType, resourceType}); auto mergeUserType = pgmBuilder.NewTupleType({mergeArgsType, emptyStructType, valueType}); auto udfTopFreq_Merge = pgmBuilder.Udf("TopFreq.TopFreq_Merge", TRuntimeNode(), mergeUserType); - + auto getArgsType = pgmBuilder.NewTupleType({resourceType, ui32Type}); auto getUserType = pgmBuilder.NewTupleType({getArgsType, emptyStructType, valueType}); auto udfTopFreq_Get = pgmBuilder.Udf("TopFreq.TopFreq_Get", TRuntimeNode(), getUserType); - + auto serializeArgsType = pgmBuilder.NewTupleType({resourceType}); auto serializeUserType = pgmBuilder.NewTupleType({serializeArgsType, emptyStructType, valueType}); auto udfTopFreq_Serialize = pgmBuilder.Udf("TopFreq.TopFreq_Serialize", TRuntimeNode(), serializeUserType); - + auto serializedType = pgmBuilder.NewTupleType({ui32Type, ui32Type, pgmBuilder.NewListType(pgmBuilder.NewTupleType({ui64Type, valueType}))}); - + auto deserializeArgsType = pgmBuilder.NewTupleType({serializedType}); auto deserializeUserType = pgmBuilder.NewTupleType({deserializeArgsType, emptyStructType, valueType}); auto udfTopFreq_Deserialize = pgmBuilder.Udf("TopFreq.TopFreq_Deserialize", TRuntimeNode(), deserializeUserType); - + static const ui64 BigNum = 20; static const ui64 BigEach = 5000; static const ui64 SmallNum = 500; diff --git a/ydb/library/yql/udfs/common/topfreq/ya.make b/ydb/library/yql/udfs/common/topfreq/ya.make index a2cead62ae..63448b297d 100644 --- a/ydb/library/yql/udfs/common/topfreq/ya.make +++ b/ydb/library/yql/udfs/common/topfreq/ya.make @@ -2,7 +2,7 @@ YQL_UDF(topfreq_udf) YQL_ABI_VERSION( 2 - 10 + 10 0 ) diff --git a/ydb/library/yql/utils/log/log_component.h b/ydb/library/yql/utils/log/log_component.h index c83497f896..d7a1e10ca1 100644 --- a/ydb/library/yql/utils/log/log_component.h +++ b/ydb/library/yql/utils/log/log_component.h @@ -23,7 +23,7 @@ enum class EComponent { Performance, Perf = Performance, Net, ProviderStat, - ProviderSolomon, + ProviderSolomon, CoreEval, CorePeepHole, ProviderDq, diff --git a/ydb/public/lib/deprecated/kicli/cpp_ut.cpp b/ydb/public/lib/deprecated/kicli/cpp_ut.cpp index d7d8052cb3..cb7dc3ed6d 100644 --- a/ydb/public/lib/deprecated/kicli/cpp_ut.cpp +++ b/ydb/public/lib/deprecated/kicli/cpp_ut.cpp @@ -21,7 +21,7 @@ Tests::TServer StartupKikimr(NMsgBusProxy::TMsgBusClientConfig& clientConfig, const ui32 port = pm.GetPort(12001); auto settings = Tests::TServerSettings(port); settings.SetLogBackend(logBackend); - settings.SetEnableSystemViews(false); + settings.SetEnableSystemViews(false); Tests::TServer Server(settings); Tests::TClient Client(settings); Client.InitRootScheme(); @@ -40,7 +40,7 @@ Tests::TServer StartupKikimr(NGRpcProxy::TGRpcClientConfig& clientConfig, auto settings = Tests::TServerSettings(msgbusPort); settings.SetLogBackend(logBackend); settings.AppConfig.CopyFrom(config); - settings.SetEnableSystemViews(false); + settings.SetEnableSystemViews(false); settings.SetEnableMvcc(false); Tests::TServer Server(settings); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp index c18e2bb9b9..4a25c372ce 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp @@ -2,7 +2,7 @@ #include <ydb/public/lib/ydb_cli/common/pretty_table.h> #include <ydb/public/lib/ydb_cli/common/query_stats.h> - + #include <util/folder/path.h> #include <util/folder/dirut.h> @@ -28,7 +28,7 @@ void TCommandExecuteYqlScript::Config(TConfig& config) { config.Opts->AddLongOption('f', "file", "[Required] Script file").RequiredArgument("PATH").StoreResult(&ScriptFile); config.Opts->AddLongOption("explain", "Explain query").Optional().StoreTrue(&Explain); config.Opts->AddLongOption("show-response-metadata", ResponseHeadersHelp).Optional().StoreTrue(&ShowHeaders); - + AddParametersOption(config); AddInputFormats(config, { @@ -36,14 +36,14 @@ void TCommandExecuteYqlScript::Config(TConfig& config) { EOutputFormat::JsonBase64 }); - AddFormats(config, { - EOutputFormat::Pretty, - EOutputFormat::JsonUnicode, + AddFormats(config, { + EOutputFormat::Pretty, + EOutputFormat::JsonUnicode, EOutputFormat::JsonUnicodeArray, EOutputFormat::JsonBase64, EOutputFormat::JsonBase64Array - }); - + }); + config.SetFreeArgsNum(0); AddCommandExamples( @@ -64,7 +64,7 @@ void TCommandExecuteYqlScript::Config(TConfig& config) { void TCommandExecuteYqlScript::Parse(TConfig& config) { TClientCommand::Parse(config); - ParseFormats(); + ParseFormats(); if (!Script && !ScriptFile) { throw TMissUseException() << "Neither \"Text of script\" (\"--script\", \"-s\") " << "nor \"Path to file with script text\" (\"--file\", \"-f\") were provided."; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index f56db9d520..2f94b2fc03 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -344,7 +344,7 @@ void TCommandExecuteQuery::Config(TConfig& config) { TTableCommand::Config(config); AddExamplesOption(config); - config.Opts->AddLongOption('t', "type", "Query type [data, scheme, scan]") + config.Opts->AddLongOption('t', "type", "Query type [data, scheme, scan]") .RequiredArgument("[String]").DefaultValue("data").StoreResult(&QueryType); config.Opts->AddLongOption("stats", "Collect statistics mode (for data & scan queries) [none, basic, full]") .RequiredArgument("[String]").StoreResult(&CollectStatsMode); @@ -354,7 +354,7 @@ void TCommandExecuteQuery::Config(TConfig& config) { config.Opts->AddLongOption('q', "query", "Text of query to execute").RequiredArgument("[String]").StoreResult(&Query); config.Opts->AddLongOption('f', "file", "Path to file with query text to execute") .RequiredArgument("PATH").StoreResult(&QueryFile); - + AddParametersOption(config, "(for data & scan queries)"); AddInputFormats(config, { @@ -363,22 +363,22 @@ void TCommandExecuteQuery::Config(TConfig& config) { }); - AddFormats(config, { - EOutputFormat::Pretty, - EOutputFormat::JsonUnicode, + AddFormats(config, { + EOutputFormat::Pretty, + EOutputFormat::JsonUnicode, EOutputFormat::JsonUnicodeArray, EOutputFormat::JsonBase64, EOutputFormat::JsonBase64Array - }); - + }); + CheckExamples(config); - + config.SetFreeArgsNum(0); } void TCommandExecuteQuery::Parse(TConfig& config) { TClientCommand::Parse(config); - ParseFormats(); + ParseFormats(); if (BasicStats && CollectStatsMode) { throw TMissUseException() << "Both mutually exclusive options \"--stats\" and \"-s\" are provided."; } @@ -398,9 +398,9 @@ int TCommandExecuteQuery::Run(TConfig& config) { if (QueryType == "scheme") { return ExecuteSchemeQuery(config); } - if (QueryType == "scan") { - return ExecuteScanQuery(config); - } + if (QueryType == "scan") { + return ExecuteScanQuery(config); + } } throw TMissUseException() << "Unknown query type."; } @@ -476,13 +476,13 @@ int TCommandExecuteQuery::ExecuteSchemeQuery(TConfig& config) { return EXIT_SUCCESS; } -int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) { - NTable::TTableClient client(CreateDriver(config)); +int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) { + NTable::TTableClient client(CreateDriver(config)); auto defaultStatsMode = BasicStats ? NTable::ECollectQueryStatsMode::Basic : NTable::ECollectQueryStatsMode::None; - NTable::TStreamExecScanQuerySettings settings; + NTable::TStreamExecScanQuerySettings settings; settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, defaultStatsMode)); - + NTable::TAsyncScanQueryPartIterator asyncResult; if (Parameters.size()) { auto validateResult = ExplainQuery(config, Query, NScripting::ExplainYqlRequestMode::Validate); @@ -496,17 +496,17 @@ int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) { } auto result = asyncResult.GetValueSync(); - ThrowOnError(result); - PrintScanQueryResponse(result); - return EXIT_SUCCESS; -} - -void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator& result) { - SetInterruptHandlers(); + ThrowOnError(result); + PrintScanQueryResponse(result); + return EXIT_SUCCESS; +} + +void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator& result) { + SetInterruptHandlers(); TStringStream statsStr; { TResultSetPrinter printer(OutputFormat, &IsInterrupted); - + while (!IsInterrupted()) { auto streamPart = result.ReadNext().GetValueSync(); if (!streamPart.IsSuccess()) { @@ -514,7 +514,7 @@ void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator break; } ThrowOnError(streamPart); - } + } if (streamPart.HasResultSet()) { printer.Print(streamPart.GetResultSet()); @@ -534,13 +534,13 @@ void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator if (statsStr.Size()) { Cout << Endl << "Statistics:" << statsStr.Str(); - } - + } + if (IsInterrupted()) { Cerr << "<INTERRUPTED>" << Endl; } -} - +} + TCommandExplain::TCommandExplain() : TTableCommand("explain", {}, "Explain query") {} @@ -688,13 +688,13 @@ void TCommandReadTable::Config(TConfig& config) { EOutputFormat::JsonBase64 }); - AddFormats(config, { - EOutputFormat::Pretty, - EOutputFormat::JsonUnicode, + AddFormats(config, { + EOutputFormat::Pretty, + EOutputFormat::JsonUnicode, EOutputFormat::JsonUnicodeArray, EOutputFormat::JsonBase64, EOutputFormat::JsonBase64Array - }); + }); // TODO: KIKIMR-8675 // Add csv format @@ -809,7 +809,7 @@ int TCommandReadTable::Run(TConfig& config) { void TCommandReadTable::PrintResponse(NTable::TTablePartIterator& result) { size_t totalRows = 0; - SetInterruptHandlers(); + SetInterruptHandlers(); TResultSetPrinter printer(OutputFormat, &IsInterrupted); while (!IsInterrupted()) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h index 66067e9673..5cd2514aa0 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h @@ -86,28 +86,28 @@ protected: void CheckQueryOptions() const; void CheckQueryFile(); -protected: +protected: TString Query; TString QueryFile; }; class TCommandExecuteQuery : public TTableCommand, TCommandQueryBase, TCommandWithParameters, - public TCommandWithFormat, public TInterruptibleCommand -{ + public TCommandWithFormat, public TInterruptibleCommand +{ public: TCommandExecuteQuery(); virtual void Config(TConfig& config) override; virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; - + int ExecuteDataQuery(TConfig& config); void PrintDataQueryResponse(NTable::TDataQueryResult& result); - + int ExecuteSchemeQuery(TConfig& config); - int ExecuteScanQuery(TConfig& config); - void PrintScanQueryResponse(NTable::TScanQueryPartIterator& result); - + int ExecuteScanQuery(TConfig& config); + void PrintScanQueryResponse(NTable::TScanQueryPartIterator& result); + private: TString CollectStatsMode; TString TxMode; @@ -129,9 +129,9 @@ private: bool Analyze = false; }; -class TCommandReadTable : public TYdbCommand, public TCommandWithPath, - public TCommandWithFormat, public TInterruptibleCommand -{ +class TCommandReadTable : public TYdbCommand, public TCommandWithPath, + public TCommandWithFormat, public TInterruptibleCommand +{ public: TCommandReadTable(); virtual void Config(TConfig& config) override; diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp index 13100b19df..7184ab41d1 100644 --- a/ydb/public/lib/ydb_cli/common/format.cpp +++ b/ydb/public/lib/ydb_cli/common/format.cpp @@ -1,11 +1,11 @@ #include "format.h" -#include "pretty_table.h" +#include "pretty_table.h" #include <util/string/vector.h> #include <library/cpp/json/json_prettifier.h> #include <ydb/public/lib/json_value/ydb_json_value.h> - + namespace NYdb { namespace NConsoleClient { @@ -142,11 +142,11 @@ void TQueryPlanPrinter::Print(const TString& plan) { default: throw TMissUseException() << "This command doesn't support " << Format << " output format"; } -} +} void TQueryPlanPrinter::PrintJson(const TString& plan) { Cout << NJson::PrettifyJson(plan, true) << Endl; -} +} void TQueryPlanPrinter::PrintPretty(const NJson::TJsonValue& plan) { if (plan.GetMapSafe().contains("Plan")) { diff --git a/ydb/public/lib/ydb_cli/common/format.h b/ydb/public/lib/ydb_cli/common/format.h index decd947689..2d8d6a2e38 100644 --- a/ydb/public/lib/ydb_cli/common/format.h +++ b/ydb/public/lib/ydb_cli/common/format.h @@ -6,7 +6,7 @@ #include <ydb/public/lib/json_value/ydb_json_value.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> - + namespace NYdb { namespace NConsoleClient { @@ -24,7 +24,7 @@ protected: void AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); void AddFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); void ParseFormats(); - + // Deprecated void AddJsonOption(TClientCommand::TConfig& config, const TString& description = "(Deprecated, will be removed soon. Use --format option instead)" diff --git a/ydb/public/sdk/python/examples/basic_example_v1/basic_example.py b/ydb/public/sdk/python/examples/basic_example_v1/basic_example.py index 6fa6361e7e..5845a18a2e 100644 --- a/ydb/public/sdk/python/examples/basic_example_v1/basic_example.py +++ b/ydb/public/sdk/python/examples/basic_example_v1/basic_example.py @@ -6,25 +6,25 @@ from . import basic_example_data FillDataQuery = """PRAGMA TablePathPrefix("{}"); -DECLARE $seriesData AS List<Struct< +DECLARE $seriesData AS List<Struct< series_id: Uint64, title: Utf8, series_info: Utf8, - release_date: Date>>; + release_date: Date>>; -DECLARE $seasonsData AS List<Struct< +DECLARE $seasonsData AS List<Struct< series_id: Uint64, season_id: Uint64, title: Utf8, first_aired: Date, - last_aired: Date>>; + last_aired: Date>>; -DECLARE $episodesData AS List<Struct< +DECLARE $episodesData AS List<Struct< series_id: Uint64, season_id: Uint64, episode_id: Uint64, title: Utf8, - air_date: Date>>; + air_date: Date>>; REPLACE INTO series SELECT diff --git a/ydb/public/sdk/python/examples/basic_example_v1/basic_example_data.py b/ydb/public/sdk/python/examples/basic_example_v1/basic_example_data.py index 06c3d9608a..967c78ead7 100644 --- a/ydb/public/sdk/python/examples/basic_example_v1/basic_example_data.py +++ b/ydb/public/sdk/python/examples/basic_example_v1/basic_example_data.py @@ -120,7 +120,7 @@ def get_episodes_data_for_bulk_upsert(): Episode(2, 2, 7, "Adult Content", "2015-05-24"), Episode(2, 2, 8, "White Hat/Black Hat", "2015-05-31"), Episode(2, 2, 9, "Binding Arbitration", "2015-06-07"), - Episode(2, 2, 10, "Two Days of the Condor", "2015-06-14"), + Episode(2, 2, 10, "Two Days of the Condor", "2015-06-14"), Episode(2, 3, 1, "Founder Friendly", "2016-04-24"), Episode(2, 3, 2, "Two in the Box", "2016-05-01"), Episode(2, 3, 3, "Meinertzhagen's Haversack", "2016-05-08"), @@ -130,7 +130,7 @@ def get_episodes_data_for_bulk_upsert(): Episode(2, 3, 7, "To Build a Better Beta", "2016-06-05"), Episode(2, 3, 8, "Bachman's Earnings Over-Ride", "2016-06-12"), Episode(2, 3, 9, "Daily Active Users", "2016-06-19"), - Episode(2, 3, 10, "The Uptick", "2016-06-26"), + Episode(2, 3, 10, "The Uptick", "2016-06-26"), Episode(2, 4, 1, "Success Failure", "2017-04-23"), Episode(2, 4, 2, "Terms of Service", "2017-04-30"), Episode(2, 4, 3, "Intellectual Property", "2017-05-07"), @@ -140,7 +140,7 @@ def get_episodes_data_for_bulk_upsert(): Episode(2, 4, 7, "The Patent Troll", "2017-06-04"), Episode(2, 4, 8, "The Keenan Vortex", "2017-06-11"), Episode(2, 4, 9, "Hooli-Con", "2017-06-18"), - Episode(2, 4, 10, "Server Error", "2017-06-25"), + Episode(2, 4, 10, "Server Error", "2017-06-25"), Episode(2, 5, 1, "Grow Fast or Die Slow", "2018-03-25"), Episode(2, 5, 2, "Reorientation", "2018-04-01"), Episode(2, 5, 3, "Chief Operating Officer", "2018-04-08"), diff --git a/ydb/services/datastreams/grpc_service.cpp b/ydb/services/datastreams/grpc_service.cpp index c8af620457..7a18cd4dc6 100644 --- a/ydb/services/datastreams/grpc_service.cpp +++ b/ydb/services/datastreams/grpc_service.cpp @@ -65,7 +65,7 @@ void TGRpcDataStreamsService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ACTION; \ }, &Ydb::DataStreams::V1::DataStreamsService::AsyncService::Request ## NAME, \ - #NAME, logger, getCounterBlock("data_streams", #NAME))->Run(); + #NAME, logger, getCounterBlock("data_streams", #NAME))->Run(); ADD_REQUEST(DescribeStream, DescribeStreamRequest, DescribeStreamResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeStreamRequest(ctx)); diff --git a/ydb/services/persqueue_v1/persqueue.cpp b/ydb/services/persqueue_v1/persqueue.cpp index 9e91c4d7cf..ccacc4d43a 100644 --- a/ydb/services/persqueue_v1/persqueue.cpp +++ b/ydb/services/persqueue_v1/persqueue.cpp @@ -86,7 +86,7 @@ void TGRpcPersQueueService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { [this](TIntrusivePtr<TStreamGRpcRequest::IContext> context) { ActorSystem->Send(GRpcRequestProxy, new NKikimr::NGRpcService::TEvStreamPQWriteRequest(context)); }, - *ActorSystem, "PersQueueService/CreateWriteSession", getCounterBlock("persistent_queue", "WriteSession", true, true), nullptr + *ActorSystem, "PersQueueService/CreateWriteSession", getCounterBlock("persistent_queue", "WriteSession", true, true), nullptr ); } @@ -106,7 +106,7 @@ void TGRpcPersQueueService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { [this](TIntrusivePtr<TStreamGRpcRequest::IContext> context) { ActorSystem->Send(GRpcRequestProxy, new NKikimr::NGRpcService::TEvStreamPQReadRequest(context)); }, - *ActorSystem, "PersQueueService/CreateReadSession", getCounterBlock("persistent_queue", "ReadSession", true, true), nullptr + *ActorSystem, "PersQueueService/CreateReadSession", getCounterBlock("persistent_queue", "ReadSession", true, true), nullptr ); } @@ -119,7 +119,7 @@ void TGRpcPersQueueService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { NGRpcService::ReportGrpcReqToMon(*ActorSystem, ctx->GetPeer()); \ ACTION; \ }, &Ydb::PersQueue::V1::SVC::AsyncService::Request ## NAME, \ - "PersQueueService/"#NAME, logger, getCounterBlock("persistent_queue", #NAME))->Run(); + "PersQueueService/"#NAME, logger, getCounterBlock("persistent_queue", #NAME))->Run(); ADD_REQUEST(GetReadSessionsInfo, PersQueueService, ReadInfoRequest, ReadInfoResponse, { ActorSystem->Send(GRpcRequestProxy, new NGRpcService::TEvPQReadInfoRequest(ctx)); diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h index bfc87e644f..c067928c78 100644 --- a/ydb/services/ydb/ydb_common_ut.h +++ b/ydb/services/ydb/ydb_common_ut.h @@ -24,7 +24,7 @@ struct TKikimrTestSettings { static constexpr bool SSL = false; static constexpr bool AUTH = false; static constexpr bool PrecreatePools = true; - static constexpr bool EnableSystemViews = true; + static constexpr bool EnableSystemViews = true; }; struct TKikimrTestWithAuth : TKikimrTestSettings { @@ -35,10 +35,10 @@ struct TKikimrTestWithAuthAndSsl : TKikimrTestWithAuth { static constexpr bool SSL = true; }; -struct TKikimrTestNoSystemViews : TKikimrTestSettings { - static constexpr bool EnableSystemViews = false; -}; - +struct TKikimrTestNoSystemViews : TKikimrTestSettings { + static constexpr bool EnableSystemViews = false; +}; + template <typename TestSettings = TKikimrTestSettings> class TBasicKikimrWithGrpcAndRootSchema { public: @@ -308,6 +308,6 @@ struct TTestOlap { using TKikimrWithGrpcAndRootSchema = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestSettings>; using TKikimrWithGrpcAndRootSchemaWithAuth = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestWithAuth>; using TKikimrWithGrpcAndRootSchemaWithAuthAndSsl = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestWithAuthAndSsl>; -using TKikimrWithGrpcAndRootSchemaNoSystemViews = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestNoSystemViews>; +using TKikimrWithGrpcAndRootSchemaNoSystemViews = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestNoSystemViews>; } diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp index e497dee61b..4861c06b92 100644 --- a/ydb/services/ydb/ydb_table_ut.cpp +++ b/ydb/services/ydb/ydb_table_ut.cpp @@ -2961,7 +2961,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015 } Y_UNIT_TEST(CopyTables) { - TKikimrWithGrpcAndRootSchemaNoSystemViews server; + TKikimrWithGrpcAndRootSchemaNoSystemViews server; server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_NOTICE); auto connection = NYdb::TDriver( diff --git a/ydb/services/ydb/ydb_ut.cpp b/ydb/services/ydb/ydb_ut.cpp index 445ba4ec1e..2d37063079 100644 --- a/ydb/services/ydb/ydb_ut.cpp +++ b/ydb/services/ydb/ydb_ut.cpp @@ -663,7 +663,7 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) { } Y_UNIT_TEST(CreateAlterUpsertDrop) { - TKikimrWithGrpcAndRootSchemaNoSystemViews server; + TKikimrWithGrpcAndRootSchemaNoSystemViews server; ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; @@ -891,7 +891,7 @@ Y_UNIT_TEST_SUITE(TGRpcYdbTest) { } Y_UNIT_TEST(MakeListRemoveDirectory) { - TKikimrWithGrpcAndRootSchemaNoSystemViews server; + TKikimrWithGrpcAndRootSchemaNoSystemViews server; ui16 grpc = server.GetPort(); std::shared_ptr<grpc::Channel> Channel_; diff --git a/ydb/tests/functional/tenants/test_db_counters.py b/ydb/tests/functional/tenants/test_db_counters.py index 2451cdaab4..b8e3c9baa3 100644 --- a/ydb/tests/functional/tenants/test_db_counters.py +++ b/ydb/tests/functional/tenants/test_db_counters.py @@ -1,131 +1,131 @@ -# -*- coding: utf-8 -*- -import logging -import os -import time -import requests - -from hamcrest import ( - assert_that, - equal_to, - greater_than, - not_none -) - +# -*- coding: utf-8 -*- +import logging +import os +import time +import requests + +from hamcrest import ( + assert_that, + equal_to, + greater_than, + not_none +) + from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.harness.util import LogLevels import ydb - -logger = logging.getLogger(__name__) - - -class BaseDbCounters(object): - @classmethod - def setup_class(cls): - cls.cluster = kikimr_cluster_factory( - KikimrConfigGenerator( - additional_log_configs={ - 'SYSTEM_VIEWS': LogLevels.DEBUG - } - ) - ) - cls.cluster.start() - - @classmethod - def teardown_class(cls): - if hasattr(cls, 'cluster'): - cls.cluster.stop() - - def setup_method(self, method=None): - self.database = "/Root/users/{class_name}_{method_name}".format( + +logger = logging.getLogger(__name__) + + +class BaseDbCounters(object): + @classmethod + def setup_class(cls): + cls.cluster = kikimr_cluster_factory( + KikimrConfigGenerator( + additional_log_configs={ + 'SYSTEM_VIEWS': LogLevels.DEBUG + } + ) + ) + cls.cluster.start() + + @classmethod + def teardown_class(cls): + if hasattr(cls, 'cluster'): + cls.cluster.stop() + + def setup_method(self, method=None): + self.database = "/Root/users/{class_name}_{method_name}".format( class_name=self.__class__.__name__, method_name=method.__name__, - ) - logger.debug("Create database %s" % self.database) - self.cluster.create_database( - self.database, - storage_pool_units_count={ - 'hdd': 1 - } - ) - + ) + logger.debug("Create database %s" % self.database) + self.cluster.create_database( + self.database, + storage_pool_units_count={ + 'hdd': 1 + } + ) + self.cluster.register_and_start_slots(self.database, count=1) self.cluster.wait_tenant_up(self.database) - def teardown_method(self, method=None): - logger.debug("Remove database %s" % self.database) - self.cluster.remove_database(self.database) - self.database = None - - def create_table(self, driver, table): - with ydb.SessionPool(driver, size=1) as pool: - with pool.checkout() as session: - session.execute_scheme( - "create table `{}` (key Int32, value String, primary key(key));".format( - table - ) - ) - - def get_db_counters(self, service, node_index=0): - mon_port = self.cluster.slots[node_index + 1].mon_port - counters_url = 'http://localhost:{}/counters/counters%3D{}/json'.format(mon_port, service) - reply = requests.get(counters_url) - if reply.status_code == 204: - return None - - assert_that(reply.status_code, equal_to(200)) - ret = reply.json() - - assert_that(ret, not_none()) - return ret - - def check_db_counters(self, sensors_to_check, group): - table = os.path.join(self.database, 'table') - - driver_config = ydb.DriverConfig( - "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), - self.database - ) - - with ydb.Driver(driver_config) as driver: - self.create_table(driver, table) - - with ydb.SessionPool(driver, size=1) as pool: - with pool.checkout() as session: - query = "select * from `{}`".format(table) - session.transaction().execute(query, commit_tx=True) - - for i in range(30): - checked = 0 - - counters = self.get_db_counters('db') - if counters: - sensors = counters['sensors'] - for sensor in sensors: - labels = sensor['labels'] - if labels['sensor'] in sensors_to_check: - assert_that(labels['group'], equal_to(group)) - assert_that(labels['database'], equal_to(self.database)) - assert_that(labels['host'], equal_to('')) - assert_that(sensor['value'], greater_than(0)) - checked = checked + 1 - - assert_that(checked, equal_to(len(sensors_to_check))) - break - - if checked > 0: - break - - time.sleep(5) - - -class TestKqpCounters(BaseDbCounters): - def test_case(self): - sensors_to_check = { - 'Requests/Bytes', - 'Requests/QueryBytes', - 'Requests/QueryExecute', - 'YdbResponses/Success', - 'Responses/Bytes', - } - self.check_db_counters(sensors_to_check, 'kqp') + def teardown_method(self, method=None): + logger.debug("Remove database %s" % self.database) + self.cluster.remove_database(self.database) + self.database = None + + def create_table(self, driver, table): + with ydb.SessionPool(driver, size=1) as pool: + with pool.checkout() as session: + session.execute_scheme( + "create table `{}` (key Int32, value String, primary key(key));".format( + table + ) + ) + + def get_db_counters(self, service, node_index=0): + mon_port = self.cluster.slots[node_index + 1].mon_port + counters_url = 'http://localhost:{}/counters/counters%3D{}/json'.format(mon_port, service) + reply = requests.get(counters_url) + if reply.status_code == 204: + return None + + assert_that(reply.status_code, equal_to(200)) + ret = reply.json() + + assert_that(ret, not_none()) + return ret + + def check_db_counters(self, sensors_to_check, group): + table = os.path.join(self.database, 'table') + + driver_config = ydb.DriverConfig( + "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), + self.database + ) + + with ydb.Driver(driver_config) as driver: + self.create_table(driver, table) + + with ydb.SessionPool(driver, size=1) as pool: + with pool.checkout() as session: + query = "select * from `{}`".format(table) + session.transaction().execute(query, commit_tx=True) + + for i in range(30): + checked = 0 + + counters = self.get_db_counters('db') + if counters: + sensors = counters['sensors'] + for sensor in sensors: + labels = sensor['labels'] + if labels['sensor'] in sensors_to_check: + assert_that(labels['group'], equal_to(group)) + assert_that(labels['database'], equal_to(self.database)) + assert_that(labels['host'], equal_to('')) + assert_that(sensor['value'], greater_than(0)) + checked = checked + 1 + + assert_that(checked, equal_to(len(sensors_to_check))) + break + + if checked > 0: + break + + time.sleep(5) + + +class TestKqpCounters(BaseDbCounters): + def test_case(self): + sensors_to_check = { + 'Requests/Bytes', + 'Requests/QueryBytes', + 'Requests/QueryExecute', + 'YdbResponses/Success', + 'Responses/Bytes', + } + self.check_db_counters(sensors_to_check, 'kqp') diff --git a/ydb/tests/functional/tenants/test_system_views.py b/ydb/tests/functional/tenants/test_system_views.py index 8c14c63d95..b0fc5640f6 100644 --- a/ydb/tests/functional/tenants/test_system_views.py +++ b/ydb/tests/functional/tenants/test_system_views.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- import logging import os -import time +import time from hamcrest import ( anything, assert_that, - greater_than, + greater_than, has_length, has_properties, ) @@ -24,9 +24,9 @@ class BaseSystemViews(object): def setup_class(cls): cls.cluster = kikimr_cluster_factory( KikimrConfigGenerator( - additional_log_configs={ - 'SYSTEM_VIEWS': LogLevels.DEBUG - } + additional_log_configs={ + 'SYSTEM_VIEWS': LogLevels.DEBUG + } ) ) cls.cluster.start() @@ -60,32 +60,32 @@ class BaseSystemViews(object): with ydb.SessionPool(driver, size=1) as pool: with pool.checkout() as session: session.execute_scheme( - "create table `{}` (key Int32, value String, primary key(key));".format( + "create table `{}` (key Int32, value String, primary key(key));".format( table ) ) def read_partition_stats(self, driver, database): table = os.path.join(database, '.sys/partition_stats') - return self._stream_query_result( - driver, - "select PathId, PartIdx, Path from `{}`;".format(table) - ) - - def read_query_metrics(self, driver, database): - table = os.path.join(database, '.sys/query_metrics_one_minute') - return self._stream_query_result( - driver, - "select Count, SumReadBytes, SumRequestUnits from `{}`;".format(table) - ) - - def read_query_stats(self, driver, database, table_name): - table = os.path.join(database, '.sys', table_name) - return self._stream_query_result( - driver, - "select Duration, ReadBytes, CPUTime, RequestUnits from `{}`;".format(table) - ) - + return self._stream_query_result( + driver, + "select PathId, PartIdx, Path from `{}`;".format(table) + ) + + def read_query_metrics(self, driver, database): + table = os.path.join(database, '.sys/query_metrics_one_minute') + return self._stream_query_result( + driver, + "select Count, SumReadBytes, SumRequestUnits from `{}`;".format(table) + ) + + def read_query_stats(self, driver, database, table_name): + table = os.path.join(database, '.sys', table_name) + return self._stream_query_result( + driver, + "select Duration, ReadBytes, CPUTime, RequestUnits from `{}`;".format(table) + ) + def _stream_query_result(self, driver, query): it = driver.table_client.scan_query(query) result = [] @@ -101,61 +101,61 @@ class BaseSystemViews(object): return result - def check_query_metrics_and_stats(self, query_count): - table = os.path.join(self.database, 'table') - - driver_config = ydb.DriverConfig( - "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), - self.database - ) - - with ydb.Driver(driver_config) as driver: - driver.wait(timeout=10) - - self.create_table(driver, table) - - with ydb.SessionPool(driver, size=1) as pool: - with pool.checkout() as session: - for i in range(query_count): - query = "select * from `{}` -- {}".format(table, i) - session.transaction().execute(query, commit_tx=True) - - time.sleep(70) - - for i in range(60): - metrics = self.read_query_metrics(driver, self.database) - if len(metrics) == query_count: - break - time.sleep(5) - - assert_that(metrics, has_length(query_count)) - assert_that(metrics[0], has_properties({ - 'Count': 1, - 'SumReadBytes': 0, - 'SumRequestUnits': greater_than(0), - })) - - for table_name in [ - 'top_queries_by_duration_one_minute', - 'top_queries_by_duration_one_hour', - 'top_queries_by_read_bytes_one_minute', - 'top_queries_by_read_bytes_one_hour', - 'top_queries_by_cpu_time_one_minute', - 'top_queries_by_cpu_time_one_hour', - 'top_queries_by_request_units_one_minute', - 'top_queries_by_request_units_one_hour' - ]: - stats = self.read_query_stats(driver, self.database, table_name) - - assert_that(stats, has_length(min(query_count, 5))) - assert_that(stats[0], has_properties({ - 'Duration': anything(), - 'ReadBytes': 0, - 'CPUTime': anything(), - 'RequestUnits': greater_than(0) - })) - - + def check_query_metrics_and_stats(self, query_count): + table = os.path.join(self.database, 'table') + + driver_config = ydb.DriverConfig( + "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), + self.database + ) + + with ydb.Driver(driver_config) as driver: + driver.wait(timeout=10) + + self.create_table(driver, table) + + with ydb.SessionPool(driver, size=1) as pool: + with pool.checkout() as session: + for i in range(query_count): + query = "select * from `{}` -- {}".format(table, i) + session.transaction().execute(query, commit_tx=True) + + time.sleep(70) + + for i in range(60): + metrics = self.read_query_metrics(driver, self.database) + if len(metrics) == query_count: + break + time.sleep(5) + + assert_that(metrics, has_length(query_count)) + assert_that(metrics[0], has_properties({ + 'Count': 1, + 'SumReadBytes': 0, + 'SumRequestUnits': greater_than(0), + })) + + for table_name in [ + 'top_queries_by_duration_one_minute', + 'top_queries_by_duration_one_hour', + 'top_queries_by_read_bytes_one_minute', + 'top_queries_by_read_bytes_one_hour', + 'top_queries_by_cpu_time_one_minute', + 'top_queries_by_cpu_time_one_hour', + 'top_queries_by_request_units_one_minute', + 'top_queries_by_request_units_one_hour' + ]: + stats = self.read_query_stats(driver, self.database, table_name) + + assert_that(stats, has_length(min(query_count, 5))) + assert_that(stats[0], has_properties({ + 'Duration': anything(), + 'ReadBytes': 0, + 'CPUTime': anything(), + 'RequestUnits': greater_than(0) + })) + + class TestPartitionStats(BaseSystemViews): def test_case(self): for database in ('/Root', self.database): @@ -178,13 +178,13 @@ class TestPartitionStats(BaseSystemViews): 'PathId': anything(), 'PartIdx': 0, })) - - -class TestQueryMetrics(BaseSystemViews): - def test_case(self): - self.check_query_metrics_and_stats(1) - - -class TestQueryMetricsUniqueQueries(BaseSystemViews): - def test_case(self): - self.check_query_metrics_and_stats(10) + + +class TestQueryMetrics(BaseSystemViews): + def test_case(self): + self.check_query_metrics_and_stats(1) + + +class TestQueryMetricsUniqueQueries(BaseSystemViews): + def test_case(self): + self.check_query_metrics_and_stats(10) diff --git a/ydb/tests/functional/tenants/ya.make b/ydb/tests/functional/tenants/ya.make index 1e5ed02250..dee204013e 100644 --- a/ydb/tests/functional/tenants/ya.make +++ b/ydb/tests/functional/tenants/ya.make @@ -3,7 +3,7 @@ PY3TEST() ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") TEST_SRCS( common.py - test_db_counters.py + test_db_counters.py test_dynamic_tenants.py test_tenants.py test_storage_config.py |