diff options
author | monster <monster@ydb.tech> | 2023-08-04 20:00:21 +0300 |
---|---|---|
committer | monster <monster@ydb.tech> | 2023-08-04 21:12:32 +0300 |
commit | 0ba0105da161f4fe59a3cc0697e1af8e0cc7fa35 (patch) | |
tree | 38b4c0187f4c13a586dfdda4bf139ad31d17cc4e | |
parent | d33597c466ba0b77f230075f06e7fb61390d8b3a (diff) | |
download | ydb-0ba0105da161f4fe59a3cc0697e1af8e0cc7fa35.tar.gz |
introduce statistics support KIKIMR-18323
34 files changed, 540 insertions, 1 deletions
diff --git a/ydb/core/CMakeLists.txt b/ydb/core/CMakeLists.txt index 474cc8c050..3436fc14bd 100644 --- a/ydb/core/CMakeLists.txt +++ b/ydb/core/CMakeLists.txt @@ -53,6 +53,7 @@ add_subdirectory(raw_socket) add_subdirectory(scheme) add_subdirectory(scheme_types) add_subdirectory(security) +add_subdirectory(statistics) add_subdirectory(sys_view) add_subdirectory(tablet) add_subdirectory(tablet_flat) diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index fcc5dad204..a97fe603da 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -164,7 +164,8 @@ struct TKikimrEvents : TEvents { ES_KQP_SCAN_EXCHANGE, ES_IC_NODE_CACHE, ES_DATA_OPERATIONS, - ES_KAFKA + ES_KAFKA, + ES_STATISTICS }; }; diff --git a/ydb/core/driver_lib/run/CMakeLists.darwin-x86_64.txt b/ydb/core/driver_lib/run/CMakeLists.darwin-x86_64.txt index 1c13f29209..7fe9caae62 100644 --- a/ydb/core/driver_lib/run/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/driver_lib/run/CMakeLists.darwin-x86_64.txt @@ -87,6 +87,7 @@ target_link_libraries(run PUBLIC ydb-core-scheme ydb-core-scheme_types ydb-core-security + ydb-core-statistics core-sys_view-processor core-sys_view-service ydb-core-tablet diff --git a/ydb/core/driver_lib/run/CMakeLists.linux-aarch64.txt b/ydb/core/driver_lib/run/CMakeLists.linux-aarch64.txt index 51fac4a558..5d41e085cf 100644 --- a/ydb/core/driver_lib/run/CMakeLists.linux-aarch64.txt +++ b/ydb/core/driver_lib/run/CMakeLists.linux-aarch64.txt @@ -88,6 +88,7 @@ target_link_libraries(run PUBLIC ydb-core-scheme ydb-core-scheme_types ydb-core-security + ydb-core-statistics core-sys_view-processor core-sys_view-service ydb-core-tablet diff --git a/ydb/core/driver_lib/run/CMakeLists.linux-x86_64.txt b/ydb/core/driver_lib/run/CMakeLists.linux-x86_64.txt index 51fac4a558..5d41e085cf 100644 --- a/ydb/core/driver_lib/run/CMakeLists.linux-x86_64.txt +++ b/ydb/core/driver_lib/run/CMakeLists.linux-x86_64.txt @@ -88,6 +88,7 @@ target_link_libraries(run PUBLIC ydb-core-scheme ydb-core-scheme_types ydb-core-security + ydb-core-statistics core-sys_view-processor core-sys_view-service ydb-core-tablet diff --git a/ydb/core/driver_lib/run/CMakeLists.windows-x86_64.txt b/ydb/core/driver_lib/run/CMakeLists.windows-x86_64.txt index 1c13f29209..7fe9caae62 100644 --- a/ydb/core/driver_lib/run/CMakeLists.windows-x86_64.txt +++ b/ydb/core/driver_lib/run/CMakeLists.windows-x86_64.txt @@ -87,6 +87,7 @@ target_link_libraries(run PUBLIC ydb-core-scheme ydb-core-scheme_types ydb-core-security + ydb-core-statistics core-sys_view-processor core-sys_view-service ydb-core-tablet diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index d219a75f16..7bec6054db 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -113,6 +113,7 @@ #include <ydb/core/sys_view/processor/processor.h> #include <ydb/core/sys_view/service/sysview_service.h> +#include <ydb/core/statistics/stat_service.h> #include <ydb/core/tablet/bootstrapper.h> #include <ydb/core/tablet/node_tablet_monitor.h> @@ -2455,6 +2456,19 @@ void TSysViewServiceInitializer::InitializeServices(NActors::TActorSystemSetup* TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->UserPoolId))); } +TStatServiceInitializer::TStatServiceInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{ +} + +void TStatServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + auto actor = NStat::CreateStatService(); + + setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + NStat::MakeStatServiceID(), + TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->UserPoolId))); +} + TMeteringWriterInitializer::TMeteringWriterInitializer(const TKikimrRunConfig &runConfig) : IKikimrServicesInitializer(runConfig) , KikimrRunConfig(runConfig) diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index cef99ac708..ee68e58eee 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -505,6 +505,13 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; +class TStatServiceInitializer : public IKikimrServicesInitializer { +public: + TStatServiceInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; + class TMeteringWriterInitializer : public IKikimrServicesInitializer { public: TMeteringWriterInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 0a6b32a1fd..50e6fa15ce 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1554,6 +1554,8 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TKafkaProxyServiceInitializer(runConfig)); } + sil->AddServiceInitializer(new TStatServiceInitializer(runConfig)); + return sil; } diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 8b0e21d25e..4635d10cfc 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -102,6 +102,7 @@ PEERDIR( ydb/core/scheme ydb/core/scheme_types ydb/core/security + ydb/core/statistics ydb/core/sys_view/processor ydb/core/sys_view/service ydb/core/tablet diff --git a/ydb/core/protos/CMakeLists.darwin-x86_64.txt b/ydb/core/protos/CMakeLists.darwin-x86_64.txt index b34e66bd7a..1dea9828e7 100644 --- a/ydb/core/protos/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/protos/CMakeLists.darwin-x86_64.txt @@ -1502,6 +1502,18 @@ get_built_tool_path( cpp_styleguide ) get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency tools/enum_parser/enum_parser @@ -1629,6 +1641,7 @@ target_proto_messages(ydb-core-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/protos/follower_group.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/ssa.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/statestorage.proto + ${CMAKE_SOURCE_DIR}/ydb/core/protos/statistics.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/stream.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/subdomains.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/table_stats.proto diff --git a/ydb/core/protos/CMakeLists.linux-aarch64.txt b/ydb/core/protos/CMakeLists.linux-aarch64.txt index 27fe911721..180e3712ac 100644 --- a/ydb/core/protos/CMakeLists.linux-aarch64.txt +++ b/ydb/core/protos/CMakeLists.linux-aarch64.txt @@ -1502,6 +1502,18 @@ get_built_tool_path( cpp_styleguide ) get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency tools/enum_parser/enum_parser @@ -1630,6 +1642,7 @@ target_proto_messages(ydb-core-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/protos/follower_group.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/ssa.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/statestorage.proto + ${CMAKE_SOURCE_DIR}/ydb/core/protos/statistics.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/stream.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/subdomains.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/table_stats.proto diff --git a/ydb/core/protos/CMakeLists.linux-x86_64.txt b/ydb/core/protos/CMakeLists.linux-x86_64.txt index 27fe911721..180e3712ac 100644 --- a/ydb/core/protos/CMakeLists.linux-x86_64.txt +++ b/ydb/core/protos/CMakeLists.linux-x86_64.txt @@ -1502,6 +1502,18 @@ get_built_tool_path( cpp_styleguide ) get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency tools/enum_parser/enum_parser @@ -1630,6 +1642,7 @@ target_proto_messages(ydb-core-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/protos/follower_group.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/ssa.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/statestorage.proto + ${CMAKE_SOURCE_DIR}/ydb/core/protos/statistics.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/stream.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/subdomains.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/table_stats.proto diff --git a/ydb/core/protos/CMakeLists.windows-x86_64.txt b/ydb/core/protos/CMakeLists.windows-x86_64.txt index b34e66bd7a..1dea9828e7 100644 --- a/ydb/core/protos/CMakeLists.windows-x86_64.txt +++ b/ydb/core/protos/CMakeLists.windows-x86_64.txt @@ -1502,6 +1502,18 @@ get_built_tool_path( cpp_styleguide ) get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency tools/enum_parser/enum_parser @@ -1629,6 +1641,7 @@ target_proto_messages(ydb-core-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/protos/follower_group.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/ssa.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/statestorage.proto + ${CMAKE_SOURCE_DIR}/ydb/core/protos/statistics.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/stream.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/subdomains.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/table_stats.proto diff --git a/ydb/core/protos/statistics.proto b/ydb/core/protos/statistics.proto new file mode 100644 index 0000000000..b81d4c486e --- /dev/null +++ b/ydb/core/protos/statistics.proto @@ -0,0 +1,21 @@ +import "ydb/core/scheme/protos/pathid.proto"; + +package NKikimrStat; + +option java_package = "ru.yandex.kikimr.proto"; + +message TEvGetStatisticsFromSS { + optional uint64 RequestId = 1; + repeated NKikimrProto.TPathID PathIds = 2; +}; + +message TEvGetStatisticsFromSSResult { + optional uint64 RequestId = 1; + message TEntry { + optional NKikimrProto.TPathID PathId = 1; + optional bool Success = 2; + optional uint64 RowCount = 3; + optional uint64 BytesSize = 4; + } + repeated TEntry Entries = 2; +}; diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index d68c5f5b7a..819d1a192e 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -97,6 +97,7 @@ SRCS( follower_group.proto ssa.proto statestorage.proto + statistics.proto stream.proto subdomains.proto table_stats.proto diff --git a/ydb/core/statistics/CMakeLists.darwin-x86_64.txt b/ydb/core/statistics/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..51a693aff4 --- /dev/null +++ b/ydb/core/statistics/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(ydb-core-statistics) +target_link_libraries(ydb-core-statistics PUBLIC + contrib-libs-cxxsupp + yutil + cpp-actors-core + ydb-core-protos + ydb-core-scheme +) +target_sources(ydb-core-statistics PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/statistics/stat_service.cpp +) diff --git a/ydb/core/statistics/CMakeLists.linux-aarch64.txt b/ydb/core/statistics/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..bc22ced4f8 --- /dev/null +++ b/ydb/core/statistics/CMakeLists.linux-aarch64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(ydb-core-statistics) +target_link_libraries(ydb-core-statistics PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-actors-core + ydb-core-protos + ydb-core-scheme +) +target_sources(ydb-core-statistics PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/statistics/stat_service.cpp +) diff --git a/ydb/core/statistics/CMakeLists.linux-x86_64.txt b/ydb/core/statistics/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..bc22ced4f8 --- /dev/null +++ b/ydb/core/statistics/CMakeLists.linux-x86_64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(ydb-core-statistics) +target_link_libraries(ydb-core-statistics PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-actors-core + ydb-core-protos + ydb-core-scheme +) +target_sources(ydb-core-statistics PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/statistics/stat_service.cpp +) diff --git a/ydb/core/statistics/CMakeLists.txt b/ydb/core/statistics/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/core/statistics/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/core/statistics/CMakeLists.windows-x86_64.txt b/ydb/core/statistics/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..51a693aff4 --- /dev/null +++ b/ydb/core/statistics/CMakeLists.windows-x86_64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(ydb-core-statistics) +target_link_libraries(ydb-core-statistics PUBLIC + contrib-libs-cxxsupp + yutil + cpp-actors-core + ydb-core-protos + ydb-core-scheme +) +target_sources(ydb-core-statistics PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/statistics/stat_service.cpp +) diff --git a/ydb/core/statistics/events.h b/ydb/core/statistics/events.h new file mode 100644 index 0000000000..5b5ffe0e0f --- /dev/null +++ b/ydb/core/statistics/events.h @@ -0,0 +1,73 @@ +#pragma once + +#include <ydb/core/base/events.h> +#include <ydb/core/scheme/scheme_pathid.h> +#include <ydb/core/protos/statistics.pb.h> +#include <library/cpp/actors/core/events.h> + +namespace NKikimr { +namespace NStat { + +struct TStatSimple { + ui64 RowCount = 0; + ui64 BytesSize = 0; +}; + +struct TStatHyperLogLog { + // TODO: +}; + +// TODO: other stats +enum EStatType { + SIMPLE = 0, + HYPER_LOG_LOG = 1, + // TODO... +}; + +struct TRequest { + EStatType StatType; + TPathId PathId; + std::optional<TString> ColumnName; // not used for simple stat +}; + +struct TResponse { + bool Success = true; + TRequest Req; + std::variant<TStatSimple, TStatHyperLogLog> Statistics; +}; + +struct TEvStatistics { + enum EEv { + EvGetStatistics = EventSpaceBegin(TKikimrEvents::ES_STATISTICS), + EvGetStatisticsResult, + + EvGetStatisticsFromSS, + EvGetStatisticsFromSSResult, + + EvEnd + }; + + struct TEvGetStatistics : public TEventLocal<TEvGetStatistics, EvGetStatistics> { + std::vector<TRequest> StatRequests; + }; + + struct TEvGetStatisticsResult : public TEventLocal<TEvGetStatisticsResult, EvGetStatisticsResult> { + bool Success = true; + std::vector<TResponse> StatResponses; + }; + + struct TEvGetStatisticsFromSS : public TEventPB< + TEvGetStatisticsFromSS, + NKikimrStat::TEvGetStatisticsFromSS, + EvGetStatisticsFromSS> + {}; + + struct TEvGetStatisticsFromSSResult : public TEventPB< + TEvGetStatisticsFromSSResult, + NKikimrStat::TEvGetStatisticsFromSSResult, + EvGetStatisticsFromSSResult> + {}; +}; + +} // NStat +} // NKikimr diff --git a/ydb/core/statistics/stat_service.cpp b/ydb/core/statistics/stat_service.cpp new file mode 100644 index 0000000000..86b7d8d409 --- /dev/null +++ b/ydb/core/statistics/stat_service.cpp @@ -0,0 +1,188 @@ +#include "stat_service.h" +#include "events.h" + +#include <ydb/library/services/services.pb.h> +#include <ydb/core/tx/scheme_cache/scheme_cache.h> +#include <ydb/core/base/tablet_pipecache.h> + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + +namespace NKikimr { +namespace NStat { + +class TStatService : public TActorBootstrapped<TStatService> { +public: + using TBase = TActorBootstrapped<TStatService>; + + static constexpr auto ActorActivityType() { + return NKikimrServices::TActivity::STAT_SERVICE; + } + + void Bootstrap() { + Become(&TStatService::StateWork); + } + + STFUNC(StateWork) { + switch(ev->GetTypeRewrite()) { + hFunc(TEvStatistics::TEvGetStatistics, Handle); + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + hFunc(TEvStatistics::TEvGetStatisticsFromSSResult, Handle); + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + cFunc(TEvents::TEvPoison::EventType, PassAway); + default: + LOG_CRIT_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "NStat::TStatService: unexpected event# " << ev->GetTypeRewrite()); + } + } + +private: + void Handle(TEvStatistics::TEvGetStatistics::TPtr& ev) { + ui64 requestId = NextRequestId++; + + auto& request = InFlight[requestId]; + request.ReplyToActorId = ev->Sender; + request.EvCookie = ev->Cookie; + request.StatRequests.swap(ev->Get()->StatRequests); + + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + auto navigate = MakeHolder<TNavigate>(); + for (const auto& req : request.StatRequests) { + TNavigate::TEntry entry; + entry.TableId = TTableId(req.PathId.OwnerId, req.PathId.LocalPathId); + entry.Operation = TNavigate::EOp::OpPath; + entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; + entry.RedirectRequired = false; + navigate->ResultSet.push_back(entry); + } + + navigate->Cookie = requestId; + + Send(MakeSchemeCacheID(), + new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.Release())); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + THolder<TNavigate> navigate(ev->Get()->Request.Release()); + + ui64 requestId = navigate->Cookie; + auto requestFound = InFlight.find(requestId); + if (requestFound == InFlight.end()) { + return; + } + auto& request = requestFound->second; + + std::unordered_set<ui64> ssIds; + for (const auto& entry : navigate->ResultSet) { + if (entry.Status != TNavigate::EStatus::Ok) { + continue; + } + ssIds.insert(entry.DomainInfo->ExtractSchemeShard()); + } + if (ssIds.size() != 1) { + ReplyFailed(requestId); + return; + } + ui64 schemeShardId = *ssIds.begin(); + + auto requestSS = MakeHolder<TEvStatistics::TEvGetStatisticsFromSS>(); + requestSS->Record.SetRequestId(requestId); + for (const auto& entry : request.StatRequests) { + auto& pathId = *requestSS->Record.AddPathIds(); + pathId.SetOwnerId(entry.PathId.OwnerId); + pathId.SetLocalId(entry.PathId.LocalPathId); + } + + Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward( + requestSS.Release(), schemeShardId, true, requestId)); + } + + void Handle(TEvStatistics::TEvGetStatisticsFromSSResult::TPtr& ev) { + const auto& record = ev->Get()->Record; + ui64 requestId = record.GetRequestId(); + auto requestFound = InFlight.find(requestId); + if (requestFound == InFlight.end()) { + return; + } + auto& request = requestFound->second; + + auto result = MakeHolder<TEvStatistics::TEvGetStatisticsResult>(); + result->Success = true; + size_t i = 0; + for (auto& req : request.StatRequests) { + const auto& entry = record.GetEntries(i++); + if (entry.GetPathId().GetOwnerId() != req.PathId.OwnerId || + entry.GetPathId().GetLocalId() != req.PathId.LocalPathId) + { + result->Success = false; + break; + } + TResponse rsp; + rsp.Success = entry.GetSuccess(); + rsp.Req = req; + TStatSimple stat; + stat.RowCount = entry.GetRowCount(); + stat.BytesSize = entry.GetBytesSize(); + rsp.Statistics = stat; + result->StatResponses.push_back(rsp); + } + + InFlight.erase(requestId); + + Send(request.ReplyToActorId, result.Release(), 0, request.EvCookie); + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { + auto upperBound = InFlight.upper_bound(ev->Cookie); + for (auto it = InFlight.begin(); it != upperBound; ++it) { + ReplyFailed(it->first, false); + } + InFlight.erase(InFlight.begin(), upperBound); + } + + void ReplyFailed(ui64 requestId, bool removeRequest = true) { + auto requestFound = InFlight.find(requestId); + if (requestFound == InFlight.end()) { + return; + } + auto& request = requestFound->second; + + auto result = MakeHolder<TEvStatistics::TEvGetStatisticsResult>(); + result->Success = false; + for (auto& req : request.StatRequests) { + TResponse rsp; + rsp.Success = false; + rsp.Req = req; + } + + if (removeRequest) { + InFlight.erase(requestId); + } + + Send(request.ReplyToActorId, std::move(result), 0, request.EvCookie); + } + + void PassAway() override { + Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + TBase::PassAway(); + } + +private: + struct TRequestState { + NActors::TActorId ReplyToActorId; + ui64 EvCookie = 0; + std::vector<TRequest> StatRequests; + }; + std::map<ui64, TRequestState> InFlight; // id -> state + ui64 NextRequestId = 1; +}; + +THolder<IActor> CreateStatService() { + return MakeHolder<TStatService>(); +} + +} // NStat +} // NKikimr + diff --git a/ydb/core/statistics/stat_service.h b/ydb/core/statistics/stat_service.h new file mode 100644 index 0000000000..c99a17727c --- /dev/null +++ b/ydb/core/statistics/stat_service.h @@ -0,0 +1,15 @@ +#pragma once + +#include <library/cpp/actors/core/actor.h> + +namespace NKikimr { +namespace NStat { + +inline NActors::TActorId MakeStatServiceID() { + return NActors::TActorId(0, TStringBuf("Statistics")); +} + +THolder<NActors::IActor> CreateStatService(); + +} // NStat +} // NKikimr diff --git a/ydb/core/statistics/ya.make b/ydb/core/statistics/ya.make new file mode 100644 index 0000000000..6496470385 --- /dev/null +++ b/ydb/core/statistics/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +SRCS( + events.h + stat_service.h + stat_service.cpp +) + +PEERDIR( + util + library/cpp/actors/core + ydb/core/protos + ydb/core/scheme +) + +END() diff --git a/ydb/core/tx/schemeshard/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/schemeshard/CMakeLists.darwin-x86_64.txt index 08540cd41c..becbb742b3 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.darwin-x86_64.txt @@ -100,6 +100,7 @@ target_link_libraries(core-tx-schemeshard PUBLIC core-persqueue-writer ydb-core-protos ydb-core-scheme + ydb-core-statistics core-sys_view-partition_stats ydb-core-tablet ydb-core-tablet_flat diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt index c495205d90..1c7c2fc30f 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt @@ -101,6 +101,7 @@ target_link_libraries(core-tx-schemeshard PUBLIC core-persqueue-writer ydb-core-protos ydb-core-scheme + ydb-core-statistics core-sys_view-partition_stats ydb-core-tablet ydb-core-tablet_flat diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux-x86_64.txt b/ydb/core/tx/schemeshard/CMakeLists.linux-x86_64.txt index c495205d90..1c7c2fc30f 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.linux-x86_64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.linux-x86_64.txt @@ -101,6 +101,7 @@ target_link_libraries(core-tx-schemeshard PUBLIC core-persqueue-writer ydb-core-protos ydb-core-scheme + ydb-core-statistics core-sys_view-partition_stats ydb-core-tablet ydb-core-tablet_flat diff --git a/ydb/core/tx/schemeshard/CMakeLists.windows-x86_64.txt b/ydb/core/tx/schemeshard/CMakeLists.windows-x86_64.txt index 63834ed7c0..5040e475df 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.windows-x86_64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.windows-x86_64.txt @@ -100,6 +100,7 @@ target_link_libraries(core-tx-schemeshard PUBLIC core-persqueue-writer ydb-core-protos ydb-core-scheme + ydb-core-statistics core-sys_view-partition_stats ydb-core-tablet ydb-core-tablet_flat diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 4814d84a53..dd3c193779 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -10,6 +10,7 @@ #include <ydb/core/base/tx_processing.h> #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/sys_view/partition_stats/partition_stats.h> +#include <ydb/core/statistics/events.h> #include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/library/yql/minikql/mkql_type_ops.h> @@ -4415,6 +4416,8 @@ void TSchemeShard::StateWork(STFUNC_SIG) { HFuncTraced(TEvPersQueue::TEvProposeTransactionAttachResult, Handle); + HFuncTraced(NStat::TEvStatistics::TEvGetStatisticsFromSS, Handle); + default: if (!HandleDefaultEvents(ev, SelfId())) { ALOG_WARN(NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -6741,5 +6744,33 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvLogin::TPtr &ev, const TActorContex Execute(CreateTxLogin(ev), ctx); } +void TSchemeShard::Handle(NStat::TEvStatistics::TEvGetStatisticsFromSS::TPtr& ev, const TActorContext& ctx) { + const auto& recordIn = ev->Get()->Record; + + auto result = MakeHolder<NStat::TEvStatistics::TEvGetStatisticsFromSSResult>(); + auto& recordOut = result->Record; + recordOut.SetRequestId(recordIn.GetRequestId()); + + for (const auto& pathIdIn : recordIn.GetPathIds()) { + TPathId pathId(pathIdIn.GetOwnerId(), pathIdIn.GetLocalId()); + auto pathFound = Tables.find(pathId); + + auto& entryOut = *recordOut.AddEntries(); + entryOut.MutablePathId()->CopyFrom(pathIdIn); + if (pathFound == Tables.end()) { + entryOut.SetSuccess(false); + entryOut.SetRowCount(0); + entryOut.SetBytesSize(0); + } else { + const auto& aggregated = pathFound->second->GetStats().Aggregated; + entryOut.SetSuccess(true); + entryOut.SetRowCount(aggregated.RowCount); + entryOut.SetBytesSize(aggregated.DataSize); + } + } + + ctx.Send(ev->Sender, result.Release(), 0, ev->Cookie); +} + } // namespace NSchemeShard } // namespace NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 5f3970d2a8..cb925265b9 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -33,6 +33,7 @@ #include <ydb/core/protos/filestore_config.pb.h> #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/sys_view/common/events.h> +#include <ydb/core/statistics/events.h> #include <ydb/core/tablet/pipe_tracker.h> #include <ydb/core/tablet/tablet_counters.h> #include <ydb/core/tablet/tablet_pipe_client_cache.h> @@ -1241,6 +1242,9 @@ public: void RemoveCdcStreamScanShardStatus(NIceDb::TNiceDb& db, const TPathId& streamPathId, const TShardIdx& shardIdx); // } // NCdcStreamScan + // simple statistics + void Handle(NStat::TEvStatistics::TEvGetStatisticsFromSS::TPtr& ev, const TActorContext& ctx); + public: void ChangeStreamShardsCount(i64 delta) override; void ChangeStreamShardsQuota(i64 delta) override; diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index 54a72e930c..15381e10a0 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -252,6 +252,7 @@ PEERDIR( ydb/core/persqueue/writer ydb/core/protos ydb/core/scheme + ydb/core/statistics ydb/core/sys_view/partition_stats ydb/core/tablet ydb/core/tablet_flat diff --git a/ydb/core/ya.make b/ydb/core/ya.make index 4450028a82..cc7fb23967 100644 --- a/ydb/core/ya.make +++ b/ydb/core/ya.make @@ -44,6 +44,7 @@ RECURSE( scheme scheme_types security + statistics sys_view tablet tablet_flat diff --git a/ydb/library/services/services.proto b/ydb/library/services/services.proto index dcbf04d226..193a125c07 100644 --- a/ydb/library/services/services.proto +++ b/ydb/library/services/services.proto @@ -373,6 +373,8 @@ enum EServiceKikimr { KAFKA_PROXY = 2200; OBJECTS_MONITORING = 2300; + + STATISTICS = 2400; }; message TActivity { @@ -997,5 +999,6 @@ message TActivity { COLUMNSHARD_CONVEYOR = 615; PERSQUEUE_READ_QUOTER = 616; KAFKA_PRODUCE_ACTOR = 617; + STAT_SERVICE = 618; }; }; |