diff options
author | dinmukhammed <dinmukhammed@yandex-team.ru> | 2022-03-01 23:05:34 +0300 |
---|---|---|
committer | dinmukhammed <dinmukhammed@yandex-team.ru> | 2022-03-01 23:05:34 +0300 |
commit | 84855f14b76adcc3c682b1d72407e4bc1b9aa191 (patch) | |
tree | a2a6630c742e226580caff65ad6547450614b191 | |
parent | f1db7e1d2a6f1e911c41352aecb7897b8cc48d74 (diff) | |
download | ydb-84855f14b76adcc3c682b1d72407e4bc1b9aa191.tar.gz |
YQ-858 Uptime metric (service + public)
Added TServiceCounters to yql folder
ref:1d878e32d544ebd4b2f00459b751bfa59cbcd25a
22 files changed, 113 insertions, 63 deletions
diff --git a/ydb/core/yq/libs/actors/CMakeLists.txt b/ydb/core/yq/libs/actors/CMakeLists.txt index f62add179b..03aca2e6e7 100644 --- a/ydb/core/yq/libs/actors/CMakeLists.txt +++ b/ydb/core/yq/libs/actors/CMakeLists.txt @@ -31,7 +31,6 @@ target_link_libraries(yq-libs-actors PUBLIC yq-libs-common yq-libs-control_plane_storage libs-control_plane_storage-events - providers-common-db_id_async_resolver yq-libs-db_schema yq-libs-events yq-libs-private_client @@ -57,6 +56,8 @@ target_link_libraries(yq-libs-actors PUBLIC providers-clickhouse-provider providers-common-codec providers-common-comp_nodes + providers-common-db_id_async_resolver + providers-common-metrics providers-common-provider common-schema-mkql providers-common-udf_resolve diff --git a/ydb/core/yq/libs/actors/nodes_manager.h b/ydb/core/yq/libs/actors/nodes_manager.h index 3377df3cb7..55a1d2300f 100644 --- a/ydb/core/yq/libs/actors/nodes_manager.h +++ b/ydb/core/yq/libs/actors/nodes_manager.h @@ -1,6 +1,6 @@ #pragma once -#include <ydb/core/yq/libs/common/service_counters.h> +#include <ydb/library/yql/providers/common/metrics/service_counters.h> #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/shared_resources/shared_resources.h> diff --git a/ydb/core/yq/libs/actors/pending_fetcher.cpp b/ydb/core/yq/libs/actors/pending_fetcher.cpp index 61359cee6d..20c03c9b37 100644 --- a/ydb/core/yq/libs/actors/pending_fetcher.cpp +++ b/ydb/core/yq/libs/actors/pending_fetcher.cpp @@ -323,7 +323,7 @@ private: task.automatic() ? (folderId ? "automatic_" + folderId : "automatic") : queryId); queryCounters.Counters = queryCounters.RootCounters; - const auto queryUptimeCounter = queryCounters.PublicCounters->GetNamedCounter("name", "query.uptime_seconds", false); + queryCounters.InitUptimeCounter(); const auto createdAt = TInstant::Now(); TRunActorParams params( @@ -355,7 +355,6 @@ private: task.query_name(), NProtoInterop::CastFromProto(task.deadline()), ClientCounters, - queryUptimeCounter, createdAt); auto runActorId = Register(CreateRunActor(SelfId(), queryCounters, std::move(params))); diff --git a/ydb/core/yq/libs/actors/pinger.cpp b/ydb/core/yq/libs/actors/pinger.cpp index 899eb7423a..30d4beecf4 100644 --- a/ydb/core/yq/libs/actors/pinger.cpp +++ b/ydb/core/yq/libs/actors/pinger.cpp @@ -150,7 +150,7 @@ public: const TActorId parent, const NConfig::TPingerConfig& config, TInstant deadline, - const NMonitoring::TDynamicCounters::TCounterPtr& queryUptime, + const ::NYq::NCommon::TServiceCounters& queryCounters, TInstant createdAt) : Config(config) , Scope(scope) @@ -160,7 +160,7 @@ public: , Client(client) , Parent(parent) , Deadline(deadline) - , QueryUptime(queryUptime) + , QueryCounters(queryCounters) , CreatedAt(createdAt) { } @@ -382,7 +382,7 @@ private: } void Ping(Yq::Private::PingTaskRequest request, ui64 cookie) { - QueryUptime->Set((TInstant::Now() - CreatedAt).Seconds()); + QueryCounters.SetUptimePublicAndServiceCounter((TInstant::Now() - CreatedAt).Seconds()); // Fill ids request.set_scope(Scope.ToString()); request.set_owner_id(OwnerId); @@ -431,7 +431,7 @@ private: const TActorId Parent; const TInstant Deadline; - const NMonitoring::TDynamicCounters::TCounterPtr QueryUptime; + const ::NYq::NCommon::TServiceCounters QueryCounters; const TInstant CreatedAt; std::deque<TForwardPingReqInfo> ForwardRequests; @@ -450,7 +450,7 @@ IActor* CreatePingerActor( const TActorId parent, const NConfig::TPingerConfig& config, TInstant deadline, - const NMonitoring::TDynamicCounters::TCounterPtr& queryUptime, + const ::NYq::NCommon::TServiceCounters& queryCounters, TInstant createdAt) { return new TPingerActor( @@ -462,7 +462,7 @@ IActor* CreatePingerActor( parent, config, deadline, - queryUptime, + queryCounters, createdAt); } diff --git a/ydb/core/yq/libs/actors/proxy.h b/ydb/core/yq/libs/actors/proxy.h index e58062b571..1d18b02548 100644 --- a/ydb/core/yq/libs/actors/proxy.h +++ b/ydb/core/yq/libs/actors/proxy.h @@ -20,7 +20,7 @@ #include <library/cpp/time_provider/time_provider.h> #include <library/cpp/random_provider/random_provider.h> -#include <ydb/core/yq/libs/common/service_counters.h> +#include <ydb/library/yql/providers/common/metrics/service_counters.h> namespace NKikimr { namespace NMiniKQL { @@ -85,7 +85,7 @@ NActors::IActor* CreatePingerActor( const NActors::TActorId parent, const NConfig::TPingerConfig& config, TInstant deadline, - const NMonitoring::TDynamicCounters::TCounterPtr& queryUptime, + const ::NYq::NCommon::TServiceCounters& queryCounters, TInstant createdAt); TString MakeInternalError(const TString& text); diff --git a/ydb/core/yq/libs/actors/run_actor.cpp b/ydb/core/yq/libs/actors/run_actor.cpp index 3e40946758..cfe604b7a0 100644 --- a/ydb/core/yq/libs/actors/run_actor.cpp +++ b/ydb/core/yq/libs/actors/run_actor.cpp @@ -96,14 +96,14 @@ public: , EnableCheckpointCoordinator(Params.QueryType == YandexQuery::QueryContent::STREAMING && Params.CheckpointCoordinatorConfig.GetEnabled()) , MaxTasksPerOperation(Params.CommonConfig.GetMaxTasksPerOperation() ? Params.CommonConfig.GetMaxTasksPerOperation() : 40) { - Params.QueryUptimeCounter->Set(0); + QueryCounters.SetUptimePublicAndServiceCounter(0); } static constexpr char ActorName[] = "YQ_RUN_ACTOR"; void Bootstrap() { LOG_D("Start run actor. Compute state: " << YandexQuery::QueryMeta::ComputeStatus_Name(Params.Status)); - Params.QueryUptimeCounter->Set((TInstant::Now() - CreatedAt).Seconds()); + QueryCounters.SetUptimePublicAndServiceCounter((TInstant::Now() - CreatedAt).Seconds()); LogReceivedParams(); Pinger = Register( CreatePingerActor( @@ -122,7 +122,7 @@ public: SelfId(), Params.PingerConfig, Params.Deadline, - Params.QueryUptimeCounter, + QueryCounters, CreatedAt )); Become(&TRunActor::StateFuncWrapper<&TRunActor::StateFunc>); diff --git a/ydb/core/yq/libs/actors/run_actor_params.cpp b/ydb/core/yq/libs/actors/run_actor_params.cpp index deb1279d62..e7477ebdc7 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.cpp +++ b/ydb/core/yq/libs/actors/run_actor_params.cpp @@ -45,7 +45,6 @@ TRunActorParams::TRunActorParams( const TString& queryName, const TInstant& deadline, const NMonitoring::TDynamicCounterPtr& clientCounters, - const NMonitoring::TDynamicCounters::TCounterPtr& queryUptimeCounter, TInstant createdAt ) : Driver(driver) @@ -88,7 +87,6 @@ TRunActorParams::TRunActorParams( , QueryName(queryName) , Deadline(deadline) , ClientCounters(clientCounters) - , QueryUptimeCounter(queryUptimeCounter) , CreatedAt(createdAt) { } diff --git a/ydb/core/yq/libs/actors/run_actor_params.h b/ydb/core/yq/libs/actors/run_actor_params.h index 1ce6ab64f2..f523eea748 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.h +++ b/ydb/core/yq/libs/actors/run_actor_params.h @@ -59,7 +59,6 @@ struct TRunActorParams { // TODO2 : Change name const TString& queryName, const TInstant& deadline, const NMonitoring::TDynamicCounterPtr& clientCounters, - const NMonitoring::TDynamicCounters::TCounterPtr& queryUptimeCounter, TInstant createdAt ); @@ -109,7 +108,6 @@ struct TRunActorParams { // TODO2 : Change name TInstant Deadline; const NMonitoring::TDynamicCounterPtr ClientCounters; - const NMonitoring::TDynamicCounters::TCounterPtr QueryUptimeCounter; const TInstant CreatedAt; }; diff --git a/ydb/core/yq/libs/actors/ya.make b/ydb/core/yq/libs/actors/ya.make index 2d1d074a85..c2f3cb7a2a 100644 --- a/ydb/core/yq/libs/actors/ya.make +++ b/ydb/core/yq/libs/actors/ya.make @@ -40,7 +40,6 @@ PEERDIR( ydb/core/yq/libs/common ydb/core/yq/libs/control_plane_storage ydb/core/yq/libs/control_plane_storage/events - ydb/library/yql/providers/common/db_id_async_resolver ydb/core/yq/libs/db_schema ydb/core/yq/libs/events ydb/core/yq/libs/private_client @@ -66,6 +65,8 @@ PEERDIR( ydb/library/yql/providers/clickhouse/provider ydb/library/yql/providers/common/codec ydb/library/yql/providers/common/comp_nodes + ydb/library/yql/providers/common/db_id_async_resolver + ydb/library/yql/providers/common/metrics ydb/library/yql/providers/common/provider ydb/library/yql/providers/common/schema/mkql ydb/library/yql/providers/common/udf_resolve diff --git a/ydb/core/yq/libs/common/CMakeLists.txt b/ydb/core/yq/libs/common/CMakeLists.txt index f8e77abd8f..275e5dc237 100644 --- a/ydb/core/yq/libs/common/CMakeLists.txt +++ b/ydb/core/yq/libs/common/CMakeLists.txt @@ -14,7 +14,6 @@ target_compile_options(yq-libs-common PRIVATE target_link_libraries(yq-libs-common PUBLIC contrib-libs-cxxsupp yutil - cpp-monlib-dynamic_counters libs-control_plane_storage-events yq-libs-events providers-common-structured_token diff --git a/ydb/core/yq/libs/common/service_counters.h b/ydb/core/yq/libs/common/service_counters.h deleted file mode 100644 index 594cbf04fc..0000000000 --- a/ydb/core/yq/libs/common/service_counters.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include <util/generic/string.h> -#include <library/cpp/monlib/dynamic_counters/counters.h> - -namespace NYq { -namespace NCommon { - -struct TServiceCounters { - NMonitoring::TDynamicCounterPtr RootCounters; // "counters/counters=yq" - root counters for service metrics - NMonitoring::TDynamicCounterPtr PublicCounters; // "counters/counters=public" - root counters for cloud user metrics - NMonitoring::TDynamicCounterPtr Counters; // "counters/counters=yq/subsystem=smth" - subsystem part, may match to RootCounters if subsystem name is empty - - explicit TServiceCounters(const NMonitoring::TDynamicCounterPtr& rootCounters, const NMonitoring::TDynamicCounterPtr& publicCounters, const TString& subsystemName = "") - : RootCounters(rootCounters) - , PublicCounters(publicCounters) - , Counters(subsystemName ? RootCounters->GetSubgroup("subsystem", subsystemName) : RootCounters) - { - } - - explicit TServiceCounters(const NMonitoring::TDynamicCounterPtr& baseCounters, const TString& subsystemName = "") - : RootCounters(baseCounters->GetSubgroup("counters", "yq")) - , PublicCounters(baseCounters->GetSubgroup("counters", "yq_public")) - , Counters(subsystemName ? RootCounters->GetSubgroup("subsystem", subsystemName) : RootCounters) - { - } - - explicit TServiceCounters(const TServiceCounters& serviceCounters, const TString& subsystemName = "") - : RootCounters(serviceCounters.RootCounters) - , PublicCounters(serviceCounters.PublicCounters) - , Counters(subsystemName ? RootCounters->GetSubgroup("subsystem", subsystemName) : serviceCounters.Counters) - { - } - - TServiceCounters& operator=(const TServiceCounters& serviceCounters) = default; -}; - -} // namespace NCommon -} // namespace NYq diff --git a/ydb/core/yq/libs/common/ya.make b/ydb/core/yq/libs/common/ya.make index d2527b6347..bbf97f05d4 100644 --- a/ydb/core/yq/libs/common/ya.make +++ b/ydb/core/yq/libs/common/ya.make @@ -8,7 +8,6 @@ SRCS( ) PEERDIR( - library/cpp/monlib/dynamic_counters ydb/core/yq/libs/control_plane_storage/events ydb/core/yq/libs/events ydb/library/yql/providers/common/structured_token diff --git a/ydb/core/yq/libs/init/CMakeLists.txt b/ydb/core/yq/libs/init/CMakeLists.txt index 3901dc1bb6..410758b2ba 100644 --- a/ydb/core/yq/libs/init/CMakeLists.txt +++ b/ydb/core/yq/libs/init/CMakeLists.txt @@ -22,6 +22,7 @@ target_link_libraries(yq-libs-init PUBLIC yq-libs-audit yq-libs-checkpoint_storage yq-libs-checkpointing + yq-libs-common yq-libs-control_plane_proxy yq-libs-control_plane_storage yq-libs-events @@ -36,6 +37,7 @@ target_link_libraries(yq-libs-init PUBLIC yql-dq-comp_nodes providers-clickhouse-actors providers-common-comp_nodes + providers-common-metrics providers-dq-actors dq-api-protos providers-dq-provider diff --git a/ydb/core/yq/libs/init/init.cpp b/ydb/core/yq/libs/init/init.cpp index a77f7f62b9..93eff49f40 100644 --- a/ydb/core/yq/libs/init/init.cpp +++ b/ydb/core/yq/libs/init/init.cpp @@ -4,7 +4,7 @@ #include <ydb/core/yq/libs/test_connection/test_connection.h> #include <ydb/core/yq/libs/audit/yq_audit_service.h> -#include <ydb/core/yq/libs/common/service_counters.h> +#include <ydb/library/yql/providers/common/metrics/service_counters.h> #include <ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.h> #include <ydb/core/yq/libs/shared_resources/shared_resources.h> #include <ydb/core/yq/libs/checkpoint_storage/storage_service.h> diff --git a/ydb/core/yq/libs/init/ya.make b/ydb/core/yq/libs/init/ya.make index 129103b024..4d70c60caf 100644 --- a/ydb/core/yq/libs/init/ya.make +++ b/ydb/core/yq/libs/init/ya.make @@ -15,6 +15,7 @@ PEERDIR( ydb/core/yq/libs/audit ydb/core/yq/libs/checkpoint_storage ydb/core/yq/libs/checkpointing + ydb/core/yq/libs/common ydb/core/yq/libs/control_plane_proxy ydb/core/yq/libs/control_plane_storage ydb/core/yq/libs/events @@ -29,6 +30,7 @@ PEERDIR( ydb/library/yql/dq/comp_nodes ydb/library/yql/providers/clickhouse/actors ydb/library/yql/providers/common/comp_nodes + ydb/library/yql/providers/common/metrics ydb/library/yql/providers/dq/actors ydb/library/yql/providers/dq/api/protos ydb/library/yql/providers/dq/provider diff --git a/ydb/library/yql/providers/common/metrics/CMakeLists.txt b/ydb/library/yql/providers/common/metrics/CMakeLists.txt index 5d9c44773f..eb1517baa5 100644 --- a/ydb/library/yql/providers/common/metrics/CMakeLists.txt +++ b/ydb/library/yql/providers/common/metrics/CMakeLists.txt @@ -18,4 +18,5 @@ target_link_libraries(providers-common-metrics PUBLIC target_sources(providers-common-metrics PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/metrics/metrics_registry.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/metrics/sensors_group.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/metrics/service_counters.cpp ) diff --git a/ydb/library/yql/providers/common/metrics/service_counters.cpp b/ydb/library/yql/providers/common/metrics/service_counters.cpp new file mode 100644 index 0000000000..57cc577cb4 --- /dev/null +++ b/ydb/library/yql/providers/common/metrics/service_counters.cpp @@ -0,0 +1,48 @@ +#include "service_counters.h" + +namespace NYq { +namespace NCommon { + +TServiceCounters::TServiceCounters( + const NMonitoring::TDynamicCounterPtr& rootCounters, + const NMonitoring::TDynamicCounterPtr& publicCounters, + const TString& subsystemName) + : RootCounters(rootCounters) + , PublicCounters(publicCounters) + , Counters(subsystemName ? RootCounters->GetSubgroup("subsystem", subsystemName) : RootCounters) +{ +} + +TServiceCounters::TServiceCounters( + const NMonitoring::TDynamicCounterPtr& baseCounters, + const TString& subsystemName) + : RootCounters(baseCounters->GetSubgroup("counters", "yq")) + , PublicCounters(baseCounters->GetSubgroup("counters", "yq_public")) + , Counters(subsystemName ? RootCounters->GetSubgroup("subsystem", subsystemName) : RootCounters) +{ +} + +TServiceCounters::TServiceCounters( + const TServiceCounters& serviceCounters, + const TString& subsystemName) + : RootCounters(serviceCounters.RootCounters) + , PublicCounters(serviceCounters.PublicCounters) + , Counters(subsystemName ? RootCounters->GetSubgroup("subsystem", subsystemName) : serviceCounters.Counters) + , UptimeRootCounter(serviceCounters.UptimeRootCounter) + , UptimePublicCounter(serviceCounters.UptimePublicCounter) +{ +} + +void TServiceCounters::InitUptimeCounter() { + UptimePublicCounter = PublicCounters->GetNamedCounter("name", "query.uptime_seconds", false); + UptimeRootCounter = RootCounters->GetNamedCounter("name", "uptime_seconds", false); +} + +void TServiceCounters::SetUptimePublicAndServiceCounter(i64 val) const { + Y_VERIFY(UptimePublicCounter && UptimeRootCounter); + UptimePublicCounter->Set(val); + UptimeRootCounter->Set(val); +} + +} // namespace NCommon +} // namespace NYq diff --git a/ydb/library/yql/providers/common/metrics/service_counters.h b/ydb/library/yql/providers/common/metrics/service_counters.h new file mode 100644 index 0000000000..47a7d28085 --- /dev/null +++ b/ydb/library/yql/providers/common/metrics/service_counters.h @@ -0,0 +1,38 @@ +#pragma once + +#include <util/generic/string.h> +#include <library/cpp/monlib/dynamic_counters/counters.h> + +namespace NYq { +namespace NCommon { + +struct TServiceCounters { + NMonitoring::TDynamicCounterPtr RootCounters; // "counters/counters=yq" - root counters for service metrics + NMonitoring::TDynamicCounterPtr PublicCounters; // "counters/counters=public" - root counters for cloud user metrics + NMonitoring::TDynamicCounterPtr Counters; // "counters/counters=yq/subsystem=smth" - subsystem part, may match to RootCounters if subsystem name is empty + + NMonitoring::TDynamicCounters::TCounterPtr UptimeRootCounter; // yq/uptime_seconds + NMonitoring::TDynamicCounters::TCounterPtr UptimePublicCounter; // yq_public/query.uptime_seconds + + explicit TServiceCounters( + const NMonitoring::TDynamicCounterPtr& rootCounters, + const NMonitoring::TDynamicCounterPtr& publicCounters, + const TString& subsystemName = ""); + + explicit TServiceCounters( + const NMonitoring::TDynamicCounterPtr& baseCounters, + const TString& subsystemName = ""); + + explicit TServiceCounters( + const TServiceCounters& serviceCounters, + const TString& subsystemName = ""); + + TServiceCounters& operator=(const TServiceCounters& serviceCounters) = default; + + void InitUptimeCounter(); + + void SetUptimePublicAndServiceCounter(i64 val) const; +}; + +} // namespace NCommon +} // namespace NYq diff --git a/ydb/library/yql/providers/common/metrics/ya.make b/ydb/library/yql/providers/common/metrics/ya.make index 19f8832060..1f447425ce 100644 --- a/ydb/library/yql/providers/common/metrics/ya.make +++ b/ydb/library/yql/providers/common/metrics/ya.make @@ -7,6 +7,7 @@ OWNER( SRCS( metrics_registry.cpp sensors_group.cpp + service_counters.cpp ) PEERDIR( diff --git a/ydb/library/yql/providers/dq/actors/CMakeLists.txt b/ydb/library/yql/providers/dq/actors/CMakeLists.txt index efe3341e1e..549efc4259 100644 --- a/ydb/library/yql/providers/dq/actors/CMakeLists.txt +++ b/ydb/library/yql/providers/dq/actors/CMakeLists.txt @@ -31,6 +31,7 @@ target_link_libraries(providers-dq-actors PUBLIC yql-dq-runtime yql-dq-tasks yql-utils-failure_injector + providers-common-metrics dq-api-grpc dq-api-protos providers-dq-common diff --git a/ydb/library/yql/providers/dq/actors/task_controller.h b/ydb/library/yql/providers/dq/actors/task_controller.h index d065ac17e7..b5eff5ccd7 100644 --- a/ydb/library/yql/providers/dq/actors/task_controller.h +++ b/ydb/library/yql/providers/dq/actors/task_controller.h @@ -4,7 +4,7 @@ #include <library/cpp/actors/core/actor.h> -#include <ydb/core/yq/libs/common/service_counters.h> +#include <ydb/library/yql/providers/common/metrics/service_counters.h> namespace NYql { diff --git a/ydb/library/yql/providers/dq/actors/ya.make b/ydb/library/yql/providers/dq/actors/ya.make index 14e93b587d..3b4fd48346 100644 --- a/ydb/library/yql/providers/dq/actors/ya.make +++ b/ydb/library/yql/providers/dq/actors/ya.make @@ -35,6 +35,7 @@ PEERDIR( ydb/library/yql/dq/runtime ydb/library/yql/dq/tasks ydb/library/yql/utils/failure_injector + ydb/library/yql/providers/common/metrics ydb/library/yql/providers/dq/api/grpc ydb/library/yql/providers/dq/api/protos ydb/library/yql/providers/dq/common |