diff options
author | aozeritsky <aozeritsky@ydb.tech> | 2023-10-24 19:48:53 +0300 |
---|---|---|
committer | aozeritsky <aozeritsky@ydb.tech> | 2023-10-24 20:23:39 +0300 |
commit | 623315504d982fbdc29eb377737d4ffcab4e5960 (patch) | |
tree | 1c50963c3406a5954083157daf2c807ce5b863f7 | |
parent | a37ccbf072fe185a1781f70b8ee55ce9b678dc7d (diff) | |
download | ydb-623315504d982fbdc29eb377737d4ffcab4e5960.tar.gz |
Move worker_node/service_node dependencies to ydb/library/yql
114 files changed, 10333 insertions, 0 deletions
diff --git a/.mapping.json b/.mapping.json index 8ea27f5354..9adbc7f04e 100644 --- a/.mapping.json +++ b/.mapping.json @@ -769,6 +769,7 @@ "contrib/libs/pdqsort/CMakeLists.linux-aarch64.txt":"", "contrib/libs/pdqsort/CMakeLists.linux-x86_64.txt":"", "contrib/libs/pdqsort/CMakeLists.txt":"", + "contrib/libs/pfr/CMakeLists.darwin-x86_64.txt":"", "contrib/libs/pfr/CMakeLists.linux-aarch64.txt":"", "contrib/libs/pfr/CMakeLists.linux-x86_64.txt":"", "contrib/libs/pfr/CMakeLists.txt":"", @@ -7507,11 +7508,21 @@ "ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt":"", "ydb/library/yql/providers/dq/actors/CMakeLists.txt":"", "ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yql/providers/dq/actors/events/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yql/providers/dq/actors/events/CMakeLists.txt":"", + "ydb/library/yql/providers/dq/actors/events/CMakeLists.windows-x86_64.txt":"", "ydb/library/yql/providers/dq/actors/ut/CMakeLists.darwin-x86_64.txt":"", "ydb/library/yql/providers/dq/actors/ut/CMakeLists.linux-aarch64.txt":"", "ydb/library/yql/providers/dq/actors/ut/CMakeLists.linux-x86_64.txt":"", "ydb/library/yql/providers/dq/actors/ut/CMakeLists.txt":"", "ydb/library/yql/providers/dq/actors/ut/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yql/providers/dq/actors/yt/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yql/providers/dq/actors/yt/CMakeLists.txt":"", + "ydb/library/yql/providers/dq/actors/yt/CMakeLists.windows-x86_64.txt":"", "ydb/library/yql/providers/dq/api/CMakeLists.txt":"", "ydb/library/yql/providers/dq/api/grpc/CMakeLists.darwin-x86_64.txt":"", "ydb/library/yql/providers/dq/api/grpc/CMakeLists.linux-aarch64.txt":"", @@ -7543,6 +7554,16 @@ "ydb/library/yql/providers/dq/expr_nodes/CMakeLists.linux-x86_64.txt":"", "ydb/library/yql/providers/dq/expr_nodes/CMakeLists.txt":"", "ydb/library/yql/providers/dq/expr_nodes/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.txt":"", + "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.windows-x86_64.txt":"", "ydb/library/yql/providers/dq/interface/CMakeLists.darwin-x86_64.txt":"", "ydb/library/yql/providers/dq/interface/CMakeLists.linux-aarch64.txt":"", "ydb/library/yql/providers/dq/interface/CMakeLists.linux-x86_64.txt":"", @@ -7587,6 +7608,16 @@ "ydb/library/yql/providers/dq/runtime/CMakeLists.linux-x86_64.txt":"", "ydb/library/yql/providers/dq/runtime/CMakeLists.txt":"", "ydb/library/yql/providers/dq/runtime/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yql/providers/dq/scheduler/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yql/providers/dq/scheduler/CMakeLists.txt":"", + "ydb/library/yql/providers/dq/scheduler/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.txt":"", + "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.windows-x86_64.txt":"", "ydb/library/yql/providers/dq/service/CMakeLists.darwin-x86_64.txt":"", "ydb/library/yql/providers/dq/service/CMakeLists.linux-aarch64.txt":"", "ydb/library/yql/providers/dq/service/CMakeLists.linux-x86_64.txt":"", @@ -9727,6 +9758,7 @@ "yt/yt/build/CMakeLists.linux-x86_64.txt":"", "yt/yt/build/CMakeLists.txt":"", "yt/yt/build/CMakeLists.windows-x86_64.txt":"", + "yt/yt/client/CMakeLists.darwin-x86_64.txt":"", "yt/yt/client/CMakeLists.linux-aarch64.txt":"", "yt/yt/client/CMakeLists.linux-x86_64.txt":"", "yt/yt/client/CMakeLists.txt":"", @@ -9736,6 +9768,7 @@ "yt/yt/client/arrow/fbs/CMakeLists.linux-aarch64.txt":"", "yt/yt/client/arrow/fbs/CMakeLists.linux-x86_64.txt":"", "yt/yt/client/arrow/fbs/CMakeLists.txt":"", + "yt/yt/client/query_tracker_client/CMakeLists.darwin-x86_64.txt":"", "yt/yt/client/query_tracker_client/CMakeLists.linux-aarch64.txt":"", "yt/yt/client/query_tracker_client/CMakeLists.linux-x86_64.txt":"", "yt/yt/client/query_tracker_client/CMakeLists.txt":"", @@ -9765,15 +9798,19 @@ "yt/yt/library/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/CMakeLists.txt":"", "yt/yt/library/CMakeLists.windows-x86_64.txt":"", + "yt/yt/library/auth/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/auth/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/auth/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/auth/CMakeLists.txt":"", + "yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/decimal/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/decimal/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/decimal/CMakeLists.txt":"", + "yt/yt/library/erasure/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/erasure/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/erasure/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/erasure/CMakeLists.txt":"", + "yt/yt/library/numeric/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/numeric/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/numeric/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/numeric/CMakeLists.txt":"", @@ -9787,9 +9824,11 @@ "yt/yt/library/profiling/resource_tracker/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/profiling/resource_tracker/CMakeLists.txt":"", "yt/yt/library/profiling/resource_tracker/CMakeLists.windows-x86_64.txt":"", + "yt/yt/library/quantile_digest/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/quantile_digest/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/quantile_digest/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/quantile_digest/CMakeLists.txt":"", + "yt/yt/library/re2/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/re2/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/re2/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/re2/CMakeLists.txt":"", @@ -9825,6 +9864,7 @@ "yt/yt_proto/yt/CMakeLists.linux-x86_64.txt":"", "yt/yt_proto/yt/CMakeLists.txt":"", "yt/yt_proto/yt/CMakeLists.windows-x86_64.txt":"", + "yt/yt_proto/yt/client/CMakeLists.darwin-x86_64.txt":"", "yt/yt_proto/yt/client/CMakeLists.linux-aarch64.txt":"", "yt/yt_proto/yt/client/CMakeLists.linux-x86_64.txt":"", "yt/yt_proto/yt/client/CMakeLists.txt":"", diff --git a/contrib/libs/CMakeLists.darwin-x86_64.txt b/contrib/libs/CMakeLists.darwin-x86_64.txt index ad248a50dc..c62e34c5ff 100644 --- a/contrib/libs/CMakeLists.darwin-x86_64.txt +++ b/contrib/libs/CMakeLists.darwin-x86_64.txt @@ -50,6 +50,7 @@ add_subdirectory(openssl) add_subdirectory(opentelemetry-proto) add_subdirectory(pcre) add_subdirectory(pdqsort) +add_subdirectory(pfr) add_subdirectory(poco) add_subdirectory(protobuf) add_subdirectory(protoc) diff --git a/contrib/libs/pfr/CMakeLists.darwin-x86_64.txt b/contrib/libs/pfr/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..66a1b47cce --- /dev/null +++ b/contrib/libs/pfr/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(contrib-libs-pfr INTERFACE) +target_include_directories(contrib-libs-pfr INTERFACE + ${CMAKE_SOURCE_DIR}/contrib/libs/pfr/include +) +target_link_libraries(contrib-libs-pfr INTERFACE + contrib-libs-cxxsupp + yutil +) diff --git a/contrib/libs/pfr/CMakeLists.txt b/contrib/libs/pfr/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/contrib/libs/pfr/CMakeLists.txt +++ b/contrib/libs/pfr/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/ydb/library/yql/providers/dq/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/CMakeLists.darwin-x86_64.txt index fae483c5c0..1b2879cd08 100644 --- a/ydb/library/yql/providers/dq/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/dq/CMakeLists.darwin-x86_64.txt @@ -12,6 +12,7 @@ add_subdirectory(common) add_subdirectory(config) add_subdirectory(counters) add_subdirectory(expr_nodes) +add_subdirectory(global_worker_manager) add_subdirectory(interface) add_subdirectory(local_gateway) add_subdirectory(mkql) @@ -19,6 +20,7 @@ add_subdirectory(opt) add_subdirectory(planner) add_subdirectory(provider) add_subdirectory(runtime) +add_subdirectory(scheduler) add_subdirectory(service) add_subdirectory(stats_collector) add_subdirectory(task_runner) diff --git a/ydb/library/yql/providers/dq/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/CMakeLists.linux-aarch64.txt index fae483c5c0..1b2879cd08 100644 --- a/ydb/library/yql/providers/dq/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/dq/CMakeLists.linux-aarch64.txt @@ -12,6 +12,7 @@ add_subdirectory(common) add_subdirectory(config) add_subdirectory(counters) add_subdirectory(expr_nodes) +add_subdirectory(global_worker_manager) add_subdirectory(interface) add_subdirectory(local_gateway) add_subdirectory(mkql) @@ -19,6 +20,7 @@ add_subdirectory(opt) add_subdirectory(planner) add_subdirectory(provider) add_subdirectory(runtime) +add_subdirectory(scheduler) add_subdirectory(service) add_subdirectory(stats_collector) add_subdirectory(task_runner) diff --git a/ydb/library/yql/providers/dq/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/CMakeLists.linux-x86_64.txt index fae483c5c0..1b2879cd08 100644 --- a/ydb/library/yql/providers/dq/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/dq/CMakeLists.linux-x86_64.txt @@ -12,6 +12,7 @@ add_subdirectory(common) add_subdirectory(config) add_subdirectory(counters) add_subdirectory(expr_nodes) +add_subdirectory(global_worker_manager) add_subdirectory(interface) add_subdirectory(local_gateway) add_subdirectory(mkql) @@ -19,6 +20,7 @@ add_subdirectory(opt) add_subdirectory(planner) add_subdirectory(provider) add_subdirectory(runtime) +add_subdirectory(scheduler) add_subdirectory(service) add_subdirectory(stats_collector) add_subdirectory(task_runner) diff --git a/ydb/library/yql/providers/dq/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/CMakeLists.windows-x86_64.txt index 8a270ca445..8a3422b0d6 100644 --- a/ydb/library/yql/providers/dq/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/dq/CMakeLists.windows-x86_64.txt @@ -12,12 +12,14 @@ add_subdirectory(common) add_subdirectory(config) add_subdirectory(counters) add_subdirectory(expr_nodes) +add_subdirectory(global_worker_manager) add_subdirectory(interface) add_subdirectory(mkql) add_subdirectory(opt) add_subdirectory(planner) add_subdirectory(provider) add_subdirectory(runtime) +add_subdirectory(scheduler) add_subdirectory(task_runner) add_subdirectory(task_runner_actor) add_subdirectory(worker_manager) diff --git a/ydb/library/yql/providers/dq/actors/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/actors/CMakeLists.darwin-x86_64.txt index 06b0b443ae..a7cb059668 100644 --- a/ydb/library/yql/providers/dq/actors/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/dq/actors/CMakeLists.darwin-x86_64.txt @@ -6,7 +6,9 @@ # original buildsystem will not be accepted. +add_subdirectory(events) add_subdirectory(ut) +add_subdirectory(yt) add_library(providers-dq-actors) target_compile_options(providers-dq-actors PRIVATE @@ -33,6 +35,7 @@ target_link_libraries(providers-dq-actors PUBLIC yql-dq-tasks yql-utils-failure_injector providers-common-metrics + dq-actors-events dq-api-grpc dq-api-protos providers-dq-common @@ -47,6 +50,8 @@ target_link_libraries(providers-dq-actors PUBLIC ) target_sources(providers-dq-actors PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/compute_actor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dummy_lock.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/executer_actor.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/execution_helpers.cpp diff --git a/ydb/library/yql/providers/dq/actors/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/actors/CMakeLists.linux-aarch64.txt index a0e19378dc..74bf95ff13 100644 --- a/ydb/library/yql/providers/dq/actors/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/dq/actors/CMakeLists.linux-aarch64.txt @@ -6,7 +6,9 @@ # original buildsystem will not be accepted. +add_subdirectory(events) add_subdirectory(ut) +add_subdirectory(yt) add_library(providers-dq-actors) target_compile_options(providers-dq-actors PRIVATE @@ -34,6 +36,7 @@ target_link_libraries(providers-dq-actors PUBLIC yql-dq-tasks yql-utils-failure_injector providers-common-metrics + dq-actors-events dq-api-grpc dq-api-protos providers-dq-common @@ -48,6 +51,8 @@ target_link_libraries(providers-dq-actors PUBLIC ) target_sources(providers-dq-actors PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/compute_actor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dummy_lock.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/executer_actor.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/execution_helpers.cpp diff --git a/ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt index a0e19378dc..74bf95ff13 100644 --- a/ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt @@ -6,7 +6,9 @@ # original buildsystem will not be accepted. +add_subdirectory(events) add_subdirectory(ut) +add_subdirectory(yt) add_library(providers-dq-actors) target_compile_options(providers-dq-actors PRIVATE @@ -34,6 +36,7 @@ target_link_libraries(providers-dq-actors PUBLIC yql-dq-tasks yql-utils-failure_injector providers-common-metrics + dq-actors-events dq-api-grpc dq-api-protos providers-dq-common @@ -48,6 +51,8 @@ target_link_libraries(providers-dq-actors PUBLIC ) target_sources(providers-dq-actors PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/compute_actor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dummy_lock.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/executer_actor.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/execution_helpers.cpp diff --git a/ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt index 06b0b443ae..a7cb059668 100644 --- a/ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt @@ -6,7 +6,9 @@ # original buildsystem will not be accepted. +add_subdirectory(events) add_subdirectory(ut) +add_subdirectory(yt) add_library(providers-dq-actors) target_compile_options(providers-dq-actors PRIVATE @@ -33,6 +35,7 @@ target_link_libraries(providers-dq-actors PUBLIC yql-dq-tasks yql-utils-failure_injector providers-common-metrics + dq-actors-events dq-api-grpc dq-api-protos providers-dq-common @@ -47,6 +50,8 @@ target_link_libraries(providers-dq-actors PUBLIC ) target_sources(providers-dq-actors PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/compute_actor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dummy_lock.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/executer_actor.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/execution_helpers.cpp diff --git a/ydb/library/yql/providers/dq/actors/dummy_lock.cpp b/ydb/library/yql/providers/dq/actors/dummy_lock.cpp new file mode 100644 index 0000000000..c81b31a7da --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/dummy_lock.cpp @@ -0,0 +1,56 @@ +#include "dummy_lock.h" + + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <library/cpp/svnversion/svnversion.h> + +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/events/events.h> + +namespace NYql { + +using namespace NActors; + +class TDummyLock: public TRichActor<TDummyLock> { +public: + static constexpr char ActorName[] = "DUMMY_LOCK"; + + TDummyLock( + const TString& lockName, + const TString& lockAttributes) + : TRichActor<TDummyLock>(&TDummyLock::Handler) + , LockName(lockName) + , Attributes(lockAttributes) + , Revision(GetProgramCommitId()) + { + Y_UNUSED(LockName); + Y_UNUSED(Attributes); + Y_UNUSED(Revision); + } + +private: + STRICT_STFUNC(Handler, { + cFunc(TEvents::TEvPoison::EventType, PassAway); + }); + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + Y_UNUSED(self); + return new IEventHandle(parentId, parentId, new TEvBecomeLeader(1, "TransactionId", Attributes), 0); + } + + TString LockName; + TString Attributes; + TString Revision; +}; + +NActors::IActor* CreateDummyLock( + const TString& lockName, + const TString& lockAttributesYson) +{ + return new TDummyLock(lockName, lockAttributesYson); +} + +} // namespace NYql + diff --git a/ydb/library/yql/providers/dq/actors/dummy_lock.h b/ydb/library/yql/providers/dq/actors/dummy_lock.h new file mode 100644 index 0000000000..18d028d4dc --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/dummy_lock.h @@ -0,0 +1,12 @@ +#pragma once + +#include <library/cpp/actors/core/actor.h> + +namespace NYql { + +NActors::IActor* CreateDummyLock( + const TString& lockName, + const TString& lockAttributesYson); + +} // namespace NYql + diff --git a/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp b/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp new file mode 100644 index 0000000000..3031f188f4 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp @@ -0,0 +1,208 @@ +#include "dynamic_nameserver.h" + +#include <ydb/library/yql/utils/log/log.h> + +#include <library/cpp/actors/interconnect/interconnect.h> +#include <library/cpp/actors/interconnect/interconnect_impl.h> +#include <library/cpp/actors/interconnect/interconnect_address.h> +#include <library/cpp/actors/interconnect/events_local.h> + +#include <library/cpp/actors/core/hfunc.h> + +#include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> + +namespace NYql::NDqs { + using namespace NActors; + + class TDynamicNameserver: public TActor<TDynamicNameserver> { + TMap<ui32, TTableNameserverSetup::TNodeInfo> NodeTable; + + public: + static constexpr EActivityType ActorActivityType() { + return EActivityType::INTERCONNECT_COMMON; + } + + TDynamicNameserver( + const TIntrusivePtr<TTableNameserverSetup>& setup, + ui32 /*resolvePoolId*/ = 0) + : TActor(&TDynamicNameserver::StateFunc) + , NodeTable(setup->StaticNodeTable) + { + Y_ABORT_UNLESS(setup->IsEntriesUnique()); + } + + STFUNC(StateFunc) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvInterconnect::TEvResolveNode, Handle); + HFunc(TEvResolveAddress, Handle); + HFunc(TEvInterconnect::TEvListNodes, Handle); + HFunc(TEvInterconnect::TEvGetNode, Handle); + HFunc(TEvInterconnect::TEvNodesInfo, UpdateNodeTable); + HFunc(TEvRegisterNode, RegisterNodeHandle); + } + } + + void PrintInfo() { + YQL_CLOG(TRACE, ProviderDq) << "Table "; + for (auto& [nodeId, node] : NodeTable) { + YQL_CLOG(TRACE, ProviderDq) << " > Node " << nodeId << " `" << node.Address << "':" << node.Port; + } + } + + bool IsNodeUpdated(const ui32 nodeId, const TString& address, const ui32 port) { + bool printInfo = false; + auto it = NodeTable.find(nodeId); + if (it == NodeTable.end()) { + YQL_CLOG(DEBUG, ProviderDq) << "New node " << nodeId << " `" << address << "':" << port; + printInfo = true; + } else if (it->second.Address != address || it->second.Port != port) { + YQL_CLOG(DEBUG, ProviderDq) << "Updated node " << nodeId << " " + << "`" << address << "':" << port + << " Was " + << "`" << it->second.Address << "':" << it->second.Port; + printInfo = true; + + Send(TActivationContext::InterconnectProxy(nodeId), new TEvInterconnect::TEvDisconnect); + } + return printInfo; + } + + void UpdateNodeTable(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext&) { + auto* req = ev->Get(); + + bool printInfo = false; + for (auto& node : req->Nodes) { + printInfo |= IsNodeUpdated(node.NodeId, node.Address, node.Port); + + NodeTable[node.NodeId] = TTableNameserverSetup::TNodeInfo( + node.Address, + node.Host, + node.ResolveHost, + node.Port, + node.Location); + } + + if (printInfo) { + PrintInfo(); + } + } + + void RegisterNodeHandle(TEvRegisterNode::TPtr& ev, const TActorContext& ctx) { + try { + TEvRegisterNode* req = ev->Get(); + auto& request = req->Record.GetRequest(); + + if (!request.GetZombie()) { + bool printInfo = IsNodeUpdated( + request.GetNodeId(), + request.GetAddress(), + request.GetPort()); + NodeTable[request.GetNodeId()] = std::make_pair( + request.GetAddress(), + request.GetPort()); + + if (printInfo) { + PrintInfo(); + } + } + + auto response = MakeHolder<TEvRegisterNodeResponse>(GetNodesInfo(), request.GetEpoch()); + *response->Record.MutableResponse()->MutableDownloadList() = request.GetDownloadList(); + ctx.Send(ev->Sender, response.Release()); + } catch (...) { + // never + Y_ABORT_UNLESS(false); + } + } + + void Handle(TEvInterconnect::TEvResolveNode::TPtr& ev, + const TActorContext& ctx) + { + const TEvInterconnect::TEvResolveNode* request = ev->Get(); + const ui32 nodeId = request->Record.GetNodeId(); + const TInstant deadline = request->Record.HasDeadline() ? TInstant::FromValue(request->Record.GetDeadline()) : TInstant::Max(); + auto it = NodeTable.find(nodeId); + + if (it == NodeTable.end()) { + auto reply = new TEvLocalNodeInfo; + reply->NodeId = nodeId; + ctx.Send(ev->Sender, reply); + return; + } + + RegisterWithSameMailbox(CreateResolveActor(it->second.ResolveHost, + it->second.Port, + nodeId, + it->second.Address, + ev->Sender, + SelfId(), + deadline)); + } + + void Handle(TEvResolveAddress::TPtr& ev, + const TActorContext& ctx) + { + Y_UNUSED(ctx); + const TEvResolveAddress* request = ev->Get(); + + RegisterWithSameMailbox(CreateResolveActor(request->Address, + request->Port, + ev->Sender, + SelfId(), + TInstant::Max())); + } + + TVector<NActors::TEvInterconnect::TNodeInfo> GetNodesInfo() + { + TVector<NActors::TEvInterconnect::TNodeInfo> nodes; + nodes.reserve(NodeTable.size()); + for (const auto& pr : NodeTable) { + nodes.emplace_back(pr.first, + pr.second.Address, pr.second.Host, pr.second.ResolveHost, + pr.second.Port, pr.second.Location); + } + return nodes; + } + + void ReplyListNodes(const NActors::TActorId sender, const TActorContext& ctx) + { + THolder<TEvInterconnect::TEvNodesInfo> + reply(new TEvInterconnect::TEvNodesInfo()); + reply->Nodes = GetNodesInfo(); + ctx.Send(sender, reply.Release()); + } + + void Handle(TEvInterconnect::TEvListNodes::TPtr& ev, + const TActorContext& ctx) { + try { + ReplyListNodes(ev->Sender, ctx); + } catch (...) { + // on error - do nothing + } + } + + void Handle(TEvInterconnect::TEvGetNode::TPtr& ev, + const TActorContext& ctx) { + try { + ui32 nodeId = ev->Get()->NodeId; + THolder<TEvInterconnect::TEvNodeInfo> + reply(new TEvInterconnect::TEvNodeInfo(nodeId)); + auto it = NodeTable.find(nodeId); + if (it != NodeTable.end()) + reply->Node.Reset(new TEvInterconnect::TNodeInfo(it->first, it->second.Address, + it->second.Host, it->second.ResolveHost, + it->second.Port, it->second.Location)); + ctx.Send(ev->Sender, reply.Release()); + } catch (...) { + // on error - do nothing + } + } + }; + + IActor* CreateDynamicNameserver( + const TIntrusivePtr<TTableNameserverSetup>& setup, + ui32 poolId) { + return new TDynamicNameserver(setup, poolId); + } + +} diff --git a/ydb/library/yql/providers/dq/actors/dynamic_nameserver.h b/ydb/library/yql/providers/dq/actors/dynamic_nameserver.h new file mode 100644 index 0000000000..df35e729a4 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/dynamic_nameserver.h @@ -0,0 +1,7 @@ +#pragma once + +#include <library/cpp/actors/interconnect/interconnect.h> + +namespace NYql::NDqs { + NActors::IActor* CreateDynamicNameserver(const TIntrusivePtr<NActors::TTableNameserverSetup>& setup, ui32 poolId = 0); +} diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..b18ae3b86f --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(dq-actors-events) +target_link_libraries(dq-actors-events PUBLIC + contrib-libs-cxxsupp + yutil + cpp-actors-core +) +target_sources(dq-actors-events PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events/empty.cpp +) diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..ab225973eb --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-aarch64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(dq-actors-events) +target_link_libraries(dq-actors-events PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-actors-core +) +target_sources(dq-actors-events PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events/empty.cpp +) diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..ab225973eb --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(dq-actors-events) +target_link_libraries(dq-actors-events PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-actors-core +) +target_sources(dq-actors-events PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events/empty.cpp +) diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..b18ae3b86f --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.windows-x86_64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(dq-actors-events) +target_link_libraries(dq-actors-events PUBLIC + contrib-libs-cxxsupp + yutil + cpp-actors-core +) +target_sources(dq-actors-events PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events/empty.cpp +) diff --git a/ydb/library/yql/providers/dq/actors/events/empty.cpp b/ydb/library/yql/providers/dq/actors/events/empty.cpp new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/events/empty.cpp diff --git a/ydb/library/yql/providers/dq/actors/events/events.h b/ydb/library/yql/providers/dq/actors/events/events.h new file mode 100644 index 0000000000..ae15f0ca40 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/events/events.h @@ -0,0 +1,59 @@ +#pragma once + +namespace NYql { + + struct TDqEvents { + enum { + ES_BECOME_LEADER = EventSpaceBegin(NActors::TEvents::EEventSpace::ES_USERSPACE) + 10100, + ES_BECOME_FOLLOWER, + + ES_TICK, + + ES_OTHER1, + ES_OTHER2, + ES_OTHER3, + ES_OTHER4 + }; + }; + + struct TEvBecomeLeader: NActors::TEventLocal<TEvBecomeLeader, TDqEvents::ES_BECOME_LEADER> { + TEvBecomeLeader() = default; + + TEvBecomeLeader(ui32 leaderEpoch, const TString& leaderTransaction, const TString& attributes) + : LeaderEpoch(leaderEpoch) + , LeaderTransaction(leaderTransaction) + , Attributes(attributes) + { } + + const ui32 LeaderEpoch; + const TString LeaderTransaction; + const TString Attributes; + }; + + struct TEvBecomeFollower: NActors::TEventLocal<TEvBecomeFollower, TDqEvents::ES_BECOME_FOLLOWER> { + TEvBecomeFollower() = default; + + TEvBecomeFollower(const TString& attributes) + : Attributes(attributes) + { } + + const TString Attributes; + }; + + struct TEvTick + : NActors::TEventLocal<TEvTick, TDqEvents::ES_TICK> { + TEvTick() = default; + }; + + struct TEvUploadComplete + : NActors::TEventLocal<TEvTick, TDqEvents::ES_OTHER2> { + TEvUploadComplete() = default; + }; + + struct TEvDownloadComplete + : NActors::TEventLocal<TEvTick, TDqEvents::ES_OTHER3> { + TVector<TString> FailedObjectIds; + TEvDownloadComplete() = default; + }; + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/events/ya.make b/ydb/library/yql/providers/dq/actors/events/ya.make new file mode 100644 index 0000000000..398ff0ecee --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/events/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +SET( + SOURCE + empty.cpp +) + +SRCS( + ${SOURCE} +) + +PEERDIR( + library/cpp/actors/core +) + +END() diff --git a/ydb/library/yql/providers/dq/actors/ya.make b/ydb/library/yql/providers/dq/actors/ya.make index 8a3c7d8b65..7d3df6fe4b 100644 --- a/ydb/library/yql/providers/dq/actors/ya.make +++ b/ydb/library/yql/providers/dq/actors/ya.make @@ -2,6 +2,8 @@ LIBRARY() SRCS( compute_actor.cpp + dummy_lock.cpp + dynamic_nameserver.cpp events.cpp executer_actor.cpp execution_helpers.cpp @@ -35,6 +37,7 @@ PEERDIR( ydb/library/yql/dq/tasks ydb/library/yql/utils/failure_injector ydb/library/yql/providers/common/metrics + ydb/library/yql/providers/dq/actors/events ydb/library/yql/providers/dq/api/grpc ydb/library/yql/providers/dq/api/protos ydb/library/yql/providers/dq/common @@ -52,6 +55,11 @@ YQL_LAST_ABI_VERSION() END() +RECURSE( + events + yt +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..12dab3d2dc --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,41 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(dq-actors-yt) +target_compile_options(dq-actors-yt PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(dq-actors-yt PUBLIC + contrib-libs-cxxsupp + yutil + cpp-actors-core + cpp-grpc-client + cpp-mapreduce-interface + providers-dq-config + yql-core-issue + providers-common-metrics + dq-api-grpc + dq-api-protos + providers-dq-common + yt-lib-log + dq-actors-events + yt-yt-client +) +target_sources(dq-actors-yt PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/lock.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp +) diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..8557c18cbd --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-aarch64.txt @@ -0,0 +1,42 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(dq-actors-yt) +target_compile_options(dq-actors-yt PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(dq-actors-yt PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-actors-core + cpp-grpc-client + cpp-mapreduce-interface + providers-dq-config + yql-core-issue + providers-common-metrics + dq-api-grpc + dq-api-protos + providers-dq-common + yt-lib-log + dq-actors-events + yt-yt-client +) +target_sources(dq-actors-yt PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/lock.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp +) diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..8557c18cbd --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-x86_64.txt @@ -0,0 +1,42 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(dq-actors-yt) +target_compile_options(dq-actors-yt PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(dq-actors-yt PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-actors-core + cpp-grpc-client + cpp-mapreduce-interface + providers-dq-config + yql-core-issue + providers-common-metrics + dq-api-grpc + dq-api-protos + providers-dq-common + yt-lib-log + dq-actors-events + yt-yt-client +) +target_sources(dq-actors-yt PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/lock.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp +) diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..e4c7a252a1 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.windows-x86_64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(dq-actors-yt) +target_compile_options(dq-actors-yt PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(dq-actors-yt PUBLIC + contrib-libs-cxxsupp + yutil + cpp-actors-core + cpp-grpc-client + cpp-mapreduce-interface + providers-dq-config + yql-core-issue + providers-common-metrics + dq-api-grpc + dq-api-protos + providers-dq-common + yt-lib-log + dq-actors-events +) +target_sources(dq-actors-yt PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp +) diff --git a/ydb/library/yql/providers/dq/actors/yt/lock.cpp b/ydb/library/yql/providers/dq/actors/yt/lock.cpp new file mode 100644 index 0000000000..991db7ea5d --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/lock.cpp @@ -0,0 +1,435 @@ +#include "lock.h" + +#include "nodeid_assigner.h" +#include "yt_wrapper.h" + +#include <ydb/library/yql/utils/log/log.h> + +#include <yt/yt/client/api/transaction.h> +#include <yt/yt/core/ytree/public.h> + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/svnversion/svnversion.h> + +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/events/events.h> + +#include <util/string/builder.h> +#include <util/system/hostname.h> +#include <util/system/getpid.h> + +namespace NYql { + +using namespace NActors; + +struct TLockRequest: public TActor<TLockRequest> { + TLockRequest( + const NYT::NApi::ITransactionPtr& transaction, + TActorId ytWrapper, + TActorId lockActorId, + TActorId parentId, + const TString prefix, + const TString lockName, + NYT::TNode attributes) + : TActor<TLockRequest>(&TLockRequest::Handler) + , Transaction(transaction) + , LockActorId(lockActorId) + , ParentId(parentId) + , Prefix(prefix) + , LockName(lockName) + , Attributes(attributes) + , Revision(GetProgramCommitId()) + , YtWrapper(ytWrapper) + { } + + ~TLockRequest() + { } + +private: + STRICT_STFUNC(Handler, { + cFunc(TEvents::TEvPoison::EventType, PassAway); + CFunc(TEvents::TEvBootstrap::EventType, CreateLockNode); + HFunc(TEvCreateNodeResponse, OnCreateLockNode); + HFunc(TEvSetNodeResponse, OnNodeLocked); + }); + + STRICT_STFUNC(ReadInfoNodeHandler, { + cFunc(TEvents::TEvPoison::EventType, PassAway); + HFunc(TEvGetNodeResponse, SendBecomeFollower); + }); + + STRICT_STFUNC(GetOrCreateEpochHandler, { + cFunc(TEvents::TEvPoison::EventType, PassAway); + HFunc(TEvGetNodeResponse, OnGetEpochNode); + HFunc(TEvCreateNodeResponse, OnCreateEpochNode); + }); + + STRICT_STFUNC(SetAttributesHandler, { + cFunc(TEvents::TEvPoison::EventType, PassAway); + HFunc(TEvSetNodeResponse, OnSetAttribute); + }); + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0); + } + + void OnCreateLockNode(TEvCreateNodeResponse::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + if (result.IsOK()) { + NYT::NApi::TLockNodeOptions options; + options.Waitable = false; + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + auto selfId = SelfId(); + try { + Transaction->LockNode("#" + ToString(result.ValueOrThrow()), NYT::NCypressClient::ELockMode::Exclusive, options).As<void>() + .Apply(BIND([actorSystem, selfId](const NYT::TErrorOr<void>& result) { + actorSystem->Send(selfId, new TEvSetNodeResponse(0, result)); + })); + } catch (...) { + Finish(ctx); + } + } else { + Finish(result, ctx); + } + } + + void OnNodeLocked(TEvSetNodeResponse::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + if (result.IsOK()) { + GetOrCreateEpoch(); + } else { + Finish(result, ctx); + } + } + + void PassAway() override { + Transaction->Abort(); + Transaction.Reset(); + Send(LockActorId, new TEvTick()); + IActor::PassAway(); + } + + void OnGetEpochNode(TEvGetNodeResponse::TPtr& ev, const TActorContext& ctx) { + auto result = std::get<0>(*ev->Get()); + try { + if (!result.IsOK() && !result.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) { + Finish(result, ctx); + } else if (result.IsOK()) { + auto epoch = static_cast<ui32>(NYT::NodeFromYsonString(result.Value().AsStringBuf()).AsUint64()); + SetInfoNodeAttributes(epoch); + } else { + NYT::NApi::TCreateNodeOptions createNodeOptions; + auto prereqId = Transaction->GetId(); + + createNodeOptions.IgnoreExisting = true; + createNodeOptions.PrerequisiteTransactionIds.push_back(prereqId); + + auto nodePath = Prefix + "/" + LockName; + Send(YtWrapper, new TEvCreateNode(nodePath, NYT::NObjectClient::EObjectType::StringNode, createNodeOptions)); + } + } catch (...) { + Finish(ctx); + } + } + + void OnCreateEpochNode(TEvCreateNodeResponse::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + if (result.IsOK()) { + SetInfoNodeAttributes(0); + } else { + Finish(result, ctx); + } + } + + void SetInfoNodeAttributes(ui32 epoch) { + epoch += 1; + Epoch = epoch; + + auto prereqId = Transaction->GetId(); + + NYT::NApi::TSetNodeOptions setNodeOptions; + setNodeOptions.PrerequisiteTransactionIds.push_back(prereqId); + + auto attributesMap = Attributes.AsMap(); + attributesMap[NCommonAttrs::EPOCH_ATTR] = NYT::TNode(epoch); + attributesMap[NCommonAttrs::REVISION_ATTR] = NYT::TNode(Revision); + + AttributesCount = attributesMap.size(); + Become(&TLockRequest::SetAttributesHandler); + + auto nodePath = Prefix + "/" + LockName; + for (const auto& [k, v]: attributesMap) { + Send(YtWrapper, new TEvSetNode( + nodePath + "/@" + k, + NYT::NYson::TYsonString(NYT::NodeToYsonString(v)), + setNodeOptions)); + } + } + + void OnSetAttribute(TEvSetNodeResponse::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + if (result.IsOK()) { + if (--AttributesCount == 0) { + SendBecomeLeader(); + Finish(result, ctx); + } + } else { + Finish(result, ctx); + } + } + + void GetOrCreateEpoch() + { + Become(&TLockRequest::GetOrCreateEpochHandler); + + auto prereqId = Transaction->GetId(); + + NYT::NApi::TGetNodeOptions getNodeOptions; + getNodeOptions.PrerequisiteTransactionIds.push_back(prereqId); + auto nodePath = Prefix + "/" + LockName; + + Send(YtWrapper, new TEvGetNode(nodePath + "/@" + NCommonAttrs::EPOCH_ATTR, getNodeOptions)); + } + + void ReadInfoNode() { + Become(&TLockRequest::ReadInfoNodeHandler); + + auto nodePath = Prefix + "/" + LockName + "/@"; + + Send(YtWrapper, new TEvGetNode(nodePath, NYT::NApi::TGetNodeOptions())); + } + + void SendBecomeLeader() { + auto attributes = NYT::NYson::TYsonString(NYT::NodeToYsonString(Attributes)); + Send(ParentId, new TEvBecomeLeader(Epoch, ToString(Transaction->GetId()), attributes.ToString())); + } + + void SendBecomeFollower(TEvGetNodeResponse::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + if (result.IsOK()) { + Send(ParentId, new TEvBecomeFollower(result.ValueOrThrow().ToString())); + } + Send(SelfId(), new TEvents::TEvPoison()); + } + + template<typename T> + void Finish(const NYT::TErrorOr<T>& result, const TActorContext& ctx) { + if (result.IsOK()) { + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + auto selfId = SelfId(); + Transaction->SubscribeAborted(BIND([actorSystem, selfId](const NYT::TError& /*error*/) { + actorSystem->Send(selfId, new TEvents::TEvPoison()); + })); + } else { + if (!result.FindMatching(NYT::NCypressClient::EErrorCode::ConcurrentTransactionLockConflict)) { + YQL_CLOG(WARN, ProviderDq) << ToString(result); + } + ReadInfoNode(); + } + } + + void Finish(const TActorContext& ctx) { + Y_UNUSED(ctx); + YQL_CLOG(WARN, ProviderDq) << CurrentExceptionMessage(); + ReadInfoNode(); + } + + void CreateLockNode(const TActorContext& ctx) { + auto lockNode = Prefix + "/" + LockName + ".lock"; + + try { + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + auto selfId = SelfId(); + Transaction->CreateNode( + lockNode, + NYT::NObjectClient::EObjectType::StringNode, + NYT::NApi::TCreateNodeOptions()) + .Apply(BIND([actorSystem, selfId](const NYT::TErrorOr<NYT::NCypressClient::TNodeId>& result) { + actorSystem->Send(selfId, new TEvCreateNodeResponse(0, result)); + })); + } catch (...) { + Finish(ctx); + } + } + + NYT::NApi::ITransactionPtr Transaction; + + const TActorId LockActorId; + const TActorId ParentId; + const TString Prefix; + const TString LockName; + const NYT::TNode Attributes; + + const TString Revision; + ui32 Epoch; + int AttributesCount = 0; + + TActorId YtWrapper; +}; + +class TYtLock: public TRichActor<TYtLock> { +public: + static constexpr char ActorName[] = "LOCK"; + + TYtLock( + const TActorId& ytWrapper, + const TString& prefix, + const TString& lockName, + const TString& lockAttributes, + bool temporary) + : TRichActor<TYtLock>(&TYtLock::Handler) + , YtWrapper(ytWrapper) + , Prefix(prefix) + , LockName(lockName) + , Attributes(NYT::NodeFromYsonString(lockAttributes)) + , Temporary(temporary) + , Revision(GetProgramCommitId()) + { } + +private: + STRICT_STFUNC(Handler, { + cFunc(TEvents::TEvPoison::EventType, PassAway); + CFunc(TEvTick::EventType, OnTick); + HFunc(TEvStartTransactionResponse, OnStartTransactionResponse); + HFunc(TEvCreateNodeResponse, OnCreateNode); + HFunc(TEvGetNodeResponse, SendBecomeFollower); + cFunc(TEvBecomeFollower::EventType, OnFollowingForever); + }); + + void DoPassAway() override { + YQL_CLOG(DEBUG, ProviderDq) << "Unlock " << LockName; + Send(Request, new TEvents::TEvPoison()); + auto nodePath = Prefix + "/" + LockName; + + if (Temporary) { + Send(YtWrapper, new TEvRemoveNode(nodePath, {})); + } + } + + static TEvStartTransaction* GetStartTransactionCommand() { + NYT::NApi::TTransactionStartOptions options; + auto attrs = NYT::NYTree::CreateEphemeralAttributes(); + attrs->Set("title", TStringBuilder{} << "Host name: " << HostName() << " Pid: " << GetPID()); + options.Attributes = attrs; + auto command = new TEvStartTransaction( + NYT::NTransactionClient::ETransactionType::Master, + options + ); + return command; + } + + void TryLock() { + NYT::NApi::TCreateNodeOptions options; + options.Recursive = true; + options.IgnoreExisting = true; + + TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way()); + + IEventBase* event; + if (FollowingMode == false) { + event = new TEvCreateNode(Prefix, NYT::NObjectClient::EObjectType::MapNode, options); + } else { + event = new TEvGetNode(Prefix + "/" + LockName + "/@", NYT::NApi::TGetNodeOptions()); + } + + TActivationContext::Schedule( + TDuration::Seconds(5), + new IEventHandle(YtWrapper, SelfId(), event, 0), TimerCookieHolder.Get()); + } + + void OnFollowingForever() { + FollowingMode = true; + YQL_CLOG(DEBUG, ProviderDq) << "Unlock and follow " << LockName; + Send(Request, new TEvents::TEvPoison()); + } + + void SendBecomeFollower(TEvGetNodeResponse::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + if (result.IsOK()) { + Send(ParentId, new TEvBecomeFollower(result.ValueOrThrow().ToString())); + } else { + YQL_CLOG(WARN, ProviderDq) << "SendBecomeFollower error: " << ToString(result); + } + + TryLock(); // update leader info + } + + void OnCreateNode(TEvCreateNodeResponse::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + if (result.IsOK()) { + Send(YtWrapper, GetStartTransactionCommand()); + } else { + TryLock(); + } + } + + void OnStartTransactionResponse(TEvStartTransactionResponse::TPtr& ev, const TActorContext& ctx) + { + auto result = std::get<0>(*ev->Get()); + + if (result.IsOK()) { + Send(Request, new TEvents::TEvPoison()); + auto requestActor = new TLockRequest( + result.Value(), + YtWrapper, + SelfId(), + ParentId, + Prefix, + LockName, + Attributes + ); + + Request = ctx.Register(requestActor); + } else { + YQL_CLOG(WARN, ProviderDq) << "OnStartTransactionResponse " << ToString(result); + + TryLock(); + } + } + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + ParentId = parentId; + return new IEventHandle(self, parentId, new TEvTick(), 0); + } + + void OnTick(const TActorContext& ctx) { + Y_UNUSED(ctx); + + TryLock(); + } + + const TActorId YtWrapper; + TActorId ParentId; + const TString Prefix; + const TString LockName; + NYT::TNode Attributes; + + TActorId Request; + const bool Temporary; + + const TString Revision; + NActors::TSchedulerCookieHolder TimerCookieHolder; + bool FollowingMode = false; +}; + +NActors::IActor* CreateYtLock( + NActors::TActorId ytWrapper, + const TString& prefix, + const TString& lockName, + const TString& lockAttributesYson, + bool temporary) +{ + return new TYtLock(ytWrapper, prefix, lockName, lockAttributesYson, temporary); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/lock.h b/ydb/library/yql/providers/dq/actors/yt/lock.h new file mode 100644 index 0000000000..dddc3fe37d --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/lock.h @@ -0,0 +1,14 @@ +#pragma once + +#include <library/cpp/actors/core/actor.h> + +namespace NYql { + +NActors::IActor* CreateYtLock( + NActors::TActorId ytWrapper, + const TString& prefix, + const TString& lockName, + const TString& lockAttributesYson, + bool temporary); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp b/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp new file mode 100644 index 0000000000..f3a2b67ac7 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp @@ -0,0 +1,116 @@ +#include "nodeid_assigner.h" + +#include <yt/cpp/mapreduce/interface/node.h> +#include <yt/cpp/mapreduce/interface/client.h> +#include <yt/cpp/mapreduce/interface/common.h> + +#include <util/system/env.h> +#include <util/generic/hash_set.h> + +namespace NYql { + +using namespace NYT; + +ui32 AssignNodeId(const TAssignNodeIdOptions& options) +{ + Y_ABORT_UNLESS(!options.ClusterName.empty()); + Y_ABORT_UNLESS(!options.NodeName.empty()); + Y_ABORT_UNLESS(!options.Prefix.empty()); + Y_ABORT_UNLESS(!options.Role.empty()); + + Y_ABORT_UNLESS(options.MinNodeId > 0); + + auto range = options.MaxNodeId - options.MinNodeId; + + Y_ABORT_UNLESS(range > 0); + + TString token = !options.Token.empty() + ? options.Token + : [] () { + TString home = GetEnv("HOME"); + TString tokenFile = home + "/.yt/token"; + return TFileInput(tokenFile).ReadLine(); + } (); + + Y_ABORT_UNLESS(!token.empty()); + + auto lockPath = options.Prefix + "/" + options.Role + "/lock"; + auto nodePath = options.Prefix + "/" + options.Role + "/" + options.NodeName; + + auto client = CreateClient(options.ClusterName, TCreateClientOptions().Token(token)); + // create std directories + client->Create(options.Prefix, ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true).Recursive(true)); + client->Create(options.Prefix + "/" + options.Role, ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true)); + client->Create(options.Prefix + "/service_node", ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true)); + client->Create(options.Prefix + "/worker_node", ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true)); + client->Create(options.Prefix + "/operations", ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true)); + client->Create(options.Prefix + "/locks", ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true)); + + if (options.NodeId) { + // Use it for debug only + return *options.NodeId; + } + + if (client->Exists(lockPath) && client->Get(lockPath).GetType() != TNode::Int64) { + client->Remove(lockPath); + } + client->Create(lockPath, ENodeType::NT_INT64, TCreateOptions().IgnoreExisting(true)); + auto transaction = client->StartTransaction(TStartTransactionOptions()); + transaction->Lock(lockPath, ELockMode::LM_EXCLUSIVE, TLockOptions().Waitable(true))->Wait(); + + ui32 nodeId = client->Get(lockPath).AsInt64(); + if (nodeId == 0) { + nodeId = options.MinNodeId; + } + + auto listResult = transaction->List( + options.Prefix + "/" + options.Role, + TListOptions() + .MaxSize(1<<18) + .AttributeFilter(TAttributeFilter().AddAttribute(NCommonAttrs::ACTOR_NODEID_ATTR)) + ); + + THashSet<ui32> allNodeIds; + allNodeIds.reserve(listResult.size()); + for (const auto& node : listResult) { + const auto& attributes = node.GetAttributes().AsMap(); + auto maybeNodeId = attributes.find(NCommonAttrs::ACTOR_NODEID_ATTR); + if (maybeNodeId != attributes.end()) { + auto nodeId = maybeNodeId->second.AsUint64(); + if (options.MinNodeId <= nodeId && nodeId < options.MaxNodeId) { + Cerr << nodeId << " "; + allNodeIds.insert(nodeId); + } + } + } + Cerr << Endl; + + Y_ABORT_UNLESS(allNodeIds.size() < range); + + while (allNodeIds.contains(nodeId)) { + nodeId = options.MinNodeId + (nodeId + 1 - options.MinNodeId) % range; + } + + if (options.NodeId.Defined()) { + nodeId = options.NodeId.GetOrElse(nodeId); + } else { + auto nextNodeId = options.MinNodeId + (nodeId + 1 - options.MinNodeId) % range; + transaction->Set(lockPath, TNode(nextNodeId)); + } + + Y_ABORT_UNLESS(options.MinNodeId <= nodeId && nodeId < options.MaxNodeId); + + // create new node + transaction->Create(nodePath, NT_STRING, TCreateOptions().IgnoreExisting(true)); + + for (const auto& [k, v] : options.Attributes) { + transaction->Set(nodePath + "/@" + k, TNode(v), TSetOptions()); + } + + transaction->Set(nodePath + "/@" + NCommonAttrs::ACTOR_NODEID_ATTR, TNode(nodeId), TSetOptions()); + transaction->Commit(); + + return nodeId; +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h b/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h new file mode 100644 index 0000000000..e6921097f7 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h @@ -0,0 +1,29 @@ +#pragma once + +#include <ydb/library/yql/providers/dq/common/attrs.h> + +#include <util/generic/string.h> +#include <util/generic/hash.h> +#include <util/generic/maybe.h> + +namespace NYql { + +struct TAssignNodeIdOptions { + TString ClusterName; + TString User; + TString Token; + TString Prefix; + TString Role; + TString NodeName; + + THashMap<TString, TString> Attributes; + + ui32 MinNodeId = 0; + ui32 MaxNodeId = 1<<18; + + TMaybe<ui32> NodeId; // for debug only +}; + +ui32 AssignNodeId(const TAssignNodeIdOptions& options); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp b/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp new file mode 100644 index 0000000000..392d8636bf --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp @@ -0,0 +1,137 @@ +#include "nodeid_cleaner.h" +#include "nodeid_assigner.h" +#include "yt_wrapper.h" + +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/event.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <library/cpp/yson/node/node.h> +#include <library/cpp/yson/node/node_io.h> + +#include <ydb/library/yql/utils/log/log.h> +#include <ydb/library/yql/utils/yql_panic.h> + +namespace NYql { + +using namespace NActors; + +class TNodeIdCleaner: public TActor<TNodeIdCleaner> { +public: + static constexpr char ActorName[] = "CLEANER"; + + TNodeIdCleaner(TActorId ytWrapper, const TNodeIdCleanerOptions& options) + : TActor(&TNodeIdCleaner::Handler) + , YtWrapper(ytWrapper) + , Options(options) + , LastUpdateTime(TInstant::Now()) + { } + + STRICT_STFUNC(Handler, { + HFunc(TEvListNodeResponse, OnListNodeResponse) + HFunc(TEvRemoveNodeResponse, OnRemoveNodeResponse) + cFunc(TEvents::TEvPoison::EventType, PassAway) + }); + +private: + TEvListNode* ListNodeCommand() { + NYT::NApi::TListNodeOptions options; + options.Attributes = {NCommonAttrs::ACTOR_NODEID_ATTR, "modification_time"}; + return new TEvListNode(Options.Prefix, options); + } + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + // TODO: TakeLock + Y_UNUSED(parentId); + return new IEventHandle(YtWrapper, self, ListNodeCommand(), 0); + } + + void MaybeReschedule(TInstant now) { + Y_UNUSED(now); + if (WaitCount == 0) { + TActivationContext::Schedule(Options.CheckPeriod, new IEventHandle(YtWrapper, SelfId(), ListNodeCommand(), 0)); + } + } + + void OnRemoveNodeResponse(TEvRemoveNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + if (!result.IsOK()) { + YQL_CLOG(WARN, ProviderDq) << "Remove error " << ToString(result); + } + + --WaitCount; + MaybeReschedule(TInstant::Now()); + } + + void ScheduleRemove(TInstant now, const TVector<NYT::TNode>& nodes, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + Y_ABORT_UNLESS(WaitCount == 0); + WaitCount = 0; + for (const auto& node : nodes) { + + const auto& attributes = node.GetAttributes().AsMap(); + auto maybeNodeId = attributes.find(NCommonAttrs::ACTOR_NODEID_ATTR); + if (maybeNodeId == attributes.end()) { + continue; + } + + auto maybeModificationTime = attributes.find("modification_time"); + YQL_ENSURE(maybeModificationTime != attributes.end()); + + auto modificationTime = TInstant::ParseIso8601(maybeModificationTime->second.AsString()); + + if (now - modificationTime > Options.Timeout) { + WaitCount ++; + auto removePath = Options.Prefix + "/" + node.AsString(); + YQL_CLOG(DEBUG, ProviderDq) << "Removing node " << removePath; + Send(YtWrapper, new TEvRemoveNode(removePath, NYT::NApi::TRemoveNodeOptions())); + } + } + } + + void OnListNodeResponse(TEvListNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + + auto now = TInstant::Now(); + auto result = std::get<0>(*ev->Get()); + auto isOk = result.IsOK(); + auto schedule = true; + + try { + if (LastListNodeOk) { + // remove only after 2't attempt to minimize Y_ABORT_UNLESS chance on registrator + ScheduleRemove(now, NYT::NodeFromYsonString(result.ValueOrThrow()).AsList(), ctx); + MaybeReschedule(now); + schedule = false; + } + } catch (...) { + WaitCount = 0; + isOk = false; + YQL_CLOG(ERROR, ProviderDq) << "Error on list node " << CurrentExceptionMessage(); + } + + if (schedule) { + TActivationContext::Schedule(Options.RetryPeriod, new IEventHandle(YtWrapper, SelfId(), ListNodeCommand(), 0)); + } + + LastListNodeOk = isOk; + LastUpdateTime = now; + } + + TActorId YtWrapper; + TNodeIdCleanerOptions Options; + TInstant LastUpdateTime; + bool LastListNodeOk = true; + int WaitCount = 0; +}; + +NActors::IActor* CreateNodeIdCleaner(TActorId ytWrapper, const TNodeIdCleanerOptions& options) +{ + Y_ABORT_UNLESS(!options.Prefix.empty()); + return new TNodeIdCleaner(ytWrapper, options); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.h b/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.h new file mode 100644 index 0000000000..173031ad12 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.h @@ -0,0 +1,18 @@ +#pragma once + +#include <library/cpp/actors/core/actor.h> + +#include <util/generic/string.h> + +namespace NYql { + +struct TNodeIdCleanerOptions { + TString Prefix; + TDuration CheckPeriod = TDuration::MilliSeconds(10000); + TDuration Timeout = TDuration::MilliSeconds(600000); + TDuration RetryPeriod = TDuration::MilliSeconds(15000); +}; + +NActors::IActor* CreateNodeIdCleaner(NActors::TActorId ytWrapper, const TNodeIdCleanerOptions& options); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp b/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp new file mode 100644 index 0000000000..c87fc0c636 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp @@ -0,0 +1,154 @@ +#include "yt_wrapper.h" + +#include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/library/yql/utils/log/log.h> + +#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h> +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h> +#include <ydb/library/yql/providers/dq/actors/events/events.h> + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/yson/node/node_io.h> + +namespace NYql { + +using namespace NActors; +using namespace NMonitoring; + +class TYtResourceCleaner: public TRichActor<TYtResourceCleaner> { +public: + static constexpr char ActorName[] = "CLEANER"; + + TYtResourceCleaner( + const TResourceManagerOptions& options, + const TIntrusivePtr<ICoordinationHelper>& coordinator) + : TRichActor<TYtResourceCleaner>(&TYtResourceCleaner::Handler) + , Coordinator(coordinator) + , Options(options) + , FilesCounter(Options.Counters->GetSubgroup("component", "cleaner")->GetCounter("files")) + , RemovesCounter(Options.Counters->GetSubgroup("component", "cleaner")->GetCounter("removes")) + , ErrorsCounter(Options.Counters->GetSubgroup("component", "cleaner")->GetCounter("errors")) + { + } + +private: + STRICT_STFUNC(Handler, { + HFunc(TEvListNodeResponse, OnListResponse) + CFunc(TEvents::TEvBootstrap::EventType, Bootstrap) + cFunc(TEvTick::EventType, Check) + cFunc(TEvents::TEvPoison::EventType, PassAway) + HFunc(TEvRemoveNodeResponse, OnRemoveNodeResponse) + }) + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0); + } + + void Bootstrap(const TActorContext& ctx) { + YtWrapper = Coordinator->GetWrapper( + ctx.ActorSystem(), + Options.YtBackend.GetClusterName(), + Options.YtBackend.GetUser(), + Options.YtBackend.GetToken()); + Check(); + } + + void Check() { + NYT::NApi::TListNodeOptions options; + options.Attributes = { + "modification_time" + }; + auto command = new TEvListNode(Options.UploadPrefix, options); + Send(YtWrapper, command); + } + + void OnRemoveNodeResponse(TEvRemoveNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + if (!result.IsOK()) { + YQL_CLOG(WARN, ProviderDq) << "Remove error " << ToString(result); + *ErrorsCounter += 1; + } + } + + void OnListResponse(TEvListNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + + auto result = std::get<0>(*ev->Get()); + + try { + auto nodes = NYT::NodeFromYsonString(result.ValueOrThrow()).AsList(); + TVector<std::pair<TInstant, TString>> pairs; + + auto filesDiff = static_cast<int>(nodes.size()) - FilesCount; + *FilesCounter += filesDiff; + FilesCount = static_cast<int>(nodes.size()); + + for (const auto& node : nodes) { + const auto& attributes = node.GetAttributes().AsMap(); + auto maybeModificationTime = attributes.find("modification_time"); + YQL_ENSURE(maybeModificationTime != attributes.end()); + auto modificationTime = TInstant::ParseIso8601(maybeModificationTime->second.AsString()); + + TString nodeStr = node.AsString(); + + if (Options.KeepFilter && nodeStr.find(*Options.KeepFilter) != TString::npos) { + continue; + } + + pairs.emplace_back(modificationTime, nodeStr); + } + + std::sort(pairs.begin(), pairs.end(), [](const auto& a, const auto& b) { + return a.first > b.first; + }); + if (Options.KeepFirst) { + for (ui32 i = *Options.KeepFirst; i < pairs.size(); ++i) { + YQL_CLOG(DEBUG, ProviderDq) << "RemoveNode " << pairs[i].second; + Send(YtWrapper, new TEvRemoveNode(Options.UploadPrefix + "/" + pairs[i].second, NYT::NApi::TRemoveNodeOptions())); + + *RemovesCounter += 1; + } + } + if (Options.DropBefore) { + auto now = TInstant::Now(); + for (const auto& [t, id] : pairs) { + if (t < now && now - t > Options.DropBefore) { + YQL_CLOG(DEBUG, ProviderDq) << "RemoveNode " << id; + Send(YtWrapper, new TEvRemoveNode(Options.UploadPrefix + "/" + id, NYT::NApi::TRemoveNodeOptions())); + + *RemovesCounter += 1; + } + } + } + } catch (...) { + YQL_CLOG(ERROR, ProviderDq) << "Error on list node " << CurrentExceptionMessage(); + } + + Schedule(TDuration::Seconds(5), new TEvTick); + } + + TActorId YtWrapper; + const TIntrusivePtr<ICoordinationHelper> Coordinator; + const TResourceManagerOptions Options; + + int FilesCount = 0; + TDynamicCounters::TCounterPtr FilesCounter; + TDynamicCounters::TCounterPtr RemovesCounter; + TDynamicCounters::TCounterPtr ErrorsCounter; +}; + +IActor* CreateYtResourceCleaner( + const TResourceManagerOptions& options, + const TIntrusivePtr<ICoordinationHelper>& coordinator) +{ + Y_ABORT_UNLESS(!options.YtBackend.GetClusterName().empty()); + Y_ABORT_UNLESS(!options.YtBackend.GetUser().empty()); + Y_ABORT_UNLESS(!options.YtBackend.GetUploadPrefix().empty()); + + return new TYtResourceCleaner(options, coordinator); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp b/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp new file mode 100644 index 0000000000..8bb58dfab1 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp @@ -0,0 +1,141 @@ + +#include "nodeid_assigner.h" +#include "yt_wrapper.h" + +#include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/library/yql/utils/log/log.h> + +#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h> +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h> +#include <ydb/library/yql/providers/dq/actors/events/events.h> + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/yson/node/node_io.h> + +#include <util/system/fs.h> + +#include <yt/cpp/mapreduce/interface/fluent.h> + +namespace NYql { + +using namespace NActors; + +class TYtResourceDownloader: public TRichActor<TYtResourceDownloader> { +public: + static constexpr char ActorName[] = "DOWNLOADER"; + + TYtResourceDownloader( + const TResourceManagerOptions& options, + const ICoordinationHelper::TPtr& coordinator) + : TRichActor<TYtResourceDownloader>(&TYtResourceDownloader::Handler) + , Options(options) + , Coordinator(coordinator) + { + } + +private: + + void DoPassAway() override { + auto ev = MakeHolder<TEvDownloadComplete>(); + for (auto it = Options.Files.begin() + CurrentFileId; it != Options.Files.end(); ++it) { + ev->FailedObjectIds.push_back(it->LocalFileName); + } + Send(ParentId, ev.Release()); + } + + STRICT_STFUNC(Handler, { + HFunc(TEvReadFileResponse, OnFileDownloaded) + CFunc(TEvents::TEvBootstrap::EventType, Bootstrap) + cFunc(TEvTick::EventType, DownloadFile) + }) + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + ParentId = parentId; + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0); + } + + void Bootstrap(const NActors::TActorContext& ctx) { + YtWrapper = Coordinator->GetWrapper( + ctx.ActorSystem(), + Options.YtBackend.GetClusterName(), + Options.YtBackend.GetUser(), + Options.YtBackend.GetToken()); + DownloadFile(); + } + + void DownloadFile() { + TString remotePath = Options.UploadPrefix + "/"; + NYT::NApi::TFileReaderOptions options; + + auto& file = Options.Files[CurrentFileId]; + remotePath += file.GetRemoteFileName(); /* md5 */ + TString localFileName = Options.TmpDir + "/" + file.GetRemoteFileName(); + + YQL_CLOG(DEBUG, ProviderDq) << "Downloading file " << remotePath << "->" << localFileName; + auto message = MakeHolder<TEvReadFile>(NYT::NYPath::TYPath(remotePath), localFileName, options); + Send(YtWrapper, message.Release()); + } + + void Tick(const NActors::TActorContext& ctx) { + ctx.Schedule(TDuration::Seconds(5), new TEvTick()); + } + + void SaveToCache() { + auto& file = Options.Files[CurrentFileId]; + Options.FileCache->AddFile( + Options.TmpDir + "/" + file.GetRemoteFileName(), /* /tmp/md5 */ + file.LocalFileName /* md5 */); + } + + void OnFileDownloaded(TEvReadFileResponse::TPtr& ev, const NActors::TActorContext& ctx) { + auto result = std::get<0>(*ev->Get()); + if (result.IsOK()) { + SaveToCache(); + + Retry = 0; + CurrentFileId++; + if (CurrentFileId == Options.Files.size()) { + // save to cache + YQL_CLOG(DEBUG, ProviderDq) << "Download complete"; + PassAway(); + } else { + DownloadFile(); + } + } else if (Options.MaxRetries == -1 || ++Retry < Options.MaxRetries) { + YQL_CLOG(DEBUG, ProviderDq) << "Retry " << ToString(result); + std::random_shuffle(Options.Files.begin() + CurrentFileId, Options.Files.end()); + Tick(ctx); + } else { + PassAway(); + } + } + + TResourceManagerOptions Options; + const ICoordinationHelper::TPtr Coordinator; + + TActorId YtWrapper; + + ui32 CurrentFileId = 0; + int Retry = 0; + TActorId ParentId; +}; + +IActor* CreateYtResourceDownloader( + const TResourceManagerOptions& options, + const ICoordinationHelper::TPtr& coordinator) +{ + Y_ABORT_UNLESS(!options.YtBackend.GetClusterName().empty()); + Y_ABORT_UNLESS(!options.YtBackend.GetUser().empty()); + Y_ABORT_UNLESS(!options.Files.empty()); + Y_ABORT_UNLESS(!options.UploadPrefix.empty()); + Y_ABORT_UNLESS(!options.TmpDir.empty()); + Y_ABORT_UNLESS(options.FileCache); + + NFs::MakeDirectoryRecursive(options.TmpDir, NFs::FP_NONSECRET_FILE, false); + + return new TYtResourceDownloader(options, coordinator); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp b/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp new file mode 100644 index 0000000000..f28ef86c0c --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp @@ -0,0 +1,69 @@ +#include "resource_manager.h" + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h> + +namespace NYql { + using namespace NActors; + + class TResourceManagerStub: public TActor<TResourceManagerStub> { + public: + TResourceManagerStub() + : TActor<TResourceManagerStub>(&TResourceManagerStub::Handler) + { } + + void Handler(STFUNC_SIG) { + Y_UNUSED(ev); + } + }; + + IActor* CreateResourceManager(const TResourceManagerOptions& options, const ICoordinationHelper::TPtr& coordinator) { + Y_UNUSED(options); +#ifdef WIN32 + return new TResourceManagerStub(); +#else + IActor* CreateYtResourceManager( + const TResourceManagerOptions& options, + const ICoordinationHelper::TPtr& coordinator); + return CreateYtResourceManager(options, coordinator); +#endif + } + + + NActors::IActor* CreateResourceUploader(const TResourceManagerOptions& options, const ICoordinationHelper::TPtr& coordinator) { + Y_UNUSED(options); +#ifdef WIN32 + return new TResourceManagerStub(); +#else + IActor* CreateYtResourceUploader( + const TResourceManagerOptions& options, + const ICoordinationHelper::TPtr& coordinator); + return CreateYtResourceUploader(options, coordinator); +#endif + } + + NActors::IActor* CreateResourceDownloader(const TResourceManagerOptions& options, const ICoordinationHelper::TPtr& coordinator) { + Y_UNUSED(options); +#ifdef WIN32 + return new TResourceManagerStub(); +#else + IActor* CreateYtResourceDownloader( + const TResourceManagerOptions& options, const ICoordinationHelper::TPtr& coordinator); + return CreateYtResourceDownloader(options, coordinator); +#endif + } + + NActors::IActor* CreateResourceCleaner(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator) { + Y_UNUSED(options); +#ifdef WIN32 + return new TResourceManagerStub(); +#else + IActor* CreateYtResourceCleaner( + const TResourceManagerOptions& options, + const TIntrusivePtr<ICoordinationHelper>& coordinator); + return CreateYtResourceCleaner(options, coordinator); +#endif + } +} diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_manager.h b/ydb/library/yql/providers/dq/actors/yt/resource_manager.h new file mode 100644 index 0000000000..2106a2e84e --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/resource_manager.h @@ -0,0 +1,90 @@ +#pragma once + +#include <util/system/file.h> + +#include <library/cpp/actors/core/actor.h> +#include <ydb/library/yql/providers/dq/config/config.pb.h> + +#include <ydb/library/yql/providers/dq/task_runner/file_cache.h> +#include <ydb/library/yql/providers/common/metrics/metrics_registry.h> + +#include <library/cpp/threading/future/future.h> +#include <library/cpp/monlib/dynamic_counters/counters.h> + +namespace NYql { + namespace NCommonJobVars { + extern const TString ACTOR_PORT; + extern const TString ACTOR_NODE_ID; + extern const TString UDFS_PATH; + extern const TString OPERATION_SIZE; + extern const TString YT_COORDINATOR; + extern const TString YT_BACKEND; + } + + class ICoordinationHelper; + + struct TResourceFile + { + TString ObjectId; + TString LocalFileName; + TMaybe<TString> RemoteFileName; + THashMap<TString, TString> Attributes; + TFile File; + + TResourceFile() { } + TResourceFile(const TString& localFileName) + : LocalFileName(localFileName) + , File(LocalFileName, RdOnly | OpenExisting) + { } + + TString GetRemoteFileName() const { + if (RemoteFileName) { + return *RemoteFileName; + } else { + auto pos = LocalFileName.rfind('/'); + return LocalFileName.substr(pos+1); + } + } + }; + + struct TResourceManagerOptions { + NYql::NProto::TDqConfig::TYtBackend YtBackend; + + TString UploadPrefix; + TString LockName; + TString LogFile; + + TString TmpDir; + IFileCache::TPtr FileCache; + + TVector<TResourceFile> Files; + + TMaybe<NThreading::TPromise<void>> Uploaded; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + + bool ExitOnPingFail = false; + + int Capabilities = 0; + int MaxRetries = -1; + + // Pinger + TString DieOnFileAbsence; // see YQL-14099 + + // Cleaner + TMaybe<int> KeepFirst; + TDuration DropBefore; + TMaybe<TString> KeepFilter; + + TMaybe<TString> AnnounceClusterName; + }; + + NActors::IActor* CreateResourceManager(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator); + + NActors::IActor* CreateResourceUploader(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator); + + NActors::IActor* CreateResourceDownloader(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator); + + NActors::IActor* CreateResourceCleaner(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp b/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp new file mode 100644 index 0000000000..07789f159f --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp @@ -0,0 +1,236 @@ + +#include "nodeid_assigner.h" +#include "yt_wrapper.h" + +#include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/library/yql/utils/log/log.h> + +#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h> +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h> +#include <ydb/library/yql/providers/dq/actors/events/events.h> + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/yson/node/node_io.h> + +#include <yt/cpp/mapreduce/interface/fluent.h> + +namespace NYql { + +using namespace NActors; +using namespace NMonitoring; + +class TYtResourceUploader: public TRichActor<TYtResourceUploader> { +public: + static constexpr char ActorName[] = "UPLOADER"; + + TYtResourceUploader( + const TResourceManagerOptions& options, + const ICoordinationHelper::TPtr& coordinator) + : TRichActor<TYtResourceUploader>(&TYtResourceUploader::Follower) + , Options(options) + , Coordinator(coordinator) + , Config(Coordinator->GetConfig()) + { + if (Options.Counters) { + FileSize = Options.Counters->GetHistogram("FileSize", ExponentialHistogram(10, 4, 10)); + FileUploadTime = Options.Counters->GetHistogram("UploadTime", ExponentialHistogram(10, 3, 1)); + Errors = Options.Counters->GetCounter("Errors"); + } + } + +private: + // States: Follower -> Upload + + void StartFollower(TEvBecomeFollower::TPtr& ev, const TActorContext& ctx) { + YQL_LOG_CTX_ROOT_SCOPE(CurrentLockName); + YQL_CLOG(DEBUG, ProviderDq) << "Current lock '" << CurrentLockName << "'"; + Y_UNUSED(ctx); + auto leaderAttributes = NYT::NodeFromYsonString(ev->Get()->Attributes).AsMap(); + + if (leaderAttributes.contains(NCommonAttrs::ACTOR_NODEID_ATTR)) { + YQL_CLOG(INFO, ProviderDq) << " Following leader: " + << leaderAttributes.at(NCommonAttrs::ACTOR_NODEID_ATTR).AsUint64(); + } + if (leaderAttributes.contains(NCommonAttrs::HOSTNAME_ATTR)) { + YQL_CLOG(INFO, ProviderDq) << " Leader hostname: " + << leaderAttributes.at(NCommonAttrs::HOSTNAME_ATTR).AsString(); + } + Become(&TYtResourceUploader::Follower); + } + + void StartLeader(TEvBecomeLeader::TPtr& ev, const TActorContext& ctx) { + YQL_LOG_CTX_ROOT_SCOPE(CurrentLockName); + YQL_CLOG(DEBUG, ProviderDq) << "Current lock '" << CurrentLockName << "'"; + Y_UNUSED(ctx); + YQL_CLOG(INFO, ProviderDq) << "Become leader, epoch=" << ev->Get()->LeaderEpoch; + Become(&TYtResourceUploader::UploadState); + UploadFile(); + } + + void Follower(STFUNC_SIG) { + switch (const ui32 etype = ev->GetTypeRewrite()) { + HFunc(TEvBecomeFollower, StartFollower) + HFunc(TEvBecomeLeader, StartLeader) + + case TEvents::TEvBootstrap::EventType: { + Bootstrap(TActivationContext::ActorContextFor(this->SelfId())); + break; + } + + default: + break; + } + } + + void DoPassAway() override { + Send(ParentId, new TEvUploadComplete()); + + if (Options.Uploaded) { + YQL_CLOG(DEBUG, ProviderDq) << "Promise SetValue"; + auto promise = *Options.Uploaded; + promise.TrySetValue(); + } + } + + void UploadState(STFUNC_SIG) { + switch (const ui32 etype = ev->GetTypeRewrite()) { + HFunc(TEvBecomeFollower, StartFollower) + HFunc(TEvWriteFileResponse, OnFileUploaded) + + case TEvTick::EventType: { + // retry + UploadFile(); + break; + } + default: + break; + } + } + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + ParentId = parentId; + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0); + } + + TString GetLockName() { + Y_ABORT_UNLESS(!Options.Files[CurrentFileId].ObjectId.empty()); + return Options.Files[CurrentFileId].ObjectId; + } + + void Lock() { + if (LockId) { + UnregisterChild(LockId); + } + CurrentLockName = Options.LockName.empty() + ? GetLockName() + : Options.LockName; + LockId = RegisterChild(Coordinator->CreateLockOnCluster(YtWrapper, Options.YtBackend.GetPrefix(), CurrentLockName)); + Become(&TYtResourceUploader::Follower); + } + + void Bootstrap(const NActors::TActorContext& ctx) { + YtWrapper = Coordinator->GetWrapper( + ctx.ActorSystem(), + Options.YtBackend.GetClusterName(), + Options.YtBackend.GetUser(), + Options.YtBackend.GetToken()); + Lock(); + } + + void UploadFile() { + TString remotePath = Options.UploadPrefix + "/"; + NYT::NApi::TFileWriterOptions options; + auto& file = Options.Files[CurrentFileId]; + THashMap<TString, NYT::TNode> attributes; + for (const auto& [k, v] : file.Attributes) { + attributes[k] = NYT::TNode(v); + } + if (Options.YtBackend.HasUploadReplicationFactor()) { + attributes["replication_factor"] = NYT::TNode(Options.YtBackend.GetUploadReplicationFactor()); + } + options.ComputeMD5 = true; + + if (FileSize && file.File.IsOpen()) { + FileSize->Collect(file.File.GetLength() / 1024 / 1024); // megabytes + } + + UploadStart = TInstant::Now(); + + remotePath += file.GetRemoteFileName(); + auto message = MakeHolder<TEvWriteFile>(file.File, NYT::NYPath::TYPath(remotePath), attributes, options); + Send(YtWrapper, message.Release()); + } + + void Tick(const NActors::TActorContext& ctx) { + ctx.Schedule(TDuration::Seconds(5), new TEvTick()); + } + + void OnFileUploaded(TEvWriteFileResponse::TPtr& ev, const NActors::TActorContext& ctx) { + YQL_LOG_CTX_ROOT_SCOPE(CurrentLockName); + YQL_CLOG(DEBUG, ProviderDq) << "Current lock '" << CurrentLockName << "'"; + auto result = std::get<0>(*ev->Get()); + if (result.IsOK()) { + if (FileUploadTime) { + FileUploadTime->Collect((TInstant::Now() - UploadStart).Seconds()); + } + CurrentFileId++; + if (CurrentFileId == Options.Files.size()) { + YQL_CLOG(DEBUG, ProviderDq) << "Upload complete"; + PassAway(); + } else { + if (Options.LockName.empty()) { + Lock(); + YQL_CLOG(DEBUG, ProviderDq) << "Lock next " << CurrentFileId << "/" << Options.Files.size(); + } else { + UploadFile(); + YQL_CLOG(DEBUG, ProviderDq) << "Upload next " << CurrentFileId << "/" << Options.Files.size(); + } + } + } else { + YQL_CLOG(DEBUG, ProviderDq) << "Retry " << ToString(result); + + if (Errors) { + *Errors += 1; + } + + std::random_shuffle(Options.Files.begin() + CurrentFileId, Options.Files.end()); + Tick(ctx); + } + } + + const NYT::NApi::IClientPtr Client; + TResourceManagerOptions Options; + + const ICoordinationHelper::TPtr Coordinator; + + const NProto::TDqConfig::TYtCoordinator Config; + + TActorId YtWrapper; + + ui32 CurrentFileId = 0; + TActorId ParentId; + TActorId LockId; + + THistogramPtr FileSize; + THistogramPtr FileUploadTime; + TDynamicCounters::TCounterPtr Errors; + TInstant UploadStart; + TString CurrentLockName; +}; + +IActor* CreateYtResourceUploader( + const TResourceManagerOptions& options, + const ICoordinationHelper::TPtr& coordinator) +{ + Y_ABORT_UNLESS(!options.YtBackend.GetClusterName().empty()); + Y_ABORT_UNLESS(!options.YtBackend.GetUser().empty()); + Y_ABORT_UNLESS(options.YtBackend.HasPrefix()); + Y_ABORT_UNLESS(!options.Files.empty()); + Y_ABORT_UNLESS(!options.UploadPrefix.empty()); + + return new TYtResourceUploader(options, coordinator); +} + +} diff --git a/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp b/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp new file mode 100644 index 0000000000..e8abfe904b --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp @@ -0,0 +1,79 @@ +#include "worker_registrator.h" +#include "nodeid_assigner.h" +#include "yt_wrapper.h" + +#include <ydb/library/yql/providers/yt/lib/log/yt_logger.h> + +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> + +#include <ydb/library/yql/utils/log/log.h> + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/event.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <library/cpp/yson/node/node.h> +#include <library/cpp/yson/node/node_io.h> + +#include <util/generic/utility.h> + +namespace NYql { + +using namespace NActors; + +class TWorkerRegistrator: public TActor<TWorkerRegistrator> { +public: + static constexpr char ActorName[] = "WORKER_REGISTRATOR"; + + TWorkerRegistrator(TActorId ytWrapper, const TWorkerRegistratorOptions& options) + : TActor(&TWorkerRegistrator::Handler) + , YtWrapper(ytWrapper) + , Options(options) + { } + + STRICT_STFUNC(Handler, { + HFunc(TEvSetNodeResponse, OnSetNodeResponse) + cFunc(TEvents::TEvPoison::EventType, PassAway) + }); + +private: + TEvSetNode* SetNodeCommand(TInstant now) + { + auto value = NYT::NYson::TYsonString(NYT::NodeToYsonString(NYT::TNode(now.ToString()))); + return new TEvSetNode(Options.Prefix + "/" + Options.NodeName + "/@" + NCommonAttrs::PINGTIME_ATTR, value, NYT::NApi::TSetNodeOptions()); + } + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + Y_UNUSED(parentId); + return new IEventHandle(YtWrapper, self, SetNodeCommand(TInstant::Now()), 0); + } + + void OnSetNodeResponse(TEvSetNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + auto result = std::get<0>(*ev->Get()); + + if (!result.IsOK()) { + YQL_CLOG(ERROR, ProviderDq) << "Error on list node " << ToString(result); + } + + if (!result.IsOK() && result.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) { + NYql::FlushYtDebugLog(); + _exit(1); + } + + YQL_CLOG(DEBUG, ProviderDq) << "OnSetNodeResponse"; + + TActivationContext::Schedule(Options.PingPeriod, new IEventHandle(YtWrapper, SelfId(), SetNodeCommand(TInstant::Now()), 0)); + } + + TActorId YtWrapper; + TWorkerRegistratorOptions Options; +}; + +IActor* CreateWorkerRegistrator(NActors::TActorId ytWrapper, const TWorkerRegistratorOptions& options) { + Y_ABORT_UNLESS(!options.Prefix.empty()); + Y_ABORT_UNLESS(!options.NodeName.empty()); + return new TWorkerRegistrator(ytWrapper, options); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/worker_registrator.h b/ydb/library/yql/providers/dq/actors/yt/worker_registrator.h new file mode 100644 index 0000000000..de714cb135 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/worker_registrator.h @@ -0,0 +1,21 @@ +#pragma once + +#include <library/cpp/actors/core/actor.h> + +#include <util/datetime/base.h> + +namespace NYql +{ + +struct TWorkerRegistratorOptions { + TString Prefix; + TString NodeName; + + TDuration PingPeriod = TDuration::MilliSeconds(5000); + TDuration RetryPeriod = TDuration::MilliSeconds(5000); +}; + +NActors::IActor* CreateWorkerRegistrator(NActors::TActorId ytWrapper, const TWorkerRegistratorOptions& options); + +} // namespace NYql + diff --git a/ydb/library/yql/providers/dq/actors/yt/ya.make b/ydb/library/yql/providers/dq/actors/yt/ya.make new file mode 100644 index 0000000000..106c3a3c8e --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/ya.make @@ -0,0 +1,56 @@ +LIBRARY() + +PEERDIR( + library/cpp/actors/core + library/cpp/grpc/client + yt/cpp/mapreduce/interface + ydb/library/yql/providers/dq/config + ydb/library/yql/core/issue + ydb/library/yql/providers/common/metrics + ydb/library/yql/providers/dq/api/grpc + ydb/library/yql/providers/dq/api/protos + ydb/library/yql/providers/dq/common + ydb/library/yql/providers/yt/lib/log + ydb/library/yql/providers/dq/actors/events +) + +IF (NOT OS_WINDOWS) + PEERDIR( + yt/yt/client + ) +ENDIF() + +SET( + SOURCE + nodeid_assigner.cpp + nodeid_assigner.h + resource_manager.cpp + resource_manager.h +) + +IF (NOT OS_WINDOWS) + SET( + SOURCE + ${SOURCE} + nodeid_cleaner.cpp + nodeid_cleaner.h + worker_registrator.cpp + worker_registrator.h + lock.cpp + lock.h + resource_uploader.cpp + resource_downloader.cpp + resource_cleaner.cpp + yt_wrapper.cpp + yt_wrapper.h + yt_resource_manager.cpp + ) +ENDIF() + +SRCS( + ${SOURCE} +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/providers/dq/actors/yt/yt_events.h b/ydb/library/yql/providers/dq/actors/yt/yt_events.h new file mode 100644 index 0000000000..9c46790b8f --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/yt_events.h @@ -0,0 +1,35 @@ +#pragma once + +namespace NYql { + + struct TYtEvents { + enum { + ES_START_OPERATION = EventSpaceBegin(NActors::TEvents::EEventSpace::ES_USERSPACE) + 10000, + ES_START_OPERATION_RESPONSE, + ES_GET_OPERATION, + ES_GET_OPERATION_RESPONSE, + ES_LIST_OPERATIONS, + ES_LIST_OPERATIONS_RESPONSE, + ES_GET_JOB, + ES_GET_JOB_RESPONSE, + ES_WRITE_FILE, + ES_WRITE_FILE_RESPONSE, + ES_READ_FILE, + ES_READ_FILE_RESPONSE, + ES_LIST_NODE, + ES_LIST_NODE_RESPONSE, + ES_CREATE_NODE, + ES_CREATE_NODE_RESPONSE, + ES_SET_NODE, + ES_SET_NODE_RESPONSE, + ES_GET_NODE, + ES_GET_NODE_RESPONSE, + ES_REMOVE_NODE, + ES_REMOVE_NODE_RESPONSE, + ES_START_TRANSACTION, + ES_START_TRANSACTION_RESPONSE, + + ES_PRINT_JOB_STDERR + }; + }; +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp b/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp new file mode 100644 index 0000000000..a19ef5b5ac --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp @@ -0,0 +1,917 @@ +#include "yt_wrapper.h" + +#include <util/thread/pool.h> +#include <util/generic/size_literals.h> +#include <util/string/strip.h> +#include <util/system/env.h> + +#include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/library/yql/utils/log/log.h> + +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/events/events.h> +#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h> +#include <ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h> +#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h> + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/yson/node/node_io.h> + +#include <yt/cpp/mapreduce/interface/fluent.h> + +#include <library/cpp/protobuf/util/pb_io.h> + +namespace NYql { + +#define RM_LOG(A) YQL_CLOG(A, ProviderDq) << ClusterName << ": " + + namespace NCommonJobVars { + const TString ACTOR_PORT("ACTOR_PORT"); + const TString ACTOR_NODE_ID("ACTOR_NODE_ID"); + const TString UDFS_PATH("UDFS_PATH"); + const TString OPERATION_SIZE("OPERATION_SIZE"); + const TString YT_COORDINATOR("YT_COORDINATOR"); + const TString YT_BACKEND("YT_BACKEND"); + } + + using namespace NActors; + + struct TEvDropOperation + : NActors::TEventLocal<TEvDropOperation, TDqEvents::ES_OTHER1> { + TEvDropOperation() = default; + TEvDropOperation(const TString& operationId, const TString& mutationId) + : OperationId(operationId) + , MutationId(mutationId) + { } + + TString OperationId; + TString MutationId; + }; + + class TYtVanillaOperation: public TActor<TYtVanillaOperation> { + public: + static constexpr char ActorName[] = "YT_OPERATION"; + + TYtVanillaOperation(const TString& clusterName, TActorId ytWrapper, TActorId parentId, TString operationId, TString mutationId, TIntrusivePtr<NMonitoring::TDynamicCounters> counters) + : TActor<TYtVanillaOperation>(&TYtVanillaOperation::Handler) + , ClusterName(clusterName) + , YtWrapper(ytWrapper) + , ParentId(parentId) + , OperationId(NYT::NScheduler::TOperationId::FromString(operationId)) + , MutationId(mutationId) + , Counters(counters) + { } + + ~TYtVanillaOperation() + { + auto counters = Counters->GetSubgroup("operation", "brief_progress"); + for (const auto& [k, v] : Status) { + *counters->GetCounter(k) += -v; + } + } + + private: + STRICT_STFUNC(Handler, { + cFunc(TEvents::TEvPoison::EventType, PassAway); + HFunc(TEvGetOperationResponse, OnGetOperationResponse); + }) + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& /*parentId*/) override { + return new IEventHandle(YtWrapper, self, new TEvGetOperation(OperationId, NYT::NApi::TGetOperationOptions()), 0); + } + + void OnGetOperationResponse(TEvGetOperationResponse::TPtr& ev, const NActors::TActorContext& ) { + auto result = std::get<0>(*ev->Get()); + bool stopWatcher = false; + if (!result.IsOK() && result.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) { + stopWatcher = true; + } + + if (result.IsOK()) { + auto attributesMap = NYT::NodeFromYsonString(result.Value()).AsMap(); + + try { + if (attributesMap.contains("result")) { + RM_LOG(DEBUG) << "Result " << NYT::NodeToYsonString(attributesMap["result"]); + stopWatcher = true; + } + + if (attributesMap.contains("brief_progress")) { + auto statusMap = attributesMap["brief_progress"].AsMap()["jobs"].AsMap(); + + auto counters = Counters->GetSubgroup("operation", "brief_progress"); + for (const auto& [k, v] : statusMap) { + auto& oldStatus = Status[k]; + auto newStatus = v.AsInt64(); + *counters->GetCounter(k) += newStatus - oldStatus; + oldStatus = newStatus; + } + } + + } catch (...) { + RM_LOG(DEBUG) << CurrentExceptionMessage(); + } + } + + if (stopWatcher) { + RM_LOG(DEBUG) << "Stop watching operation (1) " << ToString(OperationId) << " " << ToString(result); + Send(YtWrapper, new TEvPrintJobStderr(OperationId)); + Send(ParentId, new TEvDropOperation(ToString(OperationId), MutationId)); + PassAway(); + } else { + TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way()); + TActivationContext::Schedule(TDuration::Seconds(5), + new IEventHandle(YtWrapper, SelfId(), new TEvGetOperation(OperationId, NYT::NApi::TGetOperationOptions()), 0), + TimerCookieHolder.Get()); + } + } + + const TString ClusterName; + const TActorId YtWrapper; + const TActorId ParentId; + const NYT::NScheduler::TOperationId OperationId; + const TString MutationId; + NActors::TSchedulerCookieHolder TimerCookieHolder; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + THashMap<TString, i64> Status; + }; + + class TNodeIdAllocator { + public: + TNodeIdAllocator(ui32 minNodeId, ui32 maxNodeId) + : MinNodeId(minNodeId) + , MaxNodeId(maxNodeId) + { } + + void Allocate(const TVector<ui32>& res) { + // TODO: check duplicates? + for (ui32 id : res) { + Allocated.insert(id); + } + } + + void Allocate(TVector<ui32>* res, int count) { + res->reserve(count); + for (ui32 id = MinNodeId; id < MaxNodeId && static_cast<int>(res->size()) < count; id = id + 1) { + if (!Allocated.contains(id)) { + res->push_back(id); + Allocated.insert(id); + } + } + } + + void Deallocate(const TVector<ui32>& nodes) { + for (auto id : nodes) { + Allocated.erase(id); + } + } + + void Clear() { + Allocated.clear(); + } + + private: + THashSet<ui32> Allocated; + ui32 MinNodeId; + ui32 MaxNodeId; + }; + + class TYtResourceManager: public TRichActor<TYtResourceManager> { + public: + static constexpr char ActorName[] = "YTRM"; + + TYtResourceManager( + const TResourceManagerOptions& options, + const ICoordinationHelper::TPtr& coordinator) + : TRichActor<TYtResourceManager>(&TYtResourceManager::Follower) + , Options(options) + , Counters(Options.Counters) + , ClusterName(Options.YtBackend.GetClusterName()) + , Coordinator(coordinator) + , CoordinatorConfig(Coordinator->GetConfig()) + , CoordinatorWrapper(Coordinator->GetWrapper()) + , NodeIdAllocator(Options.YtBackend.GetMinNodeId(), Options.YtBackend.GetMaxNodeId()) + { + } + + private: + // States: Follower <-> (ListOperations -> Leader) + + void StartFollower(TEvBecomeFollower::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + + auto leaderAttributes = NYT::NodeFromYsonString(ev->Get()->Attributes).AsMap(); + LeaderTransactionId = NYT::NObjectClient::TTransactionId(); + + RM_LOG(TRACE) << " Following leader: " << leaderAttributes.at(NCommonAttrs::ACTOR_NODEID_ATTR).AsUint64(); + for (const auto& [k, v] : RunningOperations) { + UnregisterChild(v.ActorId); + } + RunningOperations.clear(); + NodeIdAllocator.Clear(); + Become(&TYtResourceManager::Follower); + } + + void StartLeader(TEvBecomeLeader::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + RM_LOG(INFO) << "Become leader, epoch=" << ev->Get()->LeaderEpoch; + + LeaderTransactionId = NYT::NObjectClient::TTransactionId::FromString(ev->Get()->LeaderTransaction); + + ListOperations(); + Become(&TYtResourceManager::ListOperationsState); + } + + STRICT_STFUNC(Follower, { + HFunc(TEvBecomeFollower, StartFollower) + HFunc(TEvBecomeLeader, StartLeader) + cFunc(TEvents::TEvPoison::EventType, PassAway) + CFunc(TEvents::TEvBootstrap::EventType, Bootstrap) + + IgnoreFunc(TEvTick) + IgnoreFunc(TEvDropOperation) + IgnoreFunc(TEvStartOperationResponse) + IgnoreFunc(TEvCreateNodeResponse) + IgnoreFunc(TEvSetNodeResponse) + }) + + STRICT_STFUNC(ListOperationsState, { + HFunc(TEvBecomeFollower, StartFollower) + HFunc(TEvDropOperation, OnDropOperation) + HFunc(TEvListNodeResponse, OnListOperations) + HFunc(TEvStartOperationResponse, OnStartOperationResponse) + cFunc(TEvTick::EventType, ListOperations) + cFunc(TEvents::TEvPoison::EventType, PassAway) + IgnoreFunc(TEvCreateNodeResponse) + IgnoreFunc(TEvSetNodeResponse) + }) + + STRICT_STFUNC(Leader, { + HFunc(TEvBecomeFollower, StartFollower) + HFunc(TEvDropOperation, OnDropOperation) + HFunc(TEvListNodeResponse, OnListResponse) + HFunc(TEvStartOperationResponse, OnStartOperationResponse) + cFunc(TEvents::TEvPoison::EventType, PassAway) + cFunc(TEvTick::EventType, [this]() { + ListWorkers(); + Tick(); + }) + HFunc(TEvCreateNodeResponse, OnCreateNode) + IgnoreFunc(TEvRemoveNodeResponse) + IgnoreFunc(TEvSetNodeResponse) + }) + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0); + } + + void Bootstrap(const NActors::TActorContext& ctx) { + YtWrapper = Coordinator->GetWrapper( + ctx.ActorSystem(), + Options.YtBackend.GetClusterName(), + Options.YtBackend.GetUser(), + Options.YtBackend.GetToken()); + RegisterChild(Coordinator->CreateLockOnCluster(YtWrapper, Options.YtBackend.GetPrefix(), Options.LockName, false)); + } + + void Tick() { + TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way()); + Schedule(TDuration::Seconds(5), new TEvTick(), TimerCookieHolder.Get()); + } + + void CreateCoreTable(ui32 tableNumber) + { + NYT::NApi::TCreateNodeOptions options; + options.Recursive = true; + options.IgnoreExisting = true; + + YQL_CLOG(DEBUG, ProviderDq) << "Creating core table: " << Options.UploadPrefix + "/CoreTable-" + ToString(tableNumber); + + Send(YtWrapper, new TEvCreateNode( + static_cast<ui64>(-1), + Options.UploadPrefix + "/CoreTable-" + ToString(tableNumber), + NYT::NObjectClient::EObjectType::Table, + options)); + + YQL_CLOG(DEBUG, ProviderDq) << "Creating stderr table: " << Options.UploadPrefix + "/StderrTable-" + ToString(tableNumber); + + Send(YtWrapper, new TEvCreateNode( + static_cast<ui64>(-1), + Options.UploadPrefix + "/StderrTable-" + ToString(tableNumber), + NYT::NObjectClient::EObjectType::Table, + options)); + } + + void StartOperationWatcher(const TString& operationId, const TString& mutationId, const NActors::TActorContext& ctx) + { + Y_UNUSED(ctx); + RM_LOG(DEBUG) << "StartOperationWatcher " << operationId << "|" << mutationId; + auto operation = RunningOperations.find(mutationId); + Y_ABORT_UNLESS(operation != RunningOperations.end()); + auto actorId = RegisterChild(new TYtVanillaOperation(ClusterName, YtWrapper, SelfId(), operationId, mutationId, Counters)); + operation->second.ActorId = actorId; + } + + void MaybeStartOperations(const NActors::TActorContext& ctx) + { + // to avoid races do nothing if there is PendingStartOperationRequests + if (!PendingStartOperationRequests.empty()) { + RM_LOG(DEBUG) << "PendingStartOperationRequests contains " << PendingStartOperationRequests.size() << " requests "; + for (const auto& [k, _]: PendingStartOperationRequests) { + RM_LOG(DEBUG) << "RequestId " << k; + } + return; + } + + int totalJobs = 0; + for (const auto& [k, v] : RunningOperations) { + totalJobs += v.Nodes.size(); + RM_LOG(DEBUG) << "Operation: " << k << " " << v.Nodes.size() << " "; + } + + RM_LOG(DEBUG) << "Running/Max jobs: " << totalJobs << "/" << Options.YtBackend.GetMaxJobs(); + + int needToStart = Options.YtBackend.GetMaxJobs() - totalJobs; + RM_LOG(DEBUG) << "Need to start: " << needToStart; + if (needToStart > 0) { + StartOperations(needToStart, ctx); + } + } + + void DropRunningOperation(const TString& mutationId, const TVector<ui32>& preserve = {}) { + if (RunningOperations.contains(mutationId)) { + NodeIdAllocator.Deallocate(RunningOperations[mutationId].Nodes); + } + + RunningOperations.erase(mutationId); + + if (!preserve.empty()) { + RM_LOG(DEBUG) << "Operation in unknown state, preserve mutation " << mutationId; + MutationsCache[mutationId] = preserve; + NodeIdAllocator.Allocate(preserve); + } else { + auto removePath = Options.YtBackend.GetPrefix() + "/operations/" + ClusterName + "/" + mutationId; + RM_LOG(DEBUG) << "Removing node " << removePath; + NYT::NApi::TRemoveNodeOptions removeNodeOptions; + removeNodeOptions.PrerequisiteTransactionIds.push_back(LeaderTransactionId); + Send(YtWrapper, new TEvRemoveNode(removePath, removeNodeOptions)); + } + } + + void OnDropOperation(TEvDropOperation::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + auto operationId = ev->Get()->OperationId; + auto mutationId = ev->Get()->MutationId; + auto maybeOperation = RunningOperations.find(mutationId); + if (maybeOperation != RunningOperations.end()) { + UnregisterChild(maybeOperation->second.ActorId); + + RM_LOG(DEBUG) << "Stop operation " << operationId << "|" << mutationId; + DropRunningOperation(mutationId); + } else { + RM_LOG(WARN) << "Unknown operation " << operationId << "|" << mutationId; + } + } + + void OnListResponse(TEvListNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) { + auto result = std::get<0>(*ev->Get()); + + try { + MaybeStartOperations(ctx); + } catch (...) { + RM_LOG(ERROR) << "Error on list node " << CurrentExceptionMessage(); + } + } + + void OnListOperations(TEvListNodeResponse::TPtr& ev, const TActorContext& ctx) + { + auto result = std::get<0>(*ev->Get()); + + if (!result.IsOK()) { + if (result.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) { + Become(&TYtResourceManager::Leader); + } else { + RM_LOG(ERROR) << "Error on list node " << ToString(result); + } + Tick(); + return; + } + + RM_LOG(DEBUG) << "OnListOperations "; + auto nodes = NYT::NodeFromYsonString(result.Value()).AsList(); + + for (auto node : nodes) + { + const auto& attributes = node.GetAttributes().AsMap(); + + RM_LOG(DEBUG) << "Check " << node.AsString(); + + auto mutationId = NYT::TGuid::FromString(attributes.find("yql_mutation_id")->second.AsString()); + auto command = attributes.find("yql_command")->second.AsString(); + auto file_paths = attributes.find("yql_file_paths")->second; + auto maybeNodes = attributes.find(NCommonAttrs::ACTOR_NODEID_ATTR); + + if (maybeNodes == attributes.end()) { + // unsupported + DropRunningOperation(ToString(mutationId)); + continue; + } + + TVector<ui32> nodes; + + for (auto node : maybeNodes->second.AsList()) { + nodes.push_back(node.AsUint64()); + } + + NodeIdAllocator.Allocate(nodes); + + auto maybeOperationId = attributes.find(NCommonAttrs::OPERATIONID_ATTR); + if (maybeOperationId == attributes.end()) { + RM_LOG(DEBUG) << "Start or attach to " << ToString(mutationId); + StartOrAttachOperation(mutationId, nodes, command, file_paths); + } else { + auto operationId = maybeOperationId->second.AsString(); + RM_LOG(DEBUG) << "Attach to " << operationId << "|" << ToString(mutationId); + + auto& status = RunningOperations[ToString(mutationId)]; + status.MutationId = ToString(mutationId); + status.OperationId = operationId; + status.Nodes = nodes; + + StartOperationWatcher(operationId, ToString(mutationId), ctx); + } + } + + if (PendingStartOperationRequests.empty() && CurrentStateFunc() != &TYtResourceManager::Leader) { + Become(&TYtResourceManager::Leader); + Tick(); + } + } + + void OnStartOperationResponse(TEvStartOperationResponse::TPtr& ev, const NActors::TActorContext& ctx) { + auto result = std::get<0>(*ev->Get()); + auto requestId = ev->Get()->RequestId; + + auto maybeJobs = PendingStartOperationRequests.find(requestId); + + Y_ABORT_UNLESS(maybeJobs != PendingStartOperationRequests.end()); + + auto mutationId = maybeJobs->second.MutationId; + + if (!result.IsOK()) { + // TODO: Check response code + RM_LOG(WARN) << "Failed to start operation " << ToString(result); + DropRunningOperation(mutationId, maybeJobs->second.Nodes); + } else { + auto operationId = ToString(result.Value()); + Y_ABORT_UNLESS(RunningOperations.contains(mutationId)); + + RunningOperations[mutationId].OperationId = operationId; + + NYT::NApi::TSetNodeOptions setNodeOptions; + setNodeOptions.PrerequisiteTransactionIds.push_back(LeaderTransactionId); + + Send(YtWrapper, new TEvSetNode( + Options.YtBackend.GetPrefix() + "/operations/" + ClusterName + "/" + mutationId + "/@" + NCommonAttrs::OPERATIONID_ATTR, + NYT::NYson::TYsonString(NYT::NodeToYsonString(NYT::TNode(operationId))), + setNodeOptions)); + StartOperationWatcher(operationId, mutationId, ctx); + } + + PendingStartOperationRequests.erase(maybeJobs); + + if (PendingStartOperationRequests.empty() && CurrentStateFunc() != &TYtResourceManager::Leader) { + Become(&TYtResourceManager::Leader); + Tick(); + } + } + + void ListOperations() { + NYT::NApi::TListNodeOptions options; + options.Attributes = { + "yql_mutation_id", + NCommonAttrs::OPERATIONSIZE_ATTR, + NCommonAttrs::OPERATIONID_ATTR, + NCommonAttrs::CLUSTERNAME_ATTR, + NCommonAttrs::ACTOR_NODEID_ATTR, + "yql_command", + "yql_file_paths" + }; + auto command = new TEvListNode(Options.YtBackend.GetPrefix() + "/operations/" + ClusterName, options); + RM_LOG(DEBUG) << "List " << Options.YtBackend.GetPrefix() + "/operations/" + ClusterName; + Send(YtWrapper, command); + } + + void ListWorkers() { + NYT::NApi::TListNodeOptions options; + options.Attributes = { + NCommonAttrs::ACTOR_NODEID_ATTR, + NCommonAttrs::OPERATIONID_ATTR, + NCommonAttrs::OPERATIONSIZE_ATTR, + NCommonAttrs::JOBID_ATTR, + NCommonAttrs::ROLE_ATTR, + NCommonAttrs::CLUSTERNAME_ATTR, + "modification_time" + }; + options.ReadFrom = NYT::NApi::EMasterChannelKind::Cache; + auto command = new TEvListNode(CoordinatorConfig.GetPrefix() + "/worker_node", options); + Send(CoordinatorWrapper, command); + } + + void StartOperations(int jobs, const NActors::TActorContext& ctx) { + int jobsPerOperation = Options.YtBackend.HasJobsPerOperation() + ? Options.YtBackend.GetJobsPerOperation() + : Options.YtBackend.GetMaxJobs(); + + Y_ABORT_UNLESS(jobsPerOperation > 0); + + for (int i = 0; i < jobs; i += jobsPerOperation) { + if (jobs - i >= jobsPerOperation) { + StartOperation(jobsPerOperation, ctx); + } + } + } + + TString GetOperationSpec(const TVector<ui32>& nodes, const TString& command, const TMaybe<NYT::TNode>& filePaths) const + { + int actorPort = Options.YtBackend.HasActorStartPort() + ? Options.YtBackend.GetActorStartPort() + : 31002; + + bool samePorts = Options.YtBackend.HasSameActorPorts() + ? Options.YtBackend.GetSameActorPorts() + : true; + + auto minNodeId = Options.YtBackend.GetMinNodeId(); + + Y_ABORT_UNLESS(!nodes.empty()); + + TString coordinatorStr; + TStringOutput output1(coordinatorStr); + SerializeToTextFormat(CoordinatorConfig, output1); + + TString backendStr; + TStringOutput output2(backendStr); + SerializeToTextFormat(Options.YtBackend, output2); + ui32 tableNumber = *nodes.begin(); + TString fileCache = "file_cache2"; + + TVector<std::pair<TString, TString>> initialFileList; + for (const auto& fname : Options.Files) { + initialFileList.push_back(std::make_pair(Options.UploadPrefix, fname.GetRemoteFileName())); + } + if (Options.YtBackend.HasEnablePorto()) { + for (const auto& layer : Options.YtBackend.GetPortoLayer()) { + auto pos = layer.rfind('/'); + auto baseName = layer.substr(0, pos); + auto name = layer.substr(pos+1); + initialFileList.push_back(std::make_pair(baseName, name)); + } + } + + TVector<TString> operationLayersList; + for (const auto& operationLayer : Options.YtBackend.GetOperationLayer()) { + operationLayersList.push_back(operationLayer); + } + + auto operationSpec = NYT::BuildYsonNodeFluently() + .BeginMap() + .DoIf(Options.YtBackend.GetOwner().size() > 0, [&] (NYT::TFluentMap fluent) { + fluent.Item("acl").BeginList() + .Item() + .BeginMap() + .Item("action").Value("allow") + .Item("permissions") + .BeginList() + .Item().Value("read") + .Item().Value("manage") + .EndList() + .Item("subjects") + .BeginList() + .DoFor(Options.YtBackend.GetOwner(), [&] (NYT::TFluentList fluent1, const TString& subject) { + fluent1.Item().Value(subject); + }) + .EndList() + .EndMap() + .EndList(); + }) + .Item("secure_vault") + .BeginMap() + .Item(NCommonJobVars::YT_COORDINATOR).Value(coordinatorStr) + .Item(NCommonJobVars::YT_BACKEND).Value(backendStr) + .DoFor(Options.YtBackend.GetVaultEnv(), [&] (NYT::TFluentMap fluent, const NYql::NProto::TDqConfig::TAttr& envVar) { // Добавляем env variables + TString tokenValue; + try { + tokenValue = StripString(TFileInput(envVar.GetValue()).ReadLine()); + } catch (...) { + throw yexception() << "Cannot read file " << envVar.GetValue() << " Reason: " << CurrentExceptionMessage(); + } + fluent.Item(envVar.GetName()).Value(tokenValue); + }) + .EndMap() + .Item("core_table_path").Value(Options.UploadPrefix + "/CoreTable-"+ToString(tableNumber)) + .Item("stderr_table_path").Value(Options.UploadPrefix + "/StderrTable-"+ToString(tableNumber)) + .Item("try_avoid_duplicating_jobs").Value(true) + .DoIf(!Options.YtBackend.GetPool().empty(), [&] (NYT::TFluentMap fluent) { + fluent.Item("pool").Value(Options.YtBackend.GetPool()); + }) + .DoIf(Options.YtBackend.GetPoolTrees().size() > 0, [&] (NYT::TFluentMap fluent) { + fluent.Item("pool_trees") + .BeginList() + .DoFor(Options.YtBackend.GetPoolTrees(), [&](NYT::TFluentList fluent1, const TString& subject) { + fluent1.Item().Value(subject); + }) + .EndList(); + }) + .Item("tasks") + .BeginMap() + .DoFor(nodes, [&] (NYT::TFluentMap fluent1, const auto& nodeId) { + fluent1.Item("yql_worker_" + ToString(nodeId)) + .BeginMap() + .DoIf(Options.YtBackend.GetNetworkProject().size() > 0, [&] (NYT::TFluentMap fluent) { + fluent.Item("network_project").Value(Options.YtBackend.GetNetworkProject()); + }) + .DoIf(Options.YtBackend.HasEnablePorto(), [&] (NYT::TFluentMap fluent) { + fluent.Item("enable_porto").Value(Options.YtBackend.GetEnablePorto()); + }) + .DoIf(Options.YtBackend.HasContainerCpuLimit(), [&] (NYT::TFluentMap fluent) { + fluent.Item("set_container_cpu_limit").Value(Options.YtBackend.GetContainerCpuLimit()); + }) + .Item("command").Value(command) + .DoIf(!operationLayersList.empty(), [&] (NYT::TFluentMap fluent) { + fluent.Item("layer_paths").DoListFor(operationLayersList, [&] (NYT::TFluentList list, const TString& operationLayer) { + list.Item().Value(operationLayer); + }); + }) + .Item("environment") + .BeginMap() + .Item(NCommonJobVars::ACTOR_PORT).Value(ToString( + samePorts + ? actorPort + : actorPort + nodeId - minNodeId)) + .Item(NCommonJobVars::OPERATION_SIZE).Value(ToString(nodes.size())) + .Item(NCommonJobVars::UDFS_PATH).Value(fileCache) + .Item(NCommonJobVars::ACTOR_NODE_ID).Value(ToString(nodeId)) + .DoIf(!!GetEnv("YQL_DETERMINISTIC_MODE"), [&](NYT::TFluentMap fluent) { + fluent.Item("YQL_DETERMINISTIC_MODE").Value("1"); + }) + .EndMap() + .DoIf((Options.YtBackend.GetClusterName().find("localhost") != 0) && filePaths.Empty(), [&] (NYT::TFluentMap fluent) { + fluent.Item("file_paths").DoListFor(initialFileList, [&] (NYT::TFluentList list, const std::pair<TString, TString>& item) { + auto baseName = item.second; + list.Item() + .BeginAttributes() + .Item("executable").Value(true) + .Item("file_name").Value(baseName) + .EndAttributes() + .Value(item.first + "/" + baseName); + }); + }) + .DoIf(!filePaths.Empty(), [&] (NYT::TFluentMap fluent) { + fluent.Item("file_paths").Value(*filePaths); + }) + .Item("job_count").Value(1) + .DoIf(Options.YtBackend.HasMemoryLimit(), [&] (NYT::TFluentMap fluent) { + fluent.Item("memory_limit").Value(Options.YtBackend.GetMemoryLimit()); + }) + .DoIf(Options.YtBackend.HasCpuLimit(), [&] (NYT::TFluentMap fluent) { + fluent.Item("cpu_limit").Value(Options.YtBackend.GetCpuLimit()); + }) + .DoIf(Options.YtBackend.HasUseTmpFs() && Options.YtBackend.GetUseTmpFs(), [&] (NYT::TFluentMap fluent) { + fluent.Item("tmpfs_path").Value(fileCache); + }) + .EndMap(); + }) + .EndMap() + .EndMap(); + + return NYT::NodeToYsonString(operationSpec); + } + + void StartOrAttachOperation( + const NYT::TGuid& mutationId, + const TVector<ui32>& nodes, + const TString& command, + const NYT::TNode& filePaths) + { + int jobs = static_cast<int>(nodes.size()); + RM_LOG(INFO) << "Creating " << jobs << " workers " << ToString(mutationId); + + auto operationSpec = GetOperationSpec(nodes, command, TMaybe<NYT::TNode>(filePaths)); + + auto startOperationOptions = NYT::NApi::TStartOperationOptions(); + startOperationOptions.MutationId = mutationId; + startOperationOptions.Retry = true; + + auto& state = RunningOperations[ToString(mutationId)]; + state.MutationId = ToString(mutationId); + state.Nodes = nodes; + + RM_LOG(DEBUG) << "Attaching to operation with mutationId " << ToString(mutationId); + + CreateCoreTable(*nodes.begin()); + + Send(YtWrapper, MakeHolder<TEvStartOperation>( + YtRequestId, + NYT::NScheduler::EOperationType::Vanilla, + operationSpec, + startOperationOptions).Release()); + + PendingStartOperationRequests[YtRequestId++] = {nodes, + ToString(mutationId), THolder<TEvStartOperation>()}; + } + + void StartOperation(int jobs, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + + RM_LOG(INFO) << "Creating " << jobs << " workers "; + + TString executableName = (Options.YtBackend.GetClusterName().find("localhost") == 0) + ? Options.Files[0].LocalFileName + : TString("./") + Options.Files[0].GetRemoteFileName(); + + RM_LOG(INFO) << "Executable " << executableName; + + TString command = Options.YtBackend.GetVanillaJobCommand(); + + RM_LOG(INFO) << "Executable " << command; + + TVector<ui32> nodes; + + auto startOperationOptions = NYT::NApi::TStartOperationOptions(); + + if (MutationsCache.empty()) { + startOperationOptions.MutationId = startOperationOptions.GetOrGenerateMutationId(); + NodeIdAllocator.Allocate(&nodes, jobs); + } else { + startOperationOptions.MutationId = NYT::TGuid::FromString(MutationsCache.begin()->first); + nodes = MutationsCache.begin()->second; + NodeIdAllocator.Allocate(nodes); + RM_LOG(INFO) << "Get mutation from cache " << ToString(startOperationOptions.MutationId) << "," << nodes.size(); + MutationsCache.erase(MutationsCache.begin()); + } + + if (nodes.empty()) { + RM_LOG(WARN) << "Cannot allocate node ids for " << jobs << " jobs"; + return; + } + + startOperationOptions.Retry = true; + + auto operationSpec = GetOperationSpec(nodes, command, TMaybe<NYT::TNode>()); + + auto mutationId = startOperationOptions.MutationId; + + RM_LOG(DEBUG) << "Start operation with mutationId " << ToString(mutationId) ; + + auto& state = RunningOperations[ToString(mutationId)]; + state.MutationId = ToString(mutationId); + state.Nodes = nodes; + + NYT::NApi::TCreateNodeOptions createOptions; + createOptions.IgnoreExisting = true; + createOptions.Recursive = true; + + RM_LOG(DEBUG) << "Creating operation with mutationId " << ToString(mutationId); + + auto filesAttribute = Options.Files; + if (Options.YtBackend.GetClusterName().find("localhost") == 0) { + filesAttribute.clear(); + } + + auto attributes = NYT::BuildYsonNodeFluently() + .BeginMap() + .Item("yql_mutation_id").Value(ToString(mutationId)) + .Item(NCommonAttrs::OPERATIONSIZE_ATTR).Value(jobs) + .Item("yql_command").Value(command) + .Item("yql_file_paths") + .DoListFor(filesAttribute, [&] (NYT::TFluentList list, const TResourceFile& item) { + auto baseName = item.GetRemoteFileName(); + list.Item() + .BeginAttributes() + .Item("executable").Value(true) + .Item("file_name").Value(baseName) + .EndAttributes() + .Value(Options.UploadPrefix + "/" + baseName); + }) + .Item(NCommonAttrs::ROLE_ATTR).Value("worker_node") + .Item(NCommonAttrs::ACTOR_NODEID_ATTR) + .BeginList() + .DoFor(nodes, [&] (NYT::TFluentList fluent1, const auto& nodeId) { + fluent1.Item().Value(nodeId); + }) + .EndList() + .Item(NCommonAttrs::CLUSTERNAME_ATTR).Value(ClusterName) + .EndMap(); + + createOptions.Attributes = NYT::NYTree::IAttributeDictionary::FromMap( + NYT::NYTree::ConvertToNode(NYT::NYson::TYsonString(NYT::NodeToYsonString(attributes)))->AsMap() + ); + + createOptions.PrerequisiteTransactionIds.push_back(LeaderTransactionId); + + CreateCoreTable(*nodes.begin()); + + Send(YtWrapper, new TEvCreateNode( + YtRequestId, + Options.YtBackend.GetPrefix() + "/operations/" + ClusterName + "/" + ToString(mutationId), + NYT::NObjectClient::EObjectType::StringNode, + createOptions)); + + PendingStartOperationRequests[YtRequestId] = { + nodes, + ToString(mutationId), + MakeHolder<TEvStartOperation>( + YtRequestId+1, + NYT::NScheduler::EOperationType::Vanilla, + operationSpec, + startOperationOptions) + }; + YtRequestId += 2; + } + + void OnCreateNode(TEvCreateNodeResponse::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto requestId = ev->Get()->RequestId; + auto result = std::get<0>(*ev->Get()); + if (requestId == static_cast<ui64>(-1)) { + // CoreTable + if (!result.IsOK()) { + YQL_CLOG(DEBUG, ProviderDq) << "Error on creating core table " << ToString(result); + } + return; + } + if (!PendingStartOperationRequests.contains(requestId)) { + return; + } + auto& op = PendingStartOperationRequests[requestId]; + if (result.IsOK()) { + Y_ABORT_UNLESS(!PendingStartOperationRequests.contains(requestId+1)); + PendingStartOperationRequests[requestId+1] = { + op.Nodes, + op.MutationId, + THolder<TEvStartOperation>() + }; + Send(YtWrapper, op.Ev.Release()); + } else if (RunningOperations.contains(op.MutationId)) { + YQL_CLOG(DEBUG, ProviderDq) << "Error on create node " << ToString(result); + DropRunningOperation(op.MutationId); + } + PendingStartOperationRequests.erase(requestId); + // retry in ListOperations + } + + const TResourceManagerOptions Options; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + const TString ClusterName; + + const ICoordinationHelper::TPtr Coordinator; + + const NProto::TDqConfig::TYtCoordinator CoordinatorConfig; + + TActorId YtWrapper; + const TActorId CoordinatorWrapper; + + NYT::NObjectClient::TTransactionId LeaderTransactionId; + + TNodeIdAllocator NodeIdAllocator; + + struct TOperationStatus { + TString OperationId; + TString MutationId; + TActorId ActorId; + TVector<ui32> Nodes; + }; + + // mutationId -> operation + THashMap<TString, TOperationStatus> RunningOperations; + + THashMap<TString, TVector<ui32>> MutationsCache; + + // RequestId -> Jobs + struct TPendingStartOperation { + TVector<ui32> Nodes; + TString MutationId; + THolder<TEvStartOperation> Ev; + }; + THashMap<ui64, TPendingStartOperation> PendingStartOperationRequests; + ui64 YtRequestId = 1; + NActors::TSchedulerCookieHolder TimerCookieHolder; + }; + + IActor* CreateYtResourceManager( + const TResourceManagerOptions& options, + const ICoordinationHelper::TPtr& coordinator) + { + Y_ABORT_UNLESS(!options.YtBackend.GetClusterName().empty()); + Y_ABORT_UNLESS(!options.YtBackend.GetUser().empty()); + Y_ABORT_UNLESS(options.YtBackend.HasMinNodeId()); + Y_ABORT_UNLESS(options.YtBackend.HasMaxNodeId()); + Y_ABORT_UNLESS(options.YtBackend.HasPrefix()); + Y_ABORT_UNLESS(!options.Files.empty()); + Y_ABORT_UNLESS(!options.UploadPrefix.empty()); + + return new TYtResourceManager(options, coordinator); + } +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp b/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp new file mode 100644 index 0000000000..2320f4a837 --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp @@ -0,0 +1,635 @@ +#include "yt_wrapper.h" + +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/events/events.h> + +#include <ydb/library/yql/utils/log/log.h> + +#include <library/cpp/digest/md5/md5.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/yson/node/node_io.h> + +#include <util/system/file.h> +#include <util/system/fs.h> +#include <util/stream/fwd.h> +#include <util/stream/buffered.h> +#include <util/system/mutex.h> + +#include <yt/yt/client/api/rpc_proxy/config.h> +#include <yt/yt/client/api/rpc_proxy/connection.h> +#include <yt/yt/client/api/client.h> +#include <yt/yt/client/api/file_writer.h> +#include <yt/yt/client/api/file_reader.h> + +#include <yt/yt/core/ytree/convert.h> + +using namespace NYql; +using namespace NYT; +using namespace NYT::NApi; +using namespace NActors; + +namespace NYql { + struct TRequest: public NYT::TRefCounted { + const TActorId SelfId; + const TActorId Sender; + TActorSystem* Ctx; + const ui64 RequestId; + + TRequest(const TActorId& selfId, const TActorId& sender, TActorSystem* ctx, const ui64 requestId) + : SelfId(selfId) + , Sender(sender) + , Ctx(ctx) + , RequestId(requestId) + { } + + void Complete(IEventBase* ev); + }; + + struct TWriteFileRequest: public TRequest { + IClientPtr Client; + IFileWriterPtr FileWriter; + TFile File; + i64 FileSize = -1; + TVector<char> Buffer; + TString NodePathTmp; + TString NodePath; + TFileWriterOptions WriterOptions; + TString Digest; + i64 Offset = 0; + + TWriteFileRequest(const TActorId& selfId, const TActorId& sender, TActorSystem* ctx, const ui64 requestId) + : TRequest(selfId, sender, ctx, requestId) + { } + + + TFuture<void> WriteNext() { + const ui64 chunkSize = 64 * 1024 * 1024; + Buffer.resize(chunkSize); + + i64 dataSize; + try { + if (FileSize < 0) { + FileSize = File.GetLength(); + } + dataSize = File.Pread(&Buffer[0], Buffer.size(), Offset); + Offset += dataSize; + } catch (const std::exception& ex) { + return MakeFuture(TErrorOr<void>(ex)); + } + + if (dataSize == 0) { + YQL_CLOG(DEBUG, ProviderDq) << "Closing writer"; + return FileWriter->Close() + .Apply(BIND([self = MakeWeak(this)]() { + YQL_CLOG(DEBUG, ProviderDq) << "Write finished"; + auto this_ = self.Lock(); + if (!this_) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + + return this_->Client->GetNode(this_->NodePathTmp + "/@md5") + .Apply(BIND([self](const TErrorOr<NYT::NYson::TYsonString>& err) { + auto req = self.Lock(); + if (!req) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + if (err.IsOK() && req->Digest == NYTree::ConvertTo<TString>(err.Value())) { + return VoidFuture; + } + + return MakeFuture(TErrorOr<void>(yexception() << "wrong checksum")); + })); + })); + } else { + YQL_CLOG(DEBUG, ProviderDq) << "Writing chunk " << Offset << "/" << FileSize; + return FileWriter->Write(TSharedRef(&Buffer[0], dataSize, nullptr)) + .Apply(BIND([self = MakeWeak(this)]() mutable { + auto this_ = self.Lock(); + if (!this_) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + return this_->WriteNext(); + }).AsyncVia(Client->GetConnection()->GetInvoker())); + } + } + + TFuture<void> WriteFile() + { + auto& remotePath = NodePathTmp; + YQL_CLOG(INFO, ProviderDq) << "Start writing file to " << remotePath; + FileWriter = Client->CreateFileWriter(remotePath, WriterOptions); + + return FileWriter->Open().Apply(BIND([self = MakeWeak(this)]() mutable { + auto this_ = self.Lock(); + if (!this_) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + return this_->WriteNext(); + })); + } + }; + + struct TReadFileRequest: public TRequest { + IClientPtr Client; + TString LocalPath; + IFileReaderPtr Reader; + std::shared_ptr<TFileOutput> Output; + TString Digest; + MD5 Md5; + + TReadFileRequest(const TActorId& selfId, const TActorId& sender, TActorSystem* ctx, const ui64 requestId) + : TRequest(selfId, sender, ctx, requestId) + { } + + TFuture<void> ReadNext() + { + return Reader->Read() + .Apply(BIND([self = MakeWeak(this)](const TSharedRef& blob) { + auto this_ = self.Lock(); + if (!this_) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + try { + YQL_CLOG(DEBUG, ProviderDq) << "Store " << blob.Size() << " bytes "; + if (blob.Size() > 0) { + this_->Md5.Update(blob.Begin(), blob.Size()); + this_->Output->Write(blob.Begin(), blob.Size()); + return this_->ReadNext(); + } else { + TString buf; + buf.ReserveAndResize(32); + this_->Md5.End(buf.begin()); + + if (buf == this_->Digest) { + this_->Output.reset(); + return VoidFuture; + } else { + return MakeFuture(TErrorOr<void>(yexception() << "md5 mismatch")); + } + } + } catch (...) { + return MakeFuture(TErrorOr<void>(yexception() << CurrentExceptionMessage())); + } + }).AsyncVia(Client->GetConnection()->GetInvoker())); + } + + TFuture<void> ReadFile() + { + auto pos = LocalPath.rfind('/'); + if (pos != TString::npos) { + auto dirName = LocalPath.substr(0, pos); + if (!dirName.empty()) { + NFs::MakeDirectoryRecursive(dirName, NFs::FP_NONSECRET_FILE, false); + } + } + + Output = std::make_shared<TFileOutput>(LocalPath); + return ReadNext(); + } + }; + + using TRequestPtr = NYT::TIntrusivePtr<TRequest>; + + struct TEvComplete + : NActors::TEventLocal<TEvComplete, TDqEvents::ES_OTHER1> { + TEvComplete() = default; + TEvComplete(const TRequestPtr& req) + : Request(req) + { } + + const TRequestPtr Request; + }; + + void TRequest::Complete(IEventBase* ev) { + Ctx->Send(Sender, ev); + Ctx->Send(SelfId, new TEvComplete(NYT::MakeStrong(this))); + } + + class TYtWrapper: public TActor<TYtWrapper> { + public: + static constexpr char ActorName[] = "YT_WRAPPER"; + + TYtWrapper(const IClientPtr& client) + : TActor(&TYtWrapper::Handler) + , Client(client) + { } + + private: + STRICT_STFUNC(Handler, { + HFunc(TEvStartOperation, OnStartOperation) + HFunc(TEvGetOperation, OnGetOperation) + HFunc(TEvListOperations, OnListOperations) + HFunc(TEvGetJob, OnGetJob) + HFunc(TEvWriteFile, OnFileWrite) + HFunc(TEvReadFile, OnReadFile) + HFunc(TEvListNode, OnListNode) + HFunc(TEvSetNode, OnSetNode) + HFunc(TEvGetNode, OnGetNode) + HFunc(TEvRemoveNode, OnRemoveNode) + HFunc(TEvCreateNode, OnCreateNode) + HFunc(TEvStartTransaction, OnStartTransaction) + HFunc(TEvPrintJobStderr, OnPrintJobStderr) + HFunc(TEvComplete, OnComplete) + cFunc(TEvents::TEvPoison::EventType, PassAway) + }); + + THashSet<TRequestPtr> Requests; + + void PassAway() override { + Requests.clear(); + IActor::PassAway(); + } + + template<typename T> + TWeakPtr<T> NewRequest(ui64 id, TActorId sender, const TActorContext& ctx) { + auto req = New<T>(SelfId(), sender, ctx.ExecutorThread.ActorSystem, id); + Requests.emplace(req); + return NYT::MakeWeak(req); + } + + void OnComplete(TEvComplete::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto req = ev->Get()->Request; + Requests.erase(req); + } + + void OnFileWrite(TEvWriteFile::TPtr& ev, const TActorContext& ctx) { + TFile file = std::move(std::get<0>(*ev->Get())); + NYPath::TRichYPath remotePath = std::get<1>(*ev->Get()); + THashMap<TString, NYT::TNode> attributes = std::get<2>(*ev->Get()); + TFileWriterOptions writerOptions = std::get<3>(*ev->Get()); + auto requestId = ev->Get()->RequestId; + + auto nodePathTmp = remotePath.GetPath() + ".tmp"; + auto nodePath = remotePath.GetPath(); + auto request = NewRequest<TWriteFileRequest>(requestId, ev->Sender, ctx); + writerOptions.ComputeMD5 = true; + + try { + YQL_CLOG(INFO, ProviderDq) << "Creating node " << remotePath.GetPath(); + + Y_ENSURE(file.IsOpen()); + + TString digest; + + if (writerOptions.ComputeMD5) { + char buf[32768]; + MD5 md5; + i64 size, offset = 0; + while ((size = file.Pread(buf, sizeof(buf), offset)) > 0) { + md5.Update(buf, size); + offset += size; + } + digest.ReserveAndResize(32); + md5.End(digest.begin()); + } + + if (auto req = request.Lock()) { + req->Client = Client; + req->File = std::move(file); + req->NodePathTmp = nodePathTmp; + req->NodePath = nodePath; + req->WriterOptions = writerOptions; + req->Digest = digest; + } + + Client->GetNode(nodePath + "/@md5") + .Apply(BIND([request, attributes, digest](const TErrorOr<NYT::NYson::TYsonString>& err) mutable { + auto req = request.Lock(); + if (!req) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + if (err.IsOK() && digest == NYTree::ConvertTo<TString>(err.Value())) { + YQL_CLOG(INFO, ProviderDq) << "File already uploaded"; + try { + req->Client->SetNode(req->NodePath + "/@yql_last_update", + NYT::NYson::TYsonString( + NYT::NodeToYsonString(NYT::TNode(ToString(TInstant::Now())) + ))); + } catch (...) { } + return VoidFuture; + } else if (err.IsOK() || err.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) { + TCreateNodeOptions options; + options.Recursive = true; + options.IgnoreExisting = true; + + if (err.IsOK()) { + YQL_CLOG(INFO, ProviderDq) << digest << "!=" << NYTree::ConvertTo<TString>(err.Value()); + } else { + YQL_CLOG(ERROR, ProviderDq) << ToString(err); + } + + return req->Client->CreateNode(req->NodePathTmp, NObjectClient::EObjectType::File, options).As<void>() + .Apply(BIND([request, attributes] () { + auto req = request.Lock(); + if (!req) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + TVector<NYT::TFuture<void>> futures; + futures.reserve(attributes.size()); + for (const auto& [k, v]: attributes) { + futures.push_back( + req->Client->SetNode( + req->NodePathTmp + "/@" + k, + NYT::NYson::TYsonString(NYT::NodeToYsonString(v)), + NYT::NApi::TSetNodeOptions())); + } + return NYT::AllSucceeded(futures).As<void>(); + })) + .Apply(BIND([request]() mutable { + auto req = request.Lock(); + if (!req) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + return req->WriteFile(); + })) + .Apply(BIND([request] () { + auto req = request.Lock(); + if (!req) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + auto moveOptions = NYT::NApi::TMoveNodeOptions(); + moveOptions.Force = true; + return req->Client->MoveNode(req->NodePathTmp, req->NodePath, moveOptions).As<void>(); + })); + } + + err.ThrowOnError(); + + return VoidFuture; + })) + .Apply(BIND([request, requestId](const TErrorOr<void>& err) + { + if (auto req = request.Lock()) { + req->Complete(new TEvWriteFileResponse(requestId, err)); + } + })); + } catch (const std::exception& ex) { + if (auto req = request.Lock()) { + req->Complete(new TEvWriteFileResponse(requestId, ex)); + } + } + } + + void OnReadFile(TEvReadFile::TPtr& ev, const TActorContext& ctx) { + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TReadFileRequest>(requestId, ev->Sender, ctx); + + try { + NYPath::TRichYPath remotePath = std::get<0>(*ev->Get()); + TFileReaderOptions readerOptions = std::get<2>(*ev->Get()); + if (auto req = request.Lock()) { + req->LocalPath = std::get<1>(*ev->Get()); + req->Client = Client; + } + + auto nodePath = remotePath.GetPath(); + + Client->GetNode(nodePath + "/@md5") + .Apply(BIND([request, nodePath, readerOptions](const TErrorOr<NYT::NYson::TYsonString>& err) mutable { + auto req = request.Lock(); + if (!req) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + if (!err.IsOK()) { + return MakeFuture(TErrorOr<void>(yexception() << "failed to get md5")); + } + auto digest = NYTree::ConvertTo<TString>(err.Value()); + req->Digest = digest; + + return req->Client->CreateFileReader(nodePath, readerOptions) + .Apply(BIND([request](const IFileReaderPtr& reader) { + auto req = request.Lock(); + if (!req) { + return MakeFuture(TErrorOr<void>(yexception() << "request complete")); + } + req->Reader = reader; + return req->ReadFile(); + })); + })) + .Apply(BIND([request, requestId](const TErrorOr<void>& err) { + if (auto req = request.Lock()) { + req->Complete(new TEvReadFileResponse(requestId, err)); + } + })); + } catch (const std::exception& ex) { + if (auto req = request.Lock()) { + req->Complete(new TEvReadFileResponse(requestId, ex)); + } + } + } + + void OnStartOperation(TEvStartOperation::TPtr& ev, const TActorContext& ctx) { + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + try { + NScheduler::EOperationType type = std::get<0>(*ev->Get()); + auto spec = NYT::NYson::TYsonString(std::get<1>(*ev->Get())); + TStartOperationOptions options = std::get<2>(*ev->Get()); + + Client->StartOperation(type, spec, options).Subscribe(BIND([=](const TErrorOr<NScheduler::TOperationId>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvStartOperationResponse(requestId, result)); + } + })); + } catch (const std::exception& ex) { + if (auto req = request.Lock()) { + req->Complete(new TEvStartOperationResponse(requestId, ex)); + } + } + } + + void OnGetOperation(TEvGetOperation::TPtr& ev, const TActorContext& ctx) { + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + try { + auto operationId = std::get<0>(*ev->Get()); + auto options = std::get<1>(*ev->Get()); + + Client->GetOperation(operationId, options).Apply(BIND([=](const TErrorOr<TOperation>& result) { + return NYT::NYson::ConvertToYsonString(result.ValueOrThrow()).ToString(); + })) + .Apply(BIND([=](const TErrorOr<TString>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvGetOperationResponse(requestId, result)); + } + })); + } catch (const std::exception& ex) { + if (auto req = request.Lock()) { + req->Complete(new TEvGetOperationResponse(requestId, ex)); + } + } + } + + void OnListOperations(TEvListOperations::TPtr& ev, const TActorContext& ctx) { + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + try { + auto options = std::get<0>(*ev->Get()); + + Client->ListOperations(options).Apply(BIND([=](const TErrorOr<NYT::NApi::TListOperationsResult>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvListOperationsResponse(requestId, result)); + } + })); + } catch (const std::exception& ex) { + if (auto req = request.Lock()) { + req->Complete(new TEvListOperationsResponse(requestId, ex)); + } + } + } + + void OnGetJob(TEvGetJob::TPtr& ev, const TActorContext& ctx) { + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + try { + auto operationId = std::get<0>(*ev->Get()); + auto jobId = std::get<1>(*ev->Get()); + auto options = std::get<2>(*ev->Get()); + + Client->GetJob(operationId, jobId, options).Apply(BIND([=](const TErrorOr<NYT::NYson::TYsonString>& result) { + return result.ValueOrThrow().ToString(); + })) + .Apply(BIND([=](const TErrorOr<TString>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvGetJobResponse(requestId, result)); + } + })); + } catch (const std::exception& ex) { + if (auto req = request.Lock()) { + req->Complete(new TEvGetJobResponse(requestId, ex)); + } + } + } + + void OnListNode(TEvListNode::TPtr& ev, const TActorContext& ctx) { + auto path = std::get<0>(*ev->Get()); + auto options = std::get<1>(*ev->Get()); + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + try { + Client->ListNode(path, options) + .Apply(BIND([=](const TErrorOr<NYT::NYson::TYsonString>& result) { + return result.ValueOrThrow().ToString(); + })) + .Apply(BIND([=](const TErrorOr<TString>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvListNodeResponse(requestId, result)); + } + })); + } catch (const std::exception& ex) { + if (auto req = request.Lock()) { + req->Complete(new TEvListNodeResponse(requestId, ex)); + } + } + } + + void OnSetNode(TEvSetNode::TPtr& ev, const TActorContext& ctx) { + auto path = std::get<0>(*ev->Get()); + auto value = std::get<1>(*ev->Get()); + auto options = std::get<2>(*ev->Get()); + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + Client->SetNode(path, value, options) + .Apply(BIND([=](const TErrorOr<void>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvSetNodeResponse(requestId, result)); + } + })); + } + + void OnGetNode(TEvGetNode::TPtr& ev, const TActorContext& ctx) { + auto path = std::get<0>(*ev->Get()); + auto options = std::get<1>(*ev->Get()); + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + Client->GetNode(path, options) + .Apply(BIND([=](const TErrorOr<NYT::NYson::TYsonString>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvGetNodeResponse(requestId, result)); + } + })); + } + + void OnRemoveNode(TEvRemoveNode::TPtr& ev, const TActorContext& ctx) { + auto path = std::get<0>(*ev->Get()); + auto options = std::get<1>(*ev->Get()); + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + Client->RemoveNode(path, options) + .Apply(BIND([=](const TErrorOr<void>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvRemoveNodeResponse(requestId, result)); + } + })); + } + + void OnCreateNode(TEvCreateNode::TPtr& ev, const TActorContext& ctx) { + auto path = std::get<0>(*ev->Get()); + auto type = std::get<1>(*ev->Get()); + auto options = std::get<2>(*ev->Get()); + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + Client->CreateNode(path, type, options) + .Apply(BIND([=](const TErrorOr<NYT::NCypressClient::TNodeId>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvCreateNodeResponse(requestId, result)); + } + })); + } + + void OnStartTransaction(TEvStartTransaction::TPtr& ev, const TActorContext& ctx) { + auto type = std::get<0>(*ev->Get()); + auto options = std::get<1>(*ev->Get()); + auto requestId = ev->Get()->RequestId; + auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx); + + Client->StartTransaction(type, options) + .Apply(BIND([=](const TErrorOr<ITransactionPtr>& result) { + if (auto req = request.Lock()) { + req->Complete(new TEvStartTransactionResponse(requestId, result)); + } + })); + } + + void OnPrintJobStderr(TEvPrintJobStderr::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto operationId = std::get<0>(*ev->Get()); + + YQL_CLOG(DEBUG, ProviderDq) << "Printing stderr of operation " << ToString(operationId); + + Client->ListJobs(operationId) + .Apply(BIND([operationId, client = MakeWeak(Client)](const TListJobsResult& result) { + if (auto cli = client.Lock()) { + for (const auto& job : result.Jobs) { + YQL_CLOG(DEBUG, ProviderDq) << "Printing stderr (" << ToString(operationId) << "," << ToString(job.Id) << ")"; + + cli->GetJobStderr(operationId, job.Id) + .Apply(BIND([jobId = job.Id, operationId](const TSharedRef& data) { + YQL_CLOG(DEBUG, ProviderDq) + << "Stderr (" + << ToString(operationId) << "," + << ToString(jobId) << ")" + << TString(data.Begin(), data.Size()); + })); + } + } + })); + } + + IClientPtr Client; + }; + + IActor* CreateYtWrapper(const IClientPtr& client) { + return new TYtWrapper(client); + } +} diff --git a/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h b/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h new file mode 100644 index 0000000000..3a38c683fc --- /dev/null +++ b/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h @@ -0,0 +1,88 @@ +#pragma once + +#include <util/random/random.h> +#include <util/memory/blob.h> + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/yson/node/node.h> + +#include <yt/yt/client/api/public.h> +#include <yt/yt/client/api/client.h> +#include <yt/yt/client/ypath/rich.h> + +#include <ydb/library/yql/dq/common/dq_common.h> + +#include "yt_events.h" + +namespace NYql { + + template <ui32 EventType, typename... Args> + struct TGenericYtCommand + : public std::tuple<Args...>, + NActors::TEventLocal<TGenericYtCommand<EventType, Args...>, EventType> + { + TGenericYtCommand(ui64 requestId, Args... args) + : std::tuple<Args...>(args...) + , RequestId(requestId) + { } + + TGenericYtCommand(Args... args) + : std::tuple<Args...>(args...) + , RequestId(RandomNumber<ui64>()) + { } + + TGenericYtCommand() = default; + + const ui64 RequestId; + }; + + template <ui32 EventType, typename... Args> + struct TGenericYtResponse + : public TGenericYtCommand<EventType, Args ...> + { + TGenericYtResponse(ui64 requestId, Args... args) + : TGenericYtCommand<EventType, Args ...>(requestId, args...) + { } + }; + + using TEvStartOperation = TGenericYtCommand<TYtEvents::ES_START_OPERATION, NYT::NScheduler::EOperationType, TString, NYT::NApi::TStartOperationOptions>; + using TEvStartOperationResponse = TGenericYtResponse<TYtEvents::ES_START_OPERATION_RESPONSE, NYT::TErrorOr<NYT::NScheduler::TOperationId>>; + + using TEvGetOperation = TGenericYtCommand<TYtEvents::ES_GET_OPERATION, NYT::NScheduler::TOperationIdOrAlias, NYT::NApi::TGetOperationOptions>; + using TEvGetOperationResponse = TGenericYtResponse<TYtEvents::ES_GET_OPERATION_RESPONSE, NYT::TErrorOr<TString>>; + + using TEvListOperations = TGenericYtCommand<TYtEvents::ES_LIST_OPERATIONS, NYT::NApi::TListOperationsOptions>; + using TEvListOperationsResponse = TGenericYtResponse<TYtEvents::ES_LIST_OPERATIONS_RESPONSE, NYT::TErrorOr<NYT::NApi::TListOperationsResult>>; + + using TEvGetJob = TGenericYtCommand<TYtEvents::ES_GET_JOB, NYT::NScheduler::TOperationId, NYT::NJobTrackerClient::TJobId, NYT::NApi::TGetJobOptions>; + using TEvGetJobResponse = TGenericYtResponse<TYtEvents::ES_GET_JOB_RESPONSE, NYT::TErrorOr<TString>>; + + using TEvWriteFile = TGenericYtCommand<TYtEvents::ES_WRITE_FILE, TFile, NYT::NYPath::TRichYPath, THashMap<TString, NYT::TNode>, NYT::NApi::TFileWriterOptions>; + using TEvWriteFileResponse = TGenericYtResponse<TYtEvents::ES_WRITE_FILE_RESPONSE, NYT::TErrorOr<void>>; + + using TEvReadFile = TGenericYtCommand<TYtEvents::ES_READ_FILE, NYT::NYPath::TRichYPath, TString, NYT::NApi::TFileReaderOptions>; + using TEvReadFileResponse = TGenericYtResponse<TYtEvents::ES_READ_FILE_RESPONSE, NYT::TErrorOr<void>>; + + using TEvListNode = TGenericYtCommand<TYtEvents::ES_LIST_NODE, TString, NYT::NApi::TListNodeOptions>; + using TEvListNodeResponse = TGenericYtResponse<TYtEvents::ES_LIST_NODE_RESPONSE, NYT::TErrorOr<TString>>; + + using TEvSetNode = TGenericYtCommand<TYtEvents::ES_SET_NODE, TString, NYT::NYson::TYsonString, NYT::NApi::TSetNodeOptions>; + using TEvSetNodeResponse = TGenericYtResponse<TYtEvents::ES_SET_NODE_RESPONSE, NYT::TErrorOr<void>>; + + using TEvGetNode = TGenericYtCommand<TYtEvents::ES_GET_NODE, TString, NYT::NApi::TGetNodeOptions>; + using TEvGetNodeResponse = TGenericYtResponse<TYtEvents::ES_GET_NODE_RESPONSE, NYT::TErrorOr<NYT::NYson::TYsonString>>; + + using TEvRemoveNode = TGenericYtCommand<TYtEvents::ES_REMOVE_NODE, TString, NYT::NApi::TRemoveNodeOptions>; + using TEvRemoveNodeResponse = TGenericYtResponse<TYtEvents::ES_REMOVE_NODE_RESPONSE, NYT::TErrorOr<void>>; + + using TEvCreateNode = TGenericYtCommand<TYtEvents::ES_CREATE_NODE, TString, NYT::NObjectClient::EObjectType, NYT::NApi::TCreateNodeOptions>; + using TEvCreateNodeResponse = TGenericYtResponse<TYtEvents::ES_CREATE_NODE_RESPONSE, NYT::TErrorOr<NYT::NCypressClient::TNodeId>>; + + using TEvStartTransaction = TGenericYtCommand<TYtEvents::ES_START_TRANSACTION, NYT::NTransactionClient::ETransactionType, NYT::NApi::TTransactionStartOptions>; + using TEvStartTransactionResponse = TGenericYtResponse<TYtEvents::ES_START_TRANSACTION_RESPONSE, NYT::TErrorOr<NYT::NApi::ITransactionPtr>>; + + using TEvPrintJobStderr = TGenericYtCommand<TYtEvents::ES_PRINT_JOB_STDERR, NYT::NScheduler::TOperationId>; + + NActors::IActor* CreateYtWrapper(const NYT::NApi::IClientPtr& client); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..3e5f36a8e3 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,40 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(providers-dq-global_worker_manager) +target_compile_options(providers-dq-global_worker_manager PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-dq-global_worker_manager PUBLIC + contrib-libs-cxxsupp + yutil + yql-utils-failure_injector + providers-common-config + providers-common-gateway + providers-common-metrics + providers-dq-actors + dq-api-grpc + dq-api-protos + providers-dq-config + providers-dq-counters + providers-dq-runtime + providers-dq-task_runner + dq-actors-yt + providers-dq-scheduler +) +target_sources(providers-dq-global_worker_manager PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp +) diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..f556019e59 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-aarch64.txt @@ -0,0 +1,41 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(providers-dq-global_worker_manager) +target_compile_options(providers-dq-global_worker_manager PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-dq-global_worker_manager PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yql-utils-failure_injector + providers-common-config + providers-common-gateway + providers-common-metrics + providers-dq-actors + dq-api-grpc + dq-api-protos + providers-dq-config + providers-dq-counters + providers-dq-runtime + providers-dq-task_runner + dq-actors-yt + providers-dq-scheduler +) +target_sources(providers-dq-global_worker_manager PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp +) diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..f556019e59 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-x86_64.txt @@ -0,0 +1,41 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(providers-dq-global_worker_manager) +target_compile_options(providers-dq-global_worker_manager PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-dq-global_worker_manager PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yql-utils-failure_injector + providers-common-config + providers-common-gateway + providers-common-metrics + providers-dq-actors + dq-api-grpc + dq-api-protos + providers-dq-config + providers-dq-counters + providers-dq-runtime + providers-dq-task_runner + dq-actors-yt + providers-dq-scheduler +) +target_sources(providers-dq-global_worker_manager PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp +) diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..f65dc67f83 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.windows-x86_64.txt @@ -0,0 +1,39 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(providers-dq-global_worker_manager) +target_compile_options(providers-dq-global_worker_manager PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-dq-global_worker_manager PUBLIC + contrib-libs-cxxsupp + yutil + yql-utils-failure_injector + providers-common-config + providers-common-gateway + providers-common-metrics + providers-dq-actors + dq-api-grpc + dq-api-protos + providers-dq-config + providers-dq-counters + providers-dq-runtime + providers-dq-task_runner + dq-actors-yt + providers-dq-scheduler +) +target_sources(providers-dq-global_worker_manager PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper_win.cpp +) diff --git a/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp b/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp new file mode 100644 index 0000000000..12a1ac6f15 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp @@ -0,0 +1,95 @@ +#include "benchmark.h" + +#include <ydb/library/yql/providers/dq/common/yql_dq_settings.h> +#include <ydb/library/yql/providers/dq/actors/resource_allocator.h> + +namespace NYql::NDqs { + +using namespace NActors; + +class TWorkerManagerBenchmark: public TRichActor<TWorkerManagerBenchmark> { +public: + TWorkerManagerBenchmark(NActors::TActorId workerManagerId, const TWorkerManagerBenchmarkOptions& options) + : TRichActor(&TWorkerManagerBenchmark::Handler) + , WorkerManagerId(workerManagerId) + , Options(options) + , Settings(new TDqConfiguration) + , Counters(new NMonitoring::TDynamicCounters) + { + Y_UNUSED(WorkerManagerId); + Y_UNUSED(Options); + Y_UNUSED(RunningRequests); + } + +private: + STRICT_STFUNC(Handler, { + cFunc(TEvents::TEvWakeup::EventType, Wakeup) + cFunc(TEvents::TEvPoison::EventType, PassAway) + cFunc(TEvents::TEvBootstrap::EventType, Bootstrap) + HFunc(TEvAllocateWorkersResponse, OnAllocateWorkersResponse) + }); + + void Bootstrap() { + Schedule(Options.MaxRunTimeMs, new TEvents::TEvPoison); + Wakeup(); + } + + void Wakeup() { + while ((int)RunningRequests.size() < Options.Inflight && Requests < Options.TotalRequests) { + StartRequest(); + } + + if (RunningRequests.empty()) { + PassAway(); + } + } + + void StartRequest() { + TString operationId = TStringBuilder() << "Benchmark-" << Requests; + auto resourceAllocator = RegisterChild( + CreateResourceAllocator( + WorkerManagerId, SelfId(), SelfId(), + Options.WorkerCount, + operationId, Settings, + Counters, + {})); + + auto allocateRequest = MakeHolder<TEvAllocateWorkersRequest>(Options.WorkerCount, "TestUser"); + allocateRequest->Record.SetTraceId(operationId); + + TActivationContext::Send( + new IEventHandle( + WorkerManagerId, resourceAllocator, allocateRequest.Release())); + + RunningRequests.insert(std::make_pair(resourceAllocator, TRequestInfo{TInstant::Now()})); + Requests ++; + } + + void OnAllocateWorkersResponse(TEvAllocateWorkersResponse::TPtr& ev, const NActors::TActorContext&) { + UnregisterChild(ev->Sender); + RunningRequests.erase(ev->Sender); + + Wakeup(); + } + + + const NActors::TActorId WorkerManagerId; + const TWorkerManagerBenchmarkOptions Options; + TDqConfiguration::TPtr Settings; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + + struct TRequestInfo { + TInstant StartTime; + }; + + THashMap<TActorId, TRequestInfo> RunningRequests; + int Requests = 0; +}; + +NActors::IActor* CreateWorkerManagerBenchmark(NActors::TActorId workerManagerId, const TWorkerManagerBenchmarkOptions& options) +{ + return new TWorkerManagerBenchmark(workerManagerId, options); +} + + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/global_worker_manager/benchmark.h b/ydb/library/yql/providers/dq/global_worker_manager/benchmark.h new file mode 100644 index 0000000000..04b33ede15 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/benchmark.h @@ -0,0 +1,16 @@ +#pragma once +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> + +namespace NYql::NDqs { + +struct TWorkerManagerBenchmarkOptions { + int WorkerCount = 10; + int Inflight = 10; + int TotalRequests = 100000; + TDuration MaxRunTimeMs = TDuration::Minutes(30); +}; + +NActors::IActor* CreateWorkerManagerBenchmark(NActors::TActorId workerManagerId, const TWorkerManagerBenchmarkOptions& options); + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp new file mode 100644 index 0000000000..5e15d0d470 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp @@ -0,0 +1,354 @@ +#include "coordination_helper.h" +#include "global_worker_manager.h" +#include "service_node_pinger.h" + +#include <ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h> +#include <ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.h> +#include <ydb/library/yql/providers/dq/actors/yt/worker_registrator.h> +#include <ydb/library/yql/providers/dq/actors/yt/lock.h> +#include <ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h> +#include <ydb/library/yql/providers/dq/actors/dummy_lock.h> +#include <ydb/library/yql/providers/dq/service/interconnect_helpers.h> +#include <ydb/library/yql/providers/dq/task_runner/file_cache.h> +#include <ydb/library/yql/providers/dq/runtime/runtime_data.h> + +#include <yt/yt/client/api/rpc_proxy/config.h> +#include <yt/yt/client/api/rpc_proxy/connection.h> + +#include <library/cpp/svnversion/svnversion.h> +#include <library/cpp/yson/node/node_io.h> + +#include <yt/cpp/mapreduce/interface/fluent.h> + +#include <util/system/getpid.h> +#include <util/system/env.h> + +namespace NYql { + +using namespace NActors; + +class TCoordinationHelper: public ICoordinationHelper +{ +public: + TCoordinationHelper( + const NProto::TDqConfig::TYtCoordinator& config, + const NProto::TDqConfig::TScheduler& schedulerConfig, + const TString& role, + ui16 interconnectPort) + : Config(config), SchedulerConfig(schedulerConfig) + , Role(role) + , InterconnectPort(interconnectPort) + { + std::tie(Host, Ip) = NYql::NDqs::GetLocalAddress( + config.HasHostName() ? &config.GetHostName() : nullptr + ); + } + + ui32 GetNodeId() override + { + Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1)); + return NodeId; + } + + ui32 GetNodeId( + const TMaybe<ui32> nodeId, + const TMaybe<TString>& grpcPort, + ui32 minNodeId, + ui32 maxNodeId, + const THashMap<TString, TString>& attributes) override + { + if (grpcPort) { + GrpcPort = *grpcPort; + } + + if (nodeId) { + NodeId = *nodeId; + } + + if (NodeId != static_cast<ui32>(-1)) { + return NodeId; + } + + NYql::TAssignNodeIdOptions options; + + options.ClusterName = Config.GetClusterName(); + options.User = Config.GetUser(); + options.Token = Config.GetToken(); + options.Prefix = Config.GetPrefix(); + options.Role = Role; + options.Attributes[NCommonAttrs::ROLE_ATTR] = Role; + if (grpcPort) { + options.Attributes[NCommonAttrs::GRPCPORT_ATTR] = *grpcPort; + } + options.Attributes[NCommonAttrs::INTERCONNECTPORT_ATTR] = ToString(InterconnectPort); + options.Attributes[NCommonAttrs::HOSTNAME_ATTR] = Host; + options.Attributes[NCommonAttrs::REVISION_ATTR] = GetRevision(); + + options.NodeName = Host + ":" + ToString(GetPID()) + ":" + ToString(InterconnectPort); + options.NodeId = nodeId; + options.MinNodeId = minNodeId; + options.MaxNodeId = maxNodeId; + + NodeName = options.NodeName; + + for (const auto& [k, v]: attributes) { + options.Attributes[k] = v; + } + + NodeId = AssignNodeId(options); + return NodeId; + } + + NActors::IActor* CreateLockOnCluster(NActors::TActorId ytWrapper, const TString& prefix, const TString& lockName, bool temporary) override { + Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1)); + + auto attributes = NYT::BuildYsonNodeFluently() + .BeginMap() + .Item(NCommonAttrs::ACTOR_NODEID_ATTR).Value(NodeId) + .Item(NCommonAttrs::HOSTNAME_ATTR).Value(GetHostname()) + .Item(NCommonAttrs::GRPCPORT_ATTR).Value(GrpcPort) // optional + .Item(NCommonAttrs::UPLOAD_EXECUTABLE_ATTR).Value("true") // compat + .EndMap(); + + return CreateYtLock( + ytWrapper, + prefix + "/locks", + lockName, + NYT::NodeToYsonString(attributes), + temporary); + } + + NActors::IActor* CreateLock(const TString& lockName, bool temporary) override { + return CreateLockOnCluster(GetWrapper(), Config.GetPrefix(), lockName, temporary); + } + + void StartRegistrator(NActors::TActorSystem* actorSystem) override { + TWorkerRegistratorOptions wro; + wro.NodeName = NodeName; + wro.Prefix = Config.GetPrefix() + "/" + Role; + Register(actorSystem, CreateWorkerRegistrator(GetWrapper(actorSystem), wro)); + } + + void StartCleaner(NActors::TActorSystem* actorSystem, const TMaybe<TString>& role) override { + TNodeIdCleanerOptions co; + co.Prefix = Config.GetPrefix() + "/" + role.GetOrElse(Role); + Register(actorSystem, CreateNodeIdCleaner(GetWrapper(actorSystem), co)); + } + + TString GetHostname() override { + return Host; + } + + TString GetIp() override { + return Ip; + } + + IServiceNodeResolver::TPtr CreateServiceNodeResolver( + NActors::TActorSystem* actorSystem, + const TVector<TString>& hostPortPairs) override + { + if (!hostPortPairs.empty()) { + return CreateStaticResolver(hostPortPairs); + } else { + TDynamicResolverOptions options; + options.YtWrapper = GetWrapper(actorSystem); + options.Prefix = Config.GetPrefix() + "/service_node"; + return CreateDynamicResolver(actorSystem, options); + } + } + + void StartGlobalWorker(NActors::TActorSystem* actorSystem, const TVector<TResourceManagerOptions>& resourceUploaderOptions, IMetricsRegistryPtr metricsRegistry) override + { + if (Config.GetLockType() != "dummy") { + GetWrapper(); + } + Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1)); + auto actorId = Register(actorSystem, CreateGlobalWorkerManager(this, resourceUploaderOptions, std::move(metricsRegistry), SchedulerConfig)); + actorSystem->RegisterLocalService(NDqs::MakeWorkerManagerActorID(NodeId), actorId); + } + + const NProto::TDqConfig::TYtCoordinator& GetConfig() override { + return Config; + } + + const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem, const TString& clusterName, const TString& user, const TString& token) override { + auto key = std::make_tuple(clusterName, user, token); + auto guard = Guard(Mutex); + auto it = Yt.find(key); + if (it != Yt.end()) { + return it->second; + } else { + auto client = GetYtClient(clusterName, user, token); + auto wrapper = CreateYtWrapper(client); + auto actorId = Register(actorSystem, wrapper); + Yt.emplace(key, actorId); + return actorId; + } + } + + const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem) override { + return GetWrapper(actorSystem, Config.GetClusterName(), Config.GetUser(), Config.GetToken()); + } + + const NActors::TActorId GetWrapper() override { + auto key = std::make_tuple(Config.GetClusterName(), Config.GetUser(), Config.GetToken()); + auto guard = Guard(Mutex); + auto it = Yt.find(key); + Y_ABORT_UNLESS(it != Yt.end()); + return it->second; + } + + NActors::IActor* CreateServiceNodePinger(const IServiceNodeResolver::TPtr& ptr, const TResourceManagerOptions& rmOptions, const THashMap<TString, TString>& attributes) override { + Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1)); + return ::NYql::CreateServiceNodePinger(NodeId, Ip, InterconnectPort, Role, attributes, ptr, this, rmOptions); + } + + TWorkerRuntimeData* GetRuntimeData() override { + return &RuntimeData; + } + + void Stop(NActors::TActorSystem* actorSystem) override { + for (auto id : Children) { + actorSystem->Send(id, new TEvents::TEvPoison()); + } + Children.clear(); + } + + TString GetRevision() override { + if (Config.HasRevision()) { + return Config.GetRevision(); + } else { + return GetProgramCommitId(); + } + } + +protected: + TActorId Register(TActorSystem* actorSystem, IActor* actor) { + auto id = actorSystem->Register(actor); + Children.push_back(id); + return id; + } + + NYT::NApi::IClientPtr GetYtClient(const TString& clusterName, const TString& user, const TString& token) + { + NYT::NApi::NRpcProxy::TConnectionConfigPtr config = NYT::New<NYT::NApi::NRpcProxy::TConnectionConfig>(); + config->RequestCodec = NYT::NCompression::ECodec::Lz4; + config->ClusterUrl = clusterName; + config->ConnectionType = NYT::NApi::EConnectionType::Rpc; + + auto connection = NYT::NApi::NRpcProxy::CreateConnection(config); + + NYT::NApi::TClientOptions options; + options.User = user; + options.Token = token; + + auto client = connection->CreateClient(options); + return client; + } + + NYT::NApi::IClientPtr GetYtClient() + { + return GetYtClient(Config.GetClusterName(), Config.GetUser(), Config.GetToken()); + } + + const NProto::TDqConfig::TYtCoordinator Config; + const NProto::TDqConfig::TScheduler SchedulerConfig; + + TString Role; + + TString Host; + TString Ip; + + TString NodeName; + + TMutex Mutex; + THashMap<std::tuple<TString, TString, TString>, TActorId> Yt; + + ui32 NodeId = -1; + TString GrpcPort = ""; + ui16 InterconnectPort; + + TWorkerRuntimeData RuntimeData; + IFileCache::TPtr FileCache; + + TVector<TActorId> Children; +}; + +class TCoordinationHelperWithDummyLock: public TCoordinationHelper +{ +public: + TCoordinationHelperWithDummyLock( + const NProto::TDqConfig::TYtCoordinator& config, + const NProto::TDqConfig::TScheduler& schedulerConfig, + const TString& role, + ui16 interconnectPort) + : TCoordinationHelper(config, schedulerConfig, role, interconnectPort) + { } + + NActors::IActor* CreateLockOnCluster(NActors::TActorId ytWrapper, const TString& prefix, const TString& lockName, bool temporary) override { + Y_UNUSED(ytWrapper); + Y_UNUSED(prefix); + Y_UNUSED(temporary); + + Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1)); + + auto attributes = NYT::BuildYsonNodeFluently() + .BeginMap() + .Item(NCommonAttrs::ACTOR_NODEID_ATTR).Value(NodeId) + .Item(NCommonAttrs::HOSTNAME_ATTR).Value(GetHostname()) + .Item(NCommonAttrs::GRPCPORT_ATTR).Value(GrpcPort) // optional + .Item(NCommonAttrs::UPLOAD_EXECUTABLE_ATTR).Value("true") // compat + .EndMap(); + + return CreateDummyLock( + lockName, + NYT::NodeToYsonString(attributes)); + } + + NActors::IActor* CreateLock(const TString& lockName, bool temporary) override { + return CreateLockOnCluster(NActors::TActorId(), Config.GetPrefix(), lockName, temporary); + } +}; + +ICoordinationHelper::TPtr CreateCoordiantionHelper(const NProto::TDqConfig::TYtCoordinator& cfg, const NProto::TDqConfig::TScheduler& schedulerConfig, const TString& role, ui16 interconnectPort) +{ + NProto::TDqConfig::TYtCoordinator config = cfg; + auto clusterName = config.GetClusterName(); + + TString userName; + TString token; + + if (config.HasToken()) { + // internal job + userName = config.GetUser(); + token = config.GetToken(); + } else if (!clusterName.empty()) { + std::tie(userName, token) = NDqs::GetUserToken( + config.HasUser() ? TMaybe<TString>(config.GetUser()) : TMaybe<TString>(), + config.HasTokenFile() ? TMaybe<TString>(config.GetTokenFile()) : TMaybe<TString>() + ); + + Y_ABORT_UNLESS(!token.empty()); + } + + auto prefix = config.GetPrefix(); + + if (config.GetLockType() != "dummy") { + Y_ABORT_UNLESS(!clusterName.empty()); + Y_ABORT_UNLESS(!userName.empty()); + Y_ABORT_UNLESS(!prefix.empty()); + } + + Y_ABORT_UNLESS(!role.empty()); + + config.SetUser(userName); + config.SetToken(token); + + if (config.GetLockType() == "dummy") { + return new TCoordinationHelperWithDummyLock(config, schedulerConfig, role, interconnectPort); + } else { + return new TCoordinationHelper(config, schedulerConfig, role, interconnectPort); + } +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h new file mode 100644 index 0000000000..6de1565093 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h @@ -0,0 +1,65 @@ +#pragma once + +#include <library/cpp/actors/core/actorsystem.h> + +#include <ydb/library/yql/providers/common/metrics/metrics_registry.h> +#include <ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h> +#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h> +#include <ydb/library/yql/providers/dq/config/config.pb.h> + +#include <util/generic/ptr.h> + +namespace NYql { + +struct TWorkerRuntimeData; + +class ICoordinationHelper: public TThrRefBase { +public: + using TPtr = TIntrusivePtr<ICoordinationHelper>; + + virtual ~ICoordinationHelper() = default; + + virtual ui32 GetNodeId() = 0; + virtual ui32 GetNodeId( + const TMaybe<ui32> nodeId, + const TMaybe<TString>& grpcPort, + ui32 minNodeId, + ui32 maxNodeId, + const THashMap<TString, TString>& attributes) = 0; + + virtual TString GetHostname() = 0; + virtual TString GetIp() = 0; + + virtual NActors::IActor* CreateLockOnCluster(NActors::TActorId ytWrapper, const TString& prefix, const TString& lockName, bool temporary = true) = 0; + + virtual NActors::IActor* CreateLock(const TString& lockName, bool temporary = true) = 0; + + virtual NActors::IActor* CreateServiceNodePinger(const IServiceNodeResolver::TPtr& ptr, const TResourceManagerOptions& rmOptions, const THashMap<TString, TString>& attributes = {}) = 0; + + virtual void StartRegistrator(NActors::TActorSystem* actorSystem) = 0; + + virtual void StartGlobalWorker(NActors::TActorSystem* actorSystem, const TVector<TResourceManagerOptions>& resourceUploaderOptions, IMetricsRegistryPtr metricsRegistry) = 0; + + virtual void StartCleaner(NActors::TActorSystem* actorSystem, const TMaybe<TString>& role) = 0; + + virtual IServiceNodeResolver::TPtr CreateServiceNodeResolver( + NActors::TActorSystem* actorSystem, const TVector<TString>& hostPortPairs) = 0; + + virtual const NProto::TDqConfig::TYtCoordinator& GetConfig() = 0; + + virtual const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem) = 0; + + virtual const NActors::TActorId GetWrapper() = 0; + + virtual const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem, const TString& clusterName, const TString& user, const TString& token) = 0; + + virtual TWorkerRuntimeData* GetRuntimeData() = 0; + + virtual void Stop(NActors::TActorSystem* actorSystem) = 0; + + virtual TString GetRevision() = 0; +}; + +ICoordinationHelper::TPtr CreateCoordiantionHelper(const NProto::TDqConfig::TYtCoordinator& config, const NProto::TDqConfig::TScheduler& schedulerConfig, const TString& role, ui16 interconnectPort); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper_win.cpp b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper_win.cpp new file mode 100644 index 0000000000..66ac268460 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper_win.cpp @@ -0,0 +1,13 @@ +#include "coordination_helper.h" + +namespace NYql { + +ICoordinationHelper::TPtr CreateCoordiantionHelper(const NProto::TDqConfig::TYtCoordinator& config, const TString& role) +{ + Y_UNUSED(config); + Y_UNUSED(role); + + return nullptr; +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp new file mode 100644 index 0000000000..a30946c173 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp @@ -0,0 +1,1343 @@ +#include "global_worker_manager.h" +#include "workers_storage.h" + +#include <ydb/library/yql/providers/dq/actors/events/events.h> +#include <ydb/library/yql/providers/dq/common/attrs.h> +#include <ydb/library/yql/providers/dq/scheduler/dq_scheduler.h> + +#include <ydb/library/yql/providers/dq/worker_manager/worker_manager_common.h> +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/execution_helpers.h> +#include <ydb/library/yql/providers/dq/api/grpc/api.grpc.pb.h> +#include <ydb/library/yql/providers/dq/counters/counters.h> + +#include <ydb/library/yql/utils/failure_injector/failure_injector.h> + +#include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/library/yql/utils/log/log.h> + +#include <library/cpp/grpc/client/grpc_client_low.h> +#include <library/cpp/protobuf/util/pb_io.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/interconnect/interconnect.h> +#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/svnversion/svnversion.h> + +#include <util/generic/vector.h> +#include <util/generic/guid.h> +#include <util/system/fs.h> +#include <util/system/hostname.h> +#include <util/system/getpid.h> +#include <util/generic/scope.h> +#include <util/random/shuffle.h> + +namespace NYql { + +using namespace NActors; +using namespace NDqs; +using namespace NThreading; +using namespace NMonitoring; +using EFileType = Yql::DqsProto::TFile::EFileType; +using TFileResource = Yql::DqsProto::TFile; + +union TDqResourceId { + struct { + ui32 Counter; + ui32 Epoch; + }; + ui64 Data; +}; + +static_assert(sizeof(TDqResourceId) == 8); + +class TWorkerStopFilter { +public: + TWorkerStopFilter(const Yql::DqsProto::JobStopRequest& request) + : Revision(request.GetRevision()) + , ClusterName(request.GetClusterName()) + , WorkerId(GetGuid(request.GetWorkerId())) + , NegativeRevision(request.GetNegativeRevision()) + , LastUpdate(TInstant::Now()) + { + for (const auto& attr : request.GetAttribute()) { + Attributes.emplace(attr.GetKey(), attr.GetValue()); + } + } + + bool Match(const TWorkerInfo& workerInfo) const { + return MatchInternal(workerInfo); + } + + TInstant GetLastUpdate() const { + return LastUpdate; + } + +private: + bool MatchInternal(const TWorkerInfo& workerInfo) const { + if (!Revision.empty() && NegativeRevision == (Revision == workerInfo.Revision)) { + return false; + } + if (!ClusterName.empty() && ClusterName != workerInfo.ClusterName) { + return false; + } + if (!WorkerId.IsEmpty() && WorkerId != workerInfo.WorkerId) { + return false; + } + + for (const auto& [k, v] : Attributes) { + auto i = workerInfo.Attributes.find(k); + if (i == workerInfo.Attributes.end()) { + return false; + } + if (i->second != v) { + return false; + } + } + + LastUpdate = TInstant::Now(); + + return true; + } + + const TString Revision; + const TString ClusterName; + const TGUID WorkerId; + const bool NegativeRevision; + THashMap<TString, TString> Attributes; + + mutable TInstant LastUpdate; +}; + +// used for fast state recovery on restarted jobs +class TLocalCriticalFiles: public IFileCache { +public: +// used in IsReady +// exe unsupported + void AddFile(const TString& path, const TString& objectId) override { + Y_UNUSED(path); + TGuard<TMutex> guard(Mutex); + ObjectIds.insert(objectId); + } + + void Clear() override { + TGuard<TMutex> guard(Mutex); + ObjectIds.clear(); + } + + TMaybe<TString> FindFile(const TString& objectId) override { + Y_UNUSED(objectId); + return { }; + } + + bool Contains(const TString& objectId) override { + Y_UNUSED(objectId); + return false; + } + + void Walk(const std::function<void(const TString& objectId)>& f) override { + TGuard<TMutex> guard(Mutex); + for (const auto& id : ObjectIds) { + f(id); + } + } + + ui64 FreeDiskSize() override { + return 0; + } + + ui64 UsedDiskSize() override { + return 0; + } + + TString GetDir() override { + return ""; // unused + } + +private: + TMutex Mutex; + THashSet<TString> ObjectIds; +}; + + +// used in gwm master +class TAllCriticalFiles { +public: + template<typename VectorLike> + void Add( + const TGUID& serviceNodeId, + ui32 nodeId, + const TString& revision, + const TString& address, + ui32 pid, + const VectorLike& files) + { + auto now = TInstant::Now(); + auto& t = PerServiceNode[serviceNodeId]; + t.LastUpdate = now; + t.NodeId = nodeId; + t.Revision = revision; + t.Pid = pid; + t.Address = address; + bool needUpdate = false; + for (const auto& f : files) { + needUpdate |= t.Files.insert(f.GetObjectId()).second; + } + + if (static_cast<int>(t.Files.size()) > static_cast<int>(files.size())) { + needUpdate = true; + THashSet<TString> hash; + for (const auto& f : files) { + hash.insert(f.GetObjectId()); + } + THashSet<TString> toDrop; + for (const auto& f : t.Files) { + if (!hash.contains(f)) { + toDrop.insert(f); + } + } + for (const auto& f : toDrop) { + t.Files.erase(f); + } + } + + if (needUpdate) { + t.OrderedFiles.clear(); + for (const auto& f : files) { + t.OrderedFiles.push_back(f.GetObjectId()); + } + } + + needUpdate |= ClearOld(now); + + if (needUpdate) { + Rebuild(); + } + } + + const TVector<TWorkerInfo::TFileResource>& GetResources() const { + return Resources; + } + + struct TServiceNodeFiles { + ui32 NodeId; + THashSet<TString> Files; + TVector<TString> OrderedFiles; + TInstant LastUpdate = TInstant::Now(); + TString Revision; + ui32 Pid; + TString Address; + }; + + const THashMap<TGUID, TServiceNodeFiles>& GetNodes() const { + return PerServiceNode; + } + +private: + bool ClearOld(TInstant now) { + THashSet<TGUID> toDrop; + for (const auto& [k, v] : PerServiceNode) { + if (k.IsEmpty()) { // local node + continue; + } + + if (now - v.LastUpdate > TDuration::Seconds(10)) { + toDrop.insert(k); + } + } + + for (const auto& k : toDrop) { + PerServiceNode.erase(k); + } + + return !toDrop.empty(); + } + + void Rebuild() { + Resources.clear(); + THashSet<TString> uniq; + THashSet<TString> uniqExe; + for (const auto& [_, v] : PerServiceNode) { + for (int i = 0; i < static_cast<int>(v.OrderedFiles.size()) - 1; ++i) { + const auto& f = v.OrderedFiles[i]; + uniq.insert(f); + } + if (!v.Files.empty()) { + uniqExe.insert(v.OrderedFiles.back()); + } + } + for (const auto& f : uniq) { + TWorkerInfo::TFileResource resource; + resource.SetObjectType(TWorkerInfo::TFileResource::EUDF_FILE); + resource.SetObjectId(f); + Resources.push_back(resource); + } + //exe upload controlled via DqControl + //for (const auto& f : uniqExe) { + // TWorkerInfo::TFileResource resource; + // resource.SetName("dq_vanilla_job.lite"); + // resource.SetObjectType(TWorkerInfo::TFileResource::EEXE_FILE); + // resource.SetObjectId(f); + // Resources.push_back(resource); + //} + } + + THashMap<TGUID, TServiceNodeFiles> PerServiceNode; // serviceNodeId -> List + TVector<TWorkerInfo::TFileResource> Resources; +}; + +class TGlobalWorkerManager: public TWorkerManagerCommon<TGlobalWorkerManager> { +public: + + static constexpr char ActorName[] = "GWM"; + + explicit TGlobalWorkerManager( + const ICoordinationHelper::TPtr& coordinator, + const TVector<TResourceManagerOptions>& resourceUploaderOptions, + IMetricsRegistryPtr metricsRegistry, + const NProto::TDqConfig::TScheduler& schedulerConfig) + : TWorkerManagerCommon<TGlobalWorkerManager>(&TGlobalWorkerManager::Initialization) + , Coordinator(coordinator) + , LeaderResolver(std::make_shared<TSingleNodeResolver>()) + , Metrics(metricsRegistry->GetSensors()->GetSubgroup("counters", "gwm")) + , UploaderMetrics(metricsRegistry->GetSensors()->GetSubgroup("counters", "uploader")) + , LatencyHistogram(Metrics->GetHistogram("LeaderLatency", ExponentialHistogram(10, 2, 1))) + , Workers(Coordinator->GetNodeId(), Metrics, metricsRegistry->GetSensors()->GetSubgroup("counters", "workers")) + , Scheduler(NDq::IScheduler::Make(schedulerConfig, metricsRegistry)) + , Revision(ToString(GetProgramCommitId())) + , ResourceUploaderOptions(resourceUploaderOptions) + , WaitListSize(nullptr) + { } + +private: + +#define HHFunc(TEvType, HandleFunc) \ + case TEvType::EventType: { \ + Y_SCOPE_EXIT(&) { UpdateMetrics(); }; \ + typename TEvType::TPtr* x = reinterpret_cast<typename TEvType::TPtr*>(&ev); \ + TString name(#TEvType); \ + name = name.substr(name.find_last_of(':')+1); \ + *Metrics->GetSubgroup("component", "requests")->GetCounter(name, true) += 1;\ + TInstant t = TInstant::Now(); \ + HandleFunc(*x, TActivationContext::ActorContextFor(this->SelfId())); \ + if (CurrentStateFunc() == &TGlobalWorkerManager::Leader) { \ + LatencyHistogram->Collect((TInstant::Now() - t).MilliSeconds()); \ + } \ + break; \ + } + + STRICT_STFUNC(Initialization, { + HHFunc(TEvBecomeFollower, StartFollower) + HHFunc(TEvBecomeLeader, StartLeader) + CFunc(TEvents::TEvBootstrap::EventType, Bootstrap) + + HHFunc(TEvAllocateWorkersRequest, OnAllocateWorkersRequestStub) + HHFunc(TEvFreeWorkersNotify, OnFreeWorkersStub) + HHFunc(TEvRegisterNode, OnRegisterNodeStub) + HHFunc(TEvClusterStatus, OnClusterStatusStub) + HHFunc(TEvQueryStatus, OnQueryStatusStub) + HHFunc(TEvIsReady, OnIsReadyStub) + HHFunc(TEvGetMasterRequest, OnGetMasterStub) + HHFunc(TEvConfigureFailureInjectorRequest, OnConfigureFailureInjectorStub) + HHFunc(TEvOperationStop, OnOperationStopStub) + + cFunc(TEvents::TEvPoison::EventType, PassAway) + IgnoreFunc(TEvJobStop) + HHFunc(TEvRoutesRequest, this->OnRoutesRequest) + }) + + STRICT_STFUNC(Leader, { + HHFunc(TEvAllocateWorkersRequest, OnAllocateWorkersRequest) + HHFunc(TEvFreeWorkersNotify, OnFreeWorkers) + HHFunc(TEvRegisterNode, OnRegisterNode) + + HHFunc(TEvBecomeLeader, StartLeader) + HHFunc(TEvBecomeFollower, StartFollower) + + IgnoreFunc(TEvInterconnect::TEvNodeConnected) + + HHFunc(TEvents::TEvUndelivered, OnUndelivered) + HHFunc(TEvInterconnect::TEvNodeDisconnected, OnExecuterDisconnected) + + cFunc(TEvents::TEvPoison::EventType, PassAway) + + HHFunc(TEvUploadComplete, OnFileUploaded) + HHFunc(TEvClusterStatus, OnClusterStatus) + HHFunc(TEvQueryStatus, OnQueryStatus) + HHFunc(TEvIsReady, OnIsReady) + HHFunc(TEvJobStop, OnJobStop) + HHFunc(TEvGetMasterRequest, OnGetMaster) + HHFunc(TEvConfigureFailureInjectorRequest, OnConfigureFailureInjector) + cFunc(TEvTick::EventType, OnTick) + HHFunc(TEvRoutesRequest, OnRoutesRequest) + HHFunc(TEvOperationStop, OnOperationStop) + }) + +#define FORWARD(T) \ + case T::EventType: { \ + /*YQL_CLOG(TRACE, ProviderDq) << "Forward event " << etype << " to " << LeaderId;*/ \ + typename T::TPtr& x = *(reinterpret_cast<typename T::TPtr*>(&ev)); \ + if (!x->Get()->Record.GetIsForwarded()) { \ + x->Get()->Record.SetIsForwarded(true); \ + Send(x->Forward(MakeWorkerManagerActorID(LeaderId))); \ + } \ + break; \ + } + + STRICT_STFUNC(Follower, { + HHFunc(TEvBecomeLeader, StartLeader) + HHFunc(TEvBecomeFollower, StartFollower) + HHFunc(TEvAllocateWorkersRequest, StartUploadAndForward) + FORWARD(TEvFreeWorkersNotify) + FORWARD(TEvRegisterNode) + FORWARD(TEvJobStop) + FORWARD(TEvClusterStatus) + FORWARD(TEvQueryStatus) + FORWARD(TEvOperationStop) + HHFunc(TEvIsReady, OnIsReadyForward) + HHFunc(TEvGetMasterRequest, OnGetMaster) + cFunc(TEvents::TEvPoison::EventType, PassAway) + HHFunc(TEvUploadComplete, OnFileUploaded) + IgnoreFunc(TEvTick) + HHFunc(TEvRoutesRequest, this->OnRoutesRequest) + }) + +#undef FORWARD +#undef HHFunc + + void Tick() { + Schedule(ScheduleInterval, new TEvTick); + } + + void OnTick() { + auto now = TInstant::Now(); + if (now - LastCleanTime > CleanInterval) { + Y_SCOPE_EXIT(&) { UpdateMetrics(); }; + CleanUp(now); + LastCleanTime = now; + } + TryResume(); + Tick(); + } + + void DoPassAway() override { + Y_SCOPE_EXIT(&) { UpdateMetrics(); }; + for (const auto& sender: Scheduler->Cleanup()) { + Send(sender, new TEvAllocateWorkersResponse("Shutdown in progress", NYql::NDqProto::StatusIds::UNAVAILABLE)); + } + } + + std::pair<bool, TString> CheckFiles(const TVector<TFileResource>& files) { + for (const auto& file : files) { + if (file.GetObjectType() == Yql::DqsProto::TFile::EEXE_FILE) { + if (file.GetName().empty()) { + return std::make_pair(false, "Unnamed exe file " + file.SerializeAsString()); + } + } else { + if (!NFs::Exists(file.GetLocalPath())) { + return std::make_pair(false, "Unknown file " + file.SerializeAsString()); + } + } + + if (file.GetObjectId().empty()) { + return std::make_pair(false, "Empty objectId (md5, revision) for " + file.SerializeAsString()); + } + } + + return std::make_pair(true, ""); + } + + std::pair<bool, TString> MaybeUpload(bool isForwarded, const TVector<TFileResource>& files, bool useCache = false) { + if (isForwarded) { + return std::make_pair(false, ""); + } + + auto [hasExeFile,error] = CheckFiles(files); + if (!error.empty()) { + return std::make_pair(hasExeFile,error); + } + + bool flag = false; + + TVector<TResourceFile> preparedFiles; + TVector<TString> preparedFilesIds; + TVector<TResourceFile> preparedExeFiles; + auto now = TInstant::Now(); + auto cacheTimeout = TDuration::Minutes(5); + + if (useCache) { + THashSet<TString> toremove; + for (const auto& [objectId, ts] : LastUploadCache) { + if (now - ts > cacheTimeout) { + toremove.insert(objectId); + } + } + for (const auto& objectId : toremove) { + LastUploadCache.erase(objectId); + } + } + + for (const auto& file : files) { + flag |= file.GetObjectType() == Yql::DqsProto::TFile::EEXE_FILE; + if (Uploading.contains(file.GetObjectId())) { + continue; + } + + // exe files can be without local path + if (file.GetLocalPath().empty()) { + continue; + } + auto fileName = file.GetName(); + auto objectId = file.GetObjectId(); + auto objectType = file.GetObjectType(); + auto localPath = file.GetLocalPath(); + auto size = file.GetSize(); + + if (useCache || objectType == Yql::DqsProto::TFile_EFileType_EEXE_FILE) { + if (LastUploadCache.contains(objectId) && (now - LastUploadCache[objectId] < cacheTimeout)) { + continue; + } + LastUploadCache[objectId] = now; + } + + TResourceFile preparedFile; + preparedFile.ObjectId = objectId; + preparedFile.LocalFileName = fileName; + preparedFile.Attributes["file_name"] = preparedFile.GetRemoteFileName(); + preparedFile.File = ::TFile(localPath, RdOnly | OpenExisting); + preparedFile.RemoteFileName = objectId; + preparedFiles.push_back(preparedFile); + preparedFilesIds.push_back(objectId); + + YQL_CLOG(DEBUG, ProviderDq) << "Start upload: " << fileName << "," << objectId << "," << size; + + int uploadProcesses = static_cast<int>(ResourceUploaderOptions.size()); + + if (objectType == Yql::DqsProto::TFile_EFileType_EEXE_FILE) { + // COMPAT (aozeritsky) + preparedFile.RemoteFileName = fileName; + uploadProcesses *= 2; + + for (auto options : ResourceUploaderOptions) { + options.Counters = UploaderMetrics; + options.LockName = objectId + "." + options.YtBackend.GetClusterName(); + options.Files = { preparedFile }; + options.UploadPrefix = options.UploadPrefix + "/bin/" + objectId; + auto actor = CreateResourceUploader(options, Coordinator); + UploadProcesses[RegisterChild(actor)].push_back(objectId); + } + } + + Uploading.emplace(objectId, TUploadingInfo {objectType, fileName, uploadProcesses, size}); + } + + Shuffle(preparedFiles.begin(), preparedFiles.end(), TReallyFastRng32(TInstant::Now().NanoSeconds())); + + for (auto options : ResourceUploaderOptions) { + options.Counters = UploaderMetrics; + options.LockName.clear(); + + if (!preparedFiles.empty()) { + options.Files = preparedFiles; + options.UploadPrefix = options.UploadPrefix + "/udfs"; + auto actor = CreateResourceUploader(options, Coordinator); + UploadProcesses.emplace(RegisterChild(actor), preparedFilesIds); + } + } + + return std::make_pair(flag, ""); + } + + void StartUploadAndForward(TEvAllocateWorkersRequest::TPtr& ev, const TActorContext& ctx) + { + YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId()); + auto [hasExeFile, err] = MaybeUpload(ev->Get()->Record.GetIsForwarded(), TVector<TFileResource>(ev->Get()->Record.GetFiles().begin(), ev->Get()->Record.GetFiles().end())); + if (!err.empty()) { + Send(ev->Sender, new TEvAllocateWorkersResponse(err, NYql::NDqProto::StatusIds::EXTERNAL_ERROR)); + } + + if (!hasExeFile && LeaderRevision != Revision) { + Send(ev->Sender, new TEvAllocateWorkersResponse( + Sprintf("Wrong revision %s!=%s", LeaderRevision.c_str(), Revision.c_str()), NYql::NDqProto::StatusIds::BAD_REQUEST)); + } else { + ev->Get()->Record.SetIsForwarded(true); + ctx.Send(ev->Forward(MakeWorkerManagerActorID(LeaderId))); + } + } + + void OnFileUploaded(TEvUploadComplete::TPtr& ev, const TActorContext& ctx) + { + Y_UNUSED(ctx); + auto sender = ev->Sender; + auto it = UploadProcesses.find(sender); + if (it == UploadProcesses.end()) { + return; + } + UnregisterChild(sender); + + for (const auto& objectId : it->second) { + auto jt = Uploading.find(objectId); + Y_ABORT_UNLESS(jt != Uploading.end()); + Workers.AddResource(objectId, jt->second.ObjectType, jt->second.ObjectName, jt->second.Size); + if (--jt->second.Clusters <= 0) { + Uploading.erase(jt); + } + } + UploadProcesses.erase(it); + } + + void StartLeader(TEvBecomeLeader::TPtr& ev, const TActorContext& ctx) + { + Y_UNUSED(ctx); + if (FollowingMode) { + YQL_CLOG(INFO, ProviderDq) << "Skip Leader request in following mode"; + Send(SelfId(), new TEvBecomeFollower(), /*flag=*/0, /*cookie=*/1); + return; + } + LeaderEpoch = CurrentResourceId.Epoch = ev->Get()->LeaderEpoch; + CurrentResourceId.Counter = 0; + + // update leader info (used for leader) + auto attributes = NYT::NodeFromYsonString(ev->Get()->Attributes).AsMap(); + UpdateLeaderInfo(attributes); + + YQL_CLOG(INFO, ProviderDq) << "Become leader, epoch=" << CurrentResourceId.Epoch; + YQL_CLOG(INFO, ProviderDq) << "Leader attributes leader=" << ev->Get()->Attributes; + YQL_CLOG(INFO, ProviderDq) << "Leader resolver=" << LeaderHost << ":" << LeaderPort; + + if (LeaderPinger) { + UnregisterChild(LeaderPinger); + LeaderPinger = TActorId(); + } + Become(&TGlobalWorkerManager::Leader); + Tick(); + } + + void StartFollower(TEvBecomeFollower::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + if (ev->Cookie == 1 && LockId) { + // kill from main + YQL_CLOG(INFO, ProviderDq) << "Kill from main"; + ctx.Send(ev->Forward(LockId)); + FollowingMode = true; + LocalCriticalFiles->Clear(); // disable wormup for old dq process + return; + } + auto attributes = NYT::NodeFromYsonString(ev->Get()->Attributes).AsMap(); + if (ev->Cookie == 1 && LockId) { + // kill from main + YQL_CLOG(INFO, ProviderDq) << "Kill from main"; + Send(LockId, new NActors::TEvents::TEvPoison); + LockId = TActorId(); + } + LeaderId = attributes.at(NCommonAttrs::ACTOR_NODEID_ATTR).AsUint64(); + LeaderResolver->SetLeaderHostPort( + attributes.at(NCommonAttrs::HOSTNAME_ATTR).AsString() + ":" + attributes.at(NCommonAttrs::GRPCPORT_ATTR).AsString()); + if (!LeaderPinger) { + TResourceManagerOptions rmOptions; + rmOptions.FileCache = LocalCriticalFiles; + LeaderPinger = RegisterChild(Coordinator->CreateServiceNodePinger(LeaderResolver, rmOptions)); + } + if (attributes.contains(NCommonAttrs::REVISION_ATTR)) { + LeaderRevision = attributes.at(NCommonAttrs::REVISION_ATTR).AsString(); + } + + UpdateLeaderInfo(attributes); + + YQL_CLOG(TRACE, ProviderDq) << " Following leader: " << LeaderId; + YQL_CLOG(INFO, ProviderDq) << "Leader resolver=" << LeaderHost << ":" << LeaderPort; + + WaitListSize = nullptr; + Workers.Clear(); + for (const auto& [key, value] : AllocatedResources) { + Send(value.ActorId, new TEvents::TEvPoison()); + } + for (const auto sender : Scheduler->Cleanup()) { + Send(sender, new TEvAllocateWorkersResponse("StartFollower", NYql::NDqProto::StatusIds::UNSPECIFIED)); + } + AllocatedResources.clear(); + for (auto& [k, v] : LiteralQueries) { + *v -= *v; + } + + Become(&TGlobalWorkerManager::Follower); + } + + void UpdateLeaderInfo(THashMap<TString, NYT::TNode>& attributes) { + LeaderHost = attributes.at(NCommonAttrs::HOSTNAME_ATTR).AsString(); + LeaderPort = std::stoi(attributes.at(NCommonAttrs::GRPCPORT_ATTR).AsString().Data()); + } + + void Bootstrap(const TActorContext& ctx) { + Y_UNUSED(ctx); + LockId = RegisterChild(Coordinator->CreateLock("gwm", false)); + } + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap(), 0); + } + + void OnRegisterNodeStub(TEvRegisterNode::TPtr& ev, const NActors::TActorContext&) { + Send(ev->Sender, new TEvRegisterNodeResponse()); + } + + void OnRegisterNode(TEvRegisterNode::TPtr& ev, const NActors::TActorContext& ctx) { + auto* mutableRequest = ev->Get()->Record.MutableRequest(); + auto& request = ev->Get()->Record.GetRequest(); + auto nodeId = request.GetNodeId(); + auto workerId = NYql::NDqs::NExecutionHelpers::GuidFromProto(request.GetGuid()); + auto role = request.GetRole(); + + Y_SCOPE_EXIT(&) { + ctx.Send(ev->Forward(NActors::GetNameserviceActorId())); + }; + + if (role == "worker_node" && Revision == request.GetRevision()) { + mutableRequest->SetEpoch(LeaderEpoch); + } + + if (role == "service_node") { + AllCriticalFiles.Add( + workerId, + nodeId, + request.GetRevision(), + request.GetAddress(), + request.GetPid(), + request.GetFilesOnNode()); + } + + if (role != "worker_node") { + mutableRequest->SetEpoch(LeaderEpoch); + return; + } + + int capacity = request.GetCapacity(); + if (capacity == 0) { + capacity = 1; + } + + { + TWorkerInfo::TPtr workerInfo; + bool needResume; + + auto newWorkerId = workerId; + + std::tie(workerInfo, needResume) = Workers.CreateOrUpdate(nodeId, newWorkerId, *mutableRequest); + + for (auto i = StopFilters.begin(); i != StopFilters.end(); ) { + if (workerInfo && !workerInfo->IsDead && i->Match(*workerInfo)) { + workerInfo->Stopping = true; + if (request.GetRunningWorkers() == 0) { + YQL_CLOG(DEBUG, ProviderDq) << "Stop worker on user request " << GetGuidAsString(workerId); + Send(MakeWorkerManagerActorID(nodeId), new TEvents::TEvPoison); + return; + } + } + + if ((TInstant::Now() - i->GetLastUpdate()) > TDuration::Seconds(600)) { + i = StopFilters.erase(i); + } else { + ++i; + } + } + + if (needResume) { + MarkDirty(); + } + + if (workerInfo && !workerInfo->IsDead) { + for (auto& r : AllCriticalFiles.GetResources()) { + workerInfo->AddToDownloadList(r.GetObjectId(), r); + } + + for (const auto& [_, file] : workerInfo->GetResourcesForDownloading()) { + *mutableRequest->AddDownloadList() = file; + } + } + } + } + + void CleanUp(TInstant now) { + Workers.CleanUp(now, TDuration::Seconds(15)); + } + + void OnAllocateWorkersRequestStub(TEvAllocateWorkersRequest::TPtr& ev, const NActors::TActorContext&) { + YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId()); + YQL_CLOG(DEBUG, ProviderDq) << "TGlobalWorkerManager::TEvAllocateWorkersRequest initialization"; + Send(ev->Sender, new TEvAllocateWorkersResponse("GWM: Waiting for initialization", NYql::NDqProto::StatusIds::UNAVAILABLE)); + } + + void MarkDirty(size_t count = 0U) { + ScheduleWaitCount = Min(ScheduleWaitCount, count); + } + + void TryResume() { + if (Workers.FreeSlots() >= ScheduleWaitCount) { + Scheduler->Process(Workers.Capacity(), Workers.FreeSlots(), [&] (const auto& item) { + auto maybeDead = DeadOperations.find(item.Request.GetResourceId()); + if (maybeDead != DeadOperations.end()) { + DeadOperations.erase(maybeDead); + return true; // remove from WaitList + } + auto candidates = Workers.TryAllocate(item); + if (!candidates.empty()) { + DoAllocate(item, candidates); + } + return !candidates.empty(); + }); + ScheduleWaitCount = std::numeric_limits<size_t>::max(); + DeadOperations.clear(); + } + } + + void OnAllocateWorkersRequest(TEvAllocateWorkersRequest::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId()); + YQL_CLOG(DEBUG, ProviderDq) << "TGlobalWorkerManager::TEvAllocateWorkersRequest"; + + TFailureInjector::Reach("allocate_workers_failure", [] { ::_exit(1); }); + + const auto count = ev->Get()->Record.GetCount(); + Y_ASSERT(count != 0); + + if (!Scheduler->Suspend(NDq::IScheduler::TWaitInfo(ev->Get()->Record, ev->Sender))) { + Send(ev->Sender, new TEvAllocateWorkersResponse("Too many dq operations", NYql::NDqProto::StatusIds::OVERLOADED)); + return; + } + + auto [_, error] = MaybeUpload(ev->Get()->Record.GetIsForwarded(), TVector<TFileResource>(ev->Get()->Record.GetFiles().begin(),ev->Get()->Record.GetFiles().end())); + if (!error.empty()) { + Send(ev->Sender, new TEvAllocateWorkersResponse(error, NYql::NDqProto::StatusIds::EXTERNAL_ERROR)); + return; + } + + MarkDirty(count); + } + + void DecrLiteralQueries(const TString& clusterName) { + *LiteralQueries[clusterName] += -1; + } + + void IncrLiteralQueries(const TString& clusterName) { + auto& counter = LiteralQueries[clusterName]; + if (!counter) { + counter = Metrics + ->GetSubgroup("counters", "gwm") + ->GetSubgroup("component", "lists") + ->GetSubgroup("ClusterName", clusterName) + ->GetCounter("LiteralQueries"); + } + *counter += 1; + } + + void DoAllocate(const NDq::IScheduler::TWaitInfo& waitInfo, const TVector<TWorkerInfo::TPtr>& allocated) { + YQL_LOG_CTX_ROOT_SESSION_SCOPE(waitInfo.Request.GetTraceId()); + Y_ABORT_UNLESS(allocated.size() == waitInfo.Request.GetCount()); + + THashSet<TString> clusters; + + for (const auto& workerInfo : allocated) { + YQL_CLOG(DEBUG, ProviderDq) << "Allocating worker " << GetGuidAsString(workerInfo->WorkerId); + YQL_CLOG(DEBUG, ProviderDq) << " Address : " << workerInfo->Address; + for (const auto& [k, v] : workerInfo->Attributes) { + YQL_CLOG(DEBUG, ProviderDq) << " " << k << " : " << v; + } + clusters.insert(workerInfo->ClusterName); + } + + THashSet<TString> usedFiles; + for (const auto& file : waitInfo.Request.GetFiles()) { + usedFiles.insert(file.GetObjectId()); + } + + auto resourceId = CurrentResourceId.Data; + CurrentResourceId.Counter ++; + + auto resourceAllocator = waitInfo.Sender; + + auto response = MakeHolder<TEvAllocateWorkersResponse>(resourceId, allocated); + waitInfo.Stat.FlushCounters(response->Record); + Send( + resourceAllocator, + response.Release(), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession); + + Subscribe(resourceAllocator.NodeId()); + + AllocatedResources[resourceId] = { + resourceAllocator, + allocated, + usedFiles, + clusters, + TInstant::Now() + }; + + if (allocated.size() == 1) { + Y_ABORT_UNLESS(clusters.size() == 1); + IncrLiteralQueries(*clusters.begin()); + } + } + + void DropActorOrNode(const std::function<bool(TActorId actorId)>& check, const NActors::TActorContext&) { + TVector<ui64> freeList; + for (const auto& [k, v] : AllocatedResources) { + if (check(v.ActorId)) { + freeList.push_back(k); + } + } + + for (auto resourceId : freeList) { + FreeResource(resourceId, {}); + } + + Scheduler->ProcessAll([&] (const auto& item) { + return check(item.Sender); + }); + + MarkDirty(); + } + + void OnUndelivered(TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) + { + auto deadActor = ev->Sender; + DropActorOrNode([&](TActorId actorId) { + return actorId == deadActor; + }, ctx); + } + + void OnExecuterDisconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev, const NActors::TActorContext& ctx) + { + YQL_CLOG(DEBUG, ProviderDq) << "OnExecuterDisconnected " << ev->Get()->NodeId; + + Unsubscribe(ev->Get()->NodeId); + + auto deadNode = ev->Get()->NodeId; + + DropActorOrNode([&](TActorId actorId) { + return actorId.NodeId() == deadNode; + }, ctx); + } + + void OnFreeWorkersStub(TEvFreeWorkersNotify::TPtr& ev, const NActors::TActorContext&) { + YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId()); + YQL_CLOG(DEBUG, ProviderDq) << "TGlobalWorkerManager::OnFreeWorkersStub " << ev->Sender.NodeId(); + } + + void FreeResource(ui64 resourceId, const THashSet<TGUID>& failedWorkers) { + if (!AllocatedResources.contains(resourceId)) { + return; + } + + auto& resource = AllocatedResources[resourceId]; + + auto now = TInstant::Now(); + + for (const auto& worker : resource.Workers) { + YQL_CLOG(DEBUG, ProviderDq) << "Free worker " << GetGuidAsString(worker->WorkerId); + + if (failedWorkers.contains(worker->WorkerId)) { + Workers.DropWorker(now, worker->WorkerId); + } else { + Workers.FreeWorker(now, worker); + } + } + + if (resource.Workers.size() == 1) { + DecrLiteralQueries(*resource.Clusters.begin()); + } + + Workers.UpdateResourceUseTime(now - resource.StartTime, resource.UsedFiles); + AllocatedResources.erase(resourceId); + } + + void OnFreeWorkers(TEvFreeWorkersNotify::TPtr& ev, const TActorContext&) { + auto resourceId = ev->Get()->Record.GetResourceId(); + if (AllocatedResources.contains(resourceId)) { + YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId()); + YQL_CLOG(DEBUG, ProviderDq) << "TEvFreeWorkersNotify " << resourceId; + THashSet<TGUID> failedWorkers; + for (const auto& workerInfo : ev->Get()->Record.GetFailedWorkerGuid()) { + auto guid = GetGuid(workerInfo); + YQL_CLOG(DEBUG, ProviderDq) << "Failed worker: " << GetGuidAsString(guid); + failedWorkers.insert(guid); + } + FreeResource(resourceId, failedWorkers); + } else { + DeadOperations.insert(resourceId); + } + + MarkDirty(); + } + + void OnJobStop(TEvJobStop::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + + auto request = ev->Get()->Record.GetRequest(); + TString requestStr; + TStringOutput output1(requestStr); + SerializeToTextFormat(request, output1); + + YQL_CLOG(DEBUG, ProviderDq) << "JobStop " << requestStr; + + if (request.GetForce()) { + TWorkerStopFilter filter(request); + Workers.Visit([&](const TWorkerInfo::TPtr& workerInfo) { + if (!workerInfo->IsDead && filter.Match(*workerInfo)) { + workerInfo->Stopping = true; + YQL_CLOG(DEBUG, ProviderDq) << "Force stop worker on user request " << GetGuidAsString(workerInfo->WorkerId); + Send(MakeWorkerManagerActorID(workerInfo->NodeId), new TEvents::TEvPoison); + } + }); + } else { + StopFilters.emplace_back(request); + } + } + + void OnClusterStatusStub(TEvClusterStatus::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto response = MakeHolder<TEvClusterStatusResponse>(); + Send(ev->Sender, response.Release()); + } + + void OnClusterStatus(TEvClusterStatus::TPtr& ev, const TActorContext& ctx) { + YQL_CLOG(DEBUG, ProviderDq) << "TEvClusterStatus "; + + Y_UNUSED(ctx); + + auto response = MakeHolder<TEvClusterStatusResponse>(); + + auto* r = response->Record.MutableResponse(); + + r->SetRevision(Revision); + + Workers.ClusterStatus(r); + + Scheduler->ForEach([&] (const auto& item) { + auto* info = r->AddWaitList(); + info->SetCount(item.Request.GetCount()); + info->SetOperationId(item.Request.GetTraceId()); + info->SetUserName(item.Request.GetUser()); + + THashMap<TString, Yql::DqsProto::ClusterStatusResponse::File> files; + + for (const auto& rec : item.Request.GetFiles()) { + auto& file = files[rec.GetObjectId()]; + file.SetObjectId(rec.GetObjectId()); + file.SetName(rec.GetName()); + if (item.ResLeft.contains(rec.GetObjectId())) { + file.SetLeft(item.ResLeft[rec.GetObjectId()]); + } + file.SetSize(rec.GetSize()); + } + + for (const auto& it : item.Request.GetWorkerFilterPerTask()) { + for (const auto& rec : it.GetFile()) { + auto& file = files[rec.GetObjectId()]; + file.SetObjectId(rec.GetObjectId()); + file.SetName(rec.GetName()); + file.SetCount(file.GetCount()+1); + file.SetSize(rec.GetSize()); + } + } + + for (const auto& [_, v] : files) { + *info->AddResources() = v; + } + }); + + for (const auto& i : Uploading) { + *r->AddUploading() = i.first; + } + + for (const auto& [guid, node] : AllCriticalFiles.GetNodes()) { + auto* nodeInfo = r->AddServiceNode(); + nodeInfo->SetRevision(node.Revision); + nodeInfo->SetNodeId(node.NodeId); + nodeInfo->SetPid(node.Pid); + nodeInfo->SetAddress(node.Address); + nodeInfo->SetGuid(GetGuidAsString(guid)); + for (const auto& file : node.OrderedFiles) { + *nodeInfo->AddFile() = file; + } + } + + Send(ev->Sender, response.Release()); + } + + void OnQueryStatusStub(TEvQueryStatus::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto response = MakeHolder<TEvQueryStatusResponse>(); + Send(ev->Sender, response.Release()); + } + + void OnQueryStatus(TEvQueryStatus::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + + auto sessionId = ev->Get()->Record.request().GetSession(); + + bool queued = false; + ui32 workersCount; + + Scheduler->ForEach([&queued, &sessionId, &workersCount] (const NDq::IScheduler::TWaitInfo& item) { + if (sessionId == item.Request.GetTraceId()) { + queued = true; + workersCount = item.Request.GetCount(); + } + }); + + auto response = MakeHolder<TEvQueryStatusResponse>(); + + auto* r = response->Record.MutableResponse(); + + if (queued) { + if (workersCount > Workers.FreeSlots()) { + r->SetStatus("Awaiting"); + } else { + r->SetStatus("Uploading artifacts"); + } + } else { + r->SetStatus("Executing"); + } + + Send(ev->Sender, response.Release()); + } + + void OnIsReadyStub(TEvIsReady::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + + auto response = MakeHolder<TEvIsReadyResponse>(); + Send(ev->Sender, response.Release()); + } + + void OnOperationStopStub(TEvOperationStop::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + + auto response = MakeHolder<TEvOperationStopResponse>(); + Send(ev->Sender, response.Release()); + } + + void OnOperationStop(TEvOperationStop::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + + Scheduler->Process(Workers.Capacity(), Workers.FreeSlots(), [&] (const auto& item) { + if (item.Request.GetTraceId() == ev->Get()->Record.GetRequest().GetOperationId()) { + Send(item.Sender, new TEvDqFailure(NYql::NDqProto::StatusIds::ABORTED, TIssue("Operation stopped by scheduler").SetCode(TIssuesIds::DQ_GATEWAY_ERROR, TSeverityIds::S_ERROR))); + return true; + } + return false; + }); + + auto response = MakeHolder<TEvOperationStopResponse>(); + Send(ev->Sender, response.Release()); + } + + void FillCriticalFiles(TEvIsReady::TPtr& ev) { + if (FollowingMode) { + return; + } + if (ev->Get()->Record.GetIsForwarded()) { + return; + } + LocalCriticalFiles->Clear(); + for (auto& f : ev->Get()->Record.GetRequest().GetFiles()) { + //TODO: Add support types in LocalCriticalFiles + //Don't uncomment this + //if (f.GetObjectType() == TWorkerInfo::TFileResource::EEXE_FILE) { + // LocalCriticalFiles supports only udfs + // continue; + //} + LocalCriticalFiles->AddFile("unused", f.GetObjectId()); + } + TVector<TFileResource> files; + LocalCriticalFiles->Walk([&](const TString& objectId) { + TFileResource r; + r.SetObjectId(objectId); + files.push_back(r); + }); + TFileResource r; + r.SetObjectId(Revision); + files.push_back(r); + + AllCriticalFiles.Add(TGUID(), SelfId().NodeId(), Revision, Address, Pid, files); + } + + void OnIsReadyForward(TEvIsReady::TPtr& ev, const TActorContext& ctx) { + auto [_, error] = MaybeUpload(ev->Get()->Record.GetIsForwarded(), TVector<TFileResource>(ev->Get()->Record.GetRequest().GetFiles().begin(),ev->Get()->Record.GetRequest().GetFiles().end()), true); + if (!error.empty()) { + YQL_CLOG(WARN, ProviderDq) << "TEvIsReady error on upload: " << error; + } + + if (FollowingMode) { + auto response = MakeHolder<TEvIsReadyResponse>(); + response->Record.SetIsReady(true); + Send(ev->Sender, response.Release()); + } else { + FillCriticalFiles(ev); + ev->Get()->Record.SetIsForwarded(true); + ctx.Send(ev->Forward(MakeWorkerManagerActorID(LeaderId))); + } + } + + void OnIsReady(TEvIsReady::TPtr& ev, const TActorContext& ctx) { + YQL_CLOG(DEBUG, ProviderDq) << "TEvIsReady "; + Y_UNUSED(ctx); + + FillCriticalFiles(ev); + + auto [_, error] = MaybeUpload(ev->Get()->Record.GetIsForwarded(), TVector<TFileResource>(ev->Get()->Record.GetRequest().GetFiles().begin(),ev->Get()->Record.GetRequest().GetFiles().end()), true); + if (!error.empty()) { + YQL_CLOG(WARN, ProviderDq) << "TEvIsReady error on upload: " << error; + } + THashMap<TString , std::pair<ui32, ui32>> clusterMap; + for (const auto& options : ResourceUploaderOptions) { + clusterMap.emplace(options.YtBackend.GetClusterName(), std::make_pair(options.YtBackend.GetMaxJobs(), 0)); + } + + auto resources = ev->Get()->Record.GetRequest().GetFiles(); + Workers.IsReady(TVector<TFileResource>(resources.begin(), resources.end()), clusterMap); + + // any cluster has at least 50% of workers with actual vanilla job ready + bool isReady = false; + for (const auto& [cluster,pair] : clusterMap) { + YQL_CLOG(DEBUG, ProviderDq) << cluster << " ready: " << pair.second << "/" << pair.first << " workers ready"; + isReady |= pair.second * 2 >= pair.first; + } + + YQL_CLOG(DEBUG, ProviderDq) << "IsReady status : " << isReady; + + auto response = MakeHolder<TEvIsReadyResponse>(); + response->Record.SetIsReady(isReady); + + Send(ev->Sender, response.Release()); + } + + void OnGetMasterStub(TEvGetMasterRequest::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto response = MakeHolder<TEvGetMasterResponse>(); + Send(ev->Sender, response.Release()); + } + + void OnGetMaster(TEvGetMasterRequest::TPtr& ev, const TActorContext& ctx) { + YQL_CLOG(DEBUG, ProviderDq) << "TEvGetMasterRequest "; + + Y_UNUSED(ctx); + + auto response = MakeHolder<TEvGetMasterResponse>(); + + auto* r = response->Record.MutableResponse(); + + r->SetHost(LeaderHost); + r->SetPort(LeaderPort); + + Send(ev->Sender, response.Release()); + } + + void OnConfigureFailureInjectorStub(TEvConfigureFailureInjectorRequest::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto response = MakeHolder<TEvConfigureFailureInjectorResponse>(); + Send(ev->Sender, response.Release()); + } + + void OnConfigureFailureInjector(TEvConfigureFailureInjectorRequest::TPtr& ev, const TActorContext& ctx) { + YQL_CLOG(DEBUG, ProviderDq) << "TEvConfigureFailureInjectorRequest "; + + Y_UNUSED(ctx); + + // TODO: support resending the event to all workers and gathering the results? + auto& request = ev->Get()->Record.GetRequest(); + // just forward the event to a worker node + if (request.GetNodeId() == SelfId().NodeId()) { + TFailureInjector::Set(request.GetName(), request.GetSkip(), request.GetCountOfFails()); + + auto response = MakeHolder<TEvConfigureFailureInjectorResponse>(); + auto* r = response->Record.MutableResponse(); + r->Setsuccess(true); + Send(ev->Sender, response.Release()); + } else { + ctx.Send(ev->Forward(MakeWorkerManagerActorID(request.GetNodeId()))); + } + } + + void UpdateMetrics() { + if (!WaitListSize) { + WaitListSize = Metrics->GetSubgroup("component", "lists")->GetCounter("WaitListSize"); + } + *WaitListSize = Scheduler->UpdateMetrics(); + Workers.UpdateMetrics(); + } + +private: + const ICoordinationHelper::TPtr Coordinator; + const std::shared_ptr<TSingleNodeResolver> LeaderResolver; + TActorId LeaderPinger; + TActorId LockId; + + TSensorsGroupPtr Metrics; + TSensorsGroupPtr UploaderMetrics; + THistogramPtr LatencyHistogram; + THashMap<TString,NMonitoring::TDynamicCounters::TCounterPtr> LiteralQueries; + TWorkersStorage Workers; + + struct TResourceInfo { + TActorId ActorId; + TVector<TWorkerInfo::TPtr> Workers; + THashSet<TString> UsedFiles; + THashSet<TString> Clusters; + TInstant StartTime; + }; + THashMap<ui64, TResourceInfo> AllocatedResources; + THashSet<ui64> DeadOperations; + + ui32 LeaderId = static_cast<ui32>(-1); + TString LeaderRevision = ""; + TString LeaderHost = ""; + ui32 LeaderPort = 0u; + + ui32 LeaderEpoch; + TDqResourceId CurrentResourceId; + + const NDq::IScheduler::TPtr Scheduler; + const TString Revision; + + THashMap<TActorId, TVector<TString>> UploadProcesses; // actorId -> objects + + struct TUploadingInfo { + EFileType ObjectType; + TString ObjectName; + int Clusters; + i64 Size; + }; + THashMap<TString, TUploadingInfo> Uploading; // objectId -> objectName + + TVector<TResourceManagerOptions> ResourceUploaderOptions; + + TList<TWorkerStopFilter> StopFilters; + + TDynamicCounters::TCounterPtr WaitListSize; + + // filled in IsReady + IFileCache::TPtr LocalCriticalFiles = MakeIntrusive<TLocalCriticalFiles>(); + // filled in IsReady for local files + // filled in OnRegister for other files + TAllCriticalFiles AllCriticalFiles; + + THashMap<TString, TInstant> LastUploadCache; + + // Don't reschedule too frequently to avoid GWM hanging + TDuration ScheduleInterval = TDuration::MilliSeconds(100); + size_t ScheduleWaitCount = std::numeric_limits<size_t>::max(); // max - no, 0 - any, > 0 count + TInstant LastCleanTime; + TDuration CleanInterval = TDuration::Seconds(2); + bool FollowingMode = false; + const TString Address = HostName(); + const ui32 Pid = GetPID(); +}; + +NActors::IActor* CreateGlobalWorkerManager( + const ICoordinationHelper::TPtr& coordinator, + const TVector<TResourceManagerOptions>& resourceUploaderOptions, + IMetricsRegistryPtr metricsRegistry, + const NProto::TDqConfig::TScheduler& schedulerConfig) { + return new TGlobalWorkerManager(coordinator, resourceUploaderOptions, std::move(metricsRegistry), schedulerConfig); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.h b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.h new file mode 100644 index 0000000000..8173f2c3fe --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.h @@ -0,0 +1,20 @@ +#pragma once + +#include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> +#include <ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h> +#include <ydb/library/yql/providers/dq/actors/events.h> +#include "coordination_helper.h" +#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h> +#include <ydb/library/yql/providers/common/metrics/metrics_registry.h> + +#include <library/cpp/actors/core/events.h> + +namespace NYql { + +NActors::IActor* CreateGlobalWorkerManager( + const ICoordinationHelper::TPtr& coordinator, + const TVector<TResourceManagerOptions>& resourceUploaderOptions, + IMetricsRegistryPtr metricsRegistry, + const NProto::TDqConfig::TScheduler& schedulerConfig); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp new file mode 100644 index 0000000000..85825306f7 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp @@ -0,0 +1,439 @@ +#include "global_worker_manager.h" +#include <ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/actors/testlib/test_runtime.h> +#include <library/cpp/yson/node/node_io.h> +#include <yt/cpp/mapreduce/interface/fluent.h> +#include <ydb/library/yql/providers/common/metrics/metrics_registry.h> +#include <ydb/library/yql/providers/dq/actors/events/events.h> +#include <ydb/library/yql/providers/dq/common/attrs.h> +#include <ydb/library/yql/providers/dq/actors/dynamic_nameserver.h> +#include <ydb/library/yql/providers/dq/actors/resource_allocator.h> + +using namespace NYql; +using namespace NActors; +using namespace NProto; +using namespace NDqs; + +namespace { + +class TMockLock: public TActor<TMockLock> { +public: + TMockLock() + : TActor(&TMockLock::Main) + { + } + + STATEFN(Main) { + switch (ev->GetTypeRewrite()) { + cFunc(TEvents::TEvPoison::EventType, PassAway); + } + } +}; + +class TTestCoordinationHelper: public ICoordinationHelper { +public: + + TTestCoordinationHelper(ui32 nodeId) + : NodeId(nodeId) + {} + + ui32 GetNodeId() override { + return NodeId; + } + + ui32 GetNodeId(const TMaybe<ui32> nodeId, const TMaybe<TString>& grpcPort, ui32 minNodeId, ui32 maxNodeId, const THashMap<TString, TString>& attributes) override { + Y_UNUSED(nodeId); + Y_UNUSED(grpcPort); + Y_UNUSED(minNodeId); + Y_UNUSED(maxNodeId); + Y_UNUSED(attributes); + return NodeId; + } + + TString GetHostname() override { + return "localhost"; + } + + TString GetIp() override { + return "::1"; + } + + NActors::IActor* CreateLockOnCluster(NActors::TActorId ytWrapper, const TString& prefix, const TString& lockName, bool temporary) override { + Y_UNUSED(ytWrapper); + Y_UNUSED(prefix); + Y_UNUSED(lockName); + Y_UNUSED(temporary); + + return new TMockLock(); + } + + NActors::IActor* CreateLock(const TString& lockName, bool temporary) override { + Y_UNUSED(lockName); + Y_UNUSED(temporary); + return new TMockLock(); + } + + NActors::IActor* CreateServiceNodePinger(const IServiceNodeResolver::TPtr& ptr, const TResourceManagerOptions& rmOptions, const THashMap<TString, TString>& attributes = {}) override { + Y_UNUSED(ptr); + Y_UNUSED(rmOptions); + Y_UNUSED(attributes); + return nullptr; + } + + void StartRegistrator(NActors::TActorSystem* actorSystem) override { + Y_UNUSED(actorSystem); + } + + void StartGlobalWorker(NActors::TActorSystem* actorSystem, const TVector<TResourceManagerOptions>& resourceUploaderOptions, IMetricsRegistryPtr metricsRegistry) override { + Y_UNUSED(actorSystem); + Y_UNUSED(resourceUploaderOptions); + Y_UNUSED(metricsRegistry); + } + + void StartCleaner(NActors::TActorSystem* actorSystem, const TMaybe<TString>& role) override { + Y_UNUSED(actorSystem); + Y_UNUSED(role); + } + + NYql::IServiceNodeResolver::TPtr CreateServiceNodeResolver(NActors::TActorSystem* actorSystem, const TVector<TString>& hostPortPairs) override { + Y_UNUSED(actorSystem); + Y_UNUSED(hostPortPairs); + + return NYql::IServiceNodeResolver::TPtr(); + } + + const NProto::TDqConfig::TYtCoordinator& GetConfig() override { + return YtCoordinator_; + } + + const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem) override { + Y_UNUSED(actorSystem); + return NActors::TActorId(); + } + + const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem, const TString& cluster, const TString& user, const TString& token) override { + Y_UNUSED(actorSystem); + Y_UNUSED(cluster); + Y_UNUSED(user); + Y_UNUSED(token); + return NActors::TActorId(); + } + + const NActors::TActorId GetWrapper() override { + return NActors::TActorId(); + } + + TWorkerRuntimeData* GetRuntimeData() override { + return nullptr; + } + + void Stop(NActors::TActorSystem* actorSystem) override { + Y_UNUSED(actorSystem); + } + + TString GetRevision() override { + return TString(); + } + +private: + ui32 NodeId; + NProto::TDqConfig::TYtCoordinator YtCoordinator_; +}; +} + +class TGlobalWorkerManagerTest: public TTestBase { +public: + UNIT_TEST_SUITE(TGlobalWorkerManagerTest); + UNIT_TEST(TestTasksAllocation1) + UNIT_TEST(TestTasksAllocation2) + UNIT_TEST(TestTasksAllocation3) + UNIT_TEST(TestTasksAllocation4) + UNIT_TEST(TestTasksAllocation5) + UNIT_TEST_SUITE_END(); + + void SetUp() override { + // Service node: 0 + // Worker nodes: 1..nodesNumber + const ui32 nodesNumber = 7; + ActorRuntime_.Reset(new NActors::TTestActorRuntimeBase(nodesNumber, true)); + + // Name server. + TIntrusivePtr<TTableNameserverSetup> nameserverTable = new TTableNameserverSetup(); + THashSet<ui32> staticNodeId; + + for (ui32 i = 0; i < 100; i++) { + nameserverTable->StaticNodeTable[i] = std::make_pair(ToString(i), i); + } + ActorRuntime_->AddLocalService(GetNameserviceActorId(), + TActorSetupCmd(NYql::NDqs::CreateDynamicNameserver(nameserverTable), TMailboxType::Simple, 0)); + + // GWM + TVector<TResourceManagerOptions> uploadResourcesOptions; + const TIntrusivePtr<TTestCoordinationHelper> coordPtr = new TTestCoordinationHelper(NodeId()); + const TIntrusivePtr<TSensorsGroup> sensorsPtr = new TSensorsGroup(); + const auto gwmActor = CreateGlobalWorkerManager(coordPtr, uploadResourcesOptions, CreateMetricsRegistry(sensorsPtr), NProto::TDqConfig::TScheduler()); + ActorRuntime_->AddLocalService(MakeWorkerManagerActorID(NodeId()), + TActorSetupCmd{gwmActor, TMailboxType::Simple, 0}); + + // Local WM. + for (ui32 i = 1; i < nodesNumber; i++) { + NYql::NDqs::TLocalWorkerManagerOptions lwmOptions; + lwmOptions.TaskRunnerInvokerFactory = new NDqs::TTaskRunnerInvokerFactory(); + lwmOptions.TaskRunnerActorFactory = NYql::NDq::NTaskRunnerActor::CreateTaskRunnerActorFactory( + lwmOptions.Factory, lwmOptions.TaskRunnerInvokerFactory); + auto localWM = CreateLocalWorkerManager(lwmOptions); + ActorRuntime_->AddLocalService(MakeWorkerManagerActorID(NodeId(i)), + TActorSetupCmd{localWM, TMailboxType::Simple, 0}, i); + ActorRuntime_->AddLocalService(GetNameserviceActorId(), + TActorSetupCmd(NYql::NDqs::CreateDynamicNameserver(nameserverTable), TMailboxType::Simple, 0), i); + } + + ActorRuntime_->Initialize(); + + NActors::TDispatchOptions options; + options.FinalEvents.emplace_back(NActors::TEvents::TSystem::Bootstrap, nodesNumber); + ActorRuntime_->DispatchEvents(options); + + auto attributes = NYT::BuildYsonNodeFluently() + .BeginMap() + .Item(NCommonAttrs::ACTOR_NODEID_ATTR).Value(NodeId()) + .Item(NCommonAttrs::HOSTNAME_ATTR).Value("localhost") + .Item(NCommonAttrs::GRPCPORT_ATTR).Value("1") + .Item(NCommonAttrs::UPLOAD_EXECUTABLE_ATTR).Value("true") + .EndMap(); + + // Init GWM. + auto leaderEv = MakeHolder<TEvBecomeLeader>(1, "1", NYT::NodeToYsonString(attributes)); + const auto dummyActor = ActorRuntime_->AllocateEdgeActor(); + ActorRuntime_->Send(new IEventHandle(MakeWorkerManagerActorID(NodeId()), dummyActor, leaderEv.Release())); + + auto statusEv = MakeHolder<TEvClusterStatus>(); + const auto statusSender = ActorRuntime_->AllocateEdgeActor(); + ActorRuntime_->Send(new IEventHandle(MakeWorkerManagerActorID(NodeId()), statusSender, statusEv.Release())); + auto resp = ActorRuntime_->GrabEdgeEvent<TEvClusterStatusResponse>(statusSender); + + // Fake file for GWM. + TFile outFile("/tmp/test", CreateAlways | ARW); + outFile.Close(); + } + + void TearDown() override { + ActorRuntime_.Reset(); + remove("/tmp/test"); + } + + void TestTasksAllocation1() { + TVector<TVector<TString>> filesPerNode = { + {"file0", "file11"}, + {"file0", "file22"}, + {"file0", "file33", "file333"}, + {"file0", "file44"}, + {"file0", "file55", "file555"}, + {"file0", "file66", "file6"} + }; + + TVector<TVector<TString>> filesPerTask = { + {"file333"} // Expect mapping on node 3. + }; + + TVector<ui32> expectedTasksMapping = {3}; + + CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping); + } + + void TestTasksAllocation2() { + TVector<TVector<TString>> filesPerNode = { + {"file0", "file11"}, + {"file0", "file22"}, + {"file0", "file33", "file333"}, + {"file0", "file44"}, + {"file0", "file55", "file555"}, + {"file0", "file66", "file6"} + }; + + TVector<TVector<TString>> filesPerTask = { + {"file55", "file0"}, // Expect mapping on node 5. + {"file0", "file11"} // Expect mapping on node 1. + }; + + TVector<ui32> expectedTasksMapping = {5, 1}; + + CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping); + } + + void TestTasksAllocation3() { + TVector<TVector<TString>> filesPerNode = { + {"file0", "file11"}, + {"file0", "file22"}, + {"file0", "file33", "file333"}, + {"file0", "file44"}, + {"file0", "file55", "file555"}, + {"file0", "file66", "file6"} + }; + + TVector<TVector<TString>> filesPerTask = { + {"file0", "file11"}, // Expect mapping on node 1. + {"file55", "file0"}, // Expect mapping on node 5. + {"file44", "file0"} // Expect mapping on node 4. + }; + + TVector<ui32> expectedTasksMapping = {1, 5, 4}; + + CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping); + } + + void TestTasksAllocation4() { + TVector<TVector<TString>> filesPerNode = { + {"file0", "file11"}, + {"file0", "file22"}, + {"file0", "file33", "file333"}, + {"file0", "file44"}, + {"file0", "file55", "file555"}, + {"file0", "file66", "file6"} + }; + + TVector<TVector<TString>> filesPerTask = { + {"file11"}, // Expect mapping on node 1. + {"file0", "file333", "file33"}, // Expect mapping on node 3. + {"file0", "file22"}, // Expect mapping on node 2. + {"file0", "file555"} // Expect mapping on node 5. + }; + + TVector<ui32> expectedTasksMapping = {1, 3, 2, 5}; + + CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping); + } + + void TestTasksAllocation5() { + TVector<TVector<TString>> filesPerNode = { + {"file0", "file11"}, + {"file0", "file22"}, + {"file0", "file33", "file333"}, + {"file0", "file44"}, + {"file0", "file55", "file555"}, + {"file0", "file66", "file5"} + }; + + TVector<TVector<TString>> filesPerTask = { + {"file0", "file5", "file66"}, // Expect mapping on node 6. + {"file0", "file333"}, // Expect mapping on node 3. + {"file0", "file11"}, // Expect mapping on node 1. + {"file0", "file44"}, // Expect mapping on node 4. + {"file0", "file55", "file555"} // Expect mapping on node 5. + }; + + TVector<ui32> expectedTasksMapping = {6, 3, 1, 4, 5}; + + CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping); + } + + void CheckTasksAllocation( + TVector<TVector<TString>>& filesPerNode, + TVector<TVector<TString>>& filesPerTask, + const TVector<ui32>& expectedTasksMapping + ) { + Y_ASSERT(expectedTasksMapping.size() == filesPerTask.size()); + Y_ASSERT(filesPerTask.size() <= filesPerNode.size()); + + SetupWorkerNodes(filesPerNode); + + const auto execActor = ActorRuntime_->AllocateEdgeActor(); + const auto allocatorActor = RegisterResourceAllocator(filesPerTask.size(), execActor); + + auto allocateRequest = MakeAllocationRequest(filesPerTask); + + ActorRuntime_->Send(new IEventHandle(MakeWorkerManagerActorID(NodeId()), allocatorActor, allocateRequest.Release())); + + auto resp = ActorRuntime_->GrabEdgeEvent<TEvAllocateWorkersResponse>(execActor); + + auto allocatedWorkers = resp->Get()->Record.GetWorkers(); + + UNIT_ASSERT_VALUES_EQUAL(filesPerTask.size(), allocatedWorkers.WorkerSize()); + for (ui32 task = 0; task < expectedTasksMapping.size(); ++task) { + ui32 expectedNode = expectedTasksMapping[task]; + UNIT_ASSERT_VALUES_EQUAL(NodeId(expectedNode), allocatedWorkers.worker(task).GetNodeId()); + } + } + + TActorId RegisterResourceAllocator(const ui32 workersCount, const TActorId& execActor) const { + TIntrusivePtr<NMonitoring::TDynamicCounters> counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + auto gwmActor = MakeWorkerManagerActorID(NodeId()); + auto allocator = CreateResourceAllocator(gwmActor, execActor, execActor, workersCount, "TraceId", new TDqConfiguration(), counters); + const auto allocatorId = ActorRuntime_->Register(allocator); + return allocatorId; + } + + void SetupWorkerNodes(TVector<TVector<TString>>& filesOnNodes) const { + ui32 nodeId = 1; + for (const auto& nodeFiles : filesOnNodes) { + RegisterNodeInGwm(NodeId(nodeId++), nodeFiles); + } + } + + THolder<TEvAllocateWorkersRequest> MakeAllocationRequest(TVector<TVector<TString>>& filesPerTask) const { + auto allocateRequest = MakeHolder<TEvAllocateWorkersRequest>(filesPerTask.size(), "Username"); + allocateRequest->Record.SetTraceId("TraceId"); + + THashSet<TString> allFiles; + + for (const auto& tf : filesPerTask) { + Yql::DqsProto::TWorkerFilter taskFiles; + for (const auto& f : tf) { + Yql::DqsProto::TFile file; + file.SetObjectId(f + "_id"); + file.SetLocalPath("/tmp/test"); + *taskFiles.AddFile() = file; + allFiles.emplace(f); + } + *allocateRequest->Record.AddWorkerFilterPerTask() = taskFiles; + } + + for (const auto& f : allFiles) { + Yql::DqsProto::TFile file; + file.SetObjectId(f + "_id"); + file.SetLocalPath("/tmp/test"); + *allocateRequest->Record.AddFiles() = file; + } + + return allocateRequest; + } + + ui32 NodeId(ui32 id = 0) const { + return ActorRuntime_->GetNodeId(id); + } + + void RegisterNodeInGwm(const ui32 nodeId, const TVector<TString>& files) const { + Yql::DqsProto::RegisterNodeRequest req; + req.SetCapabilities(Yql::DqsProto::RegisterNodeRequest::ECAP_RUNEXE); + req.SetNodeId(nodeId); + req.SetPort(1); + req.SetRole("worker_node"); + req.SetAddress("Address"); + req.SetRevision(ToString(GetProgramCommitId())); + req.SetRunningWorkers(1); + req.SetClusterName("plato"); + req.MutableKnownNodes()->Add(NodeId()); + req.MutableKnownNodes()->Add(nodeId); + req.MutableGuid()->SetDw0(1); + req.MutableGuid()->SetDw1(1); + req.MutableGuid()->SetDw2(1); + req.MutableGuid()->SetDw3(nodeId); + + for (const auto& file : files) { + auto fileOnNode = req.AddFilesOnNode(); + fileOnNode->SetObjectId(file + "_id"); + } + + auto ev = MakeHolder<TEvRegisterNode>(req); + const auto sender = ActorRuntime_->AllocateEdgeActor(); + ActorRuntime_->Send(new IEventHandle(MakeWorkerManagerActorID(NodeId()), sender, ev.Release())); + + ActorRuntime_->GrabEdgeEvent<TEvRegisterNodeResponse>(sender); + } + + THolder<NActors::TTestActorRuntimeBase> ActorRuntime_; +}; + +UNIT_TEST_SUITE_REGISTRATION(TGlobalWorkerManagerTest) diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp b/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp new file mode 100644 index 0000000000..7a2d192516 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp @@ -0,0 +1,398 @@ +#include "service_node_pinger.h" +#include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> + +#include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/library/yql/utils/log/log.h> + +#include <library/cpp/actors/interconnect/interconnect.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <util/generic/guid.h> +#include <util/system/getpid.h> +#include <util/system/fs.h> + +#include <ydb/library/yql/providers/dq/api/grpc/api.grpc.pb.h> +#include <ydb/library/yql/providers/dq/runtime/runtime_data.h> + +#include <ydb/library/yql/providers/dq/actors/events/events.h> +#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h> +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/execution_helpers.h> +#include <ydb/library/yql/providers/dq/task_runner/file_cache.h> + +namespace NYql { + +using namespace NActors; + +struct TEvRegisterNodeResponse + : NActors::TEventLocal<TEvRegisterNodeResponse, TDqEvents::ES_OTHER1> { + TEvRegisterNodeResponse() + : Error(true) + { } + + TEvRegisterNodeResponse(const Yql::DqsProto::RegisterNodeResponse& res) + : Error(false) + , Response(res) + { } + + bool Error; + const Yql::DqsProto::RegisterNodeResponse Response; +}; + +class TServiceNodePinger: public TActor<TServiceNodePinger> { +public: + static constexpr char ActorName[] = "PINGER"; + + TServiceNodePinger( + ui32 nodeId, + const TString& address, + ui16 port, + const TString& role, + const THashMap<TString, TString>& attributes, + const IServiceNodeResolver::TPtr& resolver, + const ICoordinationHelper::TPtr& coordinator, + const TResourceManagerOptions& options) + : TActor<TServiceNodePinger>(&TServiceNodePinger::Handler) + , NodeId(nodeId) + , Address(address) + , Port(port) + , Role(role) + , Attributes(attributes) + , Resolver(resolver) + , Revision(coordinator->GetRevision()) + , Options(options) + , RuntimeData(coordinator->GetRuntimeData()) + , Coordinator(coordinator) + { + CreateGuid(&Guid); + + RuntimeData->WorkerId = Guid; + if (Options.AnnounceClusterName) { + RuntimeData->ClusterName = *Options.AnnounceClusterName; + } else { + RuntimeData->ClusterName = Options.YtBackend.GetClusterName(); + } + + if (coordinator->GetConfig().HasHeartbeatPeriodMs()) { + HeartbeatPeriod = TDuration::MilliSeconds(coordinator->GetConfig().GetHeartbeatPeriodMs()); + } + + YQL_CLOG(DEBUG, ProviderDq) << "Node started nodeId|role|guid " << NodeId << "|" << Role << "|" << GetGuidAsString(Guid); + } + + ~TServiceNodePinger() { + Resolver->Stop(); + } + + STRICT_STFUNC(Handler, { + CFunc(TEvents::TEvBootstrap::EventType, Ping) + HFunc(TEvInterconnect::TEvNodesInfo, OnNodesInfo) + HFunc(TEvDownloadComplete, OnDownloadComplete) + HFunc(TEvRegisterNodeResponse, OnRegisterNodeResponse) + cFunc(TEvents::TEvPoison::EventType, PassAway) + }); + +private: + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0); + } + + void OnDownloadComplete(TEvDownloadComplete::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + + auto id = ev->Sender; + auto it = DownloadProcess.find(id); + if (it == DownloadProcess.end()) { + return; + } + + for (auto file : it->second) { + Downloading.erase(/*objectId = */ file.LocalFileName); + } + + DownloadProcess.erase(it); + } + + void OnNodesInfo(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + KnownNodes.clear(); + KnownNodes.reserve(ev->Get()->Nodes.size()); + for (const auto& node: ev->Get()->Nodes) { + KnownNodes.push_back(node.NodeId); + } + } + + void SchedulePing() { + auto now = TInstant::Now(); + auto delta = now - LastPingTime; + if (delta > HeartbeatPeriod) { +// YQL_CLOG(DEBUG, ProviderDq) << "Ping Now"; + Send(SelfId(), new TEvents::TEvBootstrap); + } else { + // YQL_CLOG(DEBUG, ProviderDq) << "Ping After " << (HeartbeatPeriod - delta).MilliSeconds(); + Schedule(HeartbeatPeriod - delta, new TEvents::TEvBootstrap); + } + } + + void OnRegisterNodeResponse(TEvRegisterNodeResponse::TPtr& ev, const TActorContext& ctx) { +// YQL_CLOG(DEBUG, ProviderDq) << "Pong"; + + if (ev->Get()->Error) { + Errors += 1; + Oks = 0; + if (Options.ExitOnPingFail && Errors > 3) { + YQL_CLOG(DEBUG, ProviderDq) << "ExitOnPingFail"; + _exit(-1); + } + SchedulePing(); + return; + } + + Oks += 1; + Errors = 0; + + auto& resp = ev->Get()->Response; + + THolder<TEvInterconnect::TEvNodesInfo> + reply(new TEvInterconnect::TEvNodesInfo()); + reply->Nodes.reserve(resp.GetNodes().size()); + if (resp.GetEpoch()) { + RuntimeData->Epoch = resp.GetEpoch(); + } + + for (auto& node : resp.GetNodes()) { + reply->Nodes.emplace_back( + node.GetNodeId(), + node.GetAddress(), + node.GetAddress(), + node.GetAddress(), + node.GetPort(), + NActors::TNodeLocation()); + } + + TVector<TResourceFile> downloadList; + for (auto& file : resp.GetDownloadList()) { + if (Downloading.contains(file.GetObjectId()) || (Options.FileCache && Options.FileCache->Contains(file.GetObjectId()))) + { + continue; + } + TResourceFile resource; + + switch (file.GetObjectType()) { + case Yql::DqsProto::TFile::EEXE_FILE: + resource.RemoteFileName = "bin/" + file.GetObjectId() + "/" + file.GetName(); + break; + default: + resource.RemoteFileName = "udfs/" + file.GetObjectId(); + break; + } + + resource.LocalFileName = file.GetObjectId(); + + downloadList.push_back(resource); + + YQL_CLOG(DEBUG, ProviderDq) << "Start downloading: " << file.GetObjectId(); + + Downloading.insert(file.GetObjectId()); + } + + if (!downloadList.empty() && Options.FileCache) { + auto rmOptions = Options; + rmOptions.Files = downloadList; + rmOptions.UploadPrefix = rmOptions.YtBackend.GetUploadPrefix(); + + YQL_CLOG(DEBUG, ProviderDq) << "Start downloading from " << rmOptions.YtBackend.GetClusterName(); + DownloadProcess.emplace(ctx.Register(CreateResourceDownloader(rmOptions, Coordinator)), downloadList); + } + + Send(NActors::GetNameserviceActorId(), reply.Release()); + Send(NActors::GetNameserviceActorId(), new TEvInterconnect::TEvListNodes); + SchedulePing(); + } + + void Ping(const TActorContext& ctx) { + Yql::DqsProto::RegisterNodeRequest req; + req.SetCapabilities(Yql::DqsProto::RegisterNodeRequest::ECAP_RUNEXE | Options.Capabilities); + req.SetNodeId(NodeId); + req.SetPort(Port); + req.SetRole(Role); + req.SetAddress(Address); + req.SetRevision(Revision); + req.SetPid(Pid); + auto runningWorkers = RuntimeData->GetRunningWorkers(); + int sum = 0; + for (const auto& [id, count] : runningWorkers) { + req.AddRunningOperation(id); + sum += count; + } + req.SetRunningWorkers(sum); + req.SetClusterName(RuntimeData->ClusterName); + req.SetStartTime(StartTime); + if (RuntimeData->Epoch) { + req.SetEpoch(RuntimeData->Epoch); + } + auto rusageFull = RuntimeData->GetRusage(); + req.MutableRusage()->SetStime(rusageFull.Stime.MicroSeconds()); + req.MutableRusage()->SetUtime(rusageFull.Utime.MicroSeconds()); + req.MutableRusage()->SetMajorPageFaults(rusageFull.MajorPageFaults); + + //if (Options.MetricsRegistry) { + // auto* metrics = req.MutableMetricsRegistry(); + // metrics->SetDontIncrement(true); // don't invalidate metrics + // Options.MetricsRegistry->TakeSnapshot(metrics); + //} + + if (Options.FileCache) { + req.SetUsedDiskSize(Options.FileCache->UsedDiskSize()); + req.SetFreeDiskSize(Options.FileCache->FreeDiskSize()); + } + for (const auto& [k, v] : Attributes) { + auto* attr = req.AddAttribute(); + attr->SetKey(k); + attr->SetValue(v); + } + + NDqs::NExecutionHelpers::GuidToProto(*req.MutableGuid(), Guid); + req.SetCapacity(Options.YtBackend.GetWorkerCapacity()); + + if (Options.FileCache) { + Options.FileCache->Walk([&] (const TString& objectId) { + req.AddFilesOnNode()->SetObjectId(objectId); + }); + } + + // self exe must be at the end of list + req.AddFilesOnNode()->SetObjectId(Revision); + + for (auto node : KnownNodes) { + req.AddKnownNodes(node); + } + + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + auto selfId = SelfId(); + + Resolver->GetConnection() + .Apply([actorSystem, selfId, req, maybeResolver=std::weak_ptr<IServiceNodeResolver>(Resolver), timeout=HeartbeatPeriod, lastPingTime=LastPingTime, errors=Errors, oks=Oks] (const NThreading::TFuture<IServiceNodeResolver::TConnectionResult>& resultFuture) { + const auto& result = resultFuture.GetValueSync(); + if (!result.Success()) { + YQL_CLOG(DEBUG, ProviderDq) << "Cannot resolve service node"; + actorSystem->Send(selfId, new TEvRegisterNodeResponse()); + return; + } + + if (!maybeResolver.lock()) { + return; + } + + if (!lastPingTime || errors > 1 || oks < 2 || !result.NodeId) { + // GRPC ping + NGrpc::TCallMeta meta; + meta.Timeout = timeout; + result.Connection->DoRequest<Yql::DqsProto::RegisterNodeRequest, Yql::DqsProto::RegisterNodeResponse>( + req, [=] (NGrpc::TGrpcStatus&& status, Yql::DqsProto::RegisterNodeResponse&& resp) { + if (!status.Ok()) { + YQL_CLOG(DEBUG, ProviderDq) << "Error on service node ping " << status.Msg; + if (auto resolver = maybeResolver.lock()) { + resolver->InvalidateCache(); + } + Y_ABORT_UNLESS(status.GRpcStatusCode != grpc::INVALID_ARGUMENT); + actorSystem->Send(selfId, new TEvRegisterNodeResponse()); + return; + } + + actorSystem->Send(selfId, new TEvRegisterNodeResponse(resp)); + }, &Yql::DqsProto::DqService::Stub::AsyncRegisterNode, meta, result.GRpcContext.get()); + } else { + // IC Ping + YQL_CLOG(DEBUG, ProviderDq) << "IC Ping " << result.NodeId; + auto ev = MakeHolder<NDqs::TEvRegisterNode>(req); + using ResultEv = NDqs::TEvRegisterNodeResponse; + + // Handle errors and timeouts + auto callback = MakeHolder<TRichActorFutureCallback<ResultEv>>( + [actorSystem, selfId] (TAutoPtr<TEventHandle<ResultEv>>& event) mutable { + actorSystem->Send(selfId, new TEvRegisterNodeResponse(event->Get()->Record.GetResponse())); + }, + [actorSystem, selfId, maybeResolver] () mutable { + YQL_CLOG(DEBUG, ProviderDq) << "Error on service node ping"; + if (auto resolver = maybeResolver.lock()) { + resolver->InvalidateCache(); + } + actorSystem->Send(selfId, new TEvRegisterNodeResponse()); + }, + timeout); + + TActorId callbackId = actorSystem->Register(callback.Release()); + + actorSystem->Send(new IEventHandle( + NDqs::MakeWorkerManagerActorID(result.NodeId), + callbackId, + ev.Release(), + IEventHandle::FlagTrackDelivery)); + } + }); + + LastPingTime = TInstant::Now(); + + CheckFs(); + } + + void CheckFs() { + if (Options.DieOnFileAbsence.empty()) { + return; + } + + auto now = TInstant::Now(); + + if (now - LastCheckTime < TDuration::Seconds(10)) { + return; + } + + Y_ABORT_UNLESS(NFs::Exists(Options.DieOnFileAbsence)); + + LastCheckTime = now; + } + +private: + const ui32 NodeId; + const TString Address; + const ui16 Port; + const TString Role; + const THashMap<TString, TString> Attributes; + const IServiceNodeResolver::TPtr Resolver; + TVector<ui32> KnownNodes; + const TString Revision; + const TString StartTime = ToString(TInstant::Now()); + const TProcessId Pid = GetPID(); + + TResourceManagerOptions Options; + TWorkerRuntimeData* RuntimeData; + const ICoordinationHelper::TPtr Coordinator; + + THashSet<TString> Downloading; + THashMap<TActorId, TVector<TResourceFile>> DownloadProcess; + + TDuration HeartbeatPeriod = TDuration::Seconds(5); + + TGUID Guid; + TInstant LastPingTime; + TInstant LastCheckTime; + int Errors = 0; + int Oks = 0; +}; + +IActor* CreateServiceNodePinger( + ui32 nodeId, + const TString& address, + ui16 port, + const TString& role, + const THashMap<TString, TString>& attributes, + const IServiceNodeResolver::TPtr& ptr, + const ICoordinationHelper::TPtr& coordinator, + const TResourceManagerOptions& rmOptions) +{ + return new TServiceNodePinger(nodeId, address, port, role, attributes, ptr, coordinator, rmOptions); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.h b/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.h new file mode 100644 index 0000000000..4a7efd1209 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.h @@ -0,0 +1,23 @@ +#pragma once + +#include <library/cpp/actors/core/actor.h> + +#include "service_node_resolver.h" +#include "coordination_helper.h" + +namespace NYql { + +struct TWorkerRuntimeData; +struct TResourceManagerOptions; + +NActors::IActor* CreateServiceNodePinger( + ui32 nodeId, + const TString& address, + ui16 port, + const TString& role, + const THashMap<TString, TString>& attributes, + const IServiceNodeResolver::TPtr& ptr, + const ICoordinationHelper::TPtr& coordinator, + const TResourceManagerOptions& rmOptions); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp new file mode 100644 index 0000000000..744abcdf68 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp @@ -0,0 +1,303 @@ +#include "service_node_resolver.h" + +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/yson/node/node_io.h> + +#include <ydb/library/yql/utils/log/log.h> +#include <ydb/library/yql/utils/yql_panic.h> + +#include <ydb/library/yql/providers/dq/actors/actor_helpers.h> +#include <ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h> +#include <ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h> + +#include <util/system/mutex.h> +#include <util/random/random.h> + +namespace NYql { + +using namespace NThreading; +using namespace NActors; + +struct TNodeInfo { + NGrpc::TGRpcClientConfig ClientConfig; + ui32 NodeId; +}; + +class TNodesHolder { +public: + TMaybe<TNodeInfo> GetNode() { + TGuard<TMutex> guard(Mutex); + if (Nodes.empty()) { + return {}; + } else { + auto index = RandomNumber(Min<ui32>(3, Nodes.size())); + return Nodes[index]; + } + } + + void Swap(TVector<TNodeInfo>& nodes) + { + TGuard<TMutex> guard(Mutex); + Nodes.swap(nodes); + } + +private: + TMutex Mutex; + TVector<TNodeInfo> Nodes; +}; + +class TAbstractNodeResolver: public IServiceNodeResolver { +public: + TAbstractNodeResolver() + : ClientLow(1) + , ChannelPool(NGrpc::TTcpKeepAliveSettings{true, 30, 5, 10}) + { } + + ~TAbstractNodeResolver() + { + Stop(); + } + + virtual TMaybe<TNodeInfo> GetNode() = 0; + + TFuture<TConnectionResult> GetConnection() override { + auto nodeInfo = GetNode(); + + if (!nodeInfo) { + return MakeFuture(NCommon::ResultFromError<TConnectionResult>("Unable to get node")); + } + + auto context = ClientLow.CreateContext(); + if (!context) { + return MakeFuture(NCommon::ResultFromError<TConnectionResult>("Unable to create grpc request context")); + } + + std::unique_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>> conn; + ChannelPool.GetStubsHolderLocked( + nodeInfo->ClientConfig.Locator, nodeInfo->ClientConfig, [&conn, this](NGrpc::TStubsHolder& holder) mutable { + conn.reset(ClientLow.CreateGRpcServiceConnection<Yql::DqsProto::DqService>(holder).release()); + }); + + return MakeFuture(TConnectionResult(std::move(conn), std::move(context), nodeInfo->NodeId, nodeInfo->ClientConfig.Locator)); + } + + void Stop() override { + ClientLow.Stop(true); + } + +protected: + NGrpc::TGRpcClientLow ClientLow; + NGrpc::TChannelPool ChannelPool; +}; + +class TNodeUpdater: public TActor<TNodeUpdater> { +public: + static constexpr char ActorName[] = "NODE_UPDATER"; + + TNodeUpdater(const std::shared_ptr<TNodesHolder>& holder, const TDynamicResolverOptions& options) + : TActor(&TNodeUpdater::Handler) + , Holder(holder) + , Options(options) + , LastUpdateTime(TInstant::Now()) + { } + + TEvListNode* ListNodeCommand() { + NYT::NApi::TListNodeOptions options; + options.Attributes = { + NCommonAttrs::ACTOR_NODEID_ATTR, + NCommonAttrs::ROLE_ATTR, + NCommonAttrs::HOSTNAME_ATTR, + NCommonAttrs::GRPCPORT_ATTR, + "modification_time" + }; + if (InvalidateCache) { + InvalidateCache = false; + } else { + options.ReadFrom = NYT::NApi::EMasterChannelKind::Cache; + } + return new TEvListNode(Options.Prefix, options); + } + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + Y_UNUSED(parentId); + return new IEventHandle(Options.YtWrapper, self, ListNodeCommand(), 0); + } + + STRICT_STFUNC(Handler, { + HFunc(TEvListNodeResponse, OnListNodeResponse) + cFunc(NActors::TEvents::TEvPoison::EventType, PassAway) + cFunc(NActors::TEvents::TEvWakeup::EventType, [&] () { + InvalidateCache = true; + }) + }); + + void ProcessResult(const TVector<NYT::TNode>& nodes) + { + TVector<std::tuple<TInstant, TString, ui32>> hostPort; + hostPort.reserve(nodes.size()); + for (const auto& node : nodes) { + const auto& attributes = node.GetAttributes().AsMap(); + auto maybeRole = attributes.find(NCommonAttrs::ROLE_ATTR); + auto maybeHostname = attributes.find(NCommonAttrs::HOSTNAME_ATTR); + auto maybeGrpcPort = attributes.find(NCommonAttrs::GRPCPORT_ATTR); + auto maybeNodeId = attributes.find(NCommonAttrs::ACTOR_NODEID_ATTR); + + auto maybeModificationTime = attributes.find("modification_time"); + YQL_ENSURE(maybeModificationTime != attributes.end()); + + if (maybeRole == attributes.end() + || maybeHostname == attributes.end() + || maybeGrpcPort == attributes.end() + || maybeNodeId == attributes.end()) + { + continue; + } + + if (maybeRole->second != "service_node") { + continue; + } + + ui32 nodeId = maybeNodeId->second.AsUint64(); + + auto modificationTime = TInstant::ParseIso8601(maybeModificationTime->second.AsString()); + + hostPort.emplace_back(modificationTime, maybeHostname->second.AsString() + ":" + maybeGrpcPort->second.AsString(), nodeId); + } + + std::sort(hostPort.begin(), hostPort.end()); + + TVector<TNodeInfo> locations; + locations.reserve(hostPort.size()); + for (auto it = hostPort.rbegin(); it != hostPort.rend(); ++it) { + locations.push_back({NGrpc::TGRpcClientConfig(std::get<1>(*it), TDuration::Seconds(15)), std::get<2>(*it)}); + } + + if (locations.empty()) { + YQL_CLOG(WARN, ProviderDq) << "Empty locations"; + } + + Holder->Swap(locations); + } + + void OnListNodeResponse(TEvListNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + + auto now = TInstant::Now(); + + auto result = std::get<0>(*ev->Get()); + + try { + ProcessResult(NYT::NodeFromYsonString(result.ValueOrThrow()).AsList()); + } catch (...) { + YQL_CLOG(ERROR, ProviderDq) << "Error on list node '" << ToString(result) << "' " << CurrentExceptionMessage(); + } + + TActivationContext::Schedule(Options.UpdatePeriod, new IEventHandle(Options.YtWrapper, SelfId(), ListNodeCommand(), 0)); + + LastUpdateTime = now; + } + +private: + std::shared_ptr<TNodesHolder> Holder; + TDynamicResolverOptions Options; + TInstant LastUpdateTime; + bool InvalidateCache = false; +}; + +class TDynamicResolver: public TAbstractNodeResolver { +public: + TDynamicResolver(NActors::TActorSystem* actorSystem, const TDynamicResolverOptions& options) + : Nodes(new TNodesHolder) + , ActorSystem(actorSystem) + , Options(options) + , NodeUpdater(ActorSystem->Register(new TNodeUpdater(Nodes, Options))) + { } + + ~TDynamicResolver() + { } + + TMaybe<TNodeInfo> GetNode() override { + return Nodes->GetNode(); + } + + void InvalidateCache() override { + ActorSystem->Send(NodeUpdater, new TEvents::TEvWakeup); + } + +private: + std::shared_ptr<TNodesHolder> Nodes; + + NActors::TActorSystem* ActorSystem; + TDynamicResolverOptions Options; + NActors::TActorId NodeUpdater; +}; + +class TStaticResolver: public TAbstractNodeResolver { +public: + TStaticResolver(const TVector<TString>& hostPortPairs) + { + Nodes.reserve(hostPortPairs.size()); + for (const auto& hostPort : hostPortPairs) { + Nodes.emplace_back(hostPort, TDuration::Seconds(1)); + } + } + + TMaybe<TNodeInfo> GetNode() override { + auto clientConfig = NGrpc::TGRpcClientConfig{Nodes[CurrentNodeIndex]}; + CurrentNodeIndex = (CurrentNodeIndex + 1) % Nodes.size(); + + return TNodeInfo{clientConfig, 0}; + } + + void InvalidateCache() override { + } + +private: + TVector<NGrpc::TGRpcClientConfig> Nodes; + int CurrentNodeIndex = 0; +}; + +TSingleNodeResolver::TSingleNodeResolver() + : ClientLow(1) + , ChannelPool(NGrpc::TTcpKeepAliveSettings{true, 30, 5, 10}) +{ } + +TSingleNodeResolver::~TSingleNodeResolver() +{ + ClientLow.Stop(true); +} + +TFuture<IServiceNodeResolver::TConnectionResult> TSingleNodeResolver::GetConnection() { + auto clientConfig = NGrpc::TGRpcClientConfig(LeaderHostPort); + + auto context = ClientLow.CreateContext(); + if (!context) { + return MakeFuture(NCommon::ResultFromError<IServiceNodeResolver::TConnectionResult>("Unable to create grpc request context")); + } + + std::unique_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>> conn; + ChannelPool.GetStubsHolderLocked( + clientConfig.Locator, clientConfig, [&conn, this](NGrpc::TStubsHolder& holder) mutable { + conn.reset(ClientLow.CreateGRpcServiceConnection<Yql::DqsProto::DqService>(holder).release()); + }); + + return MakeFuture(IServiceNodeResolver::TConnectionResult(std::move(conn), std::move(context))); +} + +void TSingleNodeResolver::SetLeaderHostPort(const TString& leaderHostPort) { + LeaderHostPort = leaderHostPort; +} + +IServiceNodeResolver::TPtr CreateStaticResolver(const TVector<TString>& hostPortPairs) { + return std::make_shared<TStaticResolver>(hostPortPairs); +} + +IServiceNodeResolver::TPtr CreateDynamicResolver(NActors::TActorSystem* actorSystem, const TDynamicResolverOptions& options) { + Y_ABORT_UNLESS(options.YtWrapper); + Y_ABORT_UNLESS(!options.Prefix.empty()); + + return std::make_shared<TDynamicResolver>(actorSystem, options); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h new file mode 100644 index 0000000000..c2647904b1 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h @@ -0,0 +1,80 @@ +#pragma once + +#include <ydb/library/yql/providers/common/gateway/yql_provider_gateway.h> +#include <ydb/library/yql/providers/dq/api/grpc/api.grpc.pb.h> + +#include <util/generic/string.h> + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/threading/future/future.h> +#include <library/cpp/grpc/client/grpc_client_low.h> + +namespace NYql { + +class IServiceNodeResolver { +public: + using TPtr = std::shared_ptr<IServiceNodeResolver>; + + struct TConnectionResult : public NCommon::TOperationResult { + + TConnectionResult() {} + + TConnectionResult( + std::unique_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>>&& connection, + std::shared_ptr<NGrpc::IQueueClientContext>&& ctx = nullptr, + ui32 nodeId = 0, + const TString& location = "") + : Connection(connection.release()) + , GRpcContext(std::move(ctx)) + , NodeId(nodeId) + , Location(location) + { + if (GRpcContext) { + SetSuccess(); + } + } + + std::shared_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>> Connection; + std::shared_ptr<NGrpc::IQueueClientContext> GRpcContext; + ui32 NodeId; + TString Location; + }; + + virtual ~IServiceNodeResolver() = default; + virtual NThreading::TFuture<TConnectionResult> GetConnection() = 0; + virtual void InvalidateCache() = 0; + virtual void Stop() = 0; +}; + +struct TDynamicResolverOptions { + NActors::TActorId YtWrapper; + TString Prefix; + TDuration UpdatePeriod = TDuration::Seconds(5); + TDuration RetryPeriod = TDuration::Seconds(10); +}; + +class TSingleNodeResolver: public IServiceNodeResolver { +public: + TSingleNodeResolver(); + + ~TSingleNodeResolver(); + + NThreading::TFuture<IServiceNodeResolver::TConnectionResult> GetConnection() override; + + void SetLeaderHostPort(const TString& leaderHostPort); + + void InvalidateCache() override { } + + void Stop() override { } + +private: + NGrpc::TGRpcClientLow ClientLow; + NGrpc::TChannelPool ChannelPool; + + TString LeaderHostPort; +}; + +IServiceNodeResolver::TPtr CreateStaticResolver(const TVector<TString>& hostPortPairs); +IServiceNodeResolver::TPtr CreateDynamicResolver(NActors::TActorSystem* actorSystem, const TDynamicResolverOptions& options); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..d2252b933a --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,77 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(yql-providers-dq-global_worker_manager-ut) +target_compile_options(yql-providers-dq-global_worker_manager-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(yql-providers-dq-global_worker_manager-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager +) +target_link_libraries(yql-providers-dq-global_worker_manager-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + providers-dq-global_worker_manager + cpp-actors-testlib + udf-service-stub + yql-sql-pg_dummy + dq-actors-yt + providers-dq-actors + dq-actors-compute +) +target_link_options(yql-providers-dq-global_worker_manager-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(yql-providers-dq-global_worker_manager-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp +) +set_property( + TARGET + yql-providers-dq-global_worker_manager-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + yql-providers-dq-global_worker_manager-ut + TEST_TARGET + yql-providers-dq-global_worker_manager-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + yql-providers-dq-global_worker_manager-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + yql-providers-dq-global_worker_manager-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(yql-providers-dq-global_worker_manager-ut + system_allocator +) +vcs_info(yql-providers-dq-global_worker_manager-ut) diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..5914005f21 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,81 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(yql-providers-dq-global_worker_manager-ut) +target_compile_options(yql-providers-dq-global_worker_manager-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(yql-providers-dq-global_worker_manager-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager +) +target_link_libraries(yql-providers-dq-global_worker_manager-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + providers-dq-global_worker_manager + cpp-actors-testlib + udf-service-stub + yql-sql-pg_dummy + dq-actors-yt + providers-dq-actors + dq-actors-compute +) +target_link_options(yql-providers-dq-global_worker_manager-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl + -lutil +) +target_sources(yql-providers-dq-global_worker_manager-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp +) +set_property( + TARGET + yql-providers-dq-global_worker_manager-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + yql-providers-dq-global_worker_manager-ut + TEST_TARGET + yql-providers-dq-global_worker_manager-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + yql-providers-dq-global_worker_manager-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + yql-providers-dq-global_worker_manager-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(yql-providers-dq-global_worker_manager-ut + cpp-malloc-jemalloc +) +vcs_info(yql-providers-dq-global_worker_manager-ut) diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..7a7c261c19 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,83 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(yql-providers-dq-global_worker_manager-ut) +target_compile_options(yql-providers-dq-global_worker_manager-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(yql-providers-dq-global_worker_manager-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager +) +target_link_libraries(yql-providers-dq-global_worker_manager-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + providers-dq-global_worker_manager + cpp-actors-testlib + udf-service-stub + yql-sql-pg_dummy + dq-actors-yt + providers-dq-actors + dq-actors-compute +) +target_link_options(yql-providers-dq-global_worker_manager-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl + -lutil +) +target_sources(yql-providers-dq-global_worker_manager-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp +) +set_property( + TARGET + yql-providers-dq-global_worker_manager-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + yql-providers-dq-global_worker_manager-ut + TEST_TARGET + yql-providers-dq-global_worker_manager-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + yql-providers-dq-global_worker_manager-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + yql-providers-dq-global_worker_manager-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(yql-providers-dq-global_worker_manager-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(yql-providers-dq-global_worker_manager-ut) diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..a2c865ae35 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,14 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(dq-global_worker_manager-ut INTERFACE) +target_link_libraries(dq-global_worker_manager-ut INTERFACE + contrib-libs-cxxsupp + yutil +) diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/ya.make b/ydb/library/yql/providers/dq/global_worker_manager/ut/ya.make new file mode 100644 index 0000000000..b67eba48b8 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/ya.make @@ -0,0 +1,29 @@ +IF (NOT OS_WINDOWS) + UNITTEST_FOR(ydb/library/yql/providers/dq/global_worker_manager) + + SIZE(SMALL) + + PEERDIR( + library/cpp/actors/testlib + ydb/library/yql/public/udf/service/stub + ydb/library/yql/sql/pg_dummy + ydb/library/yql/providers/dq/actors/yt + ydb/library/yql/providers/dq/actors + ydb/library/yql/dq/actors/compute + ) + + SRCS( + global_worker_manager_ut.cpp + workers_storage_ut.cpp + ) + + INCLUDE(${ARCADIA_ROOT}/ydb/tests/supp/ubsan_supp.inc) + + YQL_LAST_ABI_VERSION() + + END() +ELSE() + LIBRARY() + + END() +ENDIF() diff --git a/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp b/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp new file mode 100644 index 0000000000..8b5f443c63 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp @@ -0,0 +1,70 @@ +#include "worker_filter.h" + +namespace NYql { + +using namespace NDqs; + +TWorkerFilter::TWorkerFilter(const Yql::DqsProto::TWorkerFilter& filter) + : Filter(filter) + , FullMatch( + Filter.GetClusterName() + || (Filter.GetAddress().size() > 0) + || (Filter.GetNodeId().size() > 0) + || Filter.GetRevision()) +{ + for (const auto& address : Filter.GetAddress()) { + Addresses.insert(address); + } + for (const auto& nodeId : Filter.GetNodeId()) { + NodeIds.insert(nodeId); + } + for (const auto& nodeId : Filter.GetNodeIdHint()) { + NodeIdHints.insert(nodeId); + } +} + +TWorkerFilter::EMatchStatus TWorkerFilter::Match(const TWorkerInfo::TPtr& workerInfo, int taskId, TStats* stats) const { + bool allExists = true; + bool partial = false; + if (FullMatch) { + if (Filter.GetClusterName() && workerInfo->ClusterName != Filter.GetClusterName()) { + return EFAIL; + } + if (!Addresses.empty() && Addresses.find(workerInfo->Address) == Addresses.end()) { + return EFAIL; + } + if (!NodeIds.empty() && NodeIds.find(workerInfo->NodeId) == NodeIds.end()) { + return EFAIL; + } + } + if (Filter.GetClusterNameHint() && workerInfo->ClusterName != Filter.GetClusterNameHint()) { + partial = true; + } + if (!NodeIdHints.empty() && NodeIdHints.find(workerInfo->NodeId) == NodeIdHints.end()) { + partial = true; + } + for (const auto& file : Filter.GetFile()) { + const auto& id = file.GetObjectId(); + auto flag = workerInfo->GetResources().contains(id); + allExists &= flag; + if (stats) { + if (flag) { + (*stats->WaitingResources)[id].insert(taskId); + } else { + (*stats->WaitingResources)[id].erase(taskId); + stats->Uploaded->find(id)->second.TryCount ++; + } + } + } + return allExists + ? (partial?EPARTIAL:EOK) + : EFAIL; +} + +void TWorkerFilter::Visit(const std::function<void(const Yql::DqsProto::TFile&)>& visitor) const { + for (const auto& file : Filter.GetFile()) { + visitor(file); + } +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.h b/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.h new file mode 100644 index 0000000000..8e938e72d4 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.h @@ -0,0 +1,54 @@ +#pragma once + +#include <ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h> + +namespace NYql { + +struct TResourceStat { + TString Id; + Yql::DqsProto::TFile::EFileType ObjectType; + TString Name; + i64 Size; + TDuration UseTime; + TDuration WaitTime; + i64 UseCount = 0; + i64 TryCount = 0; + bool Uploaded = false; + TInstant LastSeenTime = TInstant::Now(); + + TResourceStat(const TString& id, Yql::DqsProto::TFile::EFileType type, const TString& name, i64 size) + : Id(id) + , ObjectType(type) + , Name(name) + , Size(size) + { } +}; + +class TWorkerFilter { +public: + struct TStats { + THashMap<TString, THashSet<int>>* WaitingResources; + THashMap<TString, TResourceStat>* Uploaded; + }; + + enum EMatchStatus { + EFAIL = 0, + EOK = 1, + EPARTIAL = 2 + }; + + TWorkerFilter(const Yql::DqsProto::TWorkerFilter& filter); + + EMatchStatus Match(const NDqs::TWorkerInfo::TPtr& workerInfo, int taskId, TStats* stats) const; + + void Visit(const std::function<void(const Yql::DqsProto::TFile&)>& visitor) const; + +private: + Yql::DqsProto::TWorkerFilter Filter; + bool FullMatch; + THashSet<TString> Addresses; + THashSet<ui64> NodeIds; + THashSet<ui64> NodeIdHints; +}; + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp new file mode 100644 index 0000000000..f2f0206b93 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp @@ -0,0 +1,580 @@ +#include "workers_storage.h" + +#include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/library/yql/utils/log/log.h> + +#include <util/generic/scope.h> + +namespace NYql { + +using namespace NDqs; +using namespace NMonitoring; + +using EFileType = Yql::DqsProto::TFile::EFileType; +using TFileResource = Yql::DqsProto::TFile; + +TWorkersStorage::TWorkersStorage(ui32 nodeId, TSensorsGroupPtr metrics, TSensorsGroupPtr workerMetrics) + : NodeId(nodeId) + , Metrics(std::move(metrics)) + , WorkerMetrics(std::move(workerMetrics)) + , ResourceCounters(Metrics->GetSubgroup("component", "ResourceCounters")) + , ResourceDownloadCounters(Metrics->GetSubgroup("component", "ResourceDownloadCounters")) + , ResourceWaitTime(Metrics + ->GetSubgroup("component", "Resource") + ->GetHistogram("WaitTime", ExponentialHistogram(10, 2, 1000))) + , WorkersSize(nullptr) + , FreeListSize(nullptr) + , GlobalResources(Metrics) +{ + NodeIds.resize(65536); +} + +void TWorkersStorage::Clear() +{ + Workers.clear(); + FreeList.clear(); + WorkersSize = nullptr; + FreeListSize = nullptr; + Metrics->RemoveSubgroup("component", "lists"); +} + +TList<TWorkerInfo::TPtr> TWorkersStorage::GetList() { + TList<TWorkerInfo::TPtr> workers; + + for (const auto& [_, workerInfo] : Workers) { + workers.push_back(workerInfo); + } + + return workers; +} + +std::tuple<TWorkerInfo::TPtr, bool> TWorkersStorage::CreateOrUpdate(ui32 nodeId, TGUID workerId, Yql::DqsProto::RegisterNodeRequest& request) { + auto& knownWorkers = request.GetKnownNodes(); + bool needResume = false; + auto maybeWorker = Workers.find(workerId); + TWorkerInfo::TPtr workerInfo; + + CheckZombie(nodeId, workerId, request); + + if (maybeWorker == Workers.end()) { + for (auto node : knownWorkers) { + if (node == NodeId) { + YQL_CLOG(DEBUG, ProviderDq) + << "Resume " << GetGuidAsString(workerId) + << " " << request.GetRevision() + << " " << request.GetRunningWorkers() + << " " << request.GetEpoch(); + + TInflightLimiter::TPtr limiter = InflightLimiters[request.GetClusterName()]; + if (!limiter) { + limiter = new NDqs::TInflightLimiter(MaxDownloadsPerFile); + InflightLimiters[request.GetClusterName()] = limiter; + } + + workerInfo = MakeIntrusive<TWorkerInfo>(StartTime, nodeId, workerId, request, WorkerMetrics, limiter, GlobalResources); + Workers[workerId] = workerInfo; + FreeList.insert(workerInfo); + needResume = true; + break; + } + } + } else { + workerInfo = maybeWorker->second; + FreeList.erase(workerInfo); + needResume = workerInfo->Update(request); + if (workerInfo->RunningRequests < workerInfo->Capacity) { + FreeList.insert(workerInfo); + } + } + + return std::make_tuple(workerInfo, needResume); +} + +void TWorkersStorage::CheckZombie(ui32 nodeId, TGUID workerId, Yql::DqsProto::RegisterNodeRequest& request) { + Y_ABORT_UNLESS(nodeId < NodeIds.size()); + + if (request.GetStartTime().empty()) { + request.SetStartTime(StartTime); + } + + if (!NodeIds[nodeId].first) { + NodeIds[nodeId] = std::make_pair(workerId, request.GetStartTime()); + return; + } + if (NodeIds[nodeId].first != workerId) { + auto curStartTime = TInstant::ParseIso8601(request.GetStartTime()); + auto prevStartTime = TInstant::ParseIso8601(NodeIds[nodeId].second); + + if (prevStartTime < curStartTime) { + // replace old nodeId + YQL_CLOG(DEBUG, ProviderDq) << "Zombie worker " << GetGuidAsString(NodeIds[nodeId].first) << " " << nodeId; + NodeIds[nodeId] = std::make_pair(workerId, request.GetStartTime()); + } else { + YQL_CLOG(DEBUG, ProviderDq) << "Zombie worker " << GetGuidAsString(workerId) << " " << nodeId; + request.SetZombie(true); + } + } +} + +void TWorkersStorage::CleanUp(TInstant now, TDuration duration) { + TVector<TGUID> deadWorkers; + THashMap<TString, TWorkerInfo::TFileResource> downloadList; + for (const auto& [k, v] : Workers) { + if (now - v->LastPingTime > duration) { + FreeList.erase(v); + + deadWorkers.push_back(k); + downloadList.insert(v->GetDownloadList().begin(), v->GetDownloadList().end()); + v->OnDead(); + } + } + + for (auto workerId : deadWorkers) { + YQL_CLOG(WARN, ProviderDq) << "Remove dead worker " << GetGuidAsString(workerId); + Workers.erase(workerId); + } + + // pass dead download list to live workers + auto it = Workers.begin(); + for (ui32 i = 0; i < deadWorkers.size() && it != Workers.end(); ++i, ++it) { + it->second->AddToDownloadList(downloadList); + } + + THashSet<TString> dropOldUploaded; + for (const auto& [k, v] : Uploaded) { + if (now - v.LastSeenTime > TDuration::Hours(1)) { + dropOldUploaded.insert(k); + } + } + for (const auto& k : dropOldUploaded) { + Uploaded.erase(k); + } +} + +void TWorkersStorage::DropWorker(TInstant now, TGUID workerId) { + Y_UNUSED(now); + auto it = Workers.find(workerId); + if (it != Workers.end()) { + FreeList.erase(it->second); + it->second->OnDead(); + Workers.erase(it); + } +} + +void TWorkersStorage::FreeWorker(TInstant now, const TWorkerInfo::TPtr& worker) { + Y_UNUSED(now); + + FreeList.erase(worker); + if (worker->Release()) { + FreeList.insert(worker); + } +} + +TWorkersStorage::TFreeList::iterator TWorkersStorage::ProcessMatched( + TWorkersStorage::TFreeList::iterator it, + TVector<NDqs::TWorkerInfo::TPtr>& result, + TFreeList* freeList, + THashMap<TWorkerInfo::TPtr, int> tasksPerWorker, + THashMap<i32, TWorkerFilter>& tasksToAllocate, + int taskId) +{ + auto workerInfo = *it; + + it = freeList->erase(it); + if (!workerInfo->Acquire()) { + freeList->insert(workerInfo); + it = freeList->begin(); + } + + tasksPerWorker[workerInfo] ++; + + Y_ASSERT(result[taskId] == nullptr); + result[taskId] = workerInfo; + tasksToAllocate.erase(taskId); + + return it; +} + +void TWorkersStorage::SearchFreeList( + TVector<TWorkerInfo::TPtr>& result, + TFreeList* freeList, + const i32 maxTasksPerWorker, + THashMap<TWorkerInfo::TPtr, int>& tasksPerWorker, + THashMap<i32, TWorkerFilter>& tasksToAllocate, + THashMap<TString, THashSet<int>>& waitingResources, + TWorkerFilter::EMatchStatus matchMode) +{ + TWorkerFilter::TStats stats; + stats.WaitingResources = &waitingResources; + stats.Uploaded = &Uploaded; + + // fresh nodes must be first + for (auto it = freeList->begin(); it != freeList->end(); ) { + // must exists + auto workerInfo = *it; + if (workerInfo->IsDead) { + YQL_CLOG(DEBUG, ProviderDq) << "Remove dead worker from free list " + << GetGuidAsString(workerInfo->WorkerId); + it = freeList->erase(it); + continue; + } + if (workerInfo->Capacity <= workerInfo->RunningRequests) { + it = freeList->erase(it); + continue; + } + + if (maxTasksPerWorker > 0 && tasksPerWorker[workerInfo] >= maxTasksPerWorker) { + ++it; + continue; + } + + if (workerInfo->Stopping) { + ++it; + continue; + } + + i32 taskId; + bool ok = false; + for (const auto& [taskId_, filter] : tasksToAllocate) { + taskId = taskId_; + if ((matchMode == filter.Match(workerInfo, taskId, &stats))) { + ok = true; + break; + } + } + + if (ok) { + it = ProcessMatched( + it, result, freeList, tasksPerWorker, tasksToAllocate, taskId); + } else { + ++it; + } + + if (tasksToAllocate.empty()) { + break; + } + } +} + +TVector<TWorkerInfo::TPtr> TWorkersStorage::TryAllocate(const NDq::IScheduler::TWaitInfo& waitInfo) noexcept +{ + auto& request = waitInfo.Request; + auto count = request.GetCount(); + TVector<TWorkerInfo::TPtr> result(count, nullptr); + auto& filterPerTask = request.GetWorkerFilterPerTask(); + THashMap<i32, TWorkerFilter> tasksToAllocate; + + YQL_LOG_CTX_ROOT_SESSION_SCOPE(waitInfo.Request.GetTraceId()); + YQL_CLOG(DEBUG, ProviderDq) << "TryAllocate, workers count: " << count << " files per task: " << filterPerTask.size(); + + if (filterPerTask.empty()) { + // COMPAT demand all files on all workers + Yql::DqsProto::TWorkerFilter commonFilter; + for (const auto& file : request.GetFiles()) { + *commonFilter.AddFile() = file; + } + for (ui32 i = 0; i < count; ++i) { + tasksToAllocate.emplace(i, commonFilter); + } + } else { + for (i32 i = 0; i < filterPerTask.size(); ++i) { + tasksToAllocate.emplace(i, filterPerTask.Get(i)); + } + } + + Y_ASSERT(tasksToAllocate.size() == count); + + auto now = TInstant::Now(); + + for (const auto& file : request.GetFiles()) { + auto id = file.GetObjectId(); + auto maybeUploaded = Uploaded.find(id); + if (maybeUploaded == Uploaded.end()) { + Uploaded.emplace(id, TResourceStat{id, file.GetObjectType(), file.GetName(), file.GetSize()}); + } else { + if (maybeUploaded->second.Name.empty()) { + maybeUploaded->second = TResourceStat{id, file.GetObjectType(), file.GetName(), file.GetSize()}; + } else { + maybeUploaded->second.LastSeenTime = now; + } + } + } + + THashMap<TString, THashSet<int>> waitingResources; // resourceId -> taskId + + TFreeList boundedList; + auto* freeList = &FreeList; + + Y_SCOPE_EXIT(&) { + for (const auto& workerInfo : boundedList) { + FreeList.insert(workerInfo); + } + }; + + i32 maxTasksPerWorker = 0; + if (request.GetWorkersCount()) { + YQL_CLOG(DEBUG, ProviderDq) << "Bounded mode"; + maxTasksPerWorker = (count + request.GetWorkersCount() - 1) / request.GetWorkersCount(); + freeList = &boundedList; + auto slots = count; slots = 0; + + for (auto it = FreeList.begin(); it != FreeList.end() && freeList->size() < request.GetWorkersCount(); ) { + auto workerInfo = *it; + if (workerInfo->IsDead || workerInfo->Capacity <= workerInfo->RunningRequests) { + it = FreeList.erase(it); + continue; + } + auto freeSlots = workerInfo->Capacity - workerInfo->RunningRequests; + if (freeSlots >= maxTasksPerWorker) { + freeList->insert(workerInfo); + slots += freeSlots; + it = FreeList.erase(it); + } else { + ++it; + } + } + + if (slots < count) { + YQL_CLOG(DEBUG, ProviderDq) << "Not enough slots"; + return { }; + } + } + + THashMap<TWorkerInfo::TPtr, int> tasksPerWorker; + SearchFreeList(result, freeList, maxTasksPerWorker, tasksPerWorker, tasksToAllocate, waitingResources, TWorkerFilter::EOK); + if (!tasksToAllocate.empty()) { + SearchFreeList(result, freeList, maxTasksPerWorker, tasksPerWorker, tasksToAllocate, waitingResources, TWorkerFilter::EPARTIAL); + } + + for (const auto& file : request.GetFiles()) { + const auto& id = file.GetObjectId(); + + if (waitingResources.contains(id)) { + waitInfo.Stat.StartCounter(id, now); + } else { + waitInfo.Stat.FlushCounter(id); + } + } + + waitInfo.ResLeft.clear(); + if (tasksToAllocate.empty()) { + auto now = TInstant::Now(); + for (const auto& workerInfo : result) { + Y_ASSERT(workerInfo != nullptr); + workerInfo->UseCount ++; + workerInfo->RequestStartTime = now; + } + for (const auto& file : request.GetFiles()) { + auto id = file.GetObjectId(); + auto it = Uploaded.find(id); + auto delta = waitInfo.Stat.Get().contains(id) + ? TDuration::MilliSeconds(waitInfo.Stat.Get().find(id)->second.Sum) + : TDuration(); + + waitInfo.Stat.Clear(); + + it->second.UseCount += count; + it->second.WaitTime += delta; + + TString resourceName; + + switch (file.GetObjectType()) { + case Yql::DqsProto::TFile::EEXE_FILE: + resourceName = "ResourceExe"; + waitInfo.Stat.AddCounter(resourceName, delta); + break; + case Yql::DqsProto::TFile::EUDF_FILE: + if (!file.GetName().empty()) { + resourceName = "ResourceUdf" + file.GetName(); + waitInfo.Stat.AddCounter(resourceName, delta); + } + waitInfo.Stat.AddCounter("ResourceUdf", delta); + break; + default: + resourceName = "ResourceFile"; + waitInfo.Stat.AddCounter(resourceName, delta); + break; + } + + ResourceWaitTime->Collect(delta.MilliSeconds()); + *ResourceCounters->GetCounter(resourceName, /*derivative=*/ true) += count; + } + + return result; + } else { + YQL_CLOG(DEBUG, ProviderDq) << "Tasks left " << tasksToAllocate.size(); + // schedule downloads + for (const auto& workerInfo : result) { + if (workerInfo == nullptr) { + continue; + } + freeList->erase(workerInfo); + if (workerInfo->Release()) { + freeList->insert(workerInfo); + } + } + for (const auto& [_, filter] : tasksToAllocate) { + filter.Visit([&](const auto& file) { + waitInfo.ResLeft[file.GetObjectId()]++; + }); + } + + auto it = freeList->begin(); + for (const auto& [_, filter] : tasksToAllocate) { + if (it == freeList->end()) { + it = freeList->begin(); + } + if (it == freeList->end()) { + // begin == end => no free workers + break; + } + const auto workerInfo = *it++; + if (workerInfo->Stopping) { + continue; + } + filter.Visit([&](const auto& file) { + if (workerInfo->AddToDownloadList(file.GetObjectId(), file)) { + YQL_CLOG(TRACE, ProviderDq) << "Added " << file.GetName() << "|" << file.GetObjectId() << " to worker's " << GetGuidAsString(workerInfo->WorkerId) << " download list" ; + TStringBuilder resourceDownloadName; + resourceDownloadName << "ResourceDownload"; + switch (file.GetObjectType()) { + case Yql::DqsProto::TFile::EEXE_FILE: + resourceDownloadName << "Exe"; + break; + case Yql::DqsProto::TFile::EUDF_FILE: + resourceDownloadName << "Udf" << file.GetName(); + break; + default: + resourceDownloadName << "File"; + break; + } + *ResourceDownloadCounters->GetCounter(resourceDownloadName, /*derivative=*/ true) += 1; + } + }); + } + + return { }; + } +} + +void TWorkersStorage::IsReady(const TVector<TFileResource>& resources, THashMap<TString, std::pair<ui32, ui32>>& clusterMap) { + for (const auto& [_,workerInfo] : Workers) { + auto count = 0; + for (const auto& r : resources){ + if (workerInfo->GetResources().contains(r.GetObjectId())) { + count++; + } + } + YQL_CLOG(TRACE, ProviderDq) << workerInfo->ClusterName << " worker " << GetGuidAsString(workerInfo->WorkerId) << " has: " << count << "/" << int(resources.size()) << " resources"; + clusterMap[workerInfo->ClusterName].second += count == int(resources.size()); + } +}; + + +void TWorkersStorage::ClusterStatus(Yql::DqsProto::ClusterStatusResponse* r) const { + for (const auto& [_, workerInfo] : Workers) { + auto* node = r->AddWorker(); + node->SetGuid(GetGuidAsString(workerInfo->WorkerId)); + node->SetNodeId(workerInfo->NodeId); + node->SetLastPingTime(workerInfo->LastPingTime.ToString()); + node->SetRevision(workerInfo->Revision); + node->SetClusterName(workerInfo->ClusterName); + node->SetAddress(workerInfo->Address); + node->SetPort(workerInfo->Port); + node->SetEpoch(workerInfo->Epoch); + node->SetStopping(workerInfo->Stopping); + node->SetStartTime(workerInfo->StartTime); + + for (const auto& f : workerInfo->GetResourcesOrdered()) { + *node->AddResource() = f.GetObjectId(); + } + + for (const auto& [k, _] : workerInfo->GetDownloadList()) { + node->AddDownloadList()->SetObjectId(k); + } + + for (const auto& [k, _] : workerInfo->GetActiveDownloads()) { + node->AddActiveDownload()->SetObjectId(k); + } + for (const auto& op : workerInfo->Operations){ + node->AddOperation(op); + } + node->SetFreeDiskSize(workerInfo->FreeDiskSize); + node->SetUsedDiskSize(workerInfo->UsedDiskSize); + node->SetCapacity(workerInfo->Capacity); + node->SetRunningWorkerActors(workerInfo->RunningWorkerActors); + node->SetRunningRequests(workerInfo->RunningRequests); + node->SetDead(workerInfo->IsDead); + + node->MutableRusage()->SetStime(workerInfo->Stime.MicroSeconds()); + node->MutableRusage()->SetUtime(workerInfo->Utime.MicroSeconds()); + node->MutableRusage()->SetMajorPageFaults(workerInfo->MajorPageFaults); + node->MutableRusage()->SetCpuSystem(workerInfo->CpuSystem); + node->MutableRusage()->SetCpuUser(workerInfo->CpuUser); + node->MutableRusage()->SetCpuTotal(workerInfo->CpuTotal); + + for (const auto& [k, v] : workerInfo->Attributes) { + auto* attr = node->AddAttribute(); + attr->SetKey(k); + attr->SetValue(v); + } + + node->SetUseCount(workerInfo->UseCount); + node->SetUseTime(workerInfo->UseTime.MilliSeconds()); + } + + for (const auto& [_, i] : Uploaded) { + auto* rr = r->AddResource(); + rr->SetId(i.Id); + rr->SetName(i.Name); + rr->SetUseTime(i.UseTime.MilliSeconds()); + rr->SetWaitTime(i.WaitTime.MilliSeconds()); + rr->SetUseCount(i.UseCount); + rr->SetTryCount(i.TryCount); + rr->SetSize(i.Size); + } + + r->SetFreeListSize(FreeList.size()); + r->SetCapacity(GlobalResources.GetCapacity()); + r->SetRunningRequests(GlobalResources.GetRunningRequests()); +} + +void TWorkersStorage::UpdateResourceUseTime(TDuration duration, const THashSet<TString>& ids) { + for (const auto& id : ids) { + auto it = Uploaded.find(id); + if (it != Uploaded.end()) { + it->second.UseTime += duration; + } + } +} + +bool TWorkersStorage::HasResource(const TString& id) const { + auto it = Uploaded.find(id); + return (it != Uploaded.end()) && it->second.Uploaded; +} + +void TWorkersStorage::AddResource(const TString& id, EFileType type, const TString& name, i64 size) { + if (!Uploaded.contains(id)) { + Uploaded.emplace(id, TResourceStat{id, type, name, size}); + } + Uploaded.find(id)->second.Uploaded = true; +} + +void TWorkersStorage::UpdateMetrics() { + if (!WorkersSize) { + WorkersSize = Metrics->GetSubgroup("component", "lists")->GetCounter("WorkersSize"); + } + if (!FreeListSize) { + FreeListSize = Metrics->GetSubgroup("component", "lists")->GetCounter("FreeListSize"); + } + *WorkersSize = Workers.size(); + *FreeListSize = FreeList.size(); +} + +void TWorkersStorage::Visit(const std::function<void(const TWorkerInfo::TPtr& workerInfo)>& f) { + for (const auto& [_, workerInfo] : Workers) { + f(workerInfo); + } +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.h b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.h new file mode 100644 index 0000000000..d40366596d --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.h @@ -0,0 +1,90 @@ +#pragma once + +#include "worker_filter.h" + +#include <ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h> + +#include <ydb/library/yql/providers/dq/scheduler/dq_scheduler.h> + +namespace NYql { + +class TWorkersStorage { +public: + TWorkersStorage(ui32 nodeId, TSensorsGroupPtr metrics, TSensorsGroupPtr workerMetrics); + + void Clear(); + + TList<NDqs::TWorkerInfo::TPtr> GetList(); + + size_t Capacity() const { return GlobalResources.GetCapacity(); } + size_t FreeSlots() const { return Max<i64>(GlobalResources.GetCapacity() - GlobalResources.GetRunningRequests(), 0); } + + std::tuple<NDqs::TWorkerInfo::TPtr, bool> CreateOrUpdate(ui32 nodeId, TGUID workerId, Yql::DqsProto::RegisterNodeRequest& request); + + void CheckZombie(ui32 nodeId, TGUID workerId, Yql::DqsProto::RegisterNodeRequest& request); + + void CleanUp(TInstant now, TDuration duration); + + void DropWorker(TInstant now, TGUID workerId); + + void FreeWorker(TInstant now, const NDqs::TWorkerInfo::TPtr& worker); + + TVector<NDqs::TWorkerInfo::TPtr> TryAllocate(const NDq::IScheduler::TWaitInfo& waitInfo) noexcept; + + void IsReady(const TVector<NDqs::TWorkerInfo::TFileResource>& resources, THashMap<TString, std::pair<ui32, ui32>>& clusterMap); + + void ClusterStatus(Yql::DqsProto::ClusterStatusResponse* r) const; + + void UpdateResourceUseTime(TDuration duration, const THashSet<TString>& ids); + + bool HasResource(const TString& id) const; + + void AddResource(const TString& id, Yql::DqsProto::TFile::EFileType type, const TString& name, i64 size); + + void UpdateMetrics(); + + void Visit(const std::function<void(const NDqs::TWorkerInfo::TPtr& workerInfo)>& f); + +private: + + const i32 MaxDownloadsPerFile = 20; + ui32 NodeId; + THashMap<TString, NDqs::TInflightLimiter::TPtr> InflightLimiters; + THashMap<TGUID, NDqs::TWorkerInfo::TPtr> Workers; + + using TFreeList = std::set<NDqs::TWorkerInfo::TPtr, NDqs::TWorkerInfoPtrComparator>; + TFreeList FreeList; + + TSensorsGroupPtr Metrics; + TSensorsGroupPtr WorkerMetrics; + TSensorsGroupPtr ResourceCounters; + TSensorsGroupPtr ResourceDownloadCounters; + NMonitoring::THistogramPtr ResourceWaitTime; + NMonitoring::TDynamicCounters::TCounterPtr WorkersSize; + NMonitoring::TDynamicCounters::TCounterPtr FreeListSize; + TString StartTime = ToString(TInstant::Now()); + TVector<std::pair<TGUID, TString>> NodeIds; + + THashMap<TString, TResourceStat> Uploaded; + + NDqs::TGlobalResources GlobalResources; + + TFreeList::iterator ProcessMatched( + TWorkersStorage::TFreeList::iterator it, + TVector<NDqs::TWorkerInfo::TPtr>& result, + TFreeList* freeList, + THashMap<NDqs::TWorkerInfo::TPtr, int> tasksPerWorker, + THashMap<i32, TWorkerFilter>& tasksToAllocate, + int taskId); + + void SearchFreeList( + TVector<NDqs::TWorkerInfo::TPtr>& result, + TFreeList* freeList, + const i32 maxTasksPerWorker, + THashMap<NDqs::TWorkerInfo::TPtr, int>& tasksPerWorker, + THashMap<i32, TWorkerFilter>& tasksToAllocate, + THashMap<TString, THashSet<int>>& waitingResources, + TWorkerFilter::EMatchStatus matchMode); +}; + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp new file mode 100644 index 0000000000..efa23c8a32 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp @@ -0,0 +1,82 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include <ydb/library/yql/providers/dq/global_worker_manager/workers_storage.h> + +using namespace NYql; +using namespace NYql::NDq; + +Y_UNIT_TEST_SUITE(WorkersBenchmark) { + Y_UNIT_TEST(Basic) { + int workers = 1000; + int i; + TWorkersStorage storage(1, new TSensorsGroup, new TSensorsGroup); + storage.Clear(); + for (i = 0; i < workers; i++) { + TGUID guid; + Yql::DqsProto::RegisterNodeRequest request; + request.SetCapacity(100); + request.AddKnownNodes(1); + CreateGuid(&guid); + storage.CreateOrUpdate(i+100, guid, request); + } + + UNIT_ASSERT_VALUES_EQUAL(workers*100, storage.FreeSlots()); + + TVector<NDqs::TWorkerInfo::TPtr> all; + TInstant now = TInstant::Now(); + while (1) { + NYql::NDqProto::TAllocateWorkersRequest request; + request.SetCount(10); + IScheduler::TWaitInfo waitInfo(request, NActors::TActorId()); + auto result = storage.TryAllocate(waitInfo); + if (result.empty()) { + break; + } + all.insert(all.end(), result.begin(), result.end()); + } + + Cerr << (TInstant::Now() - now) << Endl; + + UNIT_ASSERT_VALUES_EQUAL(0, storage.FreeSlots()); + + + now = TInstant::Now(); + for (auto& node : all) { + storage.FreeWorker(now, node); + } + Cerr << (TInstant::Now() - now) << Endl; + + UNIT_ASSERT_VALUES_EQUAL(workers*100, storage.FreeSlots()); + } + + Y_UNIT_TEST(AcquireAll) { + int workers = 1; + int i; + TWorkersStorage storage(1, new TSensorsGroup, new TSensorsGroup); + storage.Clear(); + for (i = 0; i < workers; i++) { + TGUID guid; + Yql::DqsProto::RegisterNodeRequest request; + request.SetCapacity(100); + request.AddKnownNodes(1); + CreateGuid(&guid); + storage.CreateOrUpdate(i+100, guid, request); + } + + UNIT_ASSERT_VALUES_EQUAL(workers*100, storage.FreeSlots()); + + TVector<NDqs::TWorkerInfo::TPtr> all; + while (1) { + NYql::NDqProto::TAllocateWorkersRequest request; + request.SetCount(10); + IScheduler::TWaitInfo waitInfo(request, NActors::TActorId()); + auto result = storage.TryAllocate(waitInfo); + if (result.empty()) { + break; + } + all.insert(all.end(), result.begin(), result.end()); + } + UNIT_ASSERT_VALUES_EQUAL(all.size(), 100); + UNIT_ASSERT_VALUES_EQUAL(0, storage.FreeSlots()); + } +} diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ya.make b/ydb/library/yql/providers/dq/global_worker_manager/ya.make new file mode 100644 index 0000000000..f2ce78ffd4 --- /dev/null +++ b/ydb/library/yql/providers/dq/global_worker_manager/ya.make @@ -0,0 +1,53 @@ +LIBRARY() + +PEERDIR( + ydb/library/yql/utils/failure_injector + ydb/library/yql/providers/common/config + ydb/library/yql/providers/common/gateway + ydb/library/yql/providers/common/metrics + ydb/library/yql/providers/dq/actors + ydb/library/yql/providers/dq/api/grpc + ydb/library/yql/providers/dq/api/protos + ydb/library/yql/providers/dq/config + ydb/library/yql/providers/dq/counters + ydb/library/yql/providers/dq/runtime + ydb/library/yql/providers/dq/task_runner + ydb/library/yql/providers/dq/actors/yt + ydb/library/yql/providers/dq/scheduler +) + +YQL_LAST_ABI_VERSION() + +SET( + SOURCE + benchmark.cpp + global_worker_manager.cpp + service_node_pinger.cpp + workers_storage.cpp + worker_filter.cpp +) + +IF (NOT OS_WINDOWS) + SET( + SOURCE + ${SOURCE} + service_node_resolver.cpp + coordination_helper.cpp + ) +ELSE() + SET( + SOURCE + ${SOURCE} + coordination_helper_win.cpp + ) +ENDIF() + +SRCS( + ${SOURCE} +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..450d2ffafa --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,29 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(providers-dq-scheduler) +target_compile_options(providers-dq-scheduler PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-dq-scheduler PUBLIC + contrib-libs-cxxsupp + yutil + yql-dq-common + yql-dq-proto + providers-dq-config + providers-common-proto + dq-api-protos + providers-dq-common + providers-dq-counters + cpp-actors-protos +) +target_sources(providers-dq-scheduler PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp +) diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..2479c98297 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-aarch64.txt @@ -0,0 +1,30 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(providers-dq-scheduler) +target_compile_options(providers-dq-scheduler PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-dq-scheduler PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yql-dq-common + yql-dq-proto + providers-dq-config + providers-common-proto + dq-api-protos + providers-dq-common + providers-dq-counters + cpp-actors-protos +) +target_sources(providers-dq-scheduler PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp +) diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..2479c98297 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-x86_64.txt @@ -0,0 +1,30 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(providers-dq-scheduler) +target_compile_options(providers-dq-scheduler PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-dq-scheduler PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yql-dq-common + yql-dq-proto + providers-dq-config + providers-common-proto + dq-api-protos + providers-dq-common + providers-dq-counters + cpp-actors-protos +) +target_sources(providers-dq-scheduler PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp +) diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..450d2ffafa --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.windows-x86_64.txt @@ -0,0 +1,29 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(providers-dq-scheduler) +target_compile_options(providers-dq-scheduler PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-dq-scheduler PUBLIC + contrib-libs-cxxsupp + yutil + yql-dq-common + yql-dq-proto + providers-dq-config + providers-common-proto + dq-api-protos + providers-dq-common + providers-dq-counters + cpp-actors-protos +) +target_sources(providers-dq-scheduler PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp +) diff --git a/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp b/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp new file mode 100644 index 0000000000..1291cc949f --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp @@ -0,0 +1,240 @@ +#include "dq_scheduler.h" + +#include <queue> +#include <list> +#include <unordered_map> + +#include <ydb/library/yql/utils/log/log.h> + +namespace NYql::NDq { + +IScheduler::TWaitInfo::TWaitInfo(const NYql::NDqProto::TAllocateWorkersRequest& record, const NActors::TActorId& sender) + : Request(record), Sender(sender), StartTime(TInstant::Now()) +{ } + +namespace { + +struct TMyCounters { + using TPtr = std::unique_ptr<TMyCounters>; + + const TSensorsGroupPtr Group; + const NMonitoring::TDynamicCounters::TCounterPtr QueueSizeForSmall; + const NMonitoring::TDynamicCounters::TCounterPtr QueueSizeForLarge; + const NMonitoring::TDynamicCounters::TCounterPtr IntegralQueueSizeForLarge; + const NMonitoring::TDynamicCounters::TCounterPtr AllocatedTotal; + const NMonitoring::TDynamicCounters::TCounterPtr KnownUsers; + const NMonitoring::TDynamicCounters::TCounterPtr UserLimited; + + TMyCounters(TSensorsGroupPtr&& group) + : Group(std::move(group)) + , QueueSizeForSmall(Group->GetCounter("QueueSizeForSmall")) + , QueueSizeForLarge(Group->GetCounter("QueueSizeForLarge")) + , IntegralQueueSizeForLarge(Group->GetCounter("IntegralQueueSizeForLarge")) + , AllocatedTotal(Group->GetCounter("AllocatedTotal")) + , KnownUsers(Group->GetCounter("KnownUsers")) + , UserLimited(Group->GetCounter("UserLimited")) + {} + + static TPtr Make(IMetricsRegistryPtr metricsRegistry) { + return metricsRegistry ? std::make_unique<TMyCounters>(metricsRegistry->GetSensors()->GetSubgroup("component", "scheduler")) : nullptr; + } +}; + +class TScheduler : public IScheduler { +public: + TScheduler(IMetricsRegistryPtr metricsRegistry, const NProto::TDqConfig::TScheduler& config) + : KeepReserveForLiteralRequests(config.GetKeepReserveForLiteralRequests()) + , HistoryKeepingTime(TDuration::Minutes(config.GetHistoryKeepingTime())) + , MaxOperations(config.GetMaxOperations()) + , MaxOperationsPerUser(config.GetMaxOperationsPerUser()) + , Counters(TMyCounters::Make(metricsRegistry)) + , EnableLimiter(config.GetLimitTasksPerWindow()) + , LimiterNumerator(config.GetLimiterNumerator()) + , LimiterDenumerator(config.GetLimiterDenumerator()) + {} + +private: + struct TUserInfo { + ui64 Await = 0ULL; + ui64 Allocated = 0ULL; + ui64 AwaitOperations = 0LL; + std::queue<std::pair<TInstant, ui32>> History; + struct { + NMonitoring::TDynamicCounters::TCounterPtr Await, AwaitOperations, Allocated; + } Counters; + }; + + using THistoryMap = std::unordered_map<TString, TUserInfo>; + + struct TFullWaitInfo : public TWaitInfo { + TFullWaitInfo(TWaitInfo&& info, THistoryMap::value_type* userInfo) + : TWaitInfo(std::move(info)), UserInfo(userInfo) + {} + + THistoryMap::value_type* const UserInfo; + }; + + bool Suspend(TWaitInfo&& info) final { + const auto ins = AllocationsHistory.emplace(info.Request.GetUser(), TUserInfo()); + auto& userInfo = *ins.first; + + if (info.Request.GetCount() > 1U) { + if (userInfo.second.AwaitOperations >= MaxOperationsPerUser) { + return false; + } + if (LargeWaitList.size() >= MaxOperations) { + return false; + } + } + + userInfo.second.Await += info.Request.GetCount(); + userInfo.second.AwaitOperations += 1; + if (ins.second && Counters) { + const auto group = Counters->Group->GetSubgroup("user", info.Request.GetUser()); + userInfo.second.Counters.Await = group->GetCounter("Await"); + userInfo.second.Counters.AwaitOperations = group->GetCounter("AwaitOperations"); + userInfo.second.Counters.Allocated = group->GetCounter("Allocated"); + } + (info.Request.GetCount() > 1U ? LargeWaitList : SmallWaitList).emplace_back(std::move(info), &userInfo); + return true; + } + + std::vector<NActors::TActorId> Cleanup() final { + std::vector<NActors::TActorId> senders; + senders.reserve(SmallWaitList.size() + LargeWaitList.size()); + std::transform(SmallWaitList.cbegin(), SmallWaitList.cend(), std::back_inserter(senders), [](const TWaitInfo& info) { return info.Sender; }); + std::transform(LargeWaitList.cbegin(), LargeWaitList.cend(), std::back_inserter(senders), [](const TWaitInfo& info) { return info.Sender; }); + SmallWaitList.clear(); + LargeWaitList.clear(); + + if (Counters) { + for (auto it = AllocationsHistory.cbegin(); AllocationsHistory.cend() != it; ++it) { + *it->second.Counters.Await = 0; + *it->second.Counters.AwaitOperations = 0; + *it->second.Counters.Allocated = 0; + } + } + + AllocationsHistory.clear(); + return senders; + } + + size_t UpdateMetrics() final { + if (Counters) { + auto allocated = 0ULL; + for (auto it = AllocationsHistory.cbegin(); AllocationsHistory.cend() != it; ++it) { + *it->second.Counters.Await = it->second.Await; + *it->second.Counters.AwaitOperations = it->second.AwaitOperations; + *it->second.Counters.Allocated = it->second.Allocated; + allocated += it->second.Allocated; + } + + *Counters->KnownUsers = AllocationsHistory.size(); + *Counters->AllocatedTotal = allocated; + *Counters->QueueSizeForSmall = SmallWaitList.size(); + *Counters->QueueSizeForLarge = LargeWaitList.size(); + *Counters->IntegralQueueSizeForLarge = std::accumulate(LargeWaitList.cbegin(), LargeWaitList.cend(), 0ULL, + [] (ui64 c, const TFullWaitInfo& info) { return c += info.Request.GetCount(); } + ); + } + + return SmallWaitList.size() + LargeWaitList.size(); + } + + void Process(size_t total, size_t count, const TProcessor& processor, const TInstant& now) final { + const auto from = now - HistoryKeepingTime; + for (auto& info : AllocationsHistory) { + for (auto& history = info.second.History; !history.empty() && history.front().first <= from; history.pop()) + info.second.Allocated -= history.front().second; + }; + + const auto sort = [](const TFullWaitInfo& lhs, const TFullWaitInfo& rhs) { + return lhs.UserInfo->second.Allocated < rhs.UserInfo->second.Allocated; + }; + + SmallWaitList.sort(sort); + LargeWaitList.sort(sort); + + const auto work = [&processor, &now, &total, this] (size_t& quota, std::list<TFullWaitInfo>& list) { + list.remove_if([&](const TFullWaitInfo& info) { + const auto count = info.Request.GetCount(); + if (quota < count) + return false; + + if (EnableLimiter && (info.UserInfo->second.Allocated+count) > LimiterNumerator * total / LimiterDenumerator) { + if (Counters) { + *Counters->UserLimited += 1; + } + return false; + } + + if (processor(info)) { + info.UserInfo->second.Await -= count; + info.UserInfo->second.AwaitOperations -= 1; + info.UserInfo->second.Allocated += count; + info.UserInfo->second.History.emplace(now, count); + quota -= count; + return true; + } + return false; + }); + }; + + const auto full = count; + const auto half = full >> 1U; + + if (count -= half) + work(count, SmallWaitList); + + if (KeepReserveForLiteralRequests) + count = SmallWaitList.empty() && count + half >= total >> 2U ? std::min(count + half, full - (half >> 1U)) : half; + else + count += half; + + if (count > 1U) + work(count, LargeWaitList); + + if (count && LargeWaitList.empty()) + work(count, SmallWaitList); + } + + void ProcessAll(const TProcessor& processor) final { + const auto proc = [&processor](const TFullWaitInfo& info) { + if (processor(info)) { + info.UserInfo->second.Await -= info.Request.GetCount(); + return true; + } + return false; + }; + + SmallWaitList.remove_if(proc); + LargeWaitList.remove_if(proc); + } + + void ForEach(const std::function<void(const TWaitInfo& info)>& processor) final { + std::for_each(SmallWaitList.cbegin(), SmallWaitList.cend(), processor); + std::for_each(LargeWaitList.cbegin(), LargeWaitList.cend(), processor); + } + + const bool KeepReserveForLiteralRequests; + const TDuration HistoryKeepingTime; + const size_t MaxOperations; + const size_t MaxOperationsPerUser; + + const TMyCounters::TPtr Counters; + + THistoryMap AllocationsHistory; + std::list<TFullWaitInfo> SmallWaitList, LargeWaitList; + + bool EnableLimiter; + ui32 LimiterNumerator; + ui32 LimiterDenumerator; +}; + +} + +IScheduler::TPtr IScheduler::Make(const NProto::TDqConfig::TScheduler& config, IMetricsRegistryPtr metricsRegistry) { + return std::make_unique<TScheduler>(metricsRegistry, config); +} + +} diff --git a/ydb/library/yql/providers/dq/scheduler/dq_scheduler.h b/ydb/library/yql/providers/dq/scheduler/dq_scheduler.h new file mode 100644 index 0000000000..d6a976cd5e --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/dq_scheduler.h @@ -0,0 +1,47 @@ +#pragma once + +#include <library/cpp/actors/core/events.h> + +#include <ydb/library/yql/providers/dq/counters/counters.h> +#include <ydb/library/yql/providers/dq/api/protos/dqs.pb.h> +#include <ydb/library/yql/dq/proto/dq_tasks.pb.h> +#include <ydb/library/yql/providers/dq/config/config.pb.h> +#include <ydb/library/yql/providers/common/metrics/metrics_registry.h> + +namespace NYql::NDq { + +class IScheduler { +public: + using TPtr = std::unique_ptr<IScheduler>; + + static TPtr Make(const NProto::TDqConfig::TScheduler& schedulerConfig = {}, IMetricsRegistryPtr metricsRegistry = {}); + + virtual ~IScheduler() = default; + + struct TWaitInfo { + const NYql::NDqProto::TAllocateWorkersRequest Request; + const NActors::TActorId Sender; + const TInstant StartTime; + + TCounters Stat; + mutable THashMap<TString, int> ResLeft; + + TWaitInfo(const NYql::NDqProto::TAllocateWorkersRequest& record, const NActors::TActorId& sender); + }; + + virtual bool Suspend(TWaitInfo&& info) = 0; + + virtual std::vector<NActors::TActorId> Cleanup() = 0; + + virtual size_t UpdateMetrics() = 0; + + using TProcessor = std::function<bool(const TWaitInfo& info)>; + + virtual void Process(size_t totalWorkersCount, size_t freeWorkersCount, const TProcessor& processor, const TInstant& timestamp = TInstant::Now()) = 0; + + virtual void ProcessAll(const TProcessor& processor) = 0; + + virtual void ForEach(const std::function<void(const TWaitInfo& info)>& processor) = 0; +}; + +} diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..aa754d167c --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,72 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-providers-dq-scheduler-ut) +target_compile_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler +) +target_link_libraries(ydb-library-yql-providers-dq-scheduler-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + providers-dq-scheduler + udf-service-stub + yql-utils-log +) +target_link_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp +) +set_property( + TARGET + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-providers-dq-scheduler-ut + TEST_TARGET + ydb-library-yql-providers-dq-scheduler-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-providers-dq-scheduler-ut + system_allocator +) +vcs_info(ydb-library-yql-providers-dq-scheduler-ut) diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..cdeda2cc91 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,75 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-providers-dq-scheduler-ut) +target_compile_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler +) +target_link_libraries(ydb-library-yql-providers-dq-scheduler-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + providers-dq-scheduler + udf-service-stub + yql-utils-log +) +target_link_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp +) +set_property( + TARGET + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-providers-dq-scheduler-ut + TEST_TARGET + ydb-library-yql-providers-dq-scheduler-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-providers-dq-scheduler-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-providers-dq-scheduler-ut) diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..3ee08ce188 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,77 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-providers-dq-scheduler-ut) +target_compile_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler +) +target_link_libraries(ydb-library-yql-providers-dq-scheduler-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + providers-dq-scheduler + udf-service-stub + yql-utils-log +) +target_link_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp +) +set_property( + TARGET + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-providers-dq-scheduler-ut + TEST_TARGET + ydb-library-yql-providers-dq-scheduler-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-providers-dq-scheduler-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-providers-dq-scheduler-ut) diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..e5ba4657a5 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,65 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-providers-dq-scheduler-ut) +target_compile_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler +) +target_link_libraries(ydb-library-yql-providers-dq-scheduler-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + providers-dq-scheduler + udf-service-stub + yql-utils-log +) +target_sources(ydb-library-yql-providers-dq-scheduler-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp +) +set_property( + TARGET + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-providers-dq-scheduler-ut + TEST_TARGET + ydb-library-yql-providers-dq-scheduler-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-providers-dq-scheduler-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-providers-dq-scheduler-ut + system_allocator +) +vcs_info(ydb-library-yql-providers-dq-scheduler-ut) diff --git a/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp b/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp new file mode 100644 index 0000000000..1aea552250 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp @@ -0,0 +1,311 @@ +#include "../dq_scheduler.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NYql::NDq; + +namespace { +NYql::NDqProto::TAllocateWorkersRequest MakeRequest(ui32 count, const TString& user) { + NYql::NDqProto::TAllocateWorkersRequest request; + request.SetCount(count); + request.SetUser(user); + return request; +} + +} + +Y_UNIT_TEST_SUITE(TSchedulerTest) { + Y_UNIT_TEST(SimpleFifo) { + const auto scheduler = IScheduler::Make(); + UNIT_ASSERT(scheduler); + + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + scheduler->Suspend({MakeRequest(7U, "user2"), {}}); + scheduler->Suspend({MakeRequest(5U, "user3"), {}}); + + size_t workers = 30U; + std::vector<size_t> counts; + const auto processor = [&](const IScheduler::TWaitInfo& wait) { + counts.emplace_back(wait.Request.GetCount()); + return true; + }; + + scheduler->Process(workers, workers, processor); + + const std::vector<size_t> expected = {3U, 7U, 5U}; + UNIT_ASSERT_VALUES_EQUAL(counts, expected); + } + + Y_UNIT_TEST(ReserveForSmall) { + const auto scheduler = IScheduler::Make(); + UNIT_ASSERT(scheduler); + + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + scheduler->Suspend({MakeRequest(7U, "user2"), {}}); + scheduler->Suspend({MakeRequest(5U, "user3"), {}}); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 3U); + + size_t workers = 5U; + std::vector<size_t> counts; + const auto processor = [&](const IScheduler::TWaitInfo& wait) { + counts.emplace_back(wait.Request.GetCount()); + return true; + }; + + scheduler->Process(15U, workers, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 2U); + + scheduler->Process(15U, workers = 8U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U); + + scheduler->Process(15U, workers = 8U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U); + + const std::vector<size_t> expected = {3U, 5U}; + UNIT_ASSERT_VALUES_EQUAL(counts, expected); + } + + Y_UNIT_TEST(OneUserForCluster) { + NYql::NProto::TDqConfig::TScheduler cfg; + cfg.SetHistoryKeepingTime(1); + cfg.SetLimitTasksPerWindow(true); + + const auto scheduler = IScheduler::Make(cfg); + UNIT_ASSERT(scheduler); + + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 2U); + + std::vector<size_t> counts; + const auto processor = [&](const IScheduler::TWaitInfo& wait) { + counts.emplace_back(wait.Request.GetCount()); + return true; + }; + + scheduler->Process(11U, 7U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U); + + const std::vector<size_t> expected = {3U}; + UNIT_ASSERT_VALUES_EQUAL(counts, expected); + } + + Y_UNIT_TEST(DoNotReserveForSmall) { + NYql::NProto::TDqConfig::TScheduler cfg; + cfg.SetKeepReserveForLiteralRequests(false); + + const auto scheduler = IScheduler::Make(cfg); + UNIT_ASSERT(scheduler); + + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + scheduler->Suspend({MakeRequest(7U, "user2"), {}}); + scheduler->Suspend({MakeRequest(5U, "user3"), {}}); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 3U); + + size_t workers = 5U; + std::vector<size_t> counts; + const auto processor = [&](const IScheduler::TWaitInfo& wait) { + counts.emplace_back(wait.Request.GetCount()); + return true; + }; + + scheduler->Process(30U, workers, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 2U); + + scheduler->Process(30U, workers = 8U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U); + + scheduler->Process(30U, workers = 8U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 0U); + + const std::vector<size_t> expected = {3U, 7U, 5U}; + UNIT_ASSERT_VALUES_EQUAL(counts, expected); + } + + Y_UNIT_TEST(NewbieFirst) { + const auto scheduler = IScheduler::Make(); + UNIT_ASSERT(scheduler); + + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + scheduler->Suspend({MakeRequest(7U, "user1"), {}}); + scheduler->Suspend({MakeRequest(5U, "user1"), {}}); + + size_t workers = 15U; + std::vector<size_t> counts; + const auto processor = [&](const IScheduler::TWaitInfo& wait) { + counts.emplace_back(wait.Request.GetCount()); + return true; + }; + + scheduler->Process(33U, workers, processor); + + scheduler->Suspend({MakeRequest(3U, "user2"), {}}); + scheduler->Suspend({MakeRequest(7U, "user2"), {}}); + scheduler->Suspend({MakeRequest(5U, "user2"), {}}); + + workers += 10U; + + scheduler->Process(33U, workers, processor); + workers += 10U; + + scheduler->Process(33U, workers, processor); + + const std::vector<size_t> expected = {3U, 7U, 3U, 7U, 5U, 5U}; + UNIT_ASSERT_VALUES_EQUAL(counts, expected); + } + + Y_UNIT_TEST(FifoAfterOneHour) { + const auto scheduler = IScheduler::Make(); + UNIT_ASSERT(scheduler); + + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + scheduler->Suspend({MakeRequest(7U, "user1"), {}}); + scheduler->Suspend({MakeRequest(5U, "user1"), {}}); + + size_t workers = 15U; + std::vector<size_t> counts; + const auto processor = [&](const IScheduler::TWaitInfo& wait) { + counts.emplace_back(wait.Request.GetCount()); + return true; + }; + + scheduler->Process(40U, workers, processor); + + scheduler->Suspend({MakeRequest(3U, "user2"), {}}); + scheduler->Suspend({MakeRequest(7U, "user2"), {}}); + scheduler->Suspend({MakeRequest(5U, "user2"), {}}); + + workers += 10U; + + const auto skipHour = TInstant::Now() + TDuration::Hours(1U); + + scheduler->Process(40U, workers, processor, skipHour); + workers += 10U; + + scheduler->Process(40U, workers, processor, skipHour); + + const std::vector<size_t> expected = {3U, 7U, 5U, 3U, 7U, 5U}; + UNIT_ASSERT_VALUES_EQUAL(counts, expected); + } + + Y_UNIT_TEST(HalfWorkersForSmall) { + const auto scheduler = IScheduler::Make(); + UNIT_ASSERT(scheduler); + + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + scheduler->Suspend({MakeRequest(7U, "user1"), {}}); + scheduler->Suspend({MakeRequest(5U, "user1"), {}}); + scheduler->Suspend({MakeRequest(1U, "user1"), {}}); + scheduler->Suspend({MakeRequest(1U, "user1"), {}}); + scheduler->Suspend({MakeRequest(1U, "user1"), {}}); + scheduler->Suspend({MakeRequest(1U, "user1"), {}}); + + size_t workers = 6U; + std::vector<size_t> counts; + const auto processor = [&](const IScheduler::TWaitInfo& wait) { + counts.emplace_back(wait.Request.GetCount()); + return true; + }; + + scheduler->Process(30U, workers, processor); + + workers += 15U; + + scheduler->Process(30U, workers, processor); + + const std::vector<size_t> expected = {1U, 1U, 1U, 3U, 1U, 7U, 5U}; + UNIT_ASSERT_VALUES_EQUAL(counts, expected); + } + + Y_UNIT_TEST(Use75PercentForLargeInNonOverload) { + const auto scheduler = IScheduler::Make(); + UNIT_ASSERT(scheduler); + + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + scheduler->Suspend({MakeRequest(3U, "user2"), {}}); + scheduler->Suspend({MakeRequest(3U, "user3"), {}}); + scheduler->Suspend({MakeRequest(3U, "user4"), {}}); + scheduler->Suspend({MakeRequest(3U, "user5"), {}}); + scheduler->Suspend({MakeRequest(3U, "user6"), {}}); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 6U); + + size_t workers = 4U; + std::vector<size_t> counts; + const auto processor = [&](const IScheduler::TWaitInfo& wait) { + counts.emplace_back(wait.Request.GetCount()); + return true; + }; + + scheduler->Process(16U, workers, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 5U); + + scheduler->Process(16U, workers = 4U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 4U); + + scheduler->Process(16U, workers = 4U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 3U); + + scheduler->Process(16U, workers = 4U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 2U); + + scheduler->Process(16U, workers = 4U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U); + + scheduler->Process(16U, workers = 4U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 0U); + } + + Y_UNIT_TEST(UseOnlyHalfForLargeInOverload) { + const auto scheduler = IScheduler::Make(); + UNIT_ASSERT(scheduler); + + scheduler->Suspend({MakeRequest(3U, "user1"), {}}); + scheduler->Suspend({MakeRequest(3U, "user2"), {}}); + scheduler->Suspend({MakeRequest(3U, "user3"), {}}); + scheduler->Suspend({MakeRequest(3U, "user4"), {}}); + scheduler->Suspend({MakeRequest(3U, "user5"), {}}); + scheduler->Suspend({MakeRequest(3U, "user6"), {}}); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 6U); + + size_t workers = 4U; + std::vector<size_t> counts; + const auto processor = [&](const IScheduler::TWaitInfo& wait) { + counts.emplace_back(wait.Request.GetCount()); + return true; + }; + + scheduler->Process(20U, workers, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 6U); + + scheduler->Process(20U, workers = 5U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 5U); + + scheduler->Process(20U, workers = 4U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 5U); + + scheduler->Process(20U, workers = 5U, processor); + + UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 4U); + } +} diff --git a/ydb/library/yql/providers/dq/scheduler/ut/ya.make b/ydb/library/yql/providers/dq/scheduler/ut/ya.make new file mode 100644 index 0000000000..069195b8b4 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/ut/ya.make @@ -0,0 +1,16 @@ +UNITTEST_FOR(ydb/library/yql/providers/dq/scheduler) + +SIZE(SMALL) + +SRCS( + dq_scheduler_ut.cpp +) + +PEERDIR( + ydb/library/yql/public/udf/service/stub + ydb/library/yql/utils/log +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/providers/dq/scheduler/ya.make b/ydb/library/yql/providers/dq/scheduler/ya.make new file mode 100644 index 0000000000..ed9eca8c37 --- /dev/null +++ b/ydb/library/yql/providers/dq/scheduler/ya.make @@ -0,0 +1,24 @@ +LIBRARY() + +PEERDIR( + ydb/library/yql/dq/common + ydb/library/yql/dq/proto + ydb/library/yql/providers/dq/config + ydb/library/yql/providers/common/proto + ydb/library/yql/providers/dq/api/protos + ydb/library/yql/providers/dq/common + ydb/library/yql/providers/dq/counters + library/cpp/actors/protos +) + +SRCS( + dq_scheduler.cpp +) + +YQL_LAST_ABI_VERSION() + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/providers/dq/ya.make b/ydb/library/yql/providers/dq/ya.make index e354a8a22f..b134a0eeef 100644 --- a/ydb/library/yql/providers/dq/ya.make +++ b/ydb/library/yql/providers/dq/ya.make @@ -5,6 +5,8 @@ RECURSE( config counters expr_nodes + scheduler + global_worker_manager interface mkql opt diff --git a/yt/yt/CMakeLists.darwin-x86_64.txt b/yt/yt/CMakeLists.darwin-x86_64.txt index b0414f0741..91d55b509b 100644 --- a/yt/yt/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/CMakeLists.darwin-x86_64.txt @@ -7,5 +7,6 @@ add_subdirectory(build) +add_subdirectory(client) add_subdirectory(core) add_subdirectory(library) diff --git a/yt/yt/client/CMakeLists.darwin-x86_64.txt b/yt/yt/client/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..3536fc456a --- /dev/null +++ b/yt/yt/client/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,199 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(query_tracker_client) +set( + YT_RPC_MODIFY_ROWS_STRONG_LOCKS_VERSION + 2 +) +set( + YT_RPC_PROXY_CLIENT_PROTOCOL_VERSION_MINOR + 1 +) +set( + YT_RPC_PROXY_PROTOCOL_VERSION_MAJOR + 1 +) +set( + YT_RPC_PROXY_SERVER_PROTOCOL_VERSION_MINOR + 2 +) + +add_library(yt-yt-client) +target_compile_options(yt-yt-client PRIVATE + -Wdeprecated-this-capture +) +target_include_directories(yt-yt-client PUBLIC + ${CMAKE_BINARY_DIR}/yt + ${CMAKE_BINARY_DIR}/yt/yt/client/_/api/rpc_proxy +) +target_link_libraries(yt-yt-client PUBLIC + contrib-libs-cxxsupp + yutil + yt-client-query_tracker_client + yt-yt-core + yt-core-http + yt-core-https + yt-library-auth + yt-library-decimal + yt-library-re2 + yt-library-erasure + yt-library-numeric + yt-library-quantile_digest + yt_proto-yt-client + library-cpp-json + contrib-libs-pfr +) +target_sources(yt-yt-client PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/client_common.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/client_cache.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/delegating_client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/etc_client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/journal_client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/operation_client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/security_client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/table_client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/query_tracker_client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/internal_client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/operation_archive_schema.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rowset.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/skynet.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/transaction.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/persistent_queue.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/sticky_transaction_pool.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/address_helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/client_impl.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/client_base.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/connection.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/connection_impl.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/file_reader.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/file_writer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/journal_reader.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/journal_writer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/table_mount_cache.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/table_reader.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/table_writer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/timestamp_provider.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/transaction.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/transaction_impl.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/row_stream.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/wire_row_stream.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/election/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/hive/timestamp_map.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/hydra/version.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chaos_client/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chaos_client/replication_card.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chaos_client/replication_card_cache.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chaos_client/replication_card_serialization.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/chunk_replica.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/data_statistics.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/read_limit.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/ready_event_reader_base.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/file_client/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/journal_client/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/journal_client/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/cypress_client/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/node_tracker_client/node_directory.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/node_tracker_client/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/node_tracker_client/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/object_client/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/object_client/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/scheduler/operation_id_or_alias.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/scheduler/operation_cache.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/security_client/acl.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/security_client/access_control.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/security_client/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/security_client/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/adapters.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/table_output.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/blob_reader.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/check_schema_compatibility.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/chunk_stripe_statistics.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/column_rename_descriptor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/column_sort_schema.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/comparator.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/key.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/key_bound.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/key_bound_compressor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/pipe.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/versioned_row.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/unversioned_row.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/unversioned_value.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/versioned_reader.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/row_base.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/row_batch.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/row_buffer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/schema.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/schema_serialization_helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/serialize.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/logical_type.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/name_table.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/wire_protocol.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/columnar_statistics.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/value_consumer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/table_consumer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/schemaless_row_reorderer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/unordered_schemaful_reader.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/validate_logical_type.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/composite_compare.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/columnar.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/record_codegen_cpp.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/record_helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/table_mount_cache_detail.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/table_mount_cache.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/common.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/consumer_client.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/partition_reader.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/queue_rowset.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/ypath/rich.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/ypath/parser_detail.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/batching_timestamp_provider.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/noop_timestamp_provider.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/remote_timestamp_provider.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/timestamp_provider_base.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/misc/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/misc/io_tags.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/misc/method_helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/misc/workload.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/job_tracker_client/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/job_tracker_client/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/query_client/query_builder.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/query_client/query_statistics.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/check_yson_token.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/check_type_compatibility.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/infinite_entity.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/yson_format_conversion.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/uuid_text.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/time_text.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/zookeeper/packet.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/zookeeper/protocol.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/client/zookeeper/requests.cpp +) +configure_file( + ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/protocol_version_variables.h.in + ${CMAKE_BINARY_DIR}/yt/yt/client/_/api/rpc_proxy/protocol_version_variables.h +) diff --git a/yt/yt/client/CMakeLists.txt b/yt/yt/client/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/yt/yt/client/CMakeLists.txt +++ b/yt/yt/client/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/yt/yt/client/query_tracker_client/CMakeLists.darwin-x86_64.txt b/yt/yt/client/query_tracker_client/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..3d1ea8d25a --- /dev/null +++ b/yt/yt/client/query_tracker_client/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-client-query_tracker_client) +target_compile_options(yt-client-query_tracker_client PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-client-query_tracker_client PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core +) +target_sources(yt-client-query_tracker_client PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/client/query_tracker_client/public.cpp +) diff --git a/yt/yt/client/query_tracker_client/CMakeLists.txt b/yt/yt/client/query_tracker_client/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/yt/yt/client/query_tracker_client/CMakeLists.txt +++ b/yt/yt/client/query_tracker_client/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/yt/yt/library/CMakeLists.darwin-x86_64.txt b/yt/yt/library/CMakeLists.darwin-x86_64.txt index 20f7fe76fa..d26f3bae28 100644 --- a/yt/yt/library/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/library/CMakeLists.darwin-x86_64.txt @@ -6,7 +6,13 @@ # original buildsystem will not be accepted. +add_subdirectory(auth) +add_subdirectory(decimal) +add_subdirectory(erasure) +add_subdirectory(numeric) add_subdirectory(profiling) +add_subdirectory(quantile_digest) +add_subdirectory(re2) add_subdirectory(syncmap) add_subdirectory(tracing) add_subdirectory(tvm) diff --git a/yt/yt/library/auth/CMakeLists.darwin-x86_64.txt b/yt/yt/library/auth/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..b026c98348 --- /dev/null +++ b/yt/yt/library/auth/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-auth) +target_compile_options(yt-library-auth PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-auth PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + yt-library-tvm +) +target_sources(yt-library-auth PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/auth/auth.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/auth/authentication_options.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/auth/credentials_injecting_channel.cpp +) diff --git a/yt/yt/library/auth/CMakeLists.txt b/yt/yt/library/auth/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/yt/yt/library/auth/CMakeLists.txt +++ b/yt/yt/library/auth/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt b/yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..2b82fc6b48 --- /dev/null +++ b/yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-decimal) +target_compile_options(yt-library-decimal PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-decimal PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + library-cpp-int128 +) +target_sources(yt-library-decimal PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/decimal/decimal.cpp +) diff --git a/yt/yt/library/decimal/CMakeLists.txt b/yt/yt/library/decimal/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/yt/yt/library/decimal/CMakeLists.txt +++ b/yt/yt/library/decimal/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/yt/yt/library/erasure/CMakeLists.darwin-x86_64.txt b/yt/yt/library/erasure/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..232e2fa1c0 --- /dev/null +++ b/yt/yt/library/erasure/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-erasure) +target_compile_options(yt-library-erasure PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-erasure PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core +) +target_sources(yt-library-erasure PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/erasure/public.cpp +) diff --git a/yt/yt/library/erasure/CMakeLists.txt b/yt/yt/library/erasure/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/yt/yt/library/erasure/CMakeLists.txt +++ b/yt/yt/library/erasure/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/yt/yt/library/numeric/CMakeLists.darwin-x86_64.txt b/yt/yt/library/numeric/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..9f1b4b1c1e --- /dev/null +++ b/yt/yt/library/numeric/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-numeric) +target_compile_options(yt-library-numeric PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-numeric PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-small_containers +) +target_sources(yt-library-numeric PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/numeric/piecewise_linear_function.cpp +) diff --git a/yt/yt/library/numeric/CMakeLists.txt b/yt/yt/library/numeric/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/yt/yt/library/numeric/CMakeLists.txt +++ b/yt/yt/library/numeric/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/yt/yt/library/quantile_digest/CMakeLists.darwin-x86_64.txt b/yt/yt/library/quantile_digest/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..ceac4e685a --- /dev/null +++ b/yt/yt/library/quantile_digest/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,54 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(yt-library-quantile_digest) +target_compile_options(yt-library-quantile_digest PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-quantile_digest PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-memory + library-cpp-tdigest + yt-yt-core + contrib-libs-protobuf +) +target_proto_messages(yt-library-quantile_digest PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/quantile_digest/proto/quantile_digest.proto +) +target_sources(yt-library-quantile_digest PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/quantile_digest/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/quantile_digest/quantile_digest.cpp +) +target_proto_addincls(yt-library-quantile_digest + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(yt-library-quantile_digest + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) diff --git a/yt/yt/library/quantile_digest/CMakeLists.txt b/yt/yt/library/quantile_digest/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/yt/yt/library/quantile_digest/CMakeLists.txt +++ b/yt/yt/library/quantile_digest/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/yt/yt/library/re2/CMakeLists.darwin-x86_64.txt b/yt/yt/library/re2/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..7310b7ab28 --- /dev/null +++ b/yt/yt/library/re2/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-re2) +target_compile_options(yt-library-re2 PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-re2 PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + contrib-libs-re2 +) +target_sources(yt-library-re2 PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/re2/re2.cpp +) diff --git a/yt/yt/library/re2/CMakeLists.txt b/yt/yt/library/re2/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/yt/yt/library/re2/CMakeLists.txt +++ b/yt/yt/library/re2/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/yt/yt_proto/yt/CMakeLists.darwin-x86_64.txt b/yt/yt_proto/yt/CMakeLists.darwin-x86_64.txt index d8f6bf9771..aa7a2b73aa 100644 --- a/yt/yt_proto/yt/CMakeLists.darwin-x86_64.txt +++ b/yt/yt_proto/yt/CMakeLists.darwin-x86_64.txt @@ -6,5 +6,6 @@ # original buildsystem will not be accepted. +add_subdirectory(client) add_subdirectory(core) add_subdirectory(formats) diff --git a/yt/yt_proto/yt/client/CMakeLists.darwin-x86_64.txt b/yt/yt_proto/yt/client/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..bde6ffe9e0 --- /dev/null +++ b/yt/yt_proto/yt/client/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,296 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(yt_proto-yt-client) +target_include_directories(yt_proto-yt-client PUBLIC + ${CMAKE_BINARY_DIR}/yt +) +target_link_libraries(yt_proto-yt-client PUBLIC + contrib-libs-cxxsupp + yutil + yt_proto-yt-core + contrib-libs-protobuf +) +target_proto_messages(yt_proto-yt-client PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/api/rpc_proxy/proto/discovery_service.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/cell_master/proto/cell_directory.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chaos_client/proto/replication_card.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/data_statistics.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/chunk_meta.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/read_limit.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/chunk_spec.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/confirm_chunk_replica_info.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/discovery_client/proto/discovery_client_service.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/hive/proto/timestamp_map.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/hive/proto/cluster_directory.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/node_tracker_client/proto/node.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/node_tracker_client/proto/node_directory.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/table_chunk_format/proto/chunk_meta.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/table_chunk_format/proto/column_meta.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/table_chunk_format/proto/wire_protocol.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/transaction_client/proto/timestamp_service.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/query_client/proto/query_statistics.proto + ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/misc/proto/workload.proto +) +target_proto_addincls(yt_proto-yt-client + ./yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(yt_proto-yt-client + --cpp_out=${CMAKE_BINARY_DIR}/yt + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt +) diff --git a/yt/yt_proto/yt/client/CMakeLists.txt b/yt/yt_proto/yt/client/CMakeLists.txt index 4d48dcdee6..606ff46b4b 100644 --- a/yt/yt_proto/yt/client/CMakeLists.txt +++ b/yt/yt_proto/yt/client/CMakeLists.txt @@ -8,6 +8,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() |