aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authoraozeritsky <aozeritsky@ydb.tech>2023-10-24 19:48:53 +0300
committeraozeritsky <aozeritsky@ydb.tech>2023-10-24 20:23:39 +0300
commit623315504d982fbdc29eb377737d4ffcab4e5960 (patch)
tree1c50963c3406a5954083157daf2c807ce5b863f7
parenta37ccbf072fe185a1781f70b8ee55ce9b678dc7d (diff)
downloadydb-623315504d982fbdc29eb377737d4ffcab4e5960.tar.gz
Move worker_node/service_node dependencies to ydb/library/yql
-rw-r--r--.mapping.json40
-rw-r--r--contrib/libs/CMakeLists.darwin-x86_64.txt1
-rw-r--r--contrib/libs/pfr/CMakeLists.darwin-x86_64.txt17
-rw-r--r--contrib/libs/pfr/CMakeLists.txt2
-rw-r--r--ydb/library/yql/providers/dq/CMakeLists.darwin-x86_64.txt2
-rw-r--r--ydb/library/yql/providers/dq/CMakeLists.linux-aarch64.txt2
-rw-r--r--ydb/library/yql/providers/dq/CMakeLists.linux-x86_64.txt2
-rw-r--r--ydb/library/yql/providers/dq/CMakeLists.windows-x86_64.txt2
-rw-r--r--ydb/library/yql/providers/dq/actors/CMakeLists.darwin-x86_64.txt5
-rw-r--r--ydb/library/yql/providers/dq/actors/CMakeLists.linux-aarch64.txt5
-rw-r--r--ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt5
-rw-r--r--ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt5
-rw-r--r--ydb/library/yql/providers/dq/actors/dummy_lock.cpp56
-rw-r--r--ydb/library/yql/providers/dq/actors/dummy_lock.h12
-rw-r--r--ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp208
-rw-r--r--ydb/library/yql/providers/dq/actors/dynamic_nameserver.h7
-rw-r--r--ydb/library/yql/providers/dq/actors/events/CMakeLists.darwin-x86_64.txt18
-rw-r--r--ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-aarch64.txt19
-rw-r--r--ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-x86_64.txt19
-rw-r--r--ydb/library/yql/providers/dq/actors/events/CMakeLists.txt17
-rw-r--r--ydb/library/yql/providers/dq/actors/events/CMakeLists.windows-x86_64.txt18
-rw-r--r--ydb/library/yql/providers/dq/actors/events/empty.cpp0
-rw-r--r--ydb/library/yql/providers/dq/actors/events/events.h59
-rw-r--r--ydb/library/yql/providers/dq/actors/events/ya.make16
-rw-r--r--ydb/library/yql/providers/dq/actors/ya.make8
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/CMakeLists.darwin-x86_64.txt41
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-aarch64.txt42
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-x86_64.txt42
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/CMakeLists.txt17
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/CMakeLists.windows-x86_64.txt32
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/lock.cpp435
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/lock.h14
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp116
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h29
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp137
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.h18
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp154
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp141
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp69
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/resource_manager.h90
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp236
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp79
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/worker_registrator.h21
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/ya.make56
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/yt_events.h35
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp917
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp635
-rw-r--r--ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h88
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.darwin-x86_64.txt40
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-aarch64.txt41
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-x86_64.txt41
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.txt17
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.windows-x86_64.txt39
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp95
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/benchmark.h16
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp354
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h65
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/coordination_helper_win.cpp13
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp1343
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.h20
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp439
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp398
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.h23
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp303
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h80
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.darwin-x86_64.txt77
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-aarch64.txt81
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-x86_64.txt83
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.windows-x86_64.txt14
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/ut/ya.make29
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp70
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/worker_filter.h54
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp580
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/workers_storage.h90
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp82
-rw-r--r--ydb/library/yql/providers/dq/global_worker_manager/ya.make53
-rw-r--r--ydb/library/yql/providers/dq/scheduler/CMakeLists.darwin-x86_64.txt29
-rw-r--r--ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-aarch64.txt30
-rw-r--r--ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-x86_64.txt30
-rw-r--r--ydb/library/yql/providers/dq/scheduler/CMakeLists.txt17
-rw-r--r--ydb/library/yql/providers/dq/scheduler/CMakeLists.windows-x86_64.txt29
-rw-r--r--ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp240
-rw-r--r--ydb/library/yql/providers/dq/scheduler/dq_scheduler.h47
-rw-r--r--ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.darwin-x86_64.txt72
-rw-r--r--ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-aarch64.txt75
-rw-r--r--ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-x86_64.txt77
-rw-r--r--ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.windows-x86_64.txt65
-rw-r--r--ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp311
-rw-r--r--ydb/library/yql/providers/dq/scheduler/ut/ya.make16
-rw-r--r--ydb/library/yql/providers/dq/scheduler/ya.make24
-rw-r--r--ydb/library/yql/providers/dq/ya.make2
-rw-r--r--yt/yt/CMakeLists.darwin-x86_64.txt1
-rw-r--r--yt/yt/client/CMakeLists.darwin-x86_64.txt199
-rw-r--r--yt/yt/client/CMakeLists.txt2
-rw-r--r--yt/yt/client/query_tracker_client/CMakeLists.darwin-x86_64.txt21
-rw-r--r--yt/yt/client/query_tracker_client/CMakeLists.txt2
-rw-r--r--yt/yt/library/CMakeLists.darwin-x86_64.txt6
-rw-r--r--yt/yt/library/auth/CMakeLists.darwin-x86_64.txt24
-rw-r--r--yt/yt/library/auth/CMakeLists.txt2
-rw-r--r--yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt22
-rw-r--r--yt/yt/library/decimal/CMakeLists.txt2
-rw-r--r--yt/yt/library/erasure/CMakeLists.darwin-x86_64.txt21
-rw-r--r--yt/yt/library/erasure/CMakeLists.txt2
-rw-r--r--yt/yt/library/numeric/CMakeLists.darwin-x86_64.txt21
-rw-r--r--yt/yt/library/numeric/CMakeLists.txt2
-rw-r--r--yt/yt/library/quantile_digest/CMakeLists.darwin-x86_64.txt54
-rw-r--r--yt/yt/library/quantile_digest/CMakeLists.txt2
-rw-r--r--yt/yt/library/re2/CMakeLists.darwin-x86_64.txt22
-rw-r--r--yt/yt/library/re2/CMakeLists.txt2
-rw-r--r--yt/yt_proto/yt/CMakeLists.darwin-x86_64.txt1
-rw-r--r--yt/yt_proto/yt/client/CMakeLists.darwin-x86_64.txt296
-rw-r--r--yt/yt_proto/yt/client/CMakeLists.txt2
114 files changed, 10333 insertions, 0 deletions
diff --git a/.mapping.json b/.mapping.json
index 8ea27f5354..9adbc7f04e 100644
--- a/.mapping.json
+++ b/.mapping.json
@@ -769,6 +769,7 @@
"contrib/libs/pdqsort/CMakeLists.linux-aarch64.txt":"",
"contrib/libs/pdqsort/CMakeLists.linux-x86_64.txt":"",
"contrib/libs/pdqsort/CMakeLists.txt":"",
+ "contrib/libs/pfr/CMakeLists.darwin-x86_64.txt":"",
"contrib/libs/pfr/CMakeLists.linux-aarch64.txt":"",
"contrib/libs/pfr/CMakeLists.linux-x86_64.txt":"",
"contrib/libs/pfr/CMakeLists.txt":"",
@@ -7507,11 +7508,21 @@
"ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt":"",
"ydb/library/yql/providers/dq/actors/CMakeLists.txt":"",
"ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/actors/events/CMakeLists.darwin-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-aarch64.txt":"",
+ "ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/actors/events/CMakeLists.txt":"",
+ "ydb/library/yql/providers/dq/actors/events/CMakeLists.windows-x86_64.txt":"",
"ydb/library/yql/providers/dq/actors/ut/CMakeLists.darwin-x86_64.txt":"",
"ydb/library/yql/providers/dq/actors/ut/CMakeLists.linux-aarch64.txt":"",
"ydb/library/yql/providers/dq/actors/ut/CMakeLists.linux-x86_64.txt":"",
"ydb/library/yql/providers/dq/actors/ut/CMakeLists.txt":"",
"ydb/library/yql/providers/dq/actors/ut/CMakeLists.windows-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/actors/yt/CMakeLists.darwin-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-aarch64.txt":"",
+ "ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/actors/yt/CMakeLists.txt":"",
+ "ydb/library/yql/providers/dq/actors/yt/CMakeLists.windows-x86_64.txt":"",
"ydb/library/yql/providers/dq/api/CMakeLists.txt":"",
"ydb/library/yql/providers/dq/api/grpc/CMakeLists.darwin-x86_64.txt":"",
"ydb/library/yql/providers/dq/api/grpc/CMakeLists.linux-aarch64.txt":"",
@@ -7543,6 +7554,16 @@
"ydb/library/yql/providers/dq/expr_nodes/CMakeLists.linux-x86_64.txt":"",
"ydb/library/yql/providers/dq/expr_nodes/CMakeLists.txt":"",
"ydb/library/yql/providers/dq/expr_nodes/CMakeLists.windows-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.darwin-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-aarch64.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.windows-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.darwin-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-aarch64.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.txt":"",
+ "ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.windows-x86_64.txt":"",
"ydb/library/yql/providers/dq/interface/CMakeLists.darwin-x86_64.txt":"",
"ydb/library/yql/providers/dq/interface/CMakeLists.linux-aarch64.txt":"",
"ydb/library/yql/providers/dq/interface/CMakeLists.linux-x86_64.txt":"",
@@ -7587,6 +7608,16 @@
"ydb/library/yql/providers/dq/runtime/CMakeLists.linux-x86_64.txt":"",
"ydb/library/yql/providers/dq/runtime/CMakeLists.txt":"",
"ydb/library/yql/providers/dq/runtime/CMakeLists.windows-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/CMakeLists.darwin-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-aarch64.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/CMakeLists.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/CMakeLists.windows-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.darwin-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-aarch64.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-x86_64.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.txt":"",
+ "ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.windows-x86_64.txt":"",
"ydb/library/yql/providers/dq/service/CMakeLists.darwin-x86_64.txt":"",
"ydb/library/yql/providers/dq/service/CMakeLists.linux-aarch64.txt":"",
"ydb/library/yql/providers/dq/service/CMakeLists.linux-x86_64.txt":"",
@@ -9727,6 +9758,7 @@
"yt/yt/build/CMakeLists.linux-x86_64.txt":"",
"yt/yt/build/CMakeLists.txt":"",
"yt/yt/build/CMakeLists.windows-x86_64.txt":"",
+ "yt/yt/client/CMakeLists.darwin-x86_64.txt":"",
"yt/yt/client/CMakeLists.linux-aarch64.txt":"",
"yt/yt/client/CMakeLists.linux-x86_64.txt":"",
"yt/yt/client/CMakeLists.txt":"",
@@ -9736,6 +9768,7 @@
"yt/yt/client/arrow/fbs/CMakeLists.linux-aarch64.txt":"",
"yt/yt/client/arrow/fbs/CMakeLists.linux-x86_64.txt":"",
"yt/yt/client/arrow/fbs/CMakeLists.txt":"",
+ "yt/yt/client/query_tracker_client/CMakeLists.darwin-x86_64.txt":"",
"yt/yt/client/query_tracker_client/CMakeLists.linux-aarch64.txt":"",
"yt/yt/client/query_tracker_client/CMakeLists.linux-x86_64.txt":"",
"yt/yt/client/query_tracker_client/CMakeLists.txt":"",
@@ -9765,15 +9798,19 @@
"yt/yt/library/CMakeLists.linux-x86_64.txt":"",
"yt/yt/library/CMakeLists.txt":"",
"yt/yt/library/CMakeLists.windows-x86_64.txt":"",
+ "yt/yt/library/auth/CMakeLists.darwin-x86_64.txt":"",
"yt/yt/library/auth/CMakeLists.linux-aarch64.txt":"",
"yt/yt/library/auth/CMakeLists.linux-x86_64.txt":"",
"yt/yt/library/auth/CMakeLists.txt":"",
+ "yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt":"",
"yt/yt/library/decimal/CMakeLists.linux-aarch64.txt":"",
"yt/yt/library/decimal/CMakeLists.linux-x86_64.txt":"",
"yt/yt/library/decimal/CMakeLists.txt":"",
+ "yt/yt/library/erasure/CMakeLists.darwin-x86_64.txt":"",
"yt/yt/library/erasure/CMakeLists.linux-aarch64.txt":"",
"yt/yt/library/erasure/CMakeLists.linux-x86_64.txt":"",
"yt/yt/library/erasure/CMakeLists.txt":"",
+ "yt/yt/library/numeric/CMakeLists.darwin-x86_64.txt":"",
"yt/yt/library/numeric/CMakeLists.linux-aarch64.txt":"",
"yt/yt/library/numeric/CMakeLists.linux-x86_64.txt":"",
"yt/yt/library/numeric/CMakeLists.txt":"",
@@ -9787,9 +9824,11 @@
"yt/yt/library/profiling/resource_tracker/CMakeLists.linux-x86_64.txt":"",
"yt/yt/library/profiling/resource_tracker/CMakeLists.txt":"",
"yt/yt/library/profiling/resource_tracker/CMakeLists.windows-x86_64.txt":"",
+ "yt/yt/library/quantile_digest/CMakeLists.darwin-x86_64.txt":"",
"yt/yt/library/quantile_digest/CMakeLists.linux-aarch64.txt":"",
"yt/yt/library/quantile_digest/CMakeLists.linux-x86_64.txt":"",
"yt/yt/library/quantile_digest/CMakeLists.txt":"",
+ "yt/yt/library/re2/CMakeLists.darwin-x86_64.txt":"",
"yt/yt/library/re2/CMakeLists.linux-aarch64.txt":"",
"yt/yt/library/re2/CMakeLists.linux-x86_64.txt":"",
"yt/yt/library/re2/CMakeLists.txt":"",
@@ -9825,6 +9864,7 @@
"yt/yt_proto/yt/CMakeLists.linux-x86_64.txt":"",
"yt/yt_proto/yt/CMakeLists.txt":"",
"yt/yt_proto/yt/CMakeLists.windows-x86_64.txt":"",
+ "yt/yt_proto/yt/client/CMakeLists.darwin-x86_64.txt":"",
"yt/yt_proto/yt/client/CMakeLists.linux-aarch64.txt":"",
"yt/yt_proto/yt/client/CMakeLists.linux-x86_64.txt":"",
"yt/yt_proto/yt/client/CMakeLists.txt":"",
diff --git a/contrib/libs/CMakeLists.darwin-x86_64.txt b/contrib/libs/CMakeLists.darwin-x86_64.txt
index ad248a50dc..c62e34c5ff 100644
--- a/contrib/libs/CMakeLists.darwin-x86_64.txt
+++ b/contrib/libs/CMakeLists.darwin-x86_64.txt
@@ -50,6 +50,7 @@ add_subdirectory(openssl)
add_subdirectory(opentelemetry-proto)
add_subdirectory(pcre)
add_subdirectory(pdqsort)
+add_subdirectory(pfr)
add_subdirectory(poco)
add_subdirectory(protobuf)
add_subdirectory(protoc)
diff --git a/contrib/libs/pfr/CMakeLists.darwin-x86_64.txt b/contrib/libs/pfr/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..66a1b47cce
--- /dev/null
+++ b/contrib/libs/pfr/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(contrib-libs-pfr INTERFACE)
+target_include_directories(contrib-libs-pfr INTERFACE
+ ${CMAKE_SOURCE_DIR}/contrib/libs/pfr/include
+)
+target_link_libraries(contrib-libs-pfr INTERFACE
+ contrib-libs-cxxsupp
+ yutil
+)
diff --git a/contrib/libs/pfr/CMakeLists.txt b/contrib/libs/pfr/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/contrib/libs/pfr/CMakeLists.txt
+++ b/contrib/libs/pfr/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()
diff --git a/ydb/library/yql/providers/dq/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/CMakeLists.darwin-x86_64.txt
index fae483c5c0..1b2879cd08 100644
--- a/ydb/library/yql/providers/dq/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/dq/CMakeLists.darwin-x86_64.txt
@@ -12,6 +12,7 @@ add_subdirectory(common)
add_subdirectory(config)
add_subdirectory(counters)
add_subdirectory(expr_nodes)
+add_subdirectory(global_worker_manager)
add_subdirectory(interface)
add_subdirectory(local_gateway)
add_subdirectory(mkql)
@@ -19,6 +20,7 @@ add_subdirectory(opt)
add_subdirectory(planner)
add_subdirectory(provider)
add_subdirectory(runtime)
+add_subdirectory(scheduler)
add_subdirectory(service)
add_subdirectory(stats_collector)
add_subdirectory(task_runner)
diff --git a/ydb/library/yql/providers/dq/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/CMakeLists.linux-aarch64.txt
index fae483c5c0..1b2879cd08 100644
--- a/ydb/library/yql/providers/dq/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/dq/CMakeLists.linux-aarch64.txt
@@ -12,6 +12,7 @@ add_subdirectory(common)
add_subdirectory(config)
add_subdirectory(counters)
add_subdirectory(expr_nodes)
+add_subdirectory(global_worker_manager)
add_subdirectory(interface)
add_subdirectory(local_gateway)
add_subdirectory(mkql)
@@ -19,6 +20,7 @@ add_subdirectory(opt)
add_subdirectory(planner)
add_subdirectory(provider)
add_subdirectory(runtime)
+add_subdirectory(scheduler)
add_subdirectory(service)
add_subdirectory(stats_collector)
add_subdirectory(task_runner)
diff --git a/ydb/library/yql/providers/dq/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/CMakeLists.linux-x86_64.txt
index fae483c5c0..1b2879cd08 100644
--- a/ydb/library/yql/providers/dq/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/dq/CMakeLists.linux-x86_64.txt
@@ -12,6 +12,7 @@ add_subdirectory(common)
add_subdirectory(config)
add_subdirectory(counters)
add_subdirectory(expr_nodes)
+add_subdirectory(global_worker_manager)
add_subdirectory(interface)
add_subdirectory(local_gateway)
add_subdirectory(mkql)
@@ -19,6 +20,7 @@ add_subdirectory(opt)
add_subdirectory(planner)
add_subdirectory(provider)
add_subdirectory(runtime)
+add_subdirectory(scheduler)
add_subdirectory(service)
add_subdirectory(stats_collector)
add_subdirectory(task_runner)
diff --git a/ydb/library/yql/providers/dq/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/CMakeLists.windows-x86_64.txt
index 8a270ca445..8a3422b0d6 100644
--- a/ydb/library/yql/providers/dq/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/dq/CMakeLists.windows-x86_64.txt
@@ -12,12 +12,14 @@ add_subdirectory(common)
add_subdirectory(config)
add_subdirectory(counters)
add_subdirectory(expr_nodes)
+add_subdirectory(global_worker_manager)
add_subdirectory(interface)
add_subdirectory(mkql)
add_subdirectory(opt)
add_subdirectory(planner)
add_subdirectory(provider)
add_subdirectory(runtime)
+add_subdirectory(scheduler)
add_subdirectory(task_runner)
add_subdirectory(task_runner_actor)
add_subdirectory(worker_manager)
diff --git a/ydb/library/yql/providers/dq/actors/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/actors/CMakeLists.darwin-x86_64.txt
index 06b0b443ae..a7cb059668 100644
--- a/ydb/library/yql/providers/dq/actors/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/dq/actors/CMakeLists.darwin-x86_64.txt
@@ -6,7 +6,9 @@
# original buildsystem will not be accepted.
+add_subdirectory(events)
add_subdirectory(ut)
+add_subdirectory(yt)
add_library(providers-dq-actors)
target_compile_options(providers-dq-actors PRIVATE
@@ -33,6 +35,7 @@ target_link_libraries(providers-dq-actors PUBLIC
yql-dq-tasks
yql-utils-failure_injector
providers-common-metrics
+ dq-actors-events
dq-api-grpc
dq-api-protos
providers-dq-common
@@ -47,6 +50,8 @@ target_link_libraries(providers-dq-actors PUBLIC
)
target_sources(providers-dq-actors PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/compute_actor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dummy_lock.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/executer_actor.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/execution_helpers.cpp
diff --git a/ydb/library/yql/providers/dq/actors/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/actors/CMakeLists.linux-aarch64.txt
index a0e19378dc..74bf95ff13 100644
--- a/ydb/library/yql/providers/dq/actors/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/dq/actors/CMakeLists.linux-aarch64.txt
@@ -6,7 +6,9 @@
# original buildsystem will not be accepted.
+add_subdirectory(events)
add_subdirectory(ut)
+add_subdirectory(yt)
add_library(providers-dq-actors)
target_compile_options(providers-dq-actors PRIVATE
@@ -34,6 +36,7 @@ target_link_libraries(providers-dq-actors PUBLIC
yql-dq-tasks
yql-utils-failure_injector
providers-common-metrics
+ dq-actors-events
dq-api-grpc
dq-api-protos
providers-dq-common
@@ -48,6 +51,8 @@ target_link_libraries(providers-dq-actors PUBLIC
)
target_sources(providers-dq-actors PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/compute_actor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dummy_lock.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/executer_actor.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/execution_helpers.cpp
diff --git a/ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt
index a0e19378dc..74bf95ff13 100644
--- a/ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/dq/actors/CMakeLists.linux-x86_64.txt
@@ -6,7 +6,9 @@
# original buildsystem will not be accepted.
+add_subdirectory(events)
add_subdirectory(ut)
+add_subdirectory(yt)
add_library(providers-dq-actors)
target_compile_options(providers-dq-actors PRIVATE
@@ -34,6 +36,7 @@ target_link_libraries(providers-dq-actors PUBLIC
yql-dq-tasks
yql-utils-failure_injector
providers-common-metrics
+ dq-actors-events
dq-api-grpc
dq-api-protos
providers-dq-common
@@ -48,6 +51,8 @@ target_link_libraries(providers-dq-actors PUBLIC
)
target_sources(providers-dq-actors PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/compute_actor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dummy_lock.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/executer_actor.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/execution_helpers.cpp
diff --git a/ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt
index 06b0b443ae..a7cb059668 100644
--- a/ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/dq/actors/CMakeLists.windows-x86_64.txt
@@ -6,7 +6,9 @@
# original buildsystem will not be accepted.
+add_subdirectory(events)
add_subdirectory(ut)
+add_subdirectory(yt)
add_library(providers-dq-actors)
target_compile_options(providers-dq-actors PRIVATE
@@ -33,6 +35,7 @@ target_link_libraries(providers-dq-actors PUBLIC
yql-dq-tasks
yql-utils-failure_injector
providers-common-metrics
+ dq-actors-events
dq-api-grpc
dq-api-protos
providers-dq-common
@@ -47,6 +50,8 @@ target_link_libraries(providers-dq-actors PUBLIC
)
target_sources(providers-dq-actors PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/compute_actor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dummy_lock.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/executer_actor.cpp
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/execution_helpers.cpp
diff --git a/ydb/library/yql/providers/dq/actors/dummy_lock.cpp b/ydb/library/yql/providers/dq/actors/dummy_lock.cpp
new file mode 100644
index 0000000000..c81b31a7da
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/dummy_lock.cpp
@@ -0,0 +1,56 @@
+#include "dummy_lock.h"
+
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/hfunc.h>
+
+#include <library/cpp/svnversion/svnversion.h>
+
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+
+namespace NYql {
+
+using namespace NActors;
+
+class TDummyLock: public TRichActor<TDummyLock> {
+public:
+ static constexpr char ActorName[] = "DUMMY_LOCK";
+
+ TDummyLock(
+ const TString& lockName,
+ const TString& lockAttributes)
+ : TRichActor<TDummyLock>(&TDummyLock::Handler)
+ , LockName(lockName)
+ , Attributes(lockAttributes)
+ , Revision(GetProgramCommitId())
+ {
+ Y_UNUSED(LockName);
+ Y_UNUSED(Attributes);
+ Y_UNUSED(Revision);
+ }
+
+private:
+ STRICT_STFUNC(Handler, {
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ });
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ Y_UNUSED(self);
+ return new IEventHandle(parentId, parentId, new TEvBecomeLeader(1, "TransactionId", Attributes), 0);
+ }
+
+ TString LockName;
+ TString Attributes;
+ TString Revision;
+};
+
+NActors::IActor* CreateDummyLock(
+ const TString& lockName,
+ const TString& lockAttributesYson)
+{
+ return new TDummyLock(lockName, lockAttributesYson);
+}
+
+} // namespace NYql
+
diff --git a/ydb/library/yql/providers/dq/actors/dummy_lock.h b/ydb/library/yql/providers/dq/actors/dummy_lock.h
new file mode 100644
index 0000000000..18d028d4dc
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/dummy_lock.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <library/cpp/actors/core/actor.h>
+
+namespace NYql {
+
+NActors::IActor* CreateDummyLock(
+ const TString& lockName,
+ const TString& lockAttributesYson);
+
+} // namespace NYql
+
diff --git a/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp b/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp
new file mode 100644
index 0000000000..3031f188f4
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/dynamic_nameserver.cpp
@@ -0,0 +1,208 @@
+#include "dynamic_nameserver.h"
+
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <library/cpp/actors/interconnect/interconnect.h>
+#include <library/cpp/actors/interconnect/interconnect_impl.h>
+#include <library/cpp/actors/interconnect/interconnect_address.h>
+#include <library/cpp/actors/interconnect/events_local.h>
+
+#include <library/cpp/actors/core/hfunc.h>
+
+#include <ydb/library/yql/providers/dq/worker_manager/interface/events.h>
+
+namespace NYql::NDqs {
+ using namespace NActors;
+
+ class TDynamicNameserver: public TActor<TDynamicNameserver> {
+ TMap<ui32, TTableNameserverSetup::TNodeInfo> NodeTable;
+
+ public:
+ static constexpr EActivityType ActorActivityType() {
+ return EActivityType::INTERCONNECT_COMMON;
+ }
+
+ TDynamicNameserver(
+ const TIntrusivePtr<TTableNameserverSetup>& setup,
+ ui32 /*resolvePoolId*/ = 0)
+ : TActor(&TDynamicNameserver::StateFunc)
+ , NodeTable(setup->StaticNodeTable)
+ {
+ Y_ABORT_UNLESS(setup->IsEntriesUnique());
+ }
+
+ STFUNC(StateFunc) {
+ switch (ev->GetTypeRewrite()) {
+ HFunc(TEvInterconnect::TEvResolveNode, Handle);
+ HFunc(TEvResolveAddress, Handle);
+ HFunc(TEvInterconnect::TEvListNodes, Handle);
+ HFunc(TEvInterconnect::TEvGetNode, Handle);
+ HFunc(TEvInterconnect::TEvNodesInfo, UpdateNodeTable);
+ HFunc(TEvRegisterNode, RegisterNodeHandle);
+ }
+ }
+
+ void PrintInfo() {
+ YQL_CLOG(TRACE, ProviderDq) << "Table ";
+ for (auto& [nodeId, node] : NodeTable) {
+ YQL_CLOG(TRACE, ProviderDq) << " > Node " << nodeId << " `" << node.Address << "':" << node.Port;
+ }
+ }
+
+ bool IsNodeUpdated(const ui32 nodeId, const TString& address, const ui32 port) {
+ bool printInfo = false;
+ auto it = NodeTable.find(nodeId);
+ if (it == NodeTable.end()) {
+ YQL_CLOG(DEBUG, ProviderDq) << "New node " << nodeId << " `" << address << "':" << port;
+ printInfo = true;
+ } else if (it->second.Address != address || it->second.Port != port) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Updated node " << nodeId << " "
+ << "`" << address << "':" << port
+ << " Was "
+ << "`" << it->second.Address << "':" << it->second.Port;
+ printInfo = true;
+
+ Send(TActivationContext::InterconnectProxy(nodeId), new TEvInterconnect::TEvDisconnect);
+ }
+ return printInfo;
+ }
+
+ void UpdateNodeTable(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext&) {
+ auto* req = ev->Get();
+
+ bool printInfo = false;
+ for (auto& node : req->Nodes) {
+ printInfo |= IsNodeUpdated(node.NodeId, node.Address, node.Port);
+
+ NodeTable[node.NodeId] = TTableNameserverSetup::TNodeInfo(
+ node.Address,
+ node.Host,
+ node.ResolveHost,
+ node.Port,
+ node.Location);
+ }
+
+ if (printInfo) {
+ PrintInfo();
+ }
+ }
+
+ void RegisterNodeHandle(TEvRegisterNode::TPtr& ev, const TActorContext& ctx) {
+ try {
+ TEvRegisterNode* req = ev->Get();
+ auto& request = req->Record.GetRequest();
+
+ if (!request.GetZombie()) {
+ bool printInfo = IsNodeUpdated(
+ request.GetNodeId(),
+ request.GetAddress(),
+ request.GetPort());
+ NodeTable[request.GetNodeId()] = std::make_pair(
+ request.GetAddress(),
+ request.GetPort());
+
+ if (printInfo) {
+ PrintInfo();
+ }
+ }
+
+ auto response = MakeHolder<TEvRegisterNodeResponse>(GetNodesInfo(), request.GetEpoch());
+ *response->Record.MutableResponse()->MutableDownloadList() = request.GetDownloadList();
+ ctx.Send(ev->Sender, response.Release());
+ } catch (...) {
+ // never
+ Y_ABORT_UNLESS(false);
+ }
+ }
+
+ void Handle(TEvInterconnect::TEvResolveNode::TPtr& ev,
+ const TActorContext& ctx)
+ {
+ const TEvInterconnect::TEvResolveNode* request = ev->Get();
+ const ui32 nodeId = request->Record.GetNodeId();
+ const TInstant deadline = request->Record.HasDeadline() ? TInstant::FromValue(request->Record.GetDeadline()) : TInstant::Max();
+ auto it = NodeTable.find(nodeId);
+
+ if (it == NodeTable.end()) {
+ auto reply = new TEvLocalNodeInfo;
+ reply->NodeId = nodeId;
+ ctx.Send(ev->Sender, reply);
+ return;
+ }
+
+ RegisterWithSameMailbox(CreateResolveActor(it->second.ResolveHost,
+ it->second.Port,
+ nodeId,
+ it->second.Address,
+ ev->Sender,
+ SelfId(),
+ deadline));
+ }
+
+ void Handle(TEvResolveAddress::TPtr& ev,
+ const TActorContext& ctx)
+ {
+ Y_UNUSED(ctx);
+ const TEvResolveAddress* request = ev->Get();
+
+ RegisterWithSameMailbox(CreateResolveActor(request->Address,
+ request->Port,
+ ev->Sender,
+ SelfId(),
+ TInstant::Max()));
+ }
+
+ TVector<NActors::TEvInterconnect::TNodeInfo> GetNodesInfo()
+ {
+ TVector<NActors::TEvInterconnect::TNodeInfo> nodes;
+ nodes.reserve(NodeTable.size());
+ for (const auto& pr : NodeTable) {
+ nodes.emplace_back(pr.first,
+ pr.second.Address, pr.second.Host, pr.second.ResolveHost,
+ pr.second.Port, pr.second.Location);
+ }
+ return nodes;
+ }
+
+ void ReplyListNodes(const NActors::TActorId sender, const TActorContext& ctx)
+ {
+ THolder<TEvInterconnect::TEvNodesInfo>
+ reply(new TEvInterconnect::TEvNodesInfo());
+ reply->Nodes = GetNodesInfo();
+ ctx.Send(sender, reply.Release());
+ }
+
+ void Handle(TEvInterconnect::TEvListNodes::TPtr& ev,
+ const TActorContext& ctx) {
+ try {
+ ReplyListNodes(ev->Sender, ctx);
+ } catch (...) {
+ // on error - do nothing
+ }
+ }
+
+ void Handle(TEvInterconnect::TEvGetNode::TPtr& ev,
+ const TActorContext& ctx) {
+ try {
+ ui32 nodeId = ev->Get()->NodeId;
+ THolder<TEvInterconnect::TEvNodeInfo>
+ reply(new TEvInterconnect::TEvNodeInfo(nodeId));
+ auto it = NodeTable.find(nodeId);
+ if (it != NodeTable.end())
+ reply->Node.Reset(new TEvInterconnect::TNodeInfo(it->first, it->second.Address,
+ it->second.Host, it->second.ResolveHost,
+ it->second.Port, it->second.Location));
+ ctx.Send(ev->Sender, reply.Release());
+ } catch (...) {
+ // on error - do nothing
+ }
+ }
+ };
+
+ IActor* CreateDynamicNameserver(
+ const TIntrusivePtr<TTableNameserverSetup>& setup,
+ ui32 poolId) {
+ return new TDynamicNameserver(setup, poolId);
+ }
+
+}
diff --git a/ydb/library/yql/providers/dq/actors/dynamic_nameserver.h b/ydb/library/yql/providers/dq/actors/dynamic_nameserver.h
new file mode 100644
index 0000000000..df35e729a4
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/dynamic_nameserver.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include <library/cpp/actors/interconnect/interconnect.h>
+
+namespace NYql::NDqs {
+ NActors::IActor* CreateDynamicNameserver(const TIntrusivePtr<NActors::TTableNameserverSetup>& setup, ui32 poolId = 0);
+}
diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..b18ae3b86f
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,18 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(dq-actors-events)
+target_link_libraries(dq-actors-events PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+)
+target_sources(dq-actors-events PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events/empty.cpp
+)
diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..ab225973eb
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(dq-actors-events)
+target_link_libraries(dq-actors-events PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+)
+target_sources(dq-actors-events PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events/empty.cpp
+)
diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..ab225973eb
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(dq-actors-events)
+target_link_libraries(dq-actors-events PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+)
+target_sources(dq-actors-events PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events/empty.cpp
+)
diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/dq/actors/events/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/actors/events/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..b18ae3b86f
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/events/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,18 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(dq-actors-events)
+target_link_libraries(dq-actors-events PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+)
+target_sources(dq-actors-events PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/events/empty.cpp
+)
diff --git a/ydb/library/yql/providers/dq/actors/events/empty.cpp b/ydb/library/yql/providers/dq/actors/events/empty.cpp
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/events/empty.cpp
diff --git a/ydb/library/yql/providers/dq/actors/events/events.h b/ydb/library/yql/providers/dq/actors/events/events.h
new file mode 100644
index 0000000000..ae15f0ca40
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/events/events.h
@@ -0,0 +1,59 @@
+#pragma once
+
+namespace NYql {
+
+ struct TDqEvents {
+ enum {
+ ES_BECOME_LEADER = EventSpaceBegin(NActors::TEvents::EEventSpace::ES_USERSPACE) + 10100,
+ ES_BECOME_FOLLOWER,
+
+ ES_TICK,
+
+ ES_OTHER1,
+ ES_OTHER2,
+ ES_OTHER3,
+ ES_OTHER4
+ };
+ };
+
+ struct TEvBecomeLeader: NActors::TEventLocal<TEvBecomeLeader, TDqEvents::ES_BECOME_LEADER> {
+ TEvBecomeLeader() = default;
+
+ TEvBecomeLeader(ui32 leaderEpoch, const TString& leaderTransaction, const TString& attributes)
+ : LeaderEpoch(leaderEpoch)
+ , LeaderTransaction(leaderTransaction)
+ , Attributes(attributes)
+ { }
+
+ const ui32 LeaderEpoch;
+ const TString LeaderTransaction;
+ const TString Attributes;
+ };
+
+ struct TEvBecomeFollower: NActors::TEventLocal<TEvBecomeFollower, TDqEvents::ES_BECOME_FOLLOWER> {
+ TEvBecomeFollower() = default;
+
+ TEvBecomeFollower(const TString& attributes)
+ : Attributes(attributes)
+ { }
+
+ const TString Attributes;
+ };
+
+ struct TEvTick
+ : NActors::TEventLocal<TEvTick, TDqEvents::ES_TICK> {
+ TEvTick() = default;
+ };
+
+ struct TEvUploadComplete
+ : NActors::TEventLocal<TEvTick, TDqEvents::ES_OTHER2> {
+ TEvUploadComplete() = default;
+ };
+
+ struct TEvDownloadComplete
+ : NActors::TEventLocal<TEvTick, TDqEvents::ES_OTHER3> {
+ TVector<TString> FailedObjectIds;
+ TEvDownloadComplete() = default;
+ };
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/events/ya.make b/ydb/library/yql/providers/dq/actors/events/ya.make
new file mode 100644
index 0000000000..398ff0ecee
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/events/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+SET(
+ SOURCE
+ empty.cpp
+)
+
+SRCS(
+ ${SOURCE}
+)
+
+PEERDIR(
+ library/cpp/actors/core
+)
+
+END()
diff --git a/ydb/library/yql/providers/dq/actors/ya.make b/ydb/library/yql/providers/dq/actors/ya.make
index 8a3c7d8b65..7d3df6fe4b 100644
--- a/ydb/library/yql/providers/dq/actors/ya.make
+++ b/ydb/library/yql/providers/dq/actors/ya.make
@@ -2,6 +2,8 @@ LIBRARY()
SRCS(
compute_actor.cpp
+ dummy_lock.cpp
+ dynamic_nameserver.cpp
events.cpp
executer_actor.cpp
execution_helpers.cpp
@@ -35,6 +37,7 @@ PEERDIR(
ydb/library/yql/dq/tasks
ydb/library/yql/utils/failure_injector
ydb/library/yql/providers/common/metrics
+ ydb/library/yql/providers/dq/actors/events
ydb/library/yql/providers/dq/api/grpc
ydb/library/yql/providers/dq/api/protos
ydb/library/yql/providers/dq/common
@@ -52,6 +55,11 @@ YQL_LAST_ABI_VERSION()
END()
+RECURSE(
+ events
+ yt
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..12dab3d2dc
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,41 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(dq-actors-yt)
+target_compile_options(dq-actors-yt PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(dq-actors-yt PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+ cpp-grpc-client
+ cpp-mapreduce-interface
+ providers-dq-config
+ yql-core-issue
+ providers-common-metrics
+ dq-api-grpc
+ dq-api-protos
+ providers-dq-common
+ yt-lib-log
+ dq-actors-events
+ yt-yt-client
+)
+target_sources(dq-actors-yt PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/lock.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp
+)
diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..8557c18cbd
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,42 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(dq-actors-yt)
+target_compile_options(dq-actors-yt PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(dq-actors-yt PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+ cpp-grpc-client
+ cpp-mapreduce-interface
+ providers-dq-config
+ yql-core-issue
+ providers-common-metrics
+ dq-api-grpc
+ dq-api-protos
+ providers-dq-common
+ yt-lib-log
+ dq-actors-events
+ yt-yt-client
+)
+target_sources(dq-actors-yt PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/lock.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp
+)
diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..8557c18cbd
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,42 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(dq-actors-yt)
+target_compile_options(dq-actors-yt PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(dq-actors-yt PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+ cpp-grpc-client
+ cpp-mapreduce-interface
+ providers-dq-config
+ yql-core-issue
+ providers-common-metrics
+ dq-api-grpc
+ dq-api-protos
+ providers-dq-common
+ yt-lib-log
+ dq-actors-events
+ yt-yt-client
+)
+target_sources(dq-actors-yt PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/lock.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp
+)
diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/dq/actors/yt/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..e4c7a252a1
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(dq-actors-yt)
+target_compile_options(dq-actors-yt PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(dq-actors-yt PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+ cpp-grpc-client
+ cpp-mapreduce-interface
+ providers-dq-config
+ yql-core-issue
+ providers-common-metrics
+ dq-api-grpc
+ dq-api-protos
+ providers-dq-common
+ yt-lib-log
+ dq-actors-events
+)
+target_sources(dq-actors-yt PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp
+)
diff --git a/ydb/library/yql/providers/dq/actors/yt/lock.cpp b/ydb/library/yql/providers/dq/actors/yt/lock.cpp
new file mode 100644
index 0000000000..991db7ea5d
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/lock.cpp
@@ -0,0 +1,435 @@
+#include "lock.h"
+
+#include "nodeid_assigner.h"
+#include "yt_wrapper.h"
+
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <yt/yt/client/api/transaction.h>
+#include <yt/yt/core/ytree/public.h>
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/hfunc.h>
+
+#include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/svnversion/svnversion.h>
+
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+
+#include <util/string/builder.h>
+#include <util/system/hostname.h>
+#include <util/system/getpid.h>
+
+namespace NYql {
+
+using namespace NActors;
+
+struct TLockRequest: public TActor<TLockRequest> {
+ TLockRequest(
+ const NYT::NApi::ITransactionPtr& transaction,
+ TActorId ytWrapper,
+ TActorId lockActorId,
+ TActorId parentId,
+ const TString prefix,
+ const TString lockName,
+ NYT::TNode attributes)
+ : TActor<TLockRequest>(&TLockRequest::Handler)
+ , Transaction(transaction)
+ , LockActorId(lockActorId)
+ , ParentId(parentId)
+ , Prefix(prefix)
+ , LockName(lockName)
+ , Attributes(attributes)
+ , Revision(GetProgramCommitId())
+ , YtWrapper(ytWrapper)
+ { }
+
+ ~TLockRequest()
+ { }
+
+private:
+ STRICT_STFUNC(Handler, {
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ CFunc(TEvents::TEvBootstrap::EventType, CreateLockNode);
+ HFunc(TEvCreateNodeResponse, OnCreateLockNode);
+ HFunc(TEvSetNodeResponse, OnNodeLocked);
+ });
+
+ STRICT_STFUNC(ReadInfoNodeHandler, {
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ HFunc(TEvGetNodeResponse, SendBecomeFollower);
+ });
+
+ STRICT_STFUNC(GetOrCreateEpochHandler, {
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ HFunc(TEvGetNodeResponse, OnGetEpochNode);
+ HFunc(TEvCreateNodeResponse, OnCreateEpochNode);
+ });
+
+ STRICT_STFUNC(SetAttributesHandler, {
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ HFunc(TEvSetNodeResponse, OnSetAttribute);
+ });
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0);
+ }
+
+ void OnCreateLockNode(TEvCreateNodeResponse::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+ if (result.IsOK()) {
+ NYT::NApi::TLockNodeOptions options;
+ options.Waitable = false;
+ auto* actorSystem = ctx.ExecutorThread.ActorSystem;
+ auto selfId = SelfId();
+ try {
+ Transaction->LockNode("#" + ToString(result.ValueOrThrow()), NYT::NCypressClient::ELockMode::Exclusive, options).As<void>()
+ .Apply(BIND([actorSystem, selfId](const NYT::TErrorOr<void>& result) {
+ actorSystem->Send(selfId, new TEvSetNodeResponse(0, result));
+ }));
+ } catch (...) {
+ Finish(ctx);
+ }
+ } else {
+ Finish(result, ctx);
+ }
+ }
+
+ void OnNodeLocked(TEvSetNodeResponse::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+ if (result.IsOK()) {
+ GetOrCreateEpoch();
+ } else {
+ Finish(result, ctx);
+ }
+ }
+
+ void PassAway() override {
+ Transaction->Abort();
+ Transaction.Reset();
+ Send(LockActorId, new TEvTick());
+ IActor::PassAway();
+ }
+
+ void OnGetEpochNode(TEvGetNodeResponse::TPtr& ev, const TActorContext& ctx) {
+ auto result = std::get<0>(*ev->Get());
+ try {
+ if (!result.IsOK() && !result.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) {
+ Finish(result, ctx);
+ } else if (result.IsOK()) {
+ auto epoch = static_cast<ui32>(NYT::NodeFromYsonString(result.Value().AsStringBuf()).AsUint64());
+ SetInfoNodeAttributes(epoch);
+ } else {
+ NYT::NApi::TCreateNodeOptions createNodeOptions;
+ auto prereqId = Transaction->GetId();
+
+ createNodeOptions.IgnoreExisting = true;
+ createNodeOptions.PrerequisiteTransactionIds.push_back(prereqId);
+
+ auto nodePath = Prefix + "/" + LockName;
+ Send(YtWrapper, new TEvCreateNode(nodePath, NYT::NObjectClient::EObjectType::StringNode, createNodeOptions));
+ }
+ } catch (...) {
+ Finish(ctx);
+ }
+ }
+
+ void OnCreateEpochNode(TEvCreateNodeResponse::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+ if (result.IsOK()) {
+ SetInfoNodeAttributes(0);
+ } else {
+ Finish(result, ctx);
+ }
+ }
+
+ void SetInfoNodeAttributes(ui32 epoch) {
+ epoch += 1;
+ Epoch = epoch;
+
+ auto prereqId = Transaction->GetId();
+
+ NYT::NApi::TSetNodeOptions setNodeOptions;
+ setNodeOptions.PrerequisiteTransactionIds.push_back(prereqId);
+
+ auto attributesMap = Attributes.AsMap();
+ attributesMap[NCommonAttrs::EPOCH_ATTR] = NYT::TNode(epoch);
+ attributesMap[NCommonAttrs::REVISION_ATTR] = NYT::TNode(Revision);
+
+ AttributesCount = attributesMap.size();
+ Become(&TLockRequest::SetAttributesHandler);
+
+ auto nodePath = Prefix + "/" + LockName;
+ for (const auto& [k, v]: attributesMap) {
+ Send(YtWrapper, new TEvSetNode(
+ nodePath + "/@" + k,
+ NYT::NYson::TYsonString(NYT::NodeToYsonString(v)),
+ setNodeOptions));
+ }
+ }
+
+ void OnSetAttribute(TEvSetNodeResponse::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+ if (result.IsOK()) {
+ if (--AttributesCount == 0) {
+ SendBecomeLeader();
+ Finish(result, ctx);
+ }
+ } else {
+ Finish(result, ctx);
+ }
+ }
+
+ void GetOrCreateEpoch()
+ {
+ Become(&TLockRequest::GetOrCreateEpochHandler);
+
+ auto prereqId = Transaction->GetId();
+
+ NYT::NApi::TGetNodeOptions getNodeOptions;
+ getNodeOptions.PrerequisiteTransactionIds.push_back(prereqId);
+ auto nodePath = Prefix + "/" + LockName;
+
+ Send(YtWrapper, new TEvGetNode(nodePath + "/@" + NCommonAttrs::EPOCH_ATTR, getNodeOptions));
+ }
+
+ void ReadInfoNode() {
+ Become(&TLockRequest::ReadInfoNodeHandler);
+
+ auto nodePath = Prefix + "/" + LockName + "/@";
+
+ Send(YtWrapper, new TEvGetNode(nodePath, NYT::NApi::TGetNodeOptions()));
+ }
+
+ void SendBecomeLeader() {
+ auto attributes = NYT::NYson::TYsonString(NYT::NodeToYsonString(Attributes));
+ Send(ParentId, new TEvBecomeLeader(Epoch, ToString(Transaction->GetId()), attributes.ToString()));
+ }
+
+ void SendBecomeFollower(TEvGetNodeResponse::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+ if (result.IsOK()) {
+ Send(ParentId, new TEvBecomeFollower(result.ValueOrThrow().ToString()));
+ }
+ Send(SelfId(), new TEvents::TEvPoison());
+ }
+
+ template<typename T>
+ void Finish(const NYT::TErrorOr<T>& result, const TActorContext& ctx) {
+ if (result.IsOK()) {
+ auto* actorSystem = ctx.ExecutorThread.ActorSystem;
+ auto selfId = SelfId();
+ Transaction->SubscribeAborted(BIND([actorSystem, selfId](const NYT::TError& /*error*/) {
+ actorSystem->Send(selfId, new TEvents::TEvPoison());
+ }));
+ } else {
+ if (!result.FindMatching(NYT::NCypressClient::EErrorCode::ConcurrentTransactionLockConflict)) {
+ YQL_CLOG(WARN, ProviderDq) << ToString(result);
+ }
+ ReadInfoNode();
+ }
+ }
+
+ void Finish(const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ YQL_CLOG(WARN, ProviderDq) << CurrentExceptionMessage();
+ ReadInfoNode();
+ }
+
+ void CreateLockNode(const TActorContext& ctx) {
+ auto lockNode = Prefix + "/" + LockName + ".lock";
+
+ try {
+ auto* actorSystem = ctx.ExecutorThread.ActorSystem;
+ auto selfId = SelfId();
+ Transaction->CreateNode(
+ lockNode,
+ NYT::NObjectClient::EObjectType::StringNode,
+ NYT::NApi::TCreateNodeOptions())
+ .Apply(BIND([actorSystem, selfId](const NYT::TErrorOr<NYT::NCypressClient::TNodeId>& result) {
+ actorSystem->Send(selfId, new TEvCreateNodeResponse(0, result));
+ }));
+ } catch (...) {
+ Finish(ctx);
+ }
+ }
+
+ NYT::NApi::ITransactionPtr Transaction;
+
+ const TActorId LockActorId;
+ const TActorId ParentId;
+ const TString Prefix;
+ const TString LockName;
+ const NYT::TNode Attributes;
+
+ const TString Revision;
+ ui32 Epoch;
+ int AttributesCount = 0;
+
+ TActorId YtWrapper;
+};
+
+class TYtLock: public TRichActor<TYtLock> {
+public:
+ static constexpr char ActorName[] = "LOCK";
+
+ TYtLock(
+ const TActorId& ytWrapper,
+ const TString& prefix,
+ const TString& lockName,
+ const TString& lockAttributes,
+ bool temporary)
+ : TRichActor<TYtLock>(&TYtLock::Handler)
+ , YtWrapper(ytWrapper)
+ , Prefix(prefix)
+ , LockName(lockName)
+ , Attributes(NYT::NodeFromYsonString(lockAttributes))
+ , Temporary(temporary)
+ , Revision(GetProgramCommitId())
+ { }
+
+private:
+ STRICT_STFUNC(Handler, {
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ CFunc(TEvTick::EventType, OnTick);
+ HFunc(TEvStartTransactionResponse, OnStartTransactionResponse);
+ HFunc(TEvCreateNodeResponse, OnCreateNode);
+ HFunc(TEvGetNodeResponse, SendBecomeFollower);
+ cFunc(TEvBecomeFollower::EventType, OnFollowingForever);
+ });
+
+ void DoPassAway() override {
+ YQL_CLOG(DEBUG, ProviderDq) << "Unlock " << LockName;
+ Send(Request, new TEvents::TEvPoison());
+ auto nodePath = Prefix + "/" + LockName;
+
+ if (Temporary) {
+ Send(YtWrapper, new TEvRemoveNode(nodePath, {}));
+ }
+ }
+
+ static TEvStartTransaction* GetStartTransactionCommand() {
+ NYT::NApi::TTransactionStartOptions options;
+ auto attrs = NYT::NYTree::CreateEphemeralAttributes();
+ attrs->Set("title", TStringBuilder{} << "Host name: " << HostName() << " Pid: " << GetPID());
+ options.Attributes = attrs;
+ auto command = new TEvStartTransaction(
+ NYT::NTransactionClient::ETransactionType::Master,
+ options
+ );
+ return command;
+ }
+
+ void TryLock() {
+ NYT::NApi::TCreateNodeOptions options;
+ options.Recursive = true;
+ options.IgnoreExisting = true;
+
+ TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way());
+
+ IEventBase* event;
+ if (FollowingMode == false) {
+ event = new TEvCreateNode(Prefix, NYT::NObjectClient::EObjectType::MapNode, options);
+ } else {
+ event = new TEvGetNode(Prefix + "/" + LockName + "/@", NYT::NApi::TGetNodeOptions());
+ }
+
+ TActivationContext::Schedule(
+ TDuration::Seconds(5),
+ new IEventHandle(YtWrapper, SelfId(), event, 0), TimerCookieHolder.Get());
+ }
+
+ void OnFollowingForever() {
+ FollowingMode = true;
+ YQL_CLOG(DEBUG, ProviderDq) << "Unlock and follow " << LockName;
+ Send(Request, new TEvents::TEvPoison());
+ }
+
+ void SendBecomeFollower(TEvGetNodeResponse::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+ if (result.IsOK()) {
+ Send(ParentId, new TEvBecomeFollower(result.ValueOrThrow().ToString()));
+ } else {
+ YQL_CLOG(WARN, ProviderDq) << "SendBecomeFollower error: " << ToString(result);
+ }
+
+ TryLock(); // update leader info
+ }
+
+ void OnCreateNode(TEvCreateNodeResponse::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+ if (result.IsOK()) {
+ Send(YtWrapper, GetStartTransactionCommand());
+ } else {
+ TryLock();
+ }
+ }
+
+ void OnStartTransactionResponse(TEvStartTransactionResponse::TPtr& ev, const TActorContext& ctx)
+ {
+ auto result = std::get<0>(*ev->Get());
+
+ if (result.IsOK()) {
+ Send(Request, new TEvents::TEvPoison());
+ auto requestActor = new TLockRequest(
+ result.Value(),
+ YtWrapper,
+ SelfId(),
+ ParentId,
+ Prefix,
+ LockName,
+ Attributes
+ );
+
+ Request = ctx.Register(requestActor);
+ } else {
+ YQL_CLOG(WARN, ProviderDq) << "OnStartTransactionResponse " << ToString(result);
+
+ TryLock();
+ }
+ }
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ ParentId = parentId;
+ return new IEventHandle(self, parentId, new TEvTick(), 0);
+ }
+
+ void OnTick(const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ TryLock();
+ }
+
+ const TActorId YtWrapper;
+ TActorId ParentId;
+ const TString Prefix;
+ const TString LockName;
+ NYT::TNode Attributes;
+
+ TActorId Request;
+ const bool Temporary;
+
+ const TString Revision;
+ NActors::TSchedulerCookieHolder TimerCookieHolder;
+ bool FollowingMode = false;
+};
+
+NActors::IActor* CreateYtLock(
+ NActors::TActorId ytWrapper,
+ const TString& prefix,
+ const TString& lockName,
+ const TString& lockAttributesYson,
+ bool temporary)
+{
+ return new TYtLock(ytWrapper, prefix, lockName, lockAttributesYson, temporary);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/lock.h b/ydb/library/yql/providers/dq/actors/yt/lock.h
new file mode 100644
index 0000000000..dddc3fe37d
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/lock.h
@@ -0,0 +1,14 @@
+#pragma once
+
+#include <library/cpp/actors/core/actor.h>
+
+namespace NYql {
+
+NActors::IActor* CreateYtLock(
+ NActors::TActorId ytWrapper,
+ const TString& prefix,
+ const TString& lockName,
+ const TString& lockAttributesYson,
+ bool temporary);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp b/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp
new file mode 100644
index 0000000000..f3a2b67ac7
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.cpp
@@ -0,0 +1,116 @@
+#include "nodeid_assigner.h"
+
+#include <yt/cpp/mapreduce/interface/node.h>
+#include <yt/cpp/mapreduce/interface/client.h>
+#include <yt/cpp/mapreduce/interface/common.h>
+
+#include <util/system/env.h>
+#include <util/generic/hash_set.h>
+
+namespace NYql {
+
+using namespace NYT;
+
+ui32 AssignNodeId(const TAssignNodeIdOptions& options)
+{
+ Y_ABORT_UNLESS(!options.ClusterName.empty());
+ Y_ABORT_UNLESS(!options.NodeName.empty());
+ Y_ABORT_UNLESS(!options.Prefix.empty());
+ Y_ABORT_UNLESS(!options.Role.empty());
+
+ Y_ABORT_UNLESS(options.MinNodeId > 0);
+
+ auto range = options.MaxNodeId - options.MinNodeId;
+
+ Y_ABORT_UNLESS(range > 0);
+
+ TString token = !options.Token.empty()
+ ? options.Token
+ : [] () {
+ TString home = GetEnv("HOME");
+ TString tokenFile = home + "/.yt/token";
+ return TFileInput(tokenFile).ReadLine();
+ } ();
+
+ Y_ABORT_UNLESS(!token.empty());
+
+ auto lockPath = options.Prefix + "/" + options.Role + "/lock";
+ auto nodePath = options.Prefix + "/" + options.Role + "/" + options.NodeName;
+
+ auto client = CreateClient(options.ClusterName, TCreateClientOptions().Token(token));
+ // create std directories
+ client->Create(options.Prefix, ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true).Recursive(true));
+ client->Create(options.Prefix + "/" + options.Role, ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true));
+ client->Create(options.Prefix + "/service_node", ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true));
+ client->Create(options.Prefix + "/worker_node", ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true));
+ client->Create(options.Prefix + "/operations", ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true));
+ client->Create(options.Prefix + "/locks", ENodeType::NT_MAP, TCreateOptions().IgnoreExisting(true));
+
+ if (options.NodeId) {
+ // Use it for debug only
+ return *options.NodeId;
+ }
+
+ if (client->Exists(lockPath) && client->Get(lockPath).GetType() != TNode::Int64) {
+ client->Remove(lockPath);
+ }
+ client->Create(lockPath, ENodeType::NT_INT64, TCreateOptions().IgnoreExisting(true));
+ auto transaction = client->StartTransaction(TStartTransactionOptions());
+ transaction->Lock(lockPath, ELockMode::LM_EXCLUSIVE, TLockOptions().Waitable(true))->Wait();
+
+ ui32 nodeId = client->Get(lockPath).AsInt64();
+ if (nodeId == 0) {
+ nodeId = options.MinNodeId;
+ }
+
+ auto listResult = transaction->List(
+ options.Prefix + "/" + options.Role,
+ TListOptions()
+ .MaxSize(1<<18)
+ .AttributeFilter(TAttributeFilter().AddAttribute(NCommonAttrs::ACTOR_NODEID_ATTR))
+ );
+
+ THashSet<ui32> allNodeIds;
+ allNodeIds.reserve(listResult.size());
+ for (const auto& node : listResult) {
+ const auto& attributes = node.GetAttributes().AsMap();
+ auto maybeNodeId = attributes.find(NCommonAttrs::ACTOR_NODEID_ATTR);
+ if (maybeNodeId != attributes.end()) {
+ auto nodeId = maybeNodeId->second.AsUint64();
+ if (options.MinNodeId <= nodeId && nodeId < options.MaxNodeId) {
+ Cerr << nodeId << " ";
+ allNodeIds.insert(nodeId);
+ }
+ }
+ }
+ Cerr << Endl;
+
+ Y_ABORT_UNLESS(allNodeIds.size() < range);
+
+ while (allNodeIds.contains(nodeId)) {
+ nodeId = options.MinNodeId + (nodeId + 1 - options.MinNodeId) % range;
+ }
+
+ if (options.NodeId.Defined()) {
+ nodeId = options.NodeId.GetOrElse(nodeId);
+ } else {
+ auto nextNodeId = options.MinNodeId + (nodeId + 1 - options.MinNodeId) % range;
+ transaction->Set(lockPath, TNode(nextNodeId));
+ }
+
+ Y_ABORT_UNLESS(options.MinNodeId <= nodeId && nodeId < options.MaxNodeId);
+
+ // create new node
+ transaction->Create(nodePath, NT_STRING, TCreateOptions().IgnoreExisting(true));
+
+ for (const auto& [k, v] : options.Attributes) {
+ transaction->Set(nodePath + "/@" + k, TNode(v), TSetOptions());
+ }
+
+ transaction->Set(nodePath + "/@" + NCommonAttrs::ACTOR_NODEID_ATTR, TNode(nodeId), TSetOptions());
+ transaction->Commit();
+
+ return nodeId;
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h b/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h
new file mode 100644
index 0000000000..e6921097f7
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <ydb/library/yql/providers/dq/common/attrs.h>
+
+#include <util/generic/string.h>
+#include <util/generic/hash.h>
+#include <util/generic/maybe.h>
+
+namespace NYql {
+
+struct TAssignNodeIdOptions {
+ TString ClusterName;
+ TString User;
+ TString Token;
+ TString Prefix;
+ TString Role;
+ TString NodeName;
+
+ THashMap<TString, TString> Attributes;
+
+ ui32 MinNodeId = 0;
+ ui32 MaxNodeId = 1<<18;
+
+ TMaybe<ui32> NodeId; // for debug only
+};
+
+ui32 AssignNodeId(const TAssignNodeIdOptions& options);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp b/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp
new file mode 100644
index 0000000000..392d8636bf
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.cpp
@@ -0,0 +1,137 @@
+#include "nodeid_cleaner.h"
+#include "nodeid_assigner.h"
+#include "yt_wrapper.h"
+
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/event.h>
+#include <library/cpp/actors/core/hfunc.h>
+
+#include <library/cpp/yson/node/node.h>
+#include <library/cpp/yson/node/node_io.h>
+
+#include <ydb/library/yql/utils/log/log.h>
+#include <ydb/library/yql/utils/yql_panic.h>
+
+namespace NYql {
+
+using namespace NActors;
+
+class TNodeIdCleaner: public TActor<TNodeIdCleaner> {
+public:
+ static constexpr char ActorName[] = "CLEANER";
+
+ TNodeIdCleaner(TActorId ytWrapper, const TNodeIdCleanerOptions& options)
+ : TActor(&TNodeIdCleaner::Handler)
+ , YtWrapper(ytWrapper)
+ , Options(options)
+ , LastUpdateTime(TInstant::Now())
+ { }
+
+ STRICT_STFUNC(Handler, {
+ HFunc(TEvListNodeResponse, OnListNodeResponse)
+ HFunc(TEvRemoveNodeResponse, OnRemoveNodeResponse)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ });
+
+private:
+ TEvListNode* ListNodeCommand() {
+ NYT::NApi::TListNodeOptions options;
+ options.Attributes = {NCommonAttrs::ACTOR_NODEID_ATTR, "modification_time"};
+ return new TEvListNode(Options.Prefix, options);
+ }
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ // TODO: TakeLock
+ Y_UNUSED(parentId);
+ return new IEventHandle(YtWrapper, self, ListNodeCommand(), 0);
+ }
+
+ void MaybeReschedule(TInstant now) {
+ Y_UNUSED(now);
+ if (WaitCount == 0) {
+ TActivationContext::Schedule(Options.CheckPeriod, new IEventHandle(YtWrapper, SelfId(), ListNodeCommand(), 0));
+ }
+ }
+
+ void OnRemoveNodeResponse(TEvRemoveNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+ if (!result.IsOK()) {
+ YQL_CLOG(WARN, ProviderDq) << "Remove error " << ToString(result);
+ }
+
+ --WaitCount;
+ MaybeReschedule(TInstant::Now());
+ }
+
+ void ScheduleRemove(TInstant now, const TVector<NYT::TNode>& nodes, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ Y_ABORT_UNLESS(WaitCount == 0);
+ WaitCount = 0;
+ for (const auto& node : nodes) {
+
+ const auto& attributes = node.GetAttributes().AsMap();
+ auto maybeNodeId = attributes.find(NCommonAttrs::ACTOR_NODEID_ATTR);
+ if (maybeNodeId == attributes.end()) {
+ continue;
+ }
+
+ auto maybeModificationTime = attributes.find("modification_time");
+ YQL_ENSURE(maybeModificationTime != attributes.end());
+
+ auto modificationTime = TInstant::ParseIso8601(maybeModificationTime->second.AsString());
+
+ if (now - modificationTime > Options.Timeout) {
+ WaitCount ++;
+ auto removePath = Options.Prefix + "/" + node.AsString();
+ YQL_CLOG(DEBUG, ProviderDq) << "Removing node " << removePath;
+ Send(YtWrapper, new TEvRemoveNode(removePath, NYT::NApi::TRemoveNodeOptions()));
+ }
+ }
+ }
+
+ void OnListNodeResponse(TEvListNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ auto now = TInstant::Now();
+ auto result = std::get<0>(*ev->Get());
+ auto isOk = result.IsOK();
+ auto schedule = true;
+
+ try {
+ if (LastListNodeOk) {
+ // remove only after 2't attempt to minimize Y_ABORT_UNLESS chance on registrator
+ ScheduleRemove(now, NYT::NodeFromYsonString(result.ValueOrThrow()).AsList(), ctx);
+ MaybeReschedule(now);
+ schedule = false;
+ }
+ } catch (...) {
+ WaitCount = 0;
+ isOk = false;
+ YQL_CLOG(ERROR, ProviderDq) << "Error on list node " << CurrentExceptionMessage();
+ }
+
+ if (schedule) {
+ TActivationContext::Schedule(Options.RetryPeriod, new IEventHandle(YtWrapper, SelfId(), ListNodeCommand(), 0));
+ }
+
+ LastListNodeOk = isOk;
+ LastUpdateTime = now;
+ }
+
+ TActorId YtWrapper;
+ TNodeIdCleanerOptions Options;
+ TInstant LastUpdateTime;
+ bool LastListNodeOk = true;
+ int WaitCount = 0;
+};
+
+NActors::IActor* CreateNodeIdCleaner(TActorId ytWrapper, const TNodeIdCleanerOptions& options)
+{
+ Y_ABORT_UNLESS(!options.Prefix.empty());
+ return new TNodeIdCleaner(ytWrapper, options);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.h b/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.h
new file mode 100644
index 0000000000..173031ad12
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <library/cpp/actors/core/actor.h>
+
+#include <util/generic/string.h>
+
+namespace NYql {
+
+struct TNodeIdCleanerOptions {
+ TString Prefix;
+ TDuration CheckPeriod = TDuration::MilliSeconds(10000);
+ TDuration Timeout = TDuration::MilliSeconds(600000);
+ TDuration RetryPeriod = TDuration::MilliSeconds(15000);
+};
+
+NActors::IActor* CreateNodeIdCleaner(NActors::TActorId ytWrapper, const TNodeIdCleanerOptions& options);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp b/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp
new file mode 100644
index 0000000000..c87fc0c636
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/resource_cleaner.cpp
@@ -0,0 +1,154 @@
+#include "yt_wrapper.h"
+
+#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h>
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h>
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/yson/node/node_io.h>
+
+namespace NYql {
+
+using namespace NActors;
+using namespace NMonitoring;
+
+class TYtResourceCleaner: public TRichActor<TYtResourceCleaner> {
+public:
+ static constexpr char ActorName[] = "CLEANER";
+
+ TYtResourceCleaner(
+ const TResourceManagerOptions& options,
+ const TIntrusivePtr<ICoordinationHelper>& coordinator)
+ : TRichActor<TYtResourceCleaner>(&TYtResourceCleaner::Handler)
+ , Coordinator(coordinator)
+ , Options(options)
+ , FilesCounter(Options.Counters->GetSubgroup("component", "cleaner")->GetCounter("files"))
+ , RemovesCounter(Options.Counters->GetSubgroup("component", "cleaner")->GetCounter("removes"))
+ , ErrorsCounter(Options.Counters->GetSubgroup("component", "cleaner")->GetCounter("errors"))
+ {
+ }
+
+private:
+ STRICT_STFUNC(Handler, {
+ HFunc(TEvListNodeResponse, OnListResponse)
+ CFunc(TEvents::TEvBootstrap::EventType, Bootstrap)
+ cFunc(TEvTick::EventType, Check)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ HFunc(TEvRemoveNodeResponse, OnRemoveNodeResponse)
+ })
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0);
+ }
+
+ void Bootstrap(const TActorContext& ctx) {
+ YtWrapper = Coordinator->GetWrapper(
+ ctx.ActorSystem(),
+ Options.YtBackend.GetClusterName(),
+ Options.YtBackend.GetUser(),
+ Options.YtBackend.GetToken());
+ Check();
+ }
+
+ void Check() {
+ NYT::NApi::TListNodeOptions options;
+ options.Attributes = {
+ "modification_time"
+ };
+ auto command = new TEvListNode(Options.UploadPrefix, options);
+ Send(YtWrapper, command);
+ }
+
+ void OnRemoveNodeResponse(TEvRemoveNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+ if (!result.IsOK()) {
+ YQL_CLOG(WARN, ProviderDq) << "Remove error " << ToString(result);
+ *ErrorsCounter += 1;
+ }
+ }
+
+ void OnListResponse(TEvListNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ auto result = std::get<0>(*ev->Get());
+
+ try {
+ auto nodes = NYT::NodeFromYsonString(result.ValueOrThrow()).AsList();
+ TVector<std::pair<TInstant, TString>> pairs;
+
+ auto filesDiff = static_cast<int>(nodes.size()) - FilesCount;
+ *FilesCounter += filesDiff;
+ FilesCount = static_cast<int>(nodes.size());
+
+ for (const auto& node : nodes) {
+ const auto& attributes = node.GetAttributes().AsMap();
+ auto maybeModificationTime = attributes.find("modification_time");
+ YQL_ENSURE(maybeModificationTime != attributes.end());
+ auto modificationTime = TInstant::ParseIso8601(maybeModificationTime->second.AsString());
+
+ TString nodeStr = node.AsString();
+
+ if (Options.KeepFilter && nodeStr.find(*Options.KeepFilter) != TString::npos) {
+ continue;
+ }
+
+ pairs.emplace_back(modificationTime, nodeStr);
+ }
+
+ std::sort(pairs.begin(), pairs.end(), [](const auto& a, const auto& b) {
+ return a.first > b.first;
+ });
+ if (Options.KeepFirst) {
+ for (ui32 i = *Options.KeepFirst; i < pairs.size(); ++i) {
+ YQL_CLOG(DEBUG, ProviderDq) << "RemoveNode " << pairs[i].second;
+ Send(YtWrapper, new TEvRemoveNode(Options.UploadPrefix + "/" + pairs[i].second, NYT::NApi::TRemoveNodeOptions()));
+
+ *RemovesCounter += 1;
+ }
+ }
+ if (Options.DropBefore) {
+ auto now = TInstant::Now();
+ for (const auto& [t, id] : pairs) {
+ if (t < now && now - t > Options.DropBefore) {
+ YQL_CLOG(DEBUG, ProviderDq) << "RemoveNode " << id;
+ Send(YtWrapper, new TEvRemoveNode(Options.UploadPrefix + "/" + id, NYT::NApi::TRemoveNodeOptions()));
+
+ *RemovesCounter += 1;
+ }
+ }
+ }
+ } catch (...) {
+ YQL_CLOG(ERROR, ProviderDq) << "Error on list node " << CurrentExceptionMessage();
+ }
+
+ Schedule(TDuration::Seconds(5), new TEvTick);
+ }
+
+ TActorId YtWrapper;
+ const TIntrusivePtr<ICoordinationHelper> Coordinator;
+ const TResourceManagerOptions Options;
+
+ int FilesCount = 0;
+ TDynamicCounters::TCounterPtr FilesCounter;
+ TDynamicCounters::TCounterPtr RemovesCounter;
+ TDynamicCounters::TCounterPtr ErrorsCounter;
+};
+
+IActor* CreateYtResourceCleaner(
+ const TResourceManagerOptions& options,
+ const TIntrusivePtr<ICoordinationHelper>& coordinator)
+{
+ Y_ABORT_UNLESS(!options.YtBackend.GetClusterName().empty());
+ Y_ABORT_UNLESS(!options.YtBackend.GetUser().empty());
+ Y_ABORT_UNLESS(!options.YtBackend.GetUploadPrefix().empty());
+
+ return new TYtResourceCleaner(options, coordinator);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp b/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp
new file mode 100644
index 0000000000..8bb58dfab1
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/resource_downloader.cpp
@@ -0,0 +1,141 @@
+
+#include "nodeid_assigner.h"
+#include "yt_wrapper.h"
+
+#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h>
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h>
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/yson/node/node_io.h>
+
+#include <util/system/fs.h>
+
+#include <yt/cpp/mapreduce/interface/fluent.h>
+
+namespace NYql {
+
+using namespace NActors;
+
+class TYtResourceDownloader: public TRichActor<TYtResourceDownloader> {
+public:
+ static constexpr char ActorName[] = "DOWNLOADER";
+
+ TYtResourceDownloader(
+ const TResourceManagerOptions& options,
+ const ICoordinationHelper::TPtr& coordinator)
+ : TRichActor<TYtResourceDownloader>(&TYtResourceDownloader::Handler)
+ , Options(options)
+ , Coordinator(coordinator)
+ {
+ }
+
+private:
+
+ void DoPassAway() override {
+ auto ev = MakeHolder<TEvDownloadComplete>();
+ for (auto it = Options.Files.begin() + CurrentFileId; it != Options.Files.end(); ++it) {
+ ev->FailedObjectIds.push_back(it->LocalFileName);
+ }
+ Send(ParentId, ev.Release());
+ }
+
+ STRICT_STFUNC(Handler, {
+ HFunc(TEvReadFileResponse, OnFileDownloaded)
+ CFunc(TEvents::TEvBootstrap::EventType, Bootstrap)
+ cFunc(TEvTick::EventType, DownloadFile)
+ })
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ ParentId = parentId;
+ return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0);
+ }
+
+ void Bootstrap(const NActors::TActorContext& ctx) {
+ YtWrapper = Coordinator->GetWrapper(
+ ctx.ActorSystem(),
+ Options.YtBackend.GetClusterName(),
+ Options.YtBackend.GetUser(),
+ Options.YtBackend.GetToken());
+ DownloadFile();
+ }
+
+ void DownloadFile() {
+ TString remotePath = Options.UploadPrefix + "/";
+ NYT::NApi::TFileReaderOptions options;
+
+ auto& file = Options.Files[CurrentFileId];
+ remotePath += file.GetRemoteFileName(); /* md5 */
+ TString localFileName = Options.TmpDir + "/" + file.GetRemoteFileName();
+
+ YQL_CLOG(DEBUG, ProviderDq) << "Downloading file " << remotePath << "->" << localFileName;
+ auto message = MakeHolder<TEvReadFile>(NYT::NYPath::TYPath(remotePath), localFileName, options);
+ Send(YtWrapper, message.Release());
+ }
+
+ void Tick(const NActors::TActorContext& ctx) {
+ ctx.Schedule(TDuration::Seconds(5), new TEvTick());
+ }
+
+ void SaveToCache() {
+ auto& file = Options.Files[CurrentFileId];
+ Options.FileCache->AddFile(
+ Options.TmpDir + "/" + file.GetRemoteFileName(), /* /tmp/md5 */
+ file.LocalFileName /* md5 */);
+ }
+
+ void OnFileDownloaded(TEvReadFileResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ auto result = std::get<0>(*ev->Get());
+ if (result.IsOK()) {
+ SaveToCache();
+
+ Retry = 0;
+ CurrentFileId++;
+ if (CurrentFileId == Options.Files.size()) {
+ // save to cache
+ YQL_CLOG(DEBUG, ProviderDq) << "Download complete";
+ PassAway();
+ } else {
+ DownloadFile();
+ }
+ } else if (Options.MaxRetries == -1 || ++Retry < Options.MaxRetries) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Retry " << ToString(result);
+ std::random_shuffle(Options.Files.begin() + CurrentFileId, Options.Files.end());
+ Tick(ctx);
+ } else {
+ PassAway();
+ }
+ }
+
+ TResourceManagerOptions Options;
+ const ICoordinationHelper::TPtr Coordinator;
+
+ TActorId YtWrapper;
+
+ ui32 CurrentFileId = 0;
+ int Retry = 0;
+ TActorId ParentId;
+};
+
+IActor* CreateYtResourceDownloader(
+ const TResourceManagerOptions& options,
+ const ICoordinationHelper::TPtr& coordinator)
+{
+ Y_ABORT_UNLESS(!options.YtBackend.GetClusterName().empty());
+ Y_ABORT_UNLESS(!options.YtBackend.GetUser().empty());
+ Y_ABORT_UNLESS(!options.Files.empty());
+ Y_ABORT_UNLESS(!options.UploadPrefix.empty());
+ Y_ABORT_UNLESS(!options.TmpDir.empty());
+ Y_ABORT_UNLESS(options.FileCache);
+
+ NFs::MakeDirectoryRecursive(options.TmpDir, NFs::FP_NONSECRET_FILE, false);
+
+ return new TYtResourceDownloader(options, coordinator);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp b/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp
new file mode 100644
index 0000000000..f28ef86c0c
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/resource_manager.cpp
@@ -0,0 +1,69 @@
+#include "resource_manager.h"
+
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+
+#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h>
+
+namespace NYql {
+ using namespace NActors;
+
+ class TResourceManagerStub: public TActor<TResourceManagerStub> {
+ public:
+ TResourceManagerStub()
+ : TActor<TResourceManagerStub>(&TResourceManagerStub::Handler)
+ { }
+
+ void Handler(STFUNC_SIG) {
+ Y_UNUSED(ev);
+ }
+ };
+
+ IActor* CreateResourceManager(const TResourceManagerOptions& options, const ICoordinationHelper::TPtr& coordinator) {
+ Y_UNUSED(options);
+#ifdef WIN32
+ return new TResourceManagerStub();
+#else
+ IActor* CreateYtResourceManager(
+ const TResourceManagerOptions& options,
+ const ICoordinationHelper::TPtr& coordinator);
+ return CreateYtResourceManager(options, coordinator);
+#endif
+ }
+
+
+ NActors::IActor* CreateResourceUploader(const TResourceManagerOptions& options, const ICoordinationHelper::TPtr& coordinator) {
+ Y_UNUSED(options);
+#ifdef WIN32
+ return new TResourceManagerStub();
+#else
+ IActor* CreateYtResourceUploader(
+ const TResourceManagerOptions& options,
+ const ICoordinationHelper::TPtr& coordinator);
+ return CreateYtResourceUploader(options, coordinator);
+#endif
+ }
+
+ NActors::IActor* CreateResourceDownloader(const TResourceManagerOptions& options, const ICoordinationHelper::TPtr& coordinator) {
+ Y_UNUSED(options);
+#ifdef WIN32
+ return new TResourceManagerStub();
+#else
+ IActor* CreateYtResourceDownloader(
+ const TResourceManagerOptions& options, const ICoordinationHelper::TPtr& coordinator);
+ return CreateYtResourceDownloader(options, coordinator);
+#endif
+ }
+
+ NActors::IActor* CreateResourceCleaner(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator) {
+ Y_UNUSED(options);
+#ifdef WIN32
+ return new TResourceManagerStub();
+#else
+ IActor* CreateYtResourceCleaner(
+ const TResourceManagerOptions& options,
+ const TIntrusivePtr<ICoordinationHelper>& coordinator);
+ return CreateYtResourceCleaner(options, coordinator);
+#endif
+ }
+}
diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_manager.h b/ydb/library/yql/providers/dq/actors/yt/resource_manager.h
new file mode 100644
index 0000000000..2106a2e84e
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/resource_manager.h
@@ -0,0 +1,90 @@
+#pragma once
+
+#include <util/system/file.h>
+
+#include <library/cpp/actors/core/actor.h>
+#include <ydb/library/yql/providers/dq/config/config.pb.h>
+
+#include <ydb/library/yql/providers/dq/task_runner/file_cache.h>
+#include <ydb/library/yql/providers/common/metrics/metrics_registry.h>
+
+#include <library/cpp/threading/future/future.h>
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+
+namespace NYql {
+ namespace NCommonJobVars {
+ extern const TString ACTOR_PORT;
+ extern const TString ACTOR_NODE_ID;
+ extern const TString UDFS_PATH;
+ extern const TString OPERATION_SIZE;
+ extern const TString YT_COORDINATOR;
+ extern const TString YT_BACKEND;
+ }
+
+ class ICoordinationHelper;
+
+ struct TResourceFile
+ {
+ TString ObjectId;
+ TString LocalFileName;
+ TMaybe<TString> RemoteFileName;
+ THashMap<TString, TString> Attributes;
+ TFile File;
+
+ TResourceFile() { }
+ TResourceFile(const TString& localFileName)
+ : LocalFileName(localFileName)
+ , File(LocalFileName, RdOnly | OpenExisting)
+ { }
+
+ TString GetRemoteFileName() const {
+ if (RemoteFileName) {
+ return *RemoteFileName;
+ } else {
+ auto pos = LocalFileName.rfind('/');
+ return LocalFileName.substr(pos+1);
+ }
+ }
+ };
+
+ struct TResourceManagerOptions {
+ NYql::NProto::TDqConfig::TYtBackend YtBackend;
+
+ TString UploadPrefix;
+ TString LockName;
+ TString LogFile;
+
+ TString TmpDir;
+ IFileCache::TPtr FileCache;
+
+ TVector<TResourceFile> Files;
+
+ TMaybe<NThreading::TPromise<void>> Uploaded;
+
+ TIntrusivePtr<NMonitoring::TDynamicCounters> Counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+
+ bool ExitOnPingFail = false;
+
+ int Capabilities = 0;
+ int MaxRetries = -1;
+
+ // Pinger
+ TString DieOnFileAbsence; // see YQL-14099
+
+ // Cleaner
+ TMaybe<int> KeepFirst;
+ TDuration DropBefore;
+ TMaybe<TString> KeepFilter;
+
+ TMaybe<TString> AnnounceClusterName;
+ };
+
+ NActors::IActor* CreateResourceManager(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator);
+
+ NActors::IActor* CreateResourceUploader(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator);
+
+ NActors::IActor* CreateResourceDownloader(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator);
+
+ NActors::IActor* CreateResourceCleaner(const TResourceManagerOptions& options, const TIntrusivePtr<ICoordinationHelper>& coordinator);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp b/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp
new file mode 100644
index 0000000000..07789f159f
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/resource_uploader.cpp
@@ -0,0 +1,236 @@
+
+#include "nodeid_assigner.h"
+#include "yt_wrapper.h"
+
+#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h>
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h>
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/yson/node/node_io.h>
+
+#include <yt/cpp/mapreduce/interface/fluent.h>
+
+namespace NYql {
+
+using namespace NActors;
+using namespace NMonitoring;
+
+class TYtResourceUploader: public TRichActor<TYtResourceUploader> {
+public:
+ static constexpr char ActorName[] = "UPLOADER";
+
+ TYtResourceUploader(
+ const TResourceManagerOptions& options,
+ const ICoordinationHelper::TPtr& coordinator)
+ : TRichActor<TYtResourceUploader>(&TYtResourceUploader::Follower)
+ , Options(options)
+ , Coordinator(coordinator)
+ , Config(Coordinator->GetConfig())
+ {
+ if (Options.Counters) {
+ FileSize = Options.Counters->GetHistogram("FileSize", ExponentialHistogram(10, 4, 10));
+ FileUploadTime = Options.Counters->GetHistogram("UploadTime", ExponentialHistogram(10, 3, 1));
+ Errors = Options.Counters->GetCounter("Errors");
+ }
+ }
+
+private:
+ // States: Follower -> Upload
+
+ void StartFollower(TEvBecomeFollower::TPtr& ev, const TActorContext& ctx) {
+ YQL_LOG_CTX_ROOT_SCOPE(CurrentLockName);
+ YQL_CLOG(DEBUG, ProviderDq) << "Current lock '" << CurrentLockName << "'";
+ Y_UNUSED(ctx);
+ auto leaderAttributes = NYT::NodeFromYsonString(ev->Get()->Attributes).AsMap();
+
+ if (leaderAttributes.contains(NCommonAttrs::ACTOR_NODEID_ATTR)) {
+ YQL_CLOG(INFO, ProviderDq) << " Following leader: "
+ << leaderAttributes.at(NCommonAttrs::ACTOR_NODEID_ATTR).AsUint64();
+ }
+ if (leaderAttributes.contains(NCommonAttrs::HOSTNAME_ATTR)) {
+ YQL_CLOG(INFO, ProviderDq) << " Leader hostname: "
+ << leaderAttributes.at(NCommonAttrs::HOSTNAME_ATTR).AsString();
+ }
+ Become(&TYtResourceUploader::Follower);
+ }
+
+ void StartLeader(TEvBecomeLeader::TPtr& ev, const TActorContext& ctx) {
+ YQL_LOG_CTX_ROOT_SCOPE(CurrentLockName);
+ YQL_CLOG(DEBUG, ProviderDq) << "Current lock '" << CurrentLockName << "'";
+ Y_UNUSED(ctx);
+ YQL_CLOG(INFO, ProviderDq) << "Become leader, epoch=" << ev->Get()->LeaderEpoch;
+ Become(&TYtResourceUploader::UploadState);
+ UploadFile();
+ }
+
+ void Follower(STFUNC_SIG) {
+ switch (const ui32 etype = ev->GetTypeRewrite()) {
+ HFunc(TEvBecomeFollower, StartFollower)
+ HFunc(TEvBecomeLeader, StartLeader)
+
+ case TEvents::TEvBootstrap::EventType: {
+ Bootstrap(TActivationContext::ActorContextFor(this->SelfId()));
+ break;
+ }
+
+ default:
+ break;
+ }
+ }
+
+ void DoPassAway() override {
+ Send(ParentId, new TEvUploadComplete());
+
+ if (Options.Uploaded) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Promise SetValue";
+ auto promise = *Options.Uploaded;
+ promise.TrySetValue();
+ }
+ }
+
+ void UploadState(STFUNC_SIG) {
+ switch (const ui32 etype = ev->GetTypeRewrite()) {
+ HFunc(TEvBecomeFollower, StartFollower)
+ HFunc(TEvWriteFileResponse, OnFileUploaded)
+
+ case TEvTick::EventType: {
+ // retry
+ UploadFile();
+ break;
+ }
+ default:
+ break;
+ }
+ }
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ ParentId = parentId;
+ return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0);
+ }
+
+ TString GetLockName() {
+ Y_ABORT_UNLESS(!Options.Files[CurrentFileId].ObjectId.empty());
+ return Options.Files[CurrentFileId].ObjectId;
+ }
+
+ void Lock() {
+ if (LockId) {
+ UnregisterChild(LockId);
+ }
+ CurrentLockName = Options.LockName.empty()
+ ? GetLockName()
+ : Options.LockName;
+ LockId = RegisterChild(Coordinator->CreateLockOnCluster(YtWrapper, Options.YtBackend.GetPrefix(), CurrentLockName));
+ Become(&TYtResourceUploader::Follower);
+ }
+
+ void Bootstrap(const NActors::TActorContext& ctx) {
+ YtWrapper = Coordinator->GetWrapper(
+ ctx.ActorSystem(),
+ Options.YtBackend.GetClusterName(),
+ Options.YtBackend.GetUser(),
+ Options.YtBackend.GetToken());
+ Lock();
+ }
+
+ void UploadFile() {
+ TString remotePath = Options.UploadPrefix + "/";
+ NYT::NApi::TFileWriterOptions options;
+ auto& file = Options.Files[CurrentFileId];
+ THashMap<TString, NYT::TNode> attributes;
+ for (const auto& [k, v] : file.Attributes) {
+ attributes[k] = NYT::TNode(v);
+ }
+ if (Options.YtBackend.HasUploadReplicationFactor()) {
+ attributes["replication_factor"] = NYT::TNode(Options.YtBackend.GetUploadReplicationFactor());
+ }
+ options.ComputeMD5 = true;
+
+ if (FileSize && file.File.IsOpen()) {
+ FileSize->Collect(file.File.GetLength() / 1024 / 1024); // megabytes
+ }
+
+ UploadStart = TInstant::Now();
+
+ remotePath += file.GetRemoteFileName();
+ auto message = MakeHolder<TEvWriteFile>(file.File, NYT::NYPath::TYPath(remotePath), attributes, options);
+ Send(YtWrapper, message.Release());
+ }
+
+ void Tick(const NActors::TActorContext& ctx) {
+ ctx.Schedule(TDuration::Seconds(5), new TEvTick());
+ }
+
+ void OnFileUploaded(TEvWriteFileResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ YQL_LOG_CTX_ROOT_SCOPE(CurrentLockName);
+ YQL_CLOG(DEBUG, ProviderDq) << "Current lock '" << CurrentLockName << "'";
+ auto result = std::get<0>(*ev->Get());
+ if (result.IsOK()) {
+ if (FileUploadTime) {
+ FileUploadTime->Collect((TInstant::Now() - UploadStart).Seconds());
+ }
+ CurrentFileId++;
+ if (CurrentFileId == Options.Files.size()) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Upload complete";
+ PassAway();
+ } else {
+ if (Options.LockName.empty()) {
+ Lock();
+ YQL_CLOG(DEBUG, ProviderDq) << "Lock next " << CurrentFileId << "/" << Options.Files.size();
+ } else {
+ UploadFile();
+ YQL_CLOG(DEBUG, ProviderDq) << "Upload next " << CurrentFileId << "/" << Options.Files.size();
+ }
+ }
+ } else {
+ YQL_CLOG(DEBUG, ProviderDq) << "Retry " << ToString(result);
+
+ if (Errors) {
+ *Errors += 1;
+ }
+
+ std::random_shuffle(Options.Files.begin() + CurrentFileId, Options.Files.end());
+ Tick(ctx);
+ }
+ }
+
+ const NYT::NApi::IClientPtr Client;
+ TResourceManagerOptions Options;
+
+ const ICoordinationHelper::TPtr Coordinator;
+
+ const NProto::TDqConfig::TYtCoordinator Config;
+
+ TActorId YtWrapper;
+
+ ui32 CurrentFileId = 0;
+ TActorId ParentId;
+ TActorId LockId;
+
+ THistogramPtr FileSize;
+ THistogramPtr FileUploadTime;
+ TDynamicCounters::TCounterPtr Errors;
+ TInstant UploadStart;
+ TString CurrentLockName;
+};
+
+IActor* CreateYtResourceUploader(
+ const TResourceManagerOptions& options,
+ const ICoordinationHelper::TPtr& coordinator)
+{
+ Y_ABORT_UNLESS(!options.YtBackend.GetClusterName().empty());
+ Y_ABORT_UNLESS(!options.YtBackend.GetUser().empty());
+ Y_ABORT_UNLESS(options.YtBackend.HasPrefix());
+ Y_ABORT_UNLESS(!options.Files.empty());
+ Y_ABORT_UNLESS(!options.UploadPrefix.empty());
+
+ return new TYtResourceUploader(options, coordinator);
+}
+
+}
diff --git a/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp b/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp
new file mode 100644
index 0000000000..e8abfe904b
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/worker_registrator.cpp
@@ -0,0 +1,79 @@
+#include "worker_registrator.h"
+#include "nodeid_assigner.h"
+#include "yt_wrapper.h"
+
+#include <ydb/library/yql/providers/yt/lib/log/yt_logger.h>
+
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/event.h>
+#include <library/cpp/actors/core/hfunc.h>
+
+#include <library/cpp/yson/node/node.h>
+#include <library/cpp/yson/node/node_io.h>
+
+#include <util/generic/utility.h>
+
+namespace NYql {
+
+using namespace NActors;
+
+class TWorkerRegistrator: public TActor<TWorkerRegistrator> {
+public:
+ static constexpr char ActorName[] = "WORKER_REGISTRATOR";
+
+ TWorkerRegistrator(TActorId ytWrapper, const TWorkerRegistratorOptions& options)
+ : TActor(&TWorkerRegistrator::Handler)
+ , YtWrapper(ytWrapper)
+ , Options(options)
+ { }
+
+ STRICT_STFUNC(Handler, {
+ HFunc(TEvSetNodeResponse, OnSetNodeResponse)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ });
+
+private:
+ TEvSetNode* SetNodeCommand(TInstant now)
+ {
+ auto value = NYT::NYson::TYsonString(NYT::NodeToYsonString(NYT::TNode(now.ToString())));
+ return new TEvSetNode(Options.Prefix + "/" + Options.NodeName + "/@" + NCommonAttrs::PINGTIME_ATTR, value, NYT::NApi::TSetNodeOptions());
+ }
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ Y_UNUSED(parentId);
+ return new IEventHandle(YtWrapper, self, SetNodeCommand(TInstant::Now()), 0);
+ }
+
+ void OnSetNodeResponse(TEvSetNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto result = std::get<0>(*ev->Get());
+
+ if (!result.IsOK()) {
+ YQL_CLOG(ERROR, ProviderDq) << "Error on list node " << ToString(result);
+ }
+
+ if (!result.IsOK() && result.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) {
+ NYql::FlushYtDebugLog();
+ _exit(1);
+ }
+
+ YQL_CLOG(DEBUG, ProviderDq) << "OnSetNodeResponse";
+
+ TActivationContext::Schedule(Options.PingPeriod, new IEventHandle(YtWrapper, SelfId(), SetNodeCommand(TInstant::Now()), 0));
+ }
+
+ TActorId YtWrapper;
+ TWorkerRegistratorOptions Options;
+};
+
+IActor* CreateWorkerRegistrator(NActors::TActorId ytWrapper, const TWorkerRegistratorOptions& options) {
+ Y_ABORT_UNLESS(!options.Prefix.empty());
+ Y_ABORT_UNLESS(!options.NodeName.empty());
+ return new TWorkerRegistrator(ytWrapper, options);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/worker_registrator.h b/ydb/library/yql/providers/dq/actors/yt/worker_registrator.h
new file mode 100644
index 0000000000..de714cb135
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/worker_registrator.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include <library/cpp/actors/core/actor.h>
+
+#include <util/datetime/base.h>
+
+namespace NYql
+{
+
+struct TWorkerRegistratorOptions {
+ TString Prefix;
+ TString NodeName;
+
+ TDuration PingPeriod = TDuration::MilliSeconds(5000);
+ TDuration RetryPeriod = TDuration::MilliSeconds(5000);
+};
+
+NActors::IActor* CreateWorkerRegistrator(NActors::TActorId ytWrapper, const TWorkerRegistratorOptions& options);
+
+} // namespace NYql
+
diff --git a/ydb/library/yql/providers/dq/actors/yt/ya.make b/ydb/library/yql/providers/dq/actors/yt/ya.make
new file mode 100644
index 0000000000..106c3a3c8e
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/ya.make
@@ -0,0 +1,56 @@
+LIBRARY()
+
+PEERDIR(
+ library/cpp/actors/core
+ library/cpp/grpc/client
+ yt/cpp/mapreduce/interface
+ ydb/library/yql/providers/dq/config
+ ydb/library/yql/core/issue
+ ydb/library/yql/providers/common/metrics
+ ydb/library/yql/providers/dq/api/grpc
+ ydb/library/yql/providers/dq/api/protos
+ ydb/library/yql/providers/dq/common
+ ydb/library/yql/providers/yt/lib/log
+ ydb/library/yql/providers/dq/actors/events
+)
+
+IF (NOT OS_WINDOWS)
+ PEERDIR(
+ yt/yt/client
+ )
+ENDIF()
+
+SET(
+ SOURCE
+ nodeid_assigner.cpp
+ nodeid_assigner.h
+ resource_manager.cpp
+ resource_manager.h
+)
+
+IF (NOT OS_WINDOWS)
+ SET(
+ SOURCE
+ ${SOURCE}
+ nodeid_cleaner.cpp
+ nodeid_cleaner.h
+ worker_registrator.cpp
+ worker_registrator.h
+ lock.cpp
+ lock.h
+ resource_uploader.cpp
+ resource_downloader.cpp
+ resource_cleaner.cpp
+ yt_wrapper.cpp
+ yt_wrapper.h
+ yt_resource_manager.cpp
+ )
+ENDIF()
+
+SRCS(
+ ${SOURCE}
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/providers/dq/actors/yt/yt_events.h b/ydb/library/yql/providers/dq/actors/yt/yt_events.h
new file mode 100644
index 0000000000..9c46790b8f
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/yt_events.h
@@ -0,0 +1,35 @@
+#pragma once
+
+namespace NYql {
+
+ struct TYtEvents {
+ enum {
+ ES_START_OPERATION = EventSpaceBegin(NActors::TEvents::EEventSpace::ES_USERSPACE) + 10000,
+ ES_START_OPERATION_RESPONSE,
+ ES_GET_OPERATION,
+ ES_GET_OPERATION_RESPONSE,
+ ES_LIST_OPERATIONS,
+ ES_LIST_OPERATIONS_RESPONSE,
+ ES_GET_JOB,
+ ES_GET_JOB_RESPONSE,
+ ES_WRITE_FILE,
+ ES_WRITE_FILE_RESPONSE,
+ ES_READ_FILE,
+ ES_READ_FILE_RESPONSE,
+ ES_LIST_NODE,
+ ES_LIST_NODE_RESPONSE,
+ ES_CREATE_NODE,
+ ES_CREATE_NODE_RESPONSE,
+ ES_SET_NODE,
+ ES_SET_NODE_RESPONSE,
+ ES_GET_NODE,
+ ES_GET_NODE_RESPONSE,
+ ES_REMOVE_NODE,
+ ES_REMOVE_NODE_RESPONSE,
+ ES_START_TRANSACTION,
+ ES_START_TRANSACTION_RESPONSE,
+
+ ES_PRINT_JOB_STDERR
+ };
+ };
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp b/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp
new file mode 100644
index 0000000000..a19ef5b5ac
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/yt_resource_manager.cpp
@@ -0,0 +1,917 @@
+#include "yt_wrapper.h"
+
+#include <util/thread/pool.h>
+#include <util/generic/size_literals.h>
+#include <util/string/strip.h>
+#include <util/system/env.h>
+
+#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h>
+#include <ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h>
+#include <ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h>
+
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/yson/node/node_io.h>
+
+#include <yt/cpp/mapreduce/interface/fluent.h>
+
+#include <library/cpp/protobuf/util/pb_io.h>
+
+namespace NYql {
+
+#define RM_LOG(A) YQL_CLOG(A, ProviderDq) << ClusterName << ": "
+
+ namespace NCommonJobVars {
+ const TString ACTOR_PORT("ACTOR_PORT");
+ const TString ACTOR_NODE_ID("ACTOR_NODE_ID");
+ const TString UDFS_PATH("UDFS_PATH");
+ const TString OPERATION_SIZE("OPERATION_SIZE");
+ const TString YT_COORDINATOR("YT_COORDINATOR");
+ const TString YT_BACKEND("YT_BACKEND");
+ }
+
+ using namespace NActors;
+
+ struct TEvDropOperation
+ : NActors::TEventLocal<TEvDropOperation, TDqEvents::ES_OTHER1> {
+ TEvDropOperation() = default;
+ TEvDropOperation(const TString& operationId, const TString& mutationId)
+ : OperationId(operationId)
+ , MutationId(mutationId)
+ { }
+
+ TString OperationId;
+ TString MutationId;
+ };
+
+ class TYtVanillaOperation: public TActor<TYtVanillaOperation> {
+ public:
+ static constexpr char ActorName[] = "YT_OPERATION";
+
+ TYtVanillaOperation(const TString& clusterName, TActorId ytWrapper, TActorId parentId, TString operationId, TString mutationId, TIntrusivePtr<NMonitoring::TDynamicCounters> counters)
+ : TActor<TYtVanillaOperation>(&TYtVanillaOperation::Handler)
+ , ClusterName(clusterName)
+ , YtWrapper(ytWrapper)
+ , ParentId(parentId)
+ , OperationId(NYT::NScheduler::TOperationId::FromString(operationId))
+ , MutationId(mutationId)
+ , Counters(counters)
+ { }
+
+ ~TYtVanillaOperation()
+ {
+ auto counters = Counters->GetSubgroup("operation", "brief_progress");
+ for (const auto& [k, v] : Status) {
+ *counters->GetCounter(k) += -v;
+ }
+ }
+
+ private:
+ STRICT_STFUNC(Handler, {
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ HFunc(TEvGetOperationResponse, OnGetOperationResponse);
+ })
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& /*parentId*/) override {
+ return new IEventHandle(YtWrapper, self, new TEvGetOperation(OperationId, NYT::NApi::TGetOperationOptions()), 0);
+ }
+
+ void OnGetOperationResponse(TEvGetOperationResponse::TPtr& ev, const NActors::TActorContext& ) {
+ auto result = std::get<0>(*ev->Get());
+ bool stopWatcher = false;
+ if (!result.IsOK() && result.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) {
+ stopWatcher = true;
+ }
+
+ if (result.IsOK()) {
+ auto attributesMap = NYT::NodeFromYsonString(result.Value()).AsMap();
+
+ try {
+ if (attributesMap.contains("result")) {
+ RM_LOG(DEBUG) << "Result " << NYT::NodeToYsonString(attributesMap["result"]);
+ stopWatcher = true;
+ }
+
+ if (attributesMap.contains("brief_progress")) {
+ auto statusMap = attributesMap["brief_progress"].AsMap()["jobs"].AsMap();
+
+ auto counters = Counters->GetSubgroup("operation", "brief_progress");
+ for (const auto& [k, v] : statusMap) {
+ auto& oldStatus = Status[k];
+ auto newStatus = v.AsInt64();
+ *counters->GetCounter(k) += newStatus - oldStatus;
+ oldStatus = newStatus;
+ }
+ }
+
+ } catch (...) {
+ RM_LOG(DEBUG) << CurrentExceptionMessage();
+ }
+ }
+
+ if (stopWatcher) {
+ RM_LOG(DEBUG) << "Stop watching operation (1) " << ToString(OperationId) << " " << ToString(result);
+ Send(YtWrapper, new TEvPrintJobStderr(OperationId));
+ Send(ParentId, new TEvDropOperation(ToString(OperationId), MutationId));
+ PassAway();
+ } else {
+ TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way());
+ TActivationContext::Schedule(TDuration::Seconds(5),
+ new IEventHandle(YtWrapper, SelfId(), new TEvGetOperation(OperationId, NYT::NApi::TGetOperationOptions()), 0),
+ TimerCookieHolder.Get());
+ }
+ }
+
+ const TString ClusterName;
+ const TActorId YtWrapper;
+ const TActorId ParentId;
+ const NYT::NScheduler::TOperationId OperationId;
+ const TString MutationId;
+ NActors::TSchedulerCookieHolder TimerCookieHolder;
+ TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
+ THashMap<TString, i64> Status;
+ };
+
+ class TNodeIdAllocator {
+ public:
+ TNodeIdAllocator(ui32 minNodeId, ui32 maxNodeId)
+ : MinNodeId(minNodeId)
+ , MaxNodeId(maxNodeId)
+ { }
+
+ void Allocate(const TVector<ui32>& res) {
+ // TODO: check duplicates?
+ for (ui32 id : res) {
+ Allocated.insert(id);
+ }
+ }
+
+ void Allocate(TVector<ui32>* res, int count) {
+ res->reserve(count);
+ for (ui32 id = MinNodeId; id < MaxNodeId && static_cast<int>(res->size()) < count; id = id + 1) {
+ if (!Allocated.contains(id)) {
+ res->push_back(id);
+ Allocated.insert(id);
+ }
+ }
+ }
+
+ void Deallocate(const TVector<ui32>& nodes) {
+ for (auto id : nodes) {
+ Allocated.erase(id);
+ }
+ }
+
+ void Clear() {
+ Allocated.clear();
+ }
+
+ private:
+ THashSet<ui32> Allocated;
+ ui32 MinNodeId;
+ ui32 MaxNodeId;
+ };
+
+ class TYtResourceManager: public TRichActor<TYtResourceManager> {
+ public:
+ static constexpr char ActorName[] = "YTRM";
+
+ TYtResourceManager(
+ const TResourceManagerOptions& options,
+ const ICoordinationHelper::TPtr& coordinator)
+ : TRichActor<TYtResourceManager>(&TYtResourceManager::Follower)
+ , Options(options)
+ , Counters(Options.Counters)
+ , ClusterName(Options.YtBackend.GetClusterName())
+ , Coordinator(coordinator)
+ , CoordinatorConfig(Coordinator->GetConfig())
+ , CoordinatorWrapper(Coordinator->GetWrapper())
+ , NodeIdAllocator(Options.YtBackend.GetMinNodeId(), Options.YtBackend.GetMaxNodeId())
+ {
+ }
+
+ private:
+ // States: Follower <-> (ListOperations -> Leader)
+
+ void StartFollower(TEvBecomeFollower::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ auto leaderAttributes = NYT::NodeFromYsonString(ev->Get()->Attributes).AsMap();
+ LeaderTransactionId = NYT::NObjectClient::TTransactionId();
+
+ RM_LOG(TRACE) << " Following leader: " << leaderAttributes.at(NCommonAttrs::ACTOR_NODEID_ATTR).AsUint64();
+ for (const auto& [k, v] : RunningOperations) {
+ UnregisterChild(v.ActorId);
+ }
+ RunningOperations.clear();
+ NodeIdAllocator.Clear();
+ Become(&TYtResourceManager::Follower);
+ }
+
+ void StartLeader(TEvBecomeLeader::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ RM_LOG(INFO) << "Become leader, epoch=" << ev->Get()->LeaderEpoch;
+
+ LeaderTransactionId = NYT::NObjectClient::TTransactionId::FromString(ev->Get()->LeaderTransaction);
+
+ ListOperations();
+ Become(&TYtResourceManager::ListOperationsState);
+ }
+
+ STRICT_STFUNC(Follower, {
+ HFunc(TEvBecomeFollower, StartFollower)
+ HFunc(TEvBecomeLeader, StartLeader)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ CFunc(TEvents::TEvBootstrap::EventType, Bootstrap)
+
+ IgnoreFunc(TEvTick)
+ IgnoreFunc(TEvDropOperation)
+ IgnoreFunc(TEvStartOperationResponse)
+ IgnoreFunc(TEvCreateNodeResponse)
+ IgnoreFunc(TEvSetNodeResponse)
+ })
+
+ STRICT_STFUNC(ListOperationsState, {
+ HFunc(TEvBecomeFollower, StartFollower)
+ HFunc(TEvDropOperation, OnDropOperation)
+ HFunc(TEvListNodeResponse, OnListOperations)
+ HFunc(TEvStartOperationResponse, OnStartOperationResponse)
+ cFunc(TEvTick::EventType, ListOperations)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ IgnoreFunc(TEvCreateNodeResponse)
+ IgnoreFunc(TEvSetNodeResponse)
+ })
+
+ STRICT_STFUNC(Leader, {
+ HFunc(TEvBecomeFollower, StartFollower)
+ HFunc(TEvDropOperation, OnDropOperation)
+ HFunc(TEvListNodeResponse, OnListResponse)
+ HFunc(TEvStartOperationResponse, OnStartOperationResponse)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ cFunc(TEvTick::EventType, [this]() {
+ ListWorkers();
+ Tick();
+ })
+ HFunc(TEvCreateNodeResponse, OnCreateNode)
+ IgnoreFunc(TEvRemoveNodeResponse)
+ IgnoreFunc(TEvSetNodeResponse)
+ })
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0);
+ }
+
+ void Bootstrap(const NActors::TActorContext& ctx) {
+ YtWrapper = Coordinator->GetWrapper(
+ ctx.ActorSystem(),
+ Options.YtBackend.GetClusterName(),
+ Options.YtBackend.GetUser(),
+ Options.YtBackend.GetToken());
+ RegisterChild(Coordinator->CreateLockOnCluster(YtWrapper, Options.YtBackend.GetPrefix(), Options.LockName, false));
+ }
+
+ void Tick() {
+ TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way());
+ Schedule(TDuration::Seconds(5), new TEvTick(), TimerCookieHolder.Get());
+ }
+
+ void CreateCoreTable(ui32 tableNumber)
+ {
+ NYT::NApi::TCreateNodeOptions options;
+ options.Recursive = true;
+ options.IgnoreExisting = true;
+
+ YQL_CLOG(DEBUG, ProviderDq) << "Creating core table: " << Options.UploadPrefix + "/CoreTable-" + ToString(tableNumber);
+
+ Send(YtWrapper, new TEvCreateNode(
+ static_cast<ui64>(-1),
+ Options.UploadPrefix + "/CoreTable-" + ToString(tableNumber),
+ NYT::NObjectClient::EObjectType::Table,
+ options));
+
+ YQL_CLOG(DEBUG, ProviderDq) << "Creating stderr table: " << Options.UploadPrefix + "/StderrTable-" + ToString(tableNumber);
+
+ Send(YtWrapper, new TEvCreateNode(
+ static_cast<ui64>(-1),
+ Options.UploadPrefix + "/StderrTable-" + ToString(tableNumber),
+ NYT::NObjectClient::EObjectType::Table,
+ options));
+ }
+
+ void StartOperationWatcher(const TString& operationId, const TString& mutationId, const NActors::TActorContext& ctx)
+ {
+ Y_UNUSED(ctx);
+ RM_LOG(DEBUG) << "StartOperationWatcher " << operationId << "|" << mutationId;
+ auto operation = RunningOperations.find(mutationId);
+ Y_ABORT_UNLESS(operation != RunningOperations.end());
+ auto actorId = RegisterChild(new TYtVanillaOperation(ClusterName, YtWrapper, SelfId(), operationId, mutationId, Counters));
+ operation->second.ActorId = actorId;
+ }
+
+ void MaybeStartOperations(const NActors::TActorContext& ctx)
+ {
+ // to avoid races do nothing if there is PendingStartOperationRequests
+ if (!PendingStartOperationRequests.empty()) {
+ RM_LOG(DEBUG) << "PendingStartOperationRequests contains " << PendingStartOperationRequests.size() << " requests ";
+ for (const auto& [k, _]: PendingStartOperationRequests) {
+ RM_LOG(DEBUG) << "RequestId " << k;
+ }
+ return;
+ }
+
+ int totalJobs = 0;
+ for (const auto& [k, v] : RunningOperations) {
+ totalJobs += v.Nodes.size();
+ RM_LOG(DEBUG) << "Operation: " << k << " " << v.Nodes.size() << " ";
+ }
+
+ RM_LOG(DEBUG) << "Running/Max jobs: " << totalJobs << "/" << Options.YtBackend.GetMaxJobs();
+
+ int needToStart = Options.YtBackend.GetMaxJobs() - totalJobs;
+ RM_LOG(DEBUG) << "Need to start: " << needToStart;
+ if (needToStart > 0) {
+ StartOperations(needToStart, ctx);
+ }
+ }
+
+ void DropRunningOperation(const TString& mutationId, const TVector<ui32>& preserve = {}) {
+ if (RunningOperations.contains(mutationId)) {
+ NodeIdAllocator.Deallocate(RunningOperations[mutationId].Nodes);
+ }
+
+ RunningOperations.erase(mutationId);
+
+ if (!preserve.empty()) {
+ RM_LOG(DEBUG) << "Operation in unknown state, preserve mutation " << mutationId;
+ MutationsCache[mutationId] = preserve;
+ NodeIdAllocator.Allocate(preserve);
+ } else {
+ auto removePath = Options.YtBackend.GetPrefix() + "/operations/" + ClusterName + "/" + mutationId;
+ RM_LOG(DEBUG) << "Removing node " << removePath;
+ NYT::NApi::TRemoveNodeOptions removeNodeOptions;
+ removeNodeOptions.PrerequisiteTransactionIds.push_back(LeaderTransactionId);
+ Send(YtWrapper, new TEvRemoveNode(removePath, removeNodeOptions));
+ }
+ }
+
+ void OnDropOperation(TEvDropOperation::TPtr& ev, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto operationId = ev->Get()->OperationId;
+ auto mutationId = ev->Get()->MutationId;
+ auto maybeOperation = RunningOperations.find(mutationId);
+ if (maybeOperation != RunningOperations.end()) {
+ UnregisterChild(maybeOperation->second.ActorId);
+
+ RM_LOG(DEBUG) << "Stop operation " << operationId << "|" << mutationId;
+ DropRunningOperation(mutationId);
+ } else {
+ RM_LOG(WARN) << "Unknown operation " << operationId << "|" << mutationId;
+ }
+ }
+
+ void OnListResponse(TEvListNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ auto result = std::get<0>(*ev->Get());
+
+ try {
+ MaybeStartOperations(ctx);
+ } catch (...) {
+ RM_LOG(ERROR) << "Error on list node " << CurrentExceptionMessage();
+ }
+ }
+
+ void OnListOperations(TEvListNodeResponse::TPtr& ev, const TActorContext& ctx)
+ {
+ auto result = std::get<0>(*ev->Get());
+
+ if (!result.IsOK()) {
+ if (result.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) {
+ Become(&TYtResourceManager::Leader);
+ } else {
+ RM_LOG(ERROR) << "Error on list node " << ToString(result);
+ }
+ Tick();
+ return;
+ }
+
+ RM_LOG(DEBUG) << "OnListOperations ";
+ auto nodes = NYT::NodeFromYsonString(result.Value()).AsList();
+
+ for (auto node : nodes)
+ {
+ const auto& attributes = node.GetAttributes().AsMap();
+
+ RM_LOG(DEBUG) << "Check " << node.AsString();
+
+ auto mutationId = NYT::TGuid::FromString(attributes.find("yql_mutation_id")->second.AsString());
+ auto command = attributes.find("yql_command")->second.AsString();
+ auto file_paths = attributes.find("yql_file_paths")->second;
+ auto maybeNodes = attributes.find(NCommonAttrs::ACTOR_NODEID_ATTR);
+
+ if (maybeNodes == attributes.end()) {
+ // unsupported
+ DropRunningOperation(ToString(mutationId));
+ continue;
+ }
+
+ TVector<ui32> nodes;
+
+ for (auto node : maybeNodes->second.AsList()) {
+ nodes.push_back(node.AsUint64());
+ }
+
+ NodeIdAllocator.Allocate(nodes);
+
+ auto maybeOperationId = attributes.find(NCommonAttrs::OPERATIONID_ATTR);
+ if (maybeOperationId == attributes.end()) {
+ RM_LOG(DEBUG) << "Start or attach to " << ToString(mutationId);
+ StartOrAttachOperation(mutationId, nodes, command, file_paths);
+ } else {
+ auto operationId = maybeOperationId->second.AsString();
+ RM_LOG(DEBUG) << "Attach to " << operationId << "|" << ToString(mutationId);
+
+ auto& status = RunningOperations[ToString(mutationId)];
+ status.MutationId = ToString(mutationId);
+ status.OperationId = operationId;
+ status.Nodes = nodes;
+
+ StartOperationWatcher(operationId, ToString(mutationId), ctx);
+ }
+ }
+
+ if (PendingStartOperationRequests.empty() && CurrentStateFunc() != &TYtResourceManager::Leader) {
+ Become(&TYtResourceManager::Leader);
+ Tick();
+ }
+ }
+
+ void OnStartOperationResponse(TEvStartOperationResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ auto result = std::get<0>(*ev->Get());
+ auto requestId = ev->Get()->RequestId;
+
+ auto maybeJobs = PendingStartOperationRequests.find(requestId);
+
+ Y_ABORT_UNLESS(maybeJobs != PendingStartOperationRequests.end());
+
+ auto mutationId = maybeJobs->second.MutationId;
+
+ if (!result.IsOK()) {
+ // TODO: Check response code
+ RM_LOG(WARN) << "Failed to start operation " << ToString(result);
+ DropRunningOperation(mutationId, maybeJobs->second.Nodes);
+ } else {
+ auto operationId = ToString(result.Value());
+ Y_ABORT_UNLESS(RunningOperations.contains(mutationId));
+
+ RunningOperations[mutationId].OperationId = operationId;
+
+ NYT::NApi::TSetNodeOptions setNodeOptions;
+ setNodeOptions.PrerequisiteTransactionIds.push_back(LeaderTransactionId);
+
+ Send(YtWrapper, new TEvSetNode(
+ Options.YtBackend.GetPrefix() + "/operations/" + ClusterName + "/" + mutationId + "/@" + NCommonAttrs::OPERATIONID_ATTR,
+ NYT::NYson::TYsonString(NYT::NodeToYsonString(NYT::TNode(operationId))),
+ setNodeOptions));
+ StartOperationWatcher(operationId, mutationId, ctx);
+ }
+
+ PendingStartOperationRequests.erase(maybeJobs);
+
+ if (PendingStartOperationRequests.empty() && CurrentStateFunc() != &TYtResourceManager::Leader) {
+ Become(&TYtResourceManager::Leader);
+ Tick();
+ }
+ }
+
+ void ListOperations() {
+ NYT::NApi::TListNodeOptions options;
+ options.Attributes = {
+ "yql_mutation_id",
+ NCommonAttrs::OPERATIONSIZE_ATTR,
+ NCommonAttrs::OPERATIONID_ATTR,
+ NCommonAttrs::CLUSTERNAME_ATTR,
+ NCommonAttrs::ACTOR_NODEID_ATTR,
+ "yql_command",
+ "yql_file_paths"
+ };
+ auto command = new TEvListNode(Options.YtBackend.GetPrefix() + "/operations/" + ClusterName, options);
+ RM_LOG(DEBUG) << "List " << Options.YtBackend.GetPrefix() + "/operations/" + ClusterName;
+ Send(YtWrapper, command);
+ }
+
+ void ListWorkers() {
+ NYT::NApi::TListNodeOptions options;
+ options.Attributes = {
+ NCommonAttrs::ACTOR_NODEID_ATTR,
+ NCommonAttrs::OPERATIONID_ATTR,
+ NCommonAttrs::OPERATIONSIZE_ATTR,
+ NCommonAttrs::JOBID_ATTR,
+ NCommonAttrs::ROLE_ATTR,
+ NCommonAttrs::CLUSTERNAME_ATTR,
+ "modification_time"
+ };
+ options.ReadFrom = NYT::NApi::EMasterChannelKind::Cache;
+ auto command = new TEvListNode(CoordinatorConfig.GetPrefix() + "/worker_node", options);
+ Send(CoordinatorWrapper, command);
+ }
+
+ void StartOperations(int jobs, const NActors::TActorContext& ctx) {
+ int jobsPerOperation = Options.YtBackend.HasJobsPerOperation()
+ ? Options.YtBackend.GetJobsPerOperation()
+ : Options.YtBackend.GetMaxJobs();
+
+ Y_ABORT_UNLESS(jobsPerOperation > 0);
+
+ for (int i = 0; i < jobs; i += jobsPerOperation) {
+ if (jobs - i >= jobsPerOperation) {
+ StartOperation(jobsPerOperation, ctx);
+ }
+ }
+ }
+
+ TString GetOperationSpec(const TVector<ui32>& nodes, const TString& command, const TMaybe<NYT::TNode>& filePaths) const
+ {
+ int actorPort = Options.YtBackend.HasActorStartPort()
+ ? Options.YtBackend.GetActorStartPort()
+ : 31002;
+
+ bool samePorts = Options.YtBackend.HasSameActorPorts()
+ ? Options.YtBackend.GetSameActorPorts()
+ : true;
+
+ auto minNodeId = Options.YtBackend.GetMinNodeId();
+
+ Y_ABORT_UNLESS(!nodes.empty());
+
+ TString coordinatorStr;
+ TStringOutput output1(coordinatorStr);
+ SerializeToTextFormat(CoordinatorConfig, output1);
+
+ TString backendStr;
+ TStringOutput output2(backendStr);
+ SerializeToTextFormat(Options.YtBackend, output2);
+ ui32 tableNumber = *nodes.begin();
+ TString fileCache = "file_cache2";
+
+ TVector<std::pair<TString, TString>> initialFileList;
+ for (const auto& fname : Options.Files) {
+ initialFileList.push_back(std::make_pair(Options.UploadPrefix, fname.GetRemoteFileName()));
+ }
+ if (Options.YtBackend.HasEnablePorto()) {
+ for (const auto& layer : Options.YtBackend.GetPortoLayer()) {
+ auto pos = layer.rfind('/');
+ auto baseName = layer.substr(0, pos);
+ auto name = layer.substr(pos+1);
+ initialFileList.push_back(std::make_pair(baseName, name));
+ }
+ }
+
+ TVector<TString> operationLayersList;
+ for (const auto& operationLayer : Options.YtBackend.GetOperationLayer()) {
+ operationLayersList.push_back(operationLayer);
+ }
+
+ auto operationSpec = NYT::BuildYsonNodeFluently()
+ .BeginMap()
+ .DoIf(Options.YtBackend.GetOwner().size() > 0, [&] (NYT::TFluentMap fluent) {
+ fluent.Item("acl").BeginList()
+ .Item()
+ .BeginMap()
+ .Item("action").Value("allow")
+ .Item("permissions")
+ .BeginList()
+ .Item().Value("read")
+ .Item().Value("manage")
+ .EndList()
+ .Item("subjects")
+ .BeginList()
+ .DoFor(Options.YtBackend.GetOwner(), [&] (NYT::TFluentList fluent1, const TString& subject) {
+ fluent1.Item().Value(subject);
+ })
+ .EndList()
+ .EndMap()
+ .EndList();
+ })
+ .Item("secure_vault")
+ .BeginMap()
+ .Item(NCommonJobVars::YT_COORDINATOR).Value(coordinatorStr)
+ .Item(NCommonJobVars::YT_BACKEND).Value(backendStr)
+ .DoFor(Options.YtBackend.GetVaultEnv(), [&] (NYT::TFluentMap fluent, const NYql::NProto::TDqConfig::TAttr& envVar) { // Добавляем env variables
+ TString tokenValue;
+ try {
+ tokenValue = StripString(TFileInput(envVar.GetValue()).ReadLine());
+ } catch (...) {
+ throw yexception() << "Cannot read file " << envVar.GetValue() << " Reason: " << CurrentExceptionMessage();
+ }
+ fluent.Item(envVar.GetName()).Value(tokenValue);
+ })
+ .EndMap()
+ .Item("core_table_path").Value(Options.UploadPrefix + "/CoreTable-"+ToString(tableNumber))
+ .Item("stderr_table_path").Value(Options.UploadPrefix + "/StderrTable-"+ToString(tableNumber))
+ .Item("try_avoid_duplicating_jobs").Value(true)
+ .DoIf(!Options.YtBackend.GetPool().empty(), [&] (NYT::TFluentMap fluent) {
+ fluent.Item("pool").Value(Options.YtBackend.GetPool());
+ })
+ .DoIf(Options.YtBackend.GetPoolTrees().size() > 0, [&] (NYT::TFluentMap fluent) {
+ fluent.Item("pool_trees")
+ .BeginList()
+ .DoFor(Options.YtBackend.GetPoolTrees(), [&](NYT::TFluentList fluent1, const TString& subject) {
+ fluent1.Item().Value(subject);
+ })
+ .EndList();
+ })
+ .Item("tasks")
+ .BeginMap()
+ .DoFor(nodes, [&] (NYT::TFluentMap fluent1, const auto& nodeId) {
+ fluent1.Item("yql_worker_" + ToString(nodeId))
+ .BeginMap()
+ .DoIf(Options.YtBackend.GetNetworkProject().size() > 0, [&] (NYT::TFluentMap fluent) {
+ fluent.Item("network_project").Value(Options.YtBackend.GetNetworkProject());
+ })
+ .DoIf(Options.YtBackend.HasEnablePorto(), [&] (NYT::TFluentMap fluent) {
+ fluent.Item("enable_porto").Value(Options.YtBackend.GetEnablePorto());
+ })
+ .DoIf(Options.YtBackend.HasContainerCpuLimit(), [&] (NYT::TFluentMap fluent) {
+ fluent.Item("set_container_cpu_limit").Value(Options.YtBackend.GetContainerCpuLimit());
+ })
+ .Item("command").Value(command)
+ .DoIf(!operationLayersList.empty(), [&] (NYT::TFluentMap fluent) {
+ fluent.Item("layer_paths").DoListFor(operationLayersList, [&] (NYT::TFluentList list, const TString& operationLayer) {
+ list.Item().Value(operationLayer);
+ });
+ })
+ .Item("environment")
+ .BeginMap()
+ .Item(NCommonJobVars::ACTOR_PORT).Value(ToString(
+ samePorts
+ ? actorPort
+ : actorPort + nodeId - minNodeId))
+ .Item(NCommonJobVars::OPERATION_SIZE).Value(ToString(nodes.size()))
+ .Item(NCommonJobVars::UDFS_PATH).Value(fileCache)
+ .Item(NCommonJobVars::ACTOR_NODE_ID).Value(ToString(nodeId))
+ .DoIf(!!GetEnv("YQL_DETERMINISTIC_MODE"), [&](NYT::TFluentMap fluent) {
+ fluent.Item("YQL_DETERMINISTIC_MODE").Value("1");
+ })
+ .EndMap()
+ .DoIf((Options.YtBackend.GetClusterName().find("localhost") != 0) && filePaths.Empty(), [&] (NYT::TFluentMap fluent) {
+ fluent.Item("file_paths").DoListFor(initialFileList, [&] (NYT::TFluentList list, const std::pair<TString, TString>& item) {
+ auto baseName = item.second;
+ list.Item()
+ .BeginAttributes()
+ .Item("executable").Value(true)
+ .Item("file_name").Value(baseName)
+ .EndAttributes()
+ .Value(item.first + "/" + baseName);
+ });
+ })
+ .DoIf(!filePaths.Empty(), [&] (NYT::TFluentMap fluent) {
+ fluent.Item("file_paths").Value(*filePaths);
+ })
+ .Item("job_count").Value(1)
+ .DoIf(Options.YtBackend.HasMemoryLimit(), [&] (NYT::TFluentMap fluent) {
+ fluent.Item("memory_limit").Value(Options.YtBackend.GetMemoryLimit());
+ })
+ .DoIf(Options.YtBackend.HasCpuLimit(), [&] (NYT::TFluentMap fluent) {
+ fluent.Item("cpu_limit").Value(Options.YtBackend.GetCpuLimit());
+ })
+ .DoIf(Options.YtBackend.HasUseTmpFs() && Options.YtBackend.GetUseTmpFs(), [&] (NYT::TFluentMap fluent) {
+ fluent.Item("tmpfs_path").Value(fileCache);
+ })
+ .EndMap();
+ })
+ .EndMap()
+ .EndMap();
+
+ return NYT::NodeToYsonString(operationSpec);
+ }
+
+ void StartOrAttachOperation(
+ const NYT::TGuid& mutationId,
+ const TVector<ui32>& nodes,
+ const TString& command,
+ const NYT::TNode& filePaths)
+ {
+ int jobs = static_cast<int>(nodes.size());
+ RM_LOG(INFO) << "Creating " << jobs << " workers " << ToString(mutationId);
+
+ auto operationSpec = GetOperationSpec(nodes, command, TMaybe<NYT::TNode>(filePaths));
+
+ auto startOperationOptions = NYT::NApi::TStartOperationOptions();
+ startOperationOptions.MutationId = mutationId;
+ startOperationOptions.Retry = true;
+
+ auto& state = RunningOperations[ToString(mutationId)];
+ state.MutationId = ToString(mutationId);
+ state.Nodes = nodes;
+
+ RM_LOG(DEBUG) << "Attaching to operation with mutationId " << ToString(mutationId);
+
+ CreateCoreTable(*nodes.begin());
+
+ Send(YtWrapper, MakeHolder<TEvStartOperation>(
+ YtRequestId,
+ NYT::NScheduler::EOperationType::Vanilla,
+ operationSpec,
+ startOperationOptions).Release());
+
+ PendingStartOperationRequests[YtRequestId++] = {nodes,
+ ToString(mutationId), THolder<TEvStartOperation>()};
+ }
+
+ void StartOperation(int jobs, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ RM_LOG(INFO) << "Creating " << jobs << " workers ";
+
+ TString executableName = (Options.YtBackend.GetClusterName().find("localhost") == 0)
+ ? Options.Files[0].LocalFileName
+ : TString("./") + Options.Files[0].GetRemoteFileName();
+
+ RM_LOG(INFO) << "Executable " << executableName;
+
+ TString command = Options.YtBackend.GetVanillaJobCommand();
+
+ RM_LOG(INFO) << "Executable " << command;
+
+ TVector<ui32> nodes;
+
+ auto startOperationOptions = NYT::NApi::TStartOperationOptions();
+
+ if (MutationsCache.empty()) {
+ startOperationOptions.MutationId = startOperationOptions.GetOrGenerateMutationId();
+ NodeIdAllocator.Allocate(&nodes, jobs);
+ } else {
+ startOperationOptions.MutationId = NYT::TGuid::FromString(MutationsCache.begin()->first);
+ nodes = MutationsCache.begin()->second;
+ NodeIdAllocator.Allocate(nodes);
+ RM_LOG(INFO) << "Get mutation from cache " << ToString(startOperationOptions.MutationId) << "," << nodes.size();
+ MutationsCache.erase(MutationsCache.begin());
+ }
+
+ if (nodes.empty()) {
+ RM_LOG(WARN) << "Cannot allocate node ids for " << jobs << " jobs";
+ return;
+ }
+
+ startOperationOptions.Retry = true;
+
+ auto operationSpec = GetOperationSpec(nodes, command, TMaybe<NYT::TNode>());
+
+ auto mutationId = startOperationOptions.MutationId;
+
+ RM_LOG(DEBUG) << "Start operation with mutationId " << ToString(mutationId) ;
+
+ auto& state = RunningOperations[ToString(mutationId)];
+ state.MutationId = ToString(mutationId);
+ state.Nodes = nodes;
+
+ NYT::NApi::TCreateNodeOptions createOptions;
+ createOptions.IgnoreExisting = true;
+ createOptions.Recursive = true;
+
+ RM_LOG(DEBUG) << "Creating operation with mutationId " << ToString(mutationId);
+
+ auto filesAttribute = Options.Files;
+ if (Options.YtBackend.GetClusterName().find("localhost") == 0) {
+ filesAttribute.clear();
+ }
+
+ auto attributes = NYT::BuildYsonNodeFluently()
+ .BeginMap()
+ .Item("yql_mutation_id").Value(ToString(mutationId))
+ .Item(NCommonAttrs::OPERATIONSIZE_ATTR).Value(jobs)
+ .Item("yql_command").Value(command)
+ .Item("yql_file_paths")
+ .DoListFor(filesAttribute, [&] (NYT::TFluentList list, const TResourceFile& item) {
+ auto baseName = item.GetRemoteFileName();
+ list.Item()
+ .BeginAttributes()
+ .Item("executable").Value(true)
+ .Item("file_name").Value(baseName)
+ .EndAttributes()
+ .Value(Options.UploadPrefix + "/" + baseName);
+ })
+ .Item(NCommonAttrs::ROLE_ATTR).Value("worker_node")
+ .Item(NCommonAttrs::ACTOR_NODEID_ATTR)
+ .BeginList()
+ .DoFor(nodes, [&] (NYT::TFluentList fluent1, const auto& nodeId) {
+ fluent1.Item().Value(nodeId);
+ })
+ .EndList()
+ .Item(NCommonAttrs::CLUSTERNAME_ATTR).Value(ClusterName)
+ .EndMap();
+
+ createOptions.Attributes = NYT::NYTree::IAttributeDictionary::FromMap(
+ NYT::NYTree::ConvertToNode(NYT::NYson::TYsonString(NYT::NodeToYsonString(attributes)))->AsMap()
+ );
+
+ createOptions.PrerequisiteTransactionIds.push_back(LeaderTransactionId);
+
+ CreateCoreTable(*nodes.begin());
+
+ Send(YtWrapper, new TEvCreateNode(
+ YtRequestId,
+ Options.YtBackend.GetPrefix() + "/operations/" + ClusterName + "/" + ToString(mutationId),
+ NYT::NObjectClient::EObjectType::StringNode,
+ createOptions));
+
+ PendingStartOperationRequests[YtRequestId] = {
+ nodes,
+ ToString(mutationId),
+ MakeHolder<TEvStartOperation>(
+ YtRequestId+1,
+ NYT::NScheduler::EOperationType::Vanilla,
+ operationSpec,
+ startOperationOptions)
+ };
+ YtRequestId += 2;
+ }
+
+ void OnCreateNode(TEvCreateNodeResponse::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto requestId = ev->Get()->RequestId;
+ auto result = std::get<0>(*ev->Get());
+ if (requestId == static_cast<ui64>(-1)) {
+ // CoreTable
+ if (!result.IsOK()) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Error on creating core table " << ToString(result);
+ }
+ return;
+ }
+ if (!PendingStartOperationRequests.contains(requestId)) {
+ return;
+ }
+ auto& op = PendingStartOperationRequests[requestId];
+ if (result.IsOK()) {
+ Y_ABORT_UNLESS(!PendingStartOperationRequests.contains(requestId+1));
+ PendingStartOperationRequests[requestId+1] = {
+ op.Nodes,
+ op.MutationId,
+ THolder<TEvStartOperation>()
+ };
+ Send(YtWrapper, op.Ev.Release());
+ } else if (RunningOperations.contains(op.MutationId)) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Error on create node " << ToString(result);
+ DropRunningOperation(op.MutationId);
+ }
+ PendingStartOperationRequests.erase(requestId);
+ // retry in ListOperations
+ }
+
+ const TResourceManagerOptions Options;
+ TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
+ const TString ClusterName;
+
+ const ICoordinationHelper::TPtr Coordinator;
+
+ const NProto::TDqConfig::TYtCoordinator CoordinatorConfig;
+
+ TActorId YtWrapper;
+ const TActorId CoordinatorWrapper;
+
+ NYT::NObjectClient::TTransactionId LeaderTransactionId;
+
+ TNodeIdAllocator NodeIdAllocator;
+
+ struct TOperationStatus {
+ TString OperationId;
+ TString MutationId;
+ TActorId ActorId;
+ TVector<ui32> Nodes;
+ };
+
+ // mutationId -> operation
+ THashMap<TString, TOperationStatus> RunningOperations;
+
+ THashMap<TString, TVector<ui32>> MutationsCache;
+
+ // RequestId -> Jobs
+ struct TPendingStartOperation {
+ TVector<ui32> Nodes;
+ TString MutationId;
+ THolder<TEvStartOperation> Ev;
+ };
+ THashMap<ui64, TPendingStartOperation> PendingStartOperationRequests;
+ ui64 YtRequestId = 1;
+ NActors::TSchedulerCookieHolder TimerCookieHolder;
+ };
+
+ IActor* CreateYtResourceManager(
+ const TResourceManagerOptions& options,
+ const ICoordinationHelper::TPtr& coordinator)
+ {
+ Y_ABORT_UNLESS(!options.YtBackend.GetClusterName().empty());
+ Y_ABORT_UNLESS(!options.YtBackend.GetUser().empty());
+ Y_ABORT_UNLESS(options.YtBackend.HasMinNodeId());
+ Y_ABORT_UNLESS(options.YtBackend.HasMaxNodeId());
+ Y_ABORT_UNLESS(options.YtBackend.HasPrefix());
+ Y_ABORT_UNLESS(!options.Files.empty());
+ Y_ABORT_UNLESS(!options.UploadPrefix.empty());
+
+ return new TYtResourceManager(options, coordinator);
+ }
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp b/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp
new file mode 100644
index 0000000000..2320f4a837
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.cpp
@@ -0,0 +1,635 @@
+#include "yt_wrapper.h"
+
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <library/cpp/digest/md5/md5.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/yson/node/node_io.h>
+
+#include <util/system/file.h>
+#include <util/system/fs.h>
+#include <util/stream/fwd.h>
+#include <util/stream/buffered.h>
+#include <util/system/mutex.h>
+
+#include <yt/yt/client/api/rpc_proxy/config.h>
+#include <yt/yt/client/api/rpc_proxy/connection.h>
+#include <yt/yt/client/api/client.h>
+#include <yt/yt/client/api/file_writer.h>
+#include <yt/yt/client/api/file_reader.h>
+
+#include <yt/yt/core/ytree/convert.h>
+
+using namespace NYql;
+using namespace NYT;
+using namespace NYT::NApi;
+using namespace NActors;
+
+namespace NYql {
+ struct TRequest: public NYT::TRefCounted {
+ const TActorId SelfId;
+ const TActorId Sender;
+ TActorSystem* Ctx;
+ const ui64 RequestId;
+
+ TRequest(const TActorId& selfId, const TActorId& sender, TActorSystem* ctx, const ui64 requestId)
+ : SelfId(selfId)
+ , Sender(sender)
+ , Ctx(ctx)
+ , RequestId(requestId)
+ { }
+
+ void Complete(IEventBase* ev);
+ };
+
+ struct TWriteFileRequest: public TRequest {
+ IClientPtr Client;
+ IFileWriterPtr FileWriter;
+ TFile File;
+ i64 FileSize = -1;
+ TVector<char> Buffer;
+ TString NodePathTmp;
+ TString NodePath;
+ TFileWriterOptions WriterOptions;
+ TString Digest;
+ i64 Offset = 0;
+
+ TWriteFileRequest(const TActorId& selfId, const TActorId& sender, TActorSystem* ctx, const ui64 requestId)
+ : TRequest(selfId, sender, ctx, requestId)
+ { }
+
+
+ TFuture<void> WriteNext() {
+ const ui64 chunkSize = 64 * 1024 * 1024;
+ Buffer.resize(chunkSize);
+
+ i64 dataSize;
+ try {
+ if (FileSize < 0) {
+ FileSize = File.GetLength();
+ }
+ dataSize = File.Pread(&Buffer[0], Buffer.size(), Offset);
+ Offset += dataSize;
+ } catch (const std::exception& ex) {
+ return MakeFuture(TErrorOr<void>(ex));
+ }
+
+ if (dataSize == 0) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Closing writer";
+ return FileWriter->Close()
+ .Apply(BIND([self = MakeWeak(this)]() {
+ YQL_CLOG(DEBUG, ProviderDq) << "Write finished";
+ auto this_ = self.Lock();
+ if (!this_) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+
+ return this_->Client->GetNode(this_->NodePathTmp + "/@md5")
+ .Apply(BIND([self](const TErrorOr<NYT::NYson::TYsonString>& err) {
+ auto req = self.Lock();
+ if (!req) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ if (err.IsOK() && req->Digest == NYTree::ConvertTo<TString>(err.Value())) {
+ return VoidFuture;
+ }
+
+ return MakeFuture(TErrorOr<void>(yexception() << "wrong checksum"));
+ }));
+ }));
+ } else {
+ YQL_CLOG(DEBUG, ProviderDq) << "Writing chunk " << Offset << "/" << FileSize;
+ return FileWriter->Write(TSharedRef(&Buffer[0], dataSize, nullptr))
+ .Apply(BIND([self = MakeWeak(this)]() mutable {
+ auto this_ = self.Lock();
+ if (!this_) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ return this_->WriteNext();
+ }).AsyncVia(Client->GetConnection()->GetInvoker()));
+ }
+ }
+
+ TFuture<void> WriteFile()
+ {
+ auto& remotePath = NodePathTmp;
+ YQL_CLOG(INFO, ProviderDq) << "Start writing file to " << remotePath;
+ FileWriter = Client->CreateFileWriter(remotePath, WriterOptions);
+
+ return FileWriter->Open().Apply(BIND([self = MakeWeak(this)]() mutable {
+ auto this_ = self.Lock();
+ if (!this_) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ return this_->WriteNext();
+ }));
+ }
+ };
+
+ struct TReadFileRequest: public TRequest {
+ IClientPtr Client;
+ TString LocalPath;
+ IFileReaderPtr Reader;
+ std::shared_ptr<TFileOutput> Output;
+ TString Digest;
+ MD5 Md5;
+
+ TReadFileRequest(const TActorId& selfId, const TActorId& sender, TActorSystem* ctx, const ui64 requestId)
+ : TRequest(selfId, sender, ctx, requestId)
+ { }
+
+ TFuture<void> ReadNext()
+ {
+ return Reader->Read()
+ .Apply(BIND([self = MakeWeak(this)](const TSharedRef& blob) {
+ auto this_ = self.Lock();
+ if (!this_) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ try {
+ YQL_CLOG(DEBUG, ProviderDq) << "Store " << blob.Size() << " bytes ";
+ if (blob.Size() > 0) {
+ this_->Md5.Update(blob.Begin(), blob.Size());
+ this_->Output->Write(blob.Begin(), blob.Size());
+ return this_->ReadNext();
+ } else {
+ TString buf;
+ buf.ReserveAndResize(32);
+ this_->Md5.End(buf.begin());
+
+ if (buf == this_->Digest) {
+ this_->Output.reset();
+ return VoidFuture;
+ } else {
+ return MakeFuture(TErrorOr<void>(yexception() << "md5 mismatch"));
+ }
+ }
+ } catch (...) {
+ return MakeFuture(TErrorOr<void>(yexception() << CurrentExceptionMessage()));
+ }
+ }).AsyncVia(Client->GetConnection()->GetInvoker()));
+ }
+
+ TFuture<void> ReadFile()
+ {
+ auto pos = LocalPath.rfind('/');
+ if (pos != TString::npos) {
+ auto dirName = LocalPath.substr(0, pos);
+ if (!dirName.empty()) {
+ NFs::MakeDirectoryRecursive(dirName, NFs::FP_NONSECRET_FILE, false);
+ }
+ }
+
+ Output = std::make_shared<TFileOutput>(LocalPath);
+ return ReadNext();
+ }
+ };
+
+ using TRequestPtr = NYT::TIntrusivePtr<TRequest>;
+
+ struct TEvComplete
+ : NActors::TEventLocal<TEvComplete, TDqEvents::ES_OTHER1> {
+ TEvComplete() = default;
+ TEvComplete(const TRequestPtr& req)
+ : Request(req)
+ { }
+
+ const TRequestPtr Request;
+ };
+
+ void TRequest::Complete(IEventBase* ev) {
+ Ctx->Send(Sender, ev);
+ Ctx->Send(SelfId, new TEvComplete(NYT::MakeStrong(this)));
+ }
+
+ class TYtWrapper: public TActor<TYtWrapper> {
+ public:
+ static constexpr char ActorName[] = "YT_WRAPPER";
+
+ TYtWrapper(const IClientPtr& client)
+ : TActor(&TYtWrapper::Handler)
+ , Client(client)
+ { }
+
+ private:
+ STRICT_STFUNC(Handler, {
+ HFunc(TEvStartOperation, OnStartOperation)
+ HFunc(TEvGetOperation, OnGetOperation)
+ HFunc(TEvListOperations, OnListOperations)
+ HFunc(TEvGetJob, OnGetJob)
+ HFunc(TEvWriteFile, OnFileWrite)
+ HFunc(TEvReadFile, OnReadFile)
+ HFunc(TEvListNode, OnListNode)
+ HFunc(TEvSetNode, OnSetNode)
+ HFunc(TEvGetNode, OnGetNode)
+ HFunc(TEvRemoveNode, OnRemoveNode)
+ HFunc(TEvCreateNode, OnCreateNode)
+ HFunc(TEvStartTransaction, OnStartTransaction)
+ HFunc(TEvPrintJobStderr, OnPrintJobStderr)
+ HFunc(TEvComplete, OnComplete)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ });
+
+ THashSet<TRequestPtr> Requests;
+
+ void PassAway() override {
+ Requests.clear();
+ IActor::PassAway();
+ }
+
+ template<typename T>
+ TWeakPtr<T> NewRequest(ui64 id, TActorId sender, const TActorContext& ctx) {
+ auto req = New<T>(SelfId(), sender, ctx.ExecutorThread.ActorSystem, id);
+ Requests.emplace(req);
+ return NYT::MakeWeak(req);
+ }
+
+ void OnComplete(TEvComplete::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto req = ev->Get()->Request;
+ Requests.erase(req);
+ }
+
+ void OnFileWrite(TEvWriteFile::TPtr& ev, const TActorContext& ctx) {
+ TFile file = std::move(std::get<0>(*ev->Get()));
+ NYPath::TRichYPath remotePath = std::get<1>(*ev->Get());
+ THashMap<TString, NYT::TNode> attributes = std::get<2>(*ev->Get());
+ TFileWriterOptions writerOptions = std::get<3>(*ev->Get());
+ auto requestId = ev->Get()->RequestId;
+
+ auto nodePathTmp = remotePath.GetPath() + ".tmp";
+ auto nodePath = remotePath.GetPath();
+ auto request = NewRequest<TWriteFileRequest>(requestId, ev->Sender, ctx);
+ writerOptions.ComputeMD5 = true;
+
+ try {
+ YQL_CLOG(INFO, ProviderDq) << "Creating node " << remotePath.GetPath();
+
+ Y_ENSURE(file.IsOpen());
+
+ TString digest;
+
+ if (writerOptions.ComputeMD5) {
+ char buf[32768];
+ MD5 md5;
+ i64 size, offset = 0;
+ while ((size = file.Pread(buf, sizeof(buf), offset)) > 0) {
+ md5.Update(buf, size);
+ offset += size;
+ }
+ digest.ReserveAndResize(32);
+ md5.End(digest.begin());
+ }
+
+ if (auto req = request.Lock()) {
+ req->Client = Client;
+ req->File = std::move(file);
+ req->NodePathTmp = nodePathTmp;
+ req->NodePath = nodePath;
+ req->WriterOptions = writerOptions;
+ req->Digest = digest;
+ }
+
+ Client->GetNode(nodePath + "/@md5")
+ .Apply(BIND([request, attributes, digest](const TErrorOr<NYT::NYson::TYsonString>& err) mutable {
+ auto req = request.Lock();
+ if (!req) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ if (err.IsOK() && digest == NYTree::ConvertTo<TString>(err.Value())) {
+ YQL_CLOG(INFO, ProviderDq) << "File already uploaded";
+ try {
+ req->Client->SetNode(req->NodePath + "/@yql_last_update",
+ NYT::NYson::TYsonString(
+ NYT::NodeToYsonString(NYT::TNode(ToString(TInstant::Now()))
+ )));
+ } catch (...) { }
+ return VoidFuture;
+ } else if (err.IsOK() || err.FindMatching(NYT::NYTree::EErrorCode::ResolveError)) {
+ TCreateNodeOptions options;
+ options.Recursive = true;
+ options.IgnoreExisting = true;
+
+ if (err.IsOK()) {
+ YQL_CLOG(INFO, ProviderDq) << digest << "!=" << NYTree::ConvertTo<TString>(err.Value());
+ } else {
+ YQL_CLOG(ERROR, ProviderDq) << ToString(err);
+ }
+
+ return req->Client->CreateNode(req->NodePathTmp, NObjectClient::EObjectType::File, options).As<void>()
+ .Apply(BIND([request, attributes] () {
+ auto req = request.Lock();
+ if (!req) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ TVector<NYT::TFuture<void>> futures;
+ futures.reserve(attributes.size());
+ for (const auto& [k, v]: attributes) {
+ futures.push_back(
+ req->Client->SetNode(
+ req->NodePathTmp + "/@" + k,
+ NYT::NYson::TYsonString(NYT::NodeToYsonString(v)),
+ NYT::NApi::TSetNodeOptions()));
+ }
+ return NYT::AllSucceeded(futures).As<void>();
+ }))
+ .Apply(BIND([request]() mutable {
+ auto req = request.Lock();
+ if (!req) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ return req->WriteFile();
+ }))
+ .Apply(BIND([request] () {
+ auto req = request.Lock();
+ if (!req) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ auto moveOptions = NYT::NApi::TMoveNodeOptions();
+ moveOptions.Force = true;
+ return req->Client->MoveNode(req->NodePathTmp, req->NodePath, moveOptions).As<void>();
+ }));
+ }
+
+ err.ThrowOnError();
+
+ return VoidFuture;
+ }))
+ .Apply(BIND([request, requestId](const TErrorOr<void>& err)
+ {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvWriteFileResponse(requestId, err));
+ }
+ }));
+ } catch (const std::exception& ex) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvWriteFileResponse(requestId, ex));
+ }
+ }
+ }
+
+ void OnReadFile(TEvReadFile::TPtr& ev, const TActorContext& ctx) {
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TReadFileRequest>(requestId, ev->Sender, ctx);
+
+ try {
+ NYPath::TRichYPath remotePath = std::get<0>(*ev->Get());
+ TFileReaderOptions readerOptions = std::get<2>(*ev->Get());
+ if (auto req = request.Lock()) {
+ req->LocalPath = std::get<1>(*ev->Get());
+ req->Client = Client;
+ }
+
+ auto nodePath = remotePath.GetPath();
+
+ Client->GetNode(nodePath + "/@md5")
+ .Apply(BIND([request, nodePath, readerOptions](const TErrorOr<NYT::NYson::TYsonString>& err) mutable {
+ auto req = request.Lock();
+ if (!req) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ if (!err.IsOK()) {
+ return MakeFuture(TErrorOr<void>(yexception() << "failed to get md5"));
+ }
+ auto digest = NYTree::ConvertTo<TString>(err.Value());
+ req->Digest = digest;
+
+ return req->Client->CreateFileReader(nodePath, readerOptions)
+ .Apply(BIND([request](const IFileReaderPtr& reader) {
+ auto req = request.Lock();
+ if (!req) {
+ return MakeFuture(TErrorOr<void>(yexception() << "request complete"));
+ }
+ req->Reader = reader;
+ return req->ReadFile();
+ }));
+ }))
+ .Apply(BIND([request, requestId](const TErrorOr<void>& err) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvReadFileResponse(requestId, err));
+ }
+ }));
+ } catch (const std::exception& ex) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvReadFileResponse(requestId, ex));
+ }
+ }
+ }
+
+ void OnStartOperation(TEvStartOperation::TPtr& ev, const TActorContext& ctx) {
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ try {
+ NScheduler::EOperationType type = std::get<0>(*ev->Get());
+ auto spec = NYT::NYson::TYsonString(std::get<1>(*ev->Get()));
+ TStartOperationOptions options = std::get<2>(*ev->Get());
+
+ Client->StartOperation(type, spec, options).Subscribe(BIND([=](const TErrorOr<NScheduler::TOperationId>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvStartOperationResponse(requestId, result));
+ }
+ }));
+ } catch (const std::exception& ex) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvStartOperationResponse(requestId, ex));
+ }
+ }
+ }
+
+ void OnGetOperation(TEvGetOperation::TPtr& ev, const TActorContext& ctx) {
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ try {
+ auto operationId = std::get<0>(*ev->Get());
+ auto options = std::get<1>(*ev->Get());
+
+ Client->GetOperation(operationId, options).Apply(BIND([=](const TErrorOr<TOperation>& result) {
+ return NYT::NYson::ConvertToYsonString(result.ValueOrThrow()).ToString();
+ }))
+ .Apply(BIND([=](const TErrorOr<TString>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvGetOperationResponse(requestId, result));
+ }
+ }));
+ } catch (const std::exception& ex) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvGetOperationResponse(requestId, ex));
+ }
+ }
+ }
+
+ void OnListOperations(TEvListOperations::TPtr& ev, const TActorContext& ctx) {
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ try {
+ auto options = std::get<0>(*ev->Get());
+
+ Client->ListOperations(options).Apply(BIND([=](const TErrorOr<NYT::NApi::TListOperationsResult>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvListOperationsResponse(requestId, result));
+ }
+ }));
+ } catch (const std::exception& ex) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvListOperationsResponse(requestId, ex));
+ }
+ }
+ }
+
+ void OnGetJob(TEvGetJob::TPtr& ev, const TActorContext& ctx) {
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ try {
+ auto operationId = std::get<0>(*ev->Get());
+ auto jobId = std::get<1>(*ev->Get());
+ auto options = std::get<2>(*ev->Get());
+
+ Client->GetJob(operationId, jobId, options).Apply(BIND([=](const TErrorOr<NYT::NYson::TYsonString>& result) {
+ return result.ValueOrThrow().ToString();
+ }))
+ .Apply(BIND([=](const TErrorOr<TString>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvGetJobResponse(requestId, result));
+ }
+ }));
+ } catch (const std::exception& ex) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvGetJobResponse(requestId, ex));
+ }
+ }
+ }
+
+ void OnListNode(TEvListNode::TPtr& ev, const TActorContext& ctx) {
+ auto path = std::get<0>(*ev->Get());
+ auto options = std::get<1>(*ev->Get());
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ try {
+ Client->ListNode(path, options)
+ .Apply(BIND([=](const TErrorOr<NYT::NYson::TYsonString>& result) {
+ return result.ValueOrThrow().ToString();
+ }))
+ .Apply(BIND([=](const TErrorOr<TString>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvListNodeResponse(requestId, result));
+ }
+ }));
+ } catch (const std::exception& ex) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvListNodeResponse(requestId, ex));
+ }
+ }
+ }
+
+ void OnSetNode(TEvSetNode::TPtr& ev, const TActorContext& ctx) {
+ auto path = std::get<0>(*ev->Get());
+ auto value = std::get<1>(*ev->Get());
+ auto options = std::get<2>(*ev->Get());
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ Client->SetNode(path, value, options)
+ .Apply(BIND([=](const TErrorOr<void>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvSetNodeResponse(requestId, result));
+ }
+ }));
+ }
+
+ void OnGetNode(TEvGetNode::TPtr& ev, const TActorContext& ctx) {
+ auto path = std::get<0>(*ev->Get());
+ auto options = std::get<1>(*ev->Get());
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ Client->GetNode(path, options)
+ .Apply(BIND([=](const TErrorOr<NYT::NYson::TYsonString>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvGetNodeResponse(requestId, result));
+ }
+ }));
+ }
+
+ void OnRemoveNode(TEvRemoveNode::TPtr& ev, const TActorContext& ctx) {
+ auto path = std::get<0>(*ev->Get());
+ auto options = std::get<1>(*ev->Get());
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ Client->RemoveNode(path, options)
+ .Apply(BIND([=](const TErrorOr<void>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvRemoveNodeResponse(requestId, result));
+ }
+ }));
+ }
+
+ void OnCreateNode(TEvCreateNode::TPtr& ev, const TActorContext& ctx) {
+ auto path = std::get<0>(*ev->Get());
+ auto type = std::get<1>(*ev->Get());
+ auto options = std::get<2>(*ev->Get());
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ Client->CreateNode(path, type, options)
+ .Apply(BIND([=](const TErrorOr<NYT::NCypressClient::TNodeId>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvCreateNodeResponse(requestId, result));
+ }
+ }));
+ }
+
+ void OnStartTransaction(TEvStartTransaction::TPtr& ev, const TActorContext& ctx) {
+ auto type = std::get<0>(*ev->Get());
+ auto options = std::get<1>(*ev->Get());
+ auto requestId = ev->Get()->RequestId;
+ auto request = NewRequest<TRequest>(requestId, ev->Sender, ctx);
+
+ Client->StartTransaction(type, options)
+ .Apply(BIND([=](const TErrorOr<ITransactionPtr>& result) {
+ if (auto req = request.Lock()) {
+ req->Complete(new TEvStartTransactionResponse(requestId, result));
+ }
+ }));
+ }
+
+ void OnPrintJobStderr(TEvPrintJobStderr::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto operationId = std::get<0>(*ev->Get());
+
+ YQL_CLOG(DEBUG, ProviderDq) << "Printing stderr of operation " << ToString(operationId);
+
+ Client->ListJobs(operationId)
+ .Apply(BIND([operationId, client = MakeWeak(Client)](const TListJobsResult& result) {
+ if (auto cli = client.Lock()) {
+ for (const auto& job : result.Jobs) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Printing stderr (" << ToString(operationId) << "," << ToString(job.Id) << ")";
+
+ cli->GetJobStderr(operationId, job.Id)
+ .Apply(BIND([jobId = job.Id, operationId](const TSharedRef& data) {
+ YQL_CLOG(DEBUG, ProviderDq)
+ << "Stderr ("
+ << ToString(operationId) << ","
+ << ToString(jobId) << ")"
+ << TString(data.Begin(), data.Size());
+ }));
+ }
+ }
+ }));
+ }
+
+ IClientPtr Client;
+ };
+
+ IActor* CreateYtWrapper(const IClientPtr& client) {
+ return new TYtWrapper(client);
+ }
+}
diff --git a/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h b/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h
new file mode 100644
index 0000000000..3a38c683fc
--- /dev/null
+++ b/ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h
@@ -0,0 +1,88 @@
+#pragma once
+
+#include <util/random/random.h>
+#include <util/memory/blob.h>
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/yson/node/node.h>
+
+#include <yt/yt/client/api/public.h>
+#include <yt/yt/client/api/client.h>
+#include <yt/yt/client/ypath/rich.h>
+
+#include <ydb/library/yql/dq/common/dq_common.h>
+
+#include "yt_events.h"
+
+namespace NYql {
+
+ template <ui32 EventType, typename... Args>
+ struct TGenericYtCommand
+ : public std::tuple<Args...>,
+ NActors::TEventLocal<TGenericYtCommand<EventType, Args...>, EventType>
+ {
+ TGenericYtCommand(ui64 requestId, Args... args)
+ : std::tuple<Args...>(args...)
+ , RequestId(requestId)
+ { }
+
+ TGenericYtCommand(Args... args)
+ : std::tuple<Args...>(args...)
+ , RequestId(RandomNumber<ui64>())
+ { }
+
+ TGenericYtCommand() = default;
+
+ const ui64 RequestId;
+ };
+
+ template <ui32 EventType, typename... Args>
+ struct TGenericYtResponse
+ : public TGenericYtCommand<EventType, Args ...>
+ {
+ TGenericYtResponse(ui64 requestId, Args... args)
+ : TGenericYtCommand<EventType, Args ...>(requestId, args...)
+ { }
+ };
+
+ using TEvStartOperation = TGenericYtCommand<TYtEvents::ES_START_OPERATION, NYT::NScheduler::EOperationType, TString, NYT::NApi::TStartOperationOptions>;
+ using TEvStartOperationResponse = TGenericYtResponse<TYtEvents::ES_START_OPERATION_RESPONSE, NYT::TErrorOr<NYT::NScheduler::TOperationId>>;
+
+ using TEvGetOperation = TGenericYtCommand<TYtEvents::ES_GET_OPERATION, NYT::NScheduler::TOperationIdOrAlias, NYT::NApi::TGetOperationOptions>;
+ using TEvGetOperationResponse = TGenericYtResponse<TYtEvents::ES_GET_OPERATION_RESPONSE, NYT::TErrorOr<TString>>;
+
+ using TEvListOperations = TGenericYtCommand<TYtEvents::ES_LIST_OPERATIONS, NYT::NApi::TListOperationsOptions>;
+ using TEvListOperationsResponse = TGenericYtResponse<TYtEvents::ES_LIST_OPERATIONS_RESPONSE, NYT::TErrorOr<NYT::NApi::TListOperationsResult>>;
+
+ using TEvGetJob = TGenericYtCommand<TYtEvents::ES_GET_JOB, NYT::NScheduler::TOperationId, NYT::NJobTrackerClient::TJobId, NYT::NApi::TGetJobOptions>;
+ using TEvGetJobResponse = TGenericYtResponse<TYtEvents::ES_GET_JOB_RESPONSE, NYT::TErrorOr<TString>>;
+
+ using TEvWriteFile = TGenericYtCommand<TYtEvents::ES_WRITE_FILE, TFile, NYT::NYPath::TRichYPath, THashMap<TString, NYT::TNode>, NYT::NApi::TFileWriterOptions>;
+ using TEvWriteFileResponse = TGenericYtResponse<TYtEvents::ES_WRITE_FILE_RESPONSE, NYT::TErrorOr<void>>;
+
+ using TEvReadFile = TGenericYtCommand<TYtEvents::ES_READ_FILE, NYT::NYPath::TRichYPath, TString, NYT::NApi::TFileReaderOptions>;
+ using TEvReadFileResponse = TGenericYtResponse<TYtEvents::ES_READ_FILE_RESPONSE, NYT::TErrorOr<void>>;
+
+ using TEvListNode = TGenericYtCommand<TYtEvents::ES_LIST_NODE, TString, NYT::NApi::TListNodeOptions>;
+ using TEvListNodeResponse = TGenericYtResponse<TYtEvents::ES_LIST_NODE_RESPONSE, NYT::TErrorOr<TString>>;
+
+ using TEvSetNode = TGenericYtCommand<TYtEvents::ES_SET_NODE, TString, NYT::NYson::TYsonString, NYT::NApi::TSetNodeOptions>;
+ using TEvSetNodeResponse = TGenericYtResponse<TYtEvents::ES_SET_NODE_RESPONSE, NYT::TErrorOr<void>>;
+
+ using TEvGetNode = TGenericYtCommand<TYtEvents::ES_GET_NODE, TString, NYT::NApi::TGetNodeOptions>;
+ using TEvGetNodeResponse = TGenericYtResponse<TYtEvents::ES_GET_NODE_RESPONSE, NYT::TErrorOr<NYT::NYson::TYsonString>>;
+
+ using TEvRemoveNode = TGenericYtCommand<TYtEvents::ES_REMOVE_NODE, TString, NYT::NApi::TRemoveNodeOptions>;
+ using TEvRemoveNodeResponse = TGenericYtResponse<TYtEvents::ES_REMOVE_NODE_RESPONSE, NYT::TErrorOr<void>>;
+
+ using TEvCreateNode = TGenericYtCommand<TYtEvents::ES_CREATE_NODE, TString, NYT::NObjectClient::EObjectType, NYT::NApi::TCreateNodeOptions>;
+ using TEvCreateNodeResponse = TGenericYtResponse<TYtEvents::ES_CREATE_NODE_RESPONSE, NYT::TErrorOr<NYT::NCypressClient::TNodeId>>;
+
+ using TEvStartTransaction = TGenericYtCommand<TYtEvents::ES_START_TRANSACTION, NYT::NTransactionClient::ETransactionType, NYT::NApi::TTransactionStartOptions>;
+ using TEvStartTransactionResponse = TGenericYtResponse<TYtEvents::ES_START_TRANSACTION_RESPONSE, NYT::TErrorOr<NYT::NApi::ITransactionPtr>>;
+
+ using TEvPrintJobStderr = TGenericYtCommand<TYtEvents::ES_PRINT_JOB_STDERR, NYT::NScheduler::TOperationId>;
+
+ NActors::IActor* CreateYtWrapper(const NYT::NApi::IClientPtr& client);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..3e5f36a8e3
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,40 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(providers-dq-global_worker_manager)
+target_compile_options(providers-dq-global_worker_manager PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(providers-dq-global_worker_manager PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yql-utils-failure_injector
+ providers-common-config
+ providers-common-gateway
+ providers-common-metrics
+ providers-dq-actors
+ dq-api-grpc
+ dq-api-protos
+ providers-dq-config
+ providers-dq-counters
+ providers-dq-runtime
+ providers-dq-task_runner
+ dq-actors-yt
+ providers-dq-scheduler
+)
+target_sources(providers-dq-global_worker_manager PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp
+)
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..f556019e59
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,41 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(providers-dq-global_worker_manager)
+target_compile_options(providers-dq-global_worker_manager PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(providers-dq-global_worker_manager PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yql-utils-failure_injector
+ providers-common-config
+ providers-common-gateway
+ providers-common-metrics
+ providers-dq-actors
+ dq-api-grpc
+ dq-api-protos
+ providers-dq-config
+ providers-dq-counters
+ providers-dq-runtime
+ providers-dq-task_runner
+ dq-actors-yt
+ providers-dq-scheduler
+)
+target_sources(providers-dq-global_worker_manager PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp
+)
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..f556019e59
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,41 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(providers-dq-global_worker_manager)
+target_compile_options(providers-dq-global_worker_manager PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(providers-dq-global_worker_manager PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yql-utils-failure_injector
+ providers-common-config
+ providers-common-gateway
+ providers-common-metrics
+ providers-dq-actors
+ dq-api-grpc
+ dq-api-protos
+ providers-dq-config
+ providers-dq-counters
+ providers-dq-runtime
+ providers-dq-task_runner
+ dq-actors-yt
+ providers-dq-scheduler
+)
+target_sources(providers-dq-global_worker_manager PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp
+)
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..f65dc67f83
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,39 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(providers-dq-global_worker_manager)
+target_compile_options(providers-dq-global_worker_manager PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(providers-dq-global_worker_manager PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yql-utils-failure_injector
+ providers-common-config
+ providers-common-gateway
+ providers-common-metrics
+ providers-dq-actors
+ dq-api-grpc
+ dq-api-protos
+ providers-dq-config
+ providers-dq-counters
+ providers-dq-runtime
+ providers-dq-task_runner
+ dq-actors-yt
+ providers-dq-scheduler
+)
+target_sources(providers-dq-global_worker_manager PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper_win.cpp
+)
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp b/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp
new file mode 100644
index 0000000000..12a1ac6f15
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/benchmark.cpp
@@ -0,0 +1,95 @@
+#include "benchmark.h"
+
+#include <ydb/library/yql/providers/dq/common/yql_dq_settings.h>
+#include <ydb/library/yql/providers/dq/actors/resource_allocator.h>
+
+namespace NYql::NDqs {
+
+using namespace NActors;
+
+class TWorkerManagerBenchmark: public TRichActor<TWorkerManagerBenchmark> {
+public:
+ TWorkerManagerBenchmark(NActors::TActorId workerManagerId, const TWorkerManagerBenchmarkOptions& options)
+ : TRichActor(&TWorkerManagerBenchmark::Handler)
+ , WorkerManagerId(workerManagerId)
+ , Options(options)
+ , Settings(new TDqConfiguration)
+ , Counters(new NMonitoring::TDynamicCounters)
+ {
+ Y_UNUSED(WorkerManagerId);
+ Y_UNUSED(Options);
+ Y_UNUSED(RunningRequests);
+ }
+
+private:
+ STRICT_STFUNC(Handler, {
+ cFunc(TEvents::TEvWakeup::EventType, Wakeup)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ cFunc(TEvents::TEvBootstrap::EventType, Bootstrap)
+ HFunc(TEvAllocateWorkersResponse, OnAllocateWorkersResponse)
+ });
+
+ void Bootstrap() {
+ Schedule(Options.MaxRunTimeMs, new TEvents::TEvPoison);
+ Wakeup();
+ }
+
+ void Wakeup() {
+ while ((int)RunningRequests.size() < Options.Inflight && Requests < Options.TotalRequests) {
+ StartRequest();
+ }
+
+ if (RunningRequests.empty()) {
+ PassAway();
+ }
+ }
+
+ void StartRequest() {
+ TString operationId = TStringBuilder() << "Benchmark-" << Requests;
+ auto resourceAllocator = RegisterChild(
+ CreateResourceAllocator(
+ WorkerManagerId, SelfId(), SelfId(),
+ Options.WorkerCount,
+ operationId, Settings,
+ Counters,
+ {}));
+
+ auto allocateRequest = MakeHolder<TEvAllocateWorkersRequest>(Options.WorkerCount, "TestUser");
+ allocateRequest->Record.SetTraceId(operationId);
+
+ TActivationContext::Send(
+ new IEventHandle(
+ WorkerManagerId, resourceAllocator, allocateRequest.Release()));
+
+ RunningRequests.insert(std::make_pair(resourceAllocator, TRequestInfo{TInstant::Now()}));
+ Requests ++;
+ }
+
+ void OnAllocateWorkersResponse(TEvAllocateWorkersResponse::TPtr& ev, const NActors::TActorContext&) {
+ UnregisterChild(ev->Sender);
+ RunningRequests.erase(ev->Sender);
+
+ Wakeup();
+ }
+
+
+ const NActors::TActorId WorkerManagerId;
+ const TWorkerManagerBenchmarkOptions Options;
+ TDqConfiguration::TPtr Settings;
+ TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
+
+ struct TRequestInfo {
+ TInstant StartTime;
+ };
+
+ THashMap<TActorId, TRequestInfo> RunningRequests;
+ int Requests = 0;
+};
+
+NActors::IActor* CreateWorkerManagerBenchmark(NActors::TActorId workerManagerId, const TWorkerManagerBenchmarkOptions& options)
+{
+ return new TWorkerManagerBenchmark(workerManagerId, options);
+}
+
+
+} // namespace NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/benchmark.h b/ydb/library/yql/providers/dq/global_worker_manager/benchmark.h
new file mode 100644
index 0000000000..04b33ede15
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/benchmark.h
@@ -0,0 +1,16 @@
+#pragma once
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/worker_manager/interface/events.h>
+
+namespace NYql::NDqs {
+
+struct TWorkerManagerBenchmarkOptions {
+ int WorkerCount = 10;
+ int Inflight = 10;
+ int TotalRequests = 100000;
+ TDuration MaxRunTimeMs = TDuration::Minutes(30);
+};
+
+NActors::IActor* CreateWorkerManagerBenchmark(NActors::TActorId workerManagerId, const TWorkerManagerBenchmarkOptions& options);
+
+} // namespace NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp
new file mode 100644
index 0000000000..5e15d0d470
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.cpp
@@ -0,0 +1,354 @@
+#include "coordination_helper.h"
+#include "global_worker_manager.h"
+#include "service_node_pinger.h"
+
+#include <ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h>
+#include <ydb/library/yql/providers/dq/actors/yt/nodeid_cleaner.h>
+#include <ydb/library/yql/providers/dq/actors/yt/worker_registrator.h>
+#include <ydb/library/yql/providers/dq/actors/yt/lock.h>
+#include <ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h>
+#include <ydb/library/yql/providers/dq/actors/dummy_lock.h>
+#include <ydb/library/yql/providers/dq/service/interconnect_helpers.h>
+#include <ydb/library/yql/providers/dq/task_runner/file_cache.h>
+#include <ydb/library/yql/providers/dq/runtime/runtime_data.h>
+
+#include <yt/yt/client/api/rpc_proxy/config.h>
+#include <yt/yt/client/api/rpc_proxy/connection.h>
+
+#include <library/cpp/svnversion/svnversion.h>
+#include <library/cpp/yson/node/node_io.h>
+
+#include <yt/cpp/mapreduce/interface/fluent.h>
+
+#include <util/system/getpid.h>
+#include <util/system/env.h>
+
+namespace NYql {
+
+using namespace NActors;
+
+class TCoordinationHelper: public ICoordinationHelper
+{
+public:
+ TCoordinationHelper(
+ const NProto::TDqConfig::TYtCoordinator& config,
+ const NProto::TDqConfig::TScheduler& schedulerConfig,
+ const TString& role,
+ ui16 interconnectPort)
+ : Config(config), SchedulerConfig(schedulerConfig)
+ , Role(role)
+ , InterconnectPort(interconnectPort)
+ {
+ std::tie(Host, Ip) = NYql::NDqs::GetLocalAddress(
+ config.HasHostName() ? &config.GetHostName() : nullptr
+ );
+ }
+
+ ui32 GetNodeId() override
+ {
+ Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1));
+ return NodeId;
+ }
+
+ ui32 GetNodeId(
+ const TMaybe<ui32> nodeId,
+ const TMaybe<TString>& grpcPort,
+ ui32 minNodeId,
+ ui32 maxNodeId,
+ const THashMap<TString, TString>& attributes) override
+ {
+ if (grpcPort) {
+ GrpcPort = *grpcPort;
+ }
+
+ if (nodeId) {
+ NodeId = *nodeId;
+ }
+
+ if (NodeId != static_cast<ui32>(-1)) {
+ return NodeId;
+ }
+
+ NYql::TAssignNodeIdOptions options;
+
+ options.ClusterName = Config.GetClusterName();
+ options.User = Config.GetUser();
+ options.Token = Config.GetToken();
+ options.Prefix = Config.GetPrefix();
+ options.Role = Role;
+ options.Attributes[NCommonAttrs::ROLE_ATTR] = Role;
+ if (grpcPort) {
+ options.Attributes[NCommonAttrs::GRPCPORT_ATTR] = *grpcPort;
+ }
+ options.Attributes[NCommonAttrs::INTERCONNECTPORT_ATTR] = ToString(InterconnectPort);
+ options.Attributes[NCommonAttrs::HOSTNAME_ATTR] = Host;
+ options.Attributes[NCommonAttrs::REVISION_ATTR] = GetRevision();
+
+ options.NodeName = Host + ":" + ToString(GetPID()) + ":" + ToString(InterconnectPort);
+ options.NodeId = nodeId;
+ options.MinNodeId = minNodeId;
+ options.MaxNodeId = maxNodeId;
+
+ NodeName = options.NodeName;
+
+ for (const auto& [k, v]: attributes) {
+ options.Attributes[k] = v;
+ }
+
+ NodeId = AssignNodeId(options);
+ return NodeId;
+ }
+
+ NActors::IActor* CreateLockOnCluster(NActors::TActorId ytWrapper, const TString& prefix, const TString& lockName, bool temporary) override {
+ Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1));
+
+ auto attributes = NYT::BuildYsonNodeFluently()
+ .BeginMap()
+ .Item(NCommonAttrs::ACTOR_NODEID_ATTR).Value(NodeId)
+ .Item(NCommonAttrs::HOSTNAME_ATTR).Value(GetHostname())
+ .Item(NCommonAttrs::GRPCPORT_ATTR).Value(GrpcPort) // optional
+ .Item(NCommonAttrs::UPLOAD_EXECUTABLE_ATTR).Value("true") // compat
+ .EndMap();
+
+ return CreateYtLock(
+ ytWrapper,
+ prefix + "/locks",
+ lockName,
+ NYT::NodeToYsonString(attributes),
+ temporary);
+ }
+
+ NActors::IActor* CreateLock(const TString& lockName, bool temporary) override {
+ return CreateLockOnCluster(GetWrapper(), Config.GetPrefix(), lockName, temporary);
+ }
+
+ void StartRegistrator(NActors::TActorSystem* actorSystem) override {
+ TWorkerRegistratorOptions wro;
+ wro.NodeName = NodeName;
+ wro.Prefix = Config.GetPrefix() + "/" + Role;
+ Register(actorSystem, CreateWorkerRegistrator(GetWrapper(actorSystem), wro));
+ }
+
+ void StartCleaner(NActors::TActorSystem* actorSystem, const TMaybe<TString>& role) override {
+ TNodeIdCleanerOptions co;
+ co.Prefix = Config.GetPrefix() + "/" + role.GetOrElse(Role);
+ Register(actorSystem, CreateNodeIdCleaner(GetWrapper(actorSystem), co));
+ }
+
+ TString GetHostname() override {
+ return Host;
+ }
+
+ TString GetIp() override {
+ return Ip;
+ }
+
+ IServiceNodeResolver::TPtr CreateServiceNodeResolver(
+ NActors::TActorSystem* actorSystem,
+ const TVector<TString>& hostPortPairs) override
+ {
+ if (!hostPortPairs.empty()) {
+ return CreateStaticResolver(hostPortPairs);
+ } else {
+ TDynamicResolverOptions options;
+ options.YtWrapper = GetWrapper(actorSystem);
+ options.Prefix = Config.GetPrefix() + "/service_node";
+ return CreateDynamicResolver(actorSystem, options);
+ }
+ }
+
+ void StartGlobalWorker(NActors::TActorSystem* actorSystem, const TVector<TResourceManagerOptions>& resourceUploaderOptions, IMetricsRegistryPtr metricsRegistry) override
+ {
+ if (Config.GetLockType() != "dummy") {
+ GetWrapper();
+ }
+ Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1));
+ auto actorId = Register(actorSystem, CreateGlobalWorkerManager(this, resourceUploaderOptions, std::move(metricsRegistry), SchedulerConfig));
+ actorSystem->RegisterLocalService(NDqs::MakeWorkerManagerActorID(NodeId), actorId);
+ }
+
+ const NProto::TDqConfig::TYtCoordinator& GetConfig() override {
+ return Config;
+ }
+
+ const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem, const TString& clusterName, const TString& user, const TString& token) override {
+ auto key = std::make_tuple(clusterName, user, token);
+ auto guard = Guard(Mutex);
+ auto it = Yt.find(key);
+ if (it != Yt.end()) {
+ return it->second;
+ } else {
+ auto client = GetYtClient(clusterName, user, token);
+ auto wrapper = CreateYtWrapper(client);
+ auto actorId = Register(actorSystem, wrapper);
+ Yt.emplace(key, actorId);
+ return actorId;
+ }
+ }
+
+ const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem) override {
+ return GetWrapper(actorSystem, Config.GetClusterName(), Config.GetUser(), Config.GetToken());
+ }
+
+ const NActors::TActorId GetWrapper() override {
+ auto key = std::make_tuple(Config.GetClusterName(), Config.GetUser(), Config.GetToken());
+ auto guard = Guard(Mutex);
+ auto it = Yt.find(key);
+ Y_ABORT_UNLESS(it != Yt.end());
+ return it->second;
+ }
+
+ NActors::IActor* CreateServiceNodePinger(const IServiceNodeResolver::TPtr& ptr, const TResourceManagerOptions& rmOptions, const THashMap<TString, TString>& attributes) override {
+ Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1));
+ return ::NYql::CreateServiceNodePinger(NodeId, Ip, InterconnectPort, Role, attributes, ptr, this, rmOptions);
+ }
+
+ TWorkerRuntimeData* GetRuntimeData() override {
+ return &RuntimeData;
+ }
+
+ void Stop(NActors::TActorSystem* actorSystem) override {
+ for (auto id : Children) {
+ actorSystem->Send(id, new TEvents::TEvPoison());
+ }
+ Children.clear();
+ }
+
+ TString GetRevision() override {
+ if (Config.HasRevision()) {
+ return Config.GetRevision();
+ } else {
+ return GetProgramCommitId();
+ }
+ }
+
+protected:
+ TActorId Register(TActorSystem* actorSystem, IActor* actor) {
+ auto id = actorSystem->Register(actor);
+ Children.push_back(id);
+ return id;
+ }
+
+ NYT::NApi::IClientPtr GetYtClient(const TString& clusterName, const TString& user, const TString& token)
+ {
+ NYT::NApi::NRpcProxy::TConnectionConfigPtr config = NYT::New<NYT::NApi::NRpcProxy::TConnectionConfig>();
+ config->RequestCodec = NYT::NCompression::ECodec::Lz4;
+ config->ClusterUrl = clusterName;
+ config->ConnectionType = NYT::NApi::EConnectionType::Rpc;
+
+ auto connection = NYT::NApi::NRpcProxy::CreateConnection(config);
+
+ NYT::NApi::TClientOptions options;
+ options.User = user;
+ options.Token = token;
+
+ auto client = connection->CreateClient(options);
+ return client;
+ }
+
+ NYT::NApi::IClientPtr GetYtClient()
+ {
+ return GetYtClient(Config.GetClusterName(), Config.GetUser(), Config.GetToken());
+ }
+
+ const NProto::TDqConfig::TYtCoordinator Config;
+ const NProto::TDqConfig::TScheduler SchedulerConfig;
+
+ TString Role;
+
+ TString Host;
+ TString Ip;
+
+ TString NodeName;
+
+ TMutex Mutex;
+ THashMap<std::tuple<TString, TString, TString>, TActorId> Yt;
+
+ ui32 NodeId = -1;
+ TString GrpcPort = "";
+ ui16 InterconnectPort;
+
+ TWorkerRuntimeData RuntimeData;
+ IFileCache::TPtr FileCache;
+
+ TVector<TActorId> Children;
+};
+
+class TCoordinationHelperWithDummyLock: public TCoordinationHelper
+{
+public:
+ TCoordinationHelperWithDummyLock(
+ const NProto::TDqConfig::TYtCoordinator& config,
+ const NProto::TDqConfig::TScheduler& schedulerConfig,
+ const TString& role,
+ ui16 interconnectPort)
+ : TCoordinationHelper(config, schedulerConfig, role, interconnectPort)
+ { }
+
+ NActors::IActor* CreateLockOnCluster(NActors::TActorId ytWrapper, const TString& prefix, const TString& lockName, bool temporary) override {
+ Y_UNUSED(ytWrapper);
+ Y_UNUSED(prefix);
+ Y_UNUSED(temporary);
+
+ Y_ABORT_UNLESS(NodeId != static_cast<ui32>(-1));
+
+ auto attributes = NYT::BuildYsonNodeFluently()
+ .BeginMap()
+ .Item(NCommonAttrs::ACTOR_NODEID_ATTR).Value(NodeId)
+ .Item(NCommonAttrs::HOSTNAME_ATTR).Value(GetHostname())
+ .Item(NCommonAttrs::GRPCPORT_ATTR).Value(GrpcPort) // optional
+ .Item(NCommonAttrs::UPLOAD_EXECUTABLE_ATTR).Value("true") // compat
+ .EndMap();
+
+ return CreateDummyLock(
+ lockName,
+ NYT::NodeToYsonString(attributes));
+ }
+
+ NActors::IActor* CreateLock(const TString& lockName, bool temporary) override {
+ return CreateLockOnCluster(NActors::TActorId(), Config.GetPrefix(), lockName, temporary);
+ }
+};
+
+ICoordinationHelper::TPtr CreateCoordiantionHelper(const NProto::TDqConfig::TYtCoordinator& cfg, const NProto::TDqConfig::TScheduler& schedulerConfig, const TString& role, ui16 interconnectPort)
+{
+ NProto::TDqConfig::TYtCoordinator config = cfg;
+ auto clusterName = config.GetClusterName();
+
+ TString userName;
+ TString token;
+
+ if (config.HasToken()) {
+ // internal job
+ userName = config.GetUser();
+ token = config.GetToken();
+ } else if (!clusterName.empty()) {
+ std::tie(userName, token) = NDqs::GetUserToken(
+ config.HasUser() ? TMaybe<TString>(config.GetUser()) : TMaybe<TString>(),
+ config.HasTokenFile() ? TMaybe<TString>(config.GetTokenFile()) : TMaybe<TString>()
+ );
+
+ Y_ABORT_UNLESS(!token.empty());
+ }
+
+ auto prefix = config.GetPrefix();
+
+ if (config.GetLockType() != "dummy") {
+ Y_ABORT_UNLESS(!clusterName.empty());
+ Y_ABORT_UNLESS(!userName.empty());
+ Y_ABORT_UNLESS(!prefix.empty());
+ }
+
+ Y_ABORT_UNLESS(!role.empty());
+
+ config.SetUser(userName);
+ config.SetToken(token);
+
+ if (config.GetLockType() == "dummy") {
+ return new TCoordinationHelperWithDummyLock(config, schedulerConfig, role, interconnectPort);
+ } else {
+ return new TCoordinationHelper(config, schedulerConfig, role, interconnectPort);
+ }
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h
new file mode 100644
index 0000000000..6de1565093
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper.h
@@ -0,0 +1,65 @@
+#pragma once
+
+#include <library/cpp/actors/core/actorsystem.h>
+
+#include <ydb/library/yql/providers/common/metrics/metrics_registry.h>
+#include <ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h>
+#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h>
+#include <ydb/library/yql/providers/dq/config/config.pb.h>
+
+#include <util/generic/ptr.h>
+
+namespace NYql {
+
+struct TWorkerRuntimeData;
+
+class ICoordinationHelper: public TThrRefBase {
+public:
+ using TPtr = TIntrusivePtr<ICoordinationHelper>;
+
+ virtual ~ICoordinationHelper() = default;
+
+ virtual ui32 GetNodeId() = 0;
+ virtual ui32 GetNodeId(
+ const TMaybe<ui32> nodeId,
+ const TMaybe<TString>& grpcPort,
+ ui32 minNodeId,
+ ui32 maxNodeId,
+ const THashMap<TString, TString>& attributes) = 0;
+
+ virtual TString GetHostname() = 0;
+ virtual TString GetIp() = 0;
+
+ virtual NActors::IActor* CreateLockOnCluster(NActors::TActorId ytWrapper, const TString& prefix, const TString& lockName, bool temporary = true) = 0;
+
+ virtual NActors::IActor* CreateLock(const TString& lockName, bool temporary = true) = 0;
+
+ virtual NActors::IActor* CreateServiceNodePinger(const IServiceNodeResolver::TPtr& ptr, const TResourceManagerOptions& rmOptions, const THashMap<TString, TString>& attributes = {}) = 0;
+
+ virtual void StartRegistrator(NActors::TActorSystem* actorSystem) = 0;
+
+ virtual void StartGlobalWorker(NActors::TActorSystem* actorSystem, const TVector<TResourceManagerOptions>& resourceUploaderOptions, IMetricsRegistryPtr metricsRegistry) = 0;
+
+ virtual void StartCleaner(NActors::TActorSystem* actorSystem, const TMaybe<TString>& role) = 0;
+
+ virtual IServiceNodeResolver::TPtr CreateServiceNodeResolver(
+ NActors::TActorSystem* actorSystem, const TVector<TString>& hostPortPairs) = 0;
+
+ virtual const NProto::TDqConfig::TYtCoordinator& GetConfig() = 0;
+
+ virtual const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem) = 0;
+
+ virtual const NActors::TActorId GetWrapper() = 0;
+
+ virtual const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem, const TString& clusterName, const TString& user, const TString& token) = 0;
+
+ virtual TWorkerRuntimeData* GetRuntimeData() = 0;
+
+ virtual void Stop(NActors::TActorSystem* actorSystem) = 0;
+
+ virtual TString GetRevision() = 0;
+};
+
+ICoordinationHelper::TPtr CreateCoordiantionHelper(const NProto::TDqConfig::TYtCoordinator& config, const NProto::TDqConfig::TScheduler& schedulerConfig, const TString& role, ui16 interconnectPort);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper_win.cpp b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper_win.cpp
new file mode 100644
index 0000000000..66ac268460
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/coordination_helper_win.cpp
@@ -0,0 +1,13 @@
+#include "coordination_helper.h"
+
+namespace NYql {
+
+ICoordinationHelper::TPtr CreateCoordiantionHelper(const NProto::TDqConfig::TYtCoordinator& config, const TString& role)
+{
+ Y_UNUSED(config);
+ Y_UNUSED(role);
+
+ return nullptr;
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp
new file mode 100644
index 0000000000..a30946c173
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp
@@ -0,0 +1,1343 @@
+#include "global_worker_manager.h"
+#include "workers_storage.h"
+
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+#include <ydb/library/yql/providers/dq/common/attrs.h>
+#include <ydb/library/yql/providers/dq/scheduler/dq_scheduler.h>
+
+#include <ydb/library/yql/providers/dq/worker_manager/worker_manager_common.h>
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/execution_helpers.h>
+#include <ydb/library/yql/providers/dq/api/grpc/api.grpc.pb.h>
+#include <ydb/library/yql/providers/dq/counters/counters.h>
+
+#include <ydb/library/yql/utils/failure_injector/failure_injector.h>
+
+#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <library/cpp/grpc/client/grpc_client_low.h>
+#include <library/cpp/protobuf/util/pb_io.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/interconnect/interconnect.h>
+#include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/svnversion/svnversion.h>
+
+#include <util/generic/vector.h>
+#include <util/generic/guid.h>
+#include <util/system/fs.h>
+#include <util/system/hostname.h>
+#include <util/system/getpid.h>
+#include <util/generic/scope.h>
+#include <util/random/shuffle.h>
+
+namespace NYql {
+
+using namespace NActors;
+using namespace NDqs;
+using namespace NThreading;
+using namespace NMonitoring;
+using EFileType = Yql::DqsProto::TFile::EFileType;
+using TFileResource = Yql::DqsProto::TFile;
+
+union TDqResourceId {
+ struct {
+ ui32 Counter;
+ ui32 Epoch;
+ };
+ ui64 Data;
+};
+
+static_assert(sizeof(TDqResourceId) == 8);
+
+class TWorkerStopFilter {
+public:
+ TWorkerStopFilter(const Yql::DqsProto::JobStopRequest& request)
+ : Revision(request.GetRevision())
+ , ClusterName(request.GetClusterName())
+ , WorkerId(GetGuid(request.GetWorkerId()))
+ , NegativeRevision(request.GetNegativeRevision())
+ , LastUpdate(TInstant::Now())
+ {
+ for (const auto& attr : request.GetAttribute()) {
+ Attributes.emplace(attr.GetKey(), attr.GetValue());
+ }
+ }
+
+ bool Match(const TWorkerInfo& workerInfo) const {
+ return MatchInternal(workerInfo);
+ }
+
+ TInstant GetLastUpdate() const {
+ return LastUpdate;
+ }
+
+private:
+ bool MatchInternal(const TWorkerInfo& workerInfo) const {
+ if (!Revision.empty() && NegativeRevision == (Revision == workerInfo.Revision)) {
+ return false;
+ }
+ if (!ClusterName.empty() && ClusterName != workerInfo.ClusterName) {
+ return false;
+ }
+ if (!WorkerId.IsEmpty() && WorkerId != workerInfo.WorkerId) {
+ return false;
+ }
+
+ for (const auto& [k, v] : Attributes) {
+ auto i = workerInfo.Attributes.find(k);
+ if (i == workerInfo.Attributes.end()) {
+ return false;
+ }
+ if (i->second != v) {
+ return false;
+ }
+ }
+
+ LastUpdate = TInstant::Now();
+
+ return true;
+ }
+
+ const TString Revision;
+ const TString ClusterName;
+ const TGUID WorkerId;
+ const bool NegativeRevision;
+ THashMap<TString, TString> Attributes;
+
+ mutable TInstant LastUpdate;
+};
+
+// used for fast state recovery on restarted jobs
+class TLocalCriticalFiles: public IFileCache {
+public:
+// used in IsReady
+// exe unsupported
+ void AddFile(const TString& path, const TString& objectId) override {
+ Y_UNUSED(path);
+ TGuard<TMutex> guard(Mutex);
+ ObjectIds.insert(objectId);
+ }
+
+ void Clear() override {
+ TGuard<TMutex> guard(Mutex);
+ ObjectIds.clear();
+ }
+
+ TMaybe<TString> FindFile(const TString& objectId) override {
+ Y_UNUSED(objectId);
+ return { };
+ }
+
+ bool Contains(const TString& objectId) override {
+ Y_UNUSED(objectId);
+ return false;
+ }
+
+ void Walk(const std::function<void(const TString& objectId)>& f) override {
+ TGuard<TMutex> guard(Mutex);
+ for (const auto& id : ObjectIds) {
+ f(id);
+ }
+ }
+
+ ui64 FreeDiskSize() override {
+ return 0;
+ }
+
+ ui64 UsedDiskSize() override {
+ return 0;
+ }
+
+ TString GetDir() override {
+ return ""; // unused
+ }
+
+private:
+ TMutex Mutex;
+ THashSet<TString> ObjectIds;
+};
+
+
+// used in gwm master
+class TAllCriticalFiles {
+public:
+ template<typename VectorLike>
+ void Add(
+ const TGUID& serviceNodeId,
+ ui32 nodeId,
+ const TString& revision,
+ const TString& address,
+ ui32 pid,
+ const VectorLike& files)
+ {
+ auto now = TInstant::Now();
+ auto& t = PerServiceNode[serviceNodeId];
+ t.LastUpdate = now;
+ t.NodeId = nodeId;
+ t.Revision = revision;
+ t.Pid = pid;
+ t.Address = address;
+ bool needUpdate = false;
+ for (const auto& f : files) {
+ needUpdate |= t.Files.insert(f.GetObjectId()).second;
+ }
+
+ if (static_cast<int>(t.Files.size()) > static_cast<int>(files.size())) {
+ needUpdate = true;
+ THashSet<TString> hash;
+ for (const auto& f : files) {
+ hash.insert(f.GetObjectId());
+ }
+ THashSet<TString> toDrop;
+ for (const auto& f : t.Files) {
+ if (!hash.contains(f)) {
+ toDrop.insert(f);
+ }
+ }
+ for (const auto& f : toDrop) {
+ t.Files.erase(f);
+ }
+ }
+
+ if (needUpdate) {
+ t.OrderedFiles.clear();
+ for (const auto& f : files) {
+ t.OrderedFiles.push_back(f.GetObjectId());
+ }
+ }
+
+ needUpdate |= ClearOld(now);
+
+ if (needUpdate) {
+ Rebuild();
+ }
+ }
+
+ const TVector<TWorkerInfo::TFileResource>& GetResources() const {
+ return Resources;
+ }
+
+ struct TServiceNodeFiles {
+ ui32 NodeId;
+ THashSet<TString> Files;
+ TVector<TString> OrderedFiles;
+ TInstant LastUpdate = TInstant::Now();
+ TString Revision;
+ ui32 Pid;
+ TString Address;
+ };
+
+ const THashMap<TGUID, TServiceNodeFiles>& GetNodes() const {
+ return PerServiceNode;
+ }
+
+private:
+ bool ClearOld(TInstant now) {
+ THashSet<TGUID> toDrop;
+ for (const auto& [k, v] : PerServiceNode) {
+ if (k.IsEmpty()) { // local node
+ continue;
+ }
+
+ if (now - v.LastUpdate > TDuration::Seconds(10)) {
+ toDrop.insert(k);
+ }
+ }
+
+ for (const auto& k : toDrop) {
+ PerServiceNode.erase(k);
+ }
+
+ return !toDrop.empty();
+ }
+
+ void Rebuild() {
+ Resources.clear();
+ THashSet<TString> uniq;
+ THashSet<TString> uniqExe;
+ for (const auto& [_, v] : PerServiceNode) {
+ for (int i = 0; i < static_cast<int>(v.OrderedFiles.size()) - 1; ++i) {
+ const auto& f = v.OrderedFiles[i];
+ uniq.insert(f);
+ }
+ if (!v.Files.empty()) {
+ uniqExe.insert(v.OrderedFiles.back());
+ }
+ }
+ for (const auto& f : uniq) {
+ TWorkerInfo::TFileResource resource;
+ resource.SetObjectType(TWorkerInfo::TFileResource::EUDF_FILE);
+ resource.SetObjectId(f);
+ Resources.push_back(resource);
+ }
+ //exe upload controlled via DqControl
+ //for (const auto& f : uniqExe) {
+ // TWorkerInfo::TFileResource resource;
+ // resource.SetName("dq_vanilla_job.lite");
+ // resource.SetObjectType(TWorkerInfo::TFileResource::EEXE_FILE);
+ // resource.SetObjectId(f);
+ // Resources.push_back(resource);
+ //}
+ }
+
+ THashMap<TGUID, TServiceNodeFiles> PerServiceNode; // serviceNodeId -> List
+ TVector<TWorkerInfo::TFileResource> Resources;
+};
+
+class TGlobalWorkerManager: public TWorkerManagerCommon<TGlobalWorkerManager> {
+public:
+
+ static constexpr char ActorName[] = "GWM";
+
+ explicit TGlobalWorkerManager(
+ const ICoordinationHelper::TPtr& coordinator,
+ const TVector<TResourceManagerOptions>& resourceUploaderOptions,
+ IMetricsRegistryPtr metricsRegistry,
+ const NProto::TDqConfig::TScheduler& schedulerConfig)
+ : TWorkerManagerCommon<TGlobalWorkerManager>(&TGlobalWorkerManager::Initialization)
+ , Coordinator(coordinator)
+ , LeaderResolver(std::make_shared<TSingleNodeResolver>())
+ , Metrics(metricsRegistry->GetSensors()->GetSubgroup("counters", "gwm"))
+ , UploaderMetrics(metricsRegistry->GetSensors()->GetSubgroup("counters", "uploader"))
+ , LatencyHistogram(Metrics->GetHistogram("LeaderLatency", ExponentialHistogram(10, 2, 1)))
+ , Workers(Coordinator->GetNodeId(), Metrics, metricsRegistry->GetSensors()->GetSubgroup("counters", "workers"))
+ , Scheduler(NDq::IScheduler::Make(schedulerConfig, metricsRegistry))
+ , Revision(ToString(GetProgramCommitId()))
+ , ResourceUploaderOptions(resourceUploaderOptions)
+ , WaitListSize(nullptr)
+ { }
+
+private:
+
+#define HHFunc(TEvType, HandleFunc) \
+ case TEvType::EventType: { \
+ Y_SCOPE_EXIT(&) { UpdateMetrics(); }; \
+ typename TEvType::TPtr* x = reinterpret_cast<typename TEvType::TPtr*>(&ev); \
+ TString name(#TEvType); \
+ name = name.substr(name.find_last_of(':')+1); \
+ *Metrics->GetSubgroup("component", "requests")->GetCounter(name, true) += 1;\
+ TInstant t = TInstant::Now(); \
+ HandleFunc(*x, TActivationContext::ActorContextFor(this->SelfId())); \
+ if (CurrentStateFunc() == &TGlobalWorkerManager::Leader) { \
+ LatencyHistogram->Collect((TInstant::Now() - t).MilliSeconds()); \
+ } \
+ break; \
+ }
+
+ STRICT_STFUNC(Initialization, {
+ HHFunc(TEvBecomeFollower, StartFollower)
+ HHFunc(TEvBecomeLeader, StartLeader)
+ CFunc(TEvents::TEvBootstrap::EventType, Bootstrap)
+
+ HHFunc(TEvAllocateWorkersRequest, OnAllocateWorkersRequestStub)
+ HHFunc(TEvFreeWorkersNotify, OnFreeWorkersStub)
+ HHFunc(TEvRegisterNode, OnRegisterNodeStub)
+ HHFunc(TEvClusterStatus, OnClusterStatusStub)
+ HHFunc(TEvQueryStatus, OnQueryStatusStub)
+ HHFunc(TEvIsReady, OnIsReadyStub)
+ HHFunc(TEvGetMasterRequest, OnGetMasterStub)
+ HHFunc(TEvConfigureFailureInjectorRequest, OnConfigureFailureInjectorStub)
+ HHFunc(TEvOperationStop, OnOperationStopStub)
+
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ IgnoreFunc(TEvJobStop)
+ HHFunc(TEvRoutesRequest, this->OnRoutesRequest)
+ })
+
+ STRICT_STFUNC(Leader, {
+ HHFunc(TEvAllocateWorkersRequest, OnAllocateWorkersRequest)
+ HHFunc(TEvFreeWorkersNotify, OnFreeWorkers)
+ HHFunc(TEvRegisterNode, OnRegisterNode)
+
+ HHFunc(TEvBecomeLeader, StartLeader)
+ HHFunc(TEvBecomeFollower, StartFollower)
+
+ IgnoreFunc(TEvInterconnect::TEvNodeConnected)
+
+ HHFunc(TEvents::TEvUndelivered, OnUndelivered)
+ HHFunc(TEvInterconnect::TEvNodeDisconnected, OnExecuterDisconnected)
+
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+
+ HHFunc(TEvUploadComplete, OnFileUploaded)
+ HHFunc(TEvClusterStatus, OnClusterStatus)
+ HHFunc(TEvQueryStatus, OnQueryStatus)
+ HHFunc(TEvIsReady, OnIsReady)
+ HHFunc(TEvJobStop, OnJobStop)
+ HHFunc(TEvGetMasterRequest, OnGetMaster)
+ HHFunc(TEvConfigureFailureInjectorRequest, OnConfigureFailureInjector)
+ cFunc(TEvTick::EventType, OnTick)
+ HHFunc(TEvRoutesRequest, OnRoutesRequest)
+ HHFunc(TEvOperationStop, OnOperationStop)
+ })
+
+#define FORWARD(T) \
+ case T::EventType: { \
+ /*YQL_CLOG(TRACE, ProviderDq) << "Forward event " << etype << " to " << LeaderId;*/ \
+ typename T::TPtr& x = *(reinterpret_cast<typename T::TPtr*>(&ev)); \
+ if (!x->Get()->Record.GetIsForwarded()) { \
+ x->Get()->Record.SetIsForwarded(true); \
+ Send(x->Forward(MakeWorkerManagerActorID(LeaderId))); \
+ } \
+ break; \
+ }
+
+ STRICT_STFUNC(Follower, {
+ HHFunc(TEvBecomeLeader, StartLeader)
+ HHFunc(TEvBecomeFollower, StartFollower)
+ HHFunc(TEvAllocateWorkersRequest, StartUploadAndForward)
+ FORWARD(TEvFreeWorkersNotify)
+ FORWARD(TEvRegisterNode)
+ FORWARD(TEvJobStop)
+ FORWARD(TEvClusterStatus)
+ FORWARD(TEvQueryStatus)
+ FORWARD(TEvOperationStop)
+ HHFunc(TEvIsReady, OnIsReadyForward)
+ HHFunc(TEvGetMasterRequest, OnGetMaster)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ HHFunc(TEvUploadComplete, OnFileUploaded)
+ IgnoreFunc(TEvTick)
+ HHFunc(TEvRoutesRequest, this->OnRoutesRequest)
+ })
+
+#undef FORWARD
+#undef HHFunc
+
+ void Tick() {
+ Schedule(ScheduleInterval, new TEvTick);
+ }
+
+ void OnTick() {
+ auto now = TInstant::Now();
+ if (now - LastCleanTime > CleanInterval) {
+ Y_SCOPE_EXIT(&) { UpdateMetrics(); };
+ CleanUp(now);
+ LastCleanTime = now;
+ }
+ TryResume();
+ Tick();
+ }
+
+ void DoPassAway() override {
+ Y_SCOPE_EXIT(&) { UpdateMetrics(); };
+ for (const auto& sender: Scheduler->Cleanup()) {
+ Send(sender, new TEvAllocateWorkersResponse("Shutdown in progress", NYql::NDqProto::StatusIds::UNAVAILABLE));
+ }
+ }
+
+ std::pair<bool, TString> CheckFiles(const TVector<TFileResource>& files) {
+ for (const auto& file : files) {
+ if (file.GetObjectType() == Yql::DqsProto::TFile::EEXE_FILE) {
+ if (file.GetName().empty()) {
+ return std::make_pair(false, "Unnamed exe file " + file.SerializeAsString());
+ }
+ } else {
+ if (!NFs::Exists(file.GetLocalPath())) {
+ return std::make_pair(false, "Unknown file " + file.SerializeAsString());
+ }
+ }
+
+ if (file.GetObjectId().empty()) {
+ return std::make_pair(false, "Empty objectId (md5, revision) for " + file.SerializeAsString());
+ }
+ }
+
+ return std::make_pair(true, "");
+ }
+
+ std::pair<bool, TString> MaybeUpload(bool isForwarded, const TVector<TFileResource>& files, bool useCache = false) {
+ if (isForwarded) {
+ return std::make_pair(false, "");
+ }
+
+ auto [hasExeFile,error] = CheckFiles(files);
+ if (!error.empty()) {
+ return std::make_pair(hasExeFile,error);
+ }
+
+ bool flag = false;
+
+ TVector<TResourceFile> preparedFiles;
+ TVector<TString> preparedFilesIds;
+ TVector<TResourceFile> preparedExeFiles;
+ auto now = TInstant::Now();
+ auto cacheTimeout = TDuration::Minutes(5);
+
+ if (useCache) {
+ THashSet<TString> toremove;
+ for (const auto& [objectId, ts] : LastUploadCache) {
+ if (now - ts > cacheTimeout) {
+ toremove.insert(objectId);
+ }
+ }
+ for (const auto& objectId : toremove) {
+ LastUploadCache.erase(objectId);
+ }
+ }
+
+ for (const auto& file : files) {
+ flag |= file.GetObjectType() == Yql::DqsProto::TFile::EEXE_FILE;
+ if (Uploading.contains(file.GetObjectId())) {
+ continue;
+ }
+
+ // exe files can be without local path
+ if (file.GetLocalPath().empty()) {
+ continue;
+ }
+ auto fileName = file.GetName();
+ auto objectId = file.GetObjectId();
+ auto objectType = file.GetObjectType();
+ auto localPath = file.GetLocalPath();
+ auto size = file.GetSize();
+
+ if (useCache || objectType == Yql::DqsProto::TFile_EFileType_EEXE_FILE) {
+ if (LastUploadCache.contains(objectId) && (now - LastUploadCache[objectId] < cacheTimeout)) {
+ continue;
+ }
+ LastUploadCache[objectId] = now;
+ }
+
+ TResourceFile preparedFile;
+ preparedFile.ObjectId = objectId;
+ preparedFile.LocalFileName = fileName;
+ preparedFile.Attributes["file_name"] = preparedFile.GetRemoteFileName();
+ preparedFile.File = ::TFile(localPath, RdOnly | OpenExisting);
+ preparedFile.RemoteFileName = objectId;
+ preparedFiles.push_back(preparedFile);
+ preparedFilesIds.push_back(objectId);
+
+ YQL_CLOG(DEBUG, ProviderDq) << "Start upload: " << fileName << "," << objectId << "," << size;
+
+ int uploadProcesses = static_cast<int>(ResourceUploaderOptions.size());
+
+ if (objectType == Yql::DqsProto::TFile_EFileType_EEXE_FILE) {
+ // COMPAT (aozeritsky)
+ preparedFile.RemoteFileName = fileName;
+ uploadProcesses *= 2;
+
+ for (auto options : ResourceUploaderOptions) {
+ options.Counters = UploaderMetrics;
+ options.LockName = objectId + "." + options.YtBackend.GetClusterName();
+ options.Files = { preparedFile };
+ options.UploadPrefix = options.UploadPrefix + "/bin/" + objectId;
+ auto actor = CreateResourceUploader(options, Coordinator);
+ UploadProcesses[RegisterChild(actor)].push_back(objectId);
+ }
+ }
+
+ Uploading.emplace(objectId, TUploadingInfo {objectType, fileName, uploadProcesses, size});
+ }
+
+ Shuffle(preparedFiles.begin(), preparedFiles.end(), TReallyFastRng32(TInstant::Now().NanoSeconds()));
+
+ for (auto options : ResourceUploaderOptions) {
+ options.Counters = UploaderMetrics;
+ options.LockName.clear();
+
+ if (!preparedFiles.empty()) {
+ options.Files = preparedFiles;
+ options.UploadPrefix = options.UploadPrefix + "/udfs";
+ auto actor = CreateResourceUploader(options, Coordinator);
+ UploadProcesses.emplace(RegisterChild(actor), preparedFilesIds);
+ }
+ }
+
+ return std::make_pair(flag, "");
+ }
+
+ void StartUploadAndForward(TEvAllocateWorkersRequest::TPtr& ev, const TActorContext& ctx)
+ {
+ YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId());
+ auto [hasExeFile, err] = MaybeUpload(ev->Get()->Record.GetIsForwarded(), TVector<TFileResource>(ev->Get()->Record.GetFiles().begin(), ev->Get()->Record.GetFiles().end()));
+ if (!err.empty()) {
+ Send(ev->Sender, new TEvAllocateWorkersResponse(err, NYql::NDqProto::StatusIds::EXTERNAL_ERROR));
+ }
+
+ if (!hasExeFile && LeaderRevision != Revision) {
+ Send(ev->Sender, new TEvAllocateWorkersResponse(
+ Sprintf("Wrong revision %s!=%s", LeaderRevision.c_str(), Revision.c_str()), NYql::NDqProto::StatusIds::BAD_REQUEST));
+ } else {
+ ev->Get()->Record.SetIsForwarded(true);
+ ctx.Send(ev->Forward(MakeWorkerManagerActorID(LeaderId)));
+ }
+ }
+
+ void OnFileUploaded(TEvUploadComplete::TPtr& ev, const TActorContext& ctx)
+ {
+ Y_UNUSED(ctx);
+ auto sender = ev->Sender;
+ auto it = UploadProcesses.find(sender);
+ if (it == UploadProcesses.end()) {
+ return;
+ }
+ UnregisterChild(sender);
+
+ for (const auto& objectId : it->second) {
+ auto jt = Uploading.find(objectId);
+ Y_ABORT_UNLESS(jt != Uploading.end());
+ Workers.AddResource(objectId, jt->second.ObjectType, jt->second.ObjectName, jt->second.Size);
+ if (--jt->second.Clusters <= 0) {
+ Uploading.erase(jt);
+ }
+ }
+ UploadProcesses.erase(it);
+ }
+
+ void StartLeader(TEvBecomeLeader::TPtr& ev, const TActorContext& ctx)
+ {
+ Y_UNUSED(ctx);
+ if (FollowingMode) {
+ YQL_CLOG(INFO, ProviderDq) << "Skip Leader request in following mode";
+ Send(SelfId(), new TEvBecomeFollower(), /*flag=*/0, /*cookie=*/1);
+ return;
+ }
+ LeaderEpoch = CurrentResourceId.Epoch = ev->Get()->LeaderEpoch;
+ CurrentResourceId.Counter = 0;
+
+ // update leader info (used for leader)
+ auto attributes = NYT::NodeFromYsonString(ev->Get()->Attributes).AsMap();
+ UpdateLeaderInfo(attributes);
+
+ YQL_CLOG(INFO, ProviderDq) << "Become leader, epoch=" << CurrentResourceId.Epoch;
+ YQL_CLOG(INFO, ProviderDq) << "Leader attributes leader=" << ev->Get()->Attributes;
+ YQL_CLOG(INFO, ProviderDq) << "Leader resolver=" << LeaderHost << ":" << LeaderPort;
+
+ if (LeaderPinger) {
+ UnregisterChild(LeaderPinger);
+ LeaderPinger = TActorId();
+ }
+ Become(&TGlobalWorkerManager::Leader);
+ Tick();
+ }
+
+ void StartFollower(TEvBecomeFollower::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ if (ev->Cookie == 1 && LockId) {
+ // kill from main
+ YQL_CLOG(INFO, ProviderDq) << "Kill from main";
+ ctx.Send(ev->Forward(LockId));
+ FollowingMode = true;
+ LocalCriticalFiles->Clear(); // disable wormup for old dq process
+ return;
+ }
+ auto attributes = NYT::NodeFromYsonString(ev->Get()->Attributes).AsMap();
+ if (ev->Cookie == 1 && LockId) {
+ // kill from main
+ YQL_CLOG(INFO, ProviderDq) << "Kill from main";
+ Send(LockId, new NActors::TEvents::TEvPoison);
+ LockId = TActorId();
+ }
+ LeaderId = attributes.at(NCommonAttrs::ACTOR_NODEID_ATTR).AsUint64();
+ LeaderResolver->SetLeaderHostPort(
+ attributes.at(NCommonAttrs::HOSTNAME_ATTR).AsString() + ":" + attributes.at(NCommonAttrs::GRPCPORT_ATTR).AsString());
+ if (!LeaderPinger) {
+ TResourceManagerOptions rmOptions;
+ rmOptions.FileCache = LocalCriticalFiles;
+ LeaderPinger = RegisterChild(Coordinator->CreateServiceNodePinger(LeaderResolver, rmOptions));
+ }
+ if (attributes.contains(NCommonAttrs::REVISION_ATTR)) {
+ LeaderRevision = attributes.at(NCommonAttrs::REVISION_ATTR).AsString();
+ }
+
+ UpdateLeaderInfo(attributes);
+
+ YQL_CLOG(TRACE, ProviderDq) << " Following leader: " << LeaderId;
+ YQL_CLOG(INFO, ProviderDq) << "Leader resolver=" << LeaderHost << ":" << LeaderPort;
+
+ WaitListSize = nullptr;
+ Workers.Clear();
+ for (const auto& [key, value] : AllocatedResources) {
+ Send(value.ActorId, new TEvents::TEvPoison());
+ }
+ for (const auto sender : Scheduler->Cleanup()) {
+ Send(sender, new TEvAllocateWorkersResponse("StartFollower", NYql::NDqProto::StatusIds::UNSPECIFIED));
+ }
+ AllocatedResources.clear();
+ for (auto& [k, v] : LiteralQueries) {
+ *v -= *v;
+ }
+
+ Become(&TGlobalWorkerManager::Follower);
+ }
+
+ void UpdateLeaderInfo(THashMap<TString, NYT::TNode>& attributes) {
+ LeaderHost = attributes.at(NCommonAttrs::HOSTNAME_ATTR).AsString();
+ LeaderPort = std::stoi(attributes.at(NCommonAttrs::GRPCPORT_ATTR).AsString().Data());
+ }
+
+ void Bootstrap(const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ LockId = RegisterChild(Coordinator->CreateLock("gwm", false));
+ }
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ return new IEventHandle(self, parentId, new TEvents::TEvBootstrap(), 0);
+ }
+
+ void OnRegisterNodeStub(TEvRegisterNode::TPtr& ev, const NActors::TActorContext&) {
+ Send(ev->Sender, new TEvRegisterNodeResponse());
+ }
+
+ void OnRegisterNode(TEvRegisterNode::TPtr& ev, const NActors::TActorContext& ctx) {
+ auto* mutableRequest = ev->Get()->Record.MutableRequest();
+ auto& request = ev->Get()->Record.GetRequest();
+ auto nodeId = request.GetNodeId();
+ auto workerId = NYql::NDqs::NExecutionHelpers::GuidFromProto(request.GetGuid());
+ auto role = request.GetRole();
+
+ Y_SCOPE_EXIT(&) {
+ ctx.Send(ev->Forward(NActors::GetNameserviceActorId()));
+ };
+
+ if (role == "worker_node" && Revision == request.GetRevision()) {
+ mutableRequest->SetEpoch(LeaderEpoch);
+ }
+
+ if (role == "service_node") {
+ AllCriticalFiles.Add(
+ workerId,
+ nodeId,
+ request.GetRevision(),
+ request.GetAddress(),
+ request.GetPid(),
+ request.GetFilesOnNode());
+ }
+
+ if (role != "worker_node") {
+ mutableRequest->SetEpoch(LeaderEpoch);
+ return;
+ }
+
+ int capacity = request.GetCapacity();
+ if (capacity == 0) {
+ capacity = 1;
+ }
+
+ {
+ TWorkerInfo::TPtr workerInfo;
+ bool needResume;
+
+ auto newWorkerId = workerId;
+
+ std::tie(workerInfo, needResume) = Workers.CreateOrUpdate(nodeId, newWorkerId, *mutableRequest);
+
+ for (auto i = StopFilters.begin(); i != StopFilters.end(); ) {
+ if (workerInfo && !workerInfo->IsDead && i->Match(*workerInfo)) {
+ workerInfo->Stopping = true;
+ if (request.GetRunningWorkers() == 0) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Stop worker on user request " << GetGuidAsString(workerId);
+ Send(MakeWorkerManagerActorID(nodeId), new TEvents::TEvPoison);
+ return;
+ }
+ }
+
+ if ((TInstant::Now() - i->GetLastUpdate()) > TDuration::Seconds(600)) {
+ i = StopFilters.erase(i);
+ } else {
+ ++i;
+ }
+ }
+
+ if (needResume) {
+ MarkDirty();
+ }
+
+ if (workerInfo && !workerInfo->IsDead) {
+ for (auto& r : AllCriticalFiles.GetResources()) {
+ workerInfo->AddToDownloadList(r.GetObjectId(), r);
+ }
+
+ for (const auto& [_, file] : workerInfo->GetResourcesForDownloading()) {
+ *mutableRequest->AddDownloadList() = file;
+ }
+ }
+ }
+ }
+
+ void CleanUp(TInstant now) {
+ Workers.CleanUp(now, TDuration::Seconds(15));
+ }
+
+ void OnAllocateWorkersRequestStub(TEvAllocateWorkersRequest::TPtr& ev, const NActors::TActorContext&) {
+ YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId());
+ YQL_CLOG(DEBUG, ProviderDq) << "TGlobalWorkerManager::TEvAllocateWorkersRequest initialization";
+ Send(ev->Sender, new TEvAllocateWorkersResponse("GWM: Waiting for initialization", NYql::NDqProto::StatusIds::UNAVAILABLE));
+ }
+
+ void MarkDirty(size_t count = 0U) {
+ ScheduleWaitCount = Min(ScheduleWaitCount, count);
+ }
+
+ void TryResume() {
+ if (Workers.FreeSlots() >= ScheduleWaitCount) {
+ Scheduler->Process(Workers.Capacity(), Workers.FreeSlots(), [&] (const auto& item) {
+ auto maybeDead = DeadOperations.find(item.Request.GetResourceId());
+ if (maybeDead != DeadOperations.end()) {
+ DeadOperations.erase(maybeDead);
+ return true; // remove from WaitList
+ }
+ auto candidates = Workers.TryAllocate(item);
+ if (!candidates.empty()) {
+ DoAllocate(item, candidates);
+ }
+ return !candidates.empty();
+ });
+ ScheduleWaitCount = std::numeric_limits<size_t>::max();
+ DeadOperations.clear();
+ }
+ }
+
+ void OnAllocateWorkersRequest(TEvAllocateWorkersRequest::TPtr& ev, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId());
+ YQL_CLOG(DEBUG, ProviderDq) << "TGlobalWorkerManager::TEvAllocateWorkersRequest";
+
+ TFailureInjector::Reach("allocate_workers_failure", [] { ::_exit(1); });
+
+ const auto count = ev->Get()->Record.GetCount();
+ Y_ASSERT(count != 0);
+
+ if (!Scheduler->Suspend(NDq::IScheduler::TWaitInfo(ev->Get()->Record, ev->Sender))) {
+ Send(ev->Sender, new TEvAllocateWorkersResponse("Too many dq operations", NYql::NDqProto::StatusIds::OVERLOADED));
+ return;
+ }
+
+ auto [_, error] = MaybeUpload(ev->Get()->Record.GetIsForwarded(), TVector<TFileResource>(ev->Get()->Record.GetFiles().begin(),ev->Get()->Record.GetFiles().end()));
+ if (!error.empty()) {
+ Send(ev->Sender, new TEvAllocateWorkersResponse(error, NYql::NDqProto::StatusIds::EXTERNAL_ERROR));
+ return;
+ }
+
+ MarkDirty(count);
+ }
+
+ void DecrLiteralQueries(const TString& clusterName) {
+ *LiteralQueries[clusterName] += -1;
+ }
+
+ void IncrLiteralQueries(const TString& clusterName) {
+ auto& counter = LiteralQueries[clusterName];
+ if (!counter) {
+ counter = Metrics
+ ->GetSubgroup("counters", "gwm")
+ ->GetSubgroup("component", "lists")
+ ->GetSubgroup("ClusterName", clusterName)
+ ->GetCounter("LiteralQueries");
+ }
+ *counter += 1;
+ }
+
+ void DoAllocate(const NDq::IScheduler::TWaitInfo& waitInfo, const TVector<TWorkerInfo::TPtr>& allocated) {
+ YQL_LOG_CTX_ROOT_SESSION_SCOPE(waitInfo.Request.GetTraceId());
+ Y_ABORT_UNLESS(allocated.size() == waitInfo.Request.GetCount());
+
+ THashSet<TString> clusters;
+
+ for (const auto& workerInfo : allocated) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Allocating worker " << GetGuidAsString(workerInfo->WorkerId);
+ YQL_CLOG(DEBUG, ProviderDq) << " Address : " << workerInfo->Address;
+ for (const auto& [k, v] : workerInfo->Attributes) {
+ YQL_CLOG(DEBUG, ProviderDq) << " " << k << " : " << v;
+ }
+ clusters.insert(workerInfo->ClusterName);
+ }
+
+ THashSet<TString> usedFiles;
+ for (const auto& file : waitInfo.Request.GetFiles()) {
+ usedFiles.insert(file.GetObjectId());
+ }
+
+ auto resourceId = CurrentResourceId.Data;
+ CurrentResourceId.Counter ++;
+
+ auto resourceAllocator = waitInfo.Sender;
+
+ auto response = MakeHolder<TEvAllocateWorkersResponse>(resourceId, allocated);
+ waitInfo.Stat.FlushCounters(response->Record);
+ Send(
+ resourceAllocator,
+ response.Release(),
+ IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession);
+
+ Subscribe(resourceAllocator.NodeId());
+
+ AllocatedResources[resourceId] = {
+ resourceAllocator,
+ allocated,
+ usedFiles,
+ clusters,
+ TInstant::Now()
+ };
+
+ if (allocated.size() == 1) {
+ Y_ABORT_UNLESS(clusters.size() == 1);
+ IncrLiteralQueries(*clusters.begin());
+ }
+ }
+
+ void DropActorOrNode(const std::function<bool(TActorId actorId)>& check, const NActors::TActorContext&) {
+ TVector<ui64> freeList;
+ for (const auto& [k, v] : AllocatedResources) {
+ if (check(v.ActorId)) {
+ freeList.push_back(k);
+ }
+ }
+
+ for (auto resourceId : freeList) {
+ FreeResource(resourceId, {});
+ }
+
+ Scheduler->ProcessAll([&] (const auto& item) {
+ return check(item.Sender);
+ });
+
+ MarkDirty();
+ }
+
+ void OnUndelivered(TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx)
+ {
+ auto deadActor = ev->Sender;
+ DropActorOrNode([&](TActorId actorId) {
+ return actorId == deadActor;
+ }, ctx);
+ }
+
+ void OnExecuterDisconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev, const NActors::TActorContext& ctx)
+ {
+ YQL_CLOG(DEBUG, ProviderDq) << "OnExecuterDisconnected " << ev->Get()->NodeId;
+
+ Unsubscribe(ev->Get()->NodeId);
+
+ auto deadNode = ev->Get()->NodeId;
+
+ DropActorOrNode([&](TActorId actorId) {
+ return actorId.NodeId() == deadNode;
+ }, ctx);
+ }
+
+ void OnFreeWorkersStub(TEvFreeWorkersNotify::TPtr& ev, const NActors::TActorContext&) {
+ YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId());
+ YQL_CLOG(DEBUG, ProviderDq) << "TGlobalWorkerManager::OnFreeWorkersStub " << ev->Sender.NodeId();
+ }
+
+ void FreeResource(ui64 resourceId, const THashSet<TGUID>& failedWorkers) {
+ if (!AllocatedResources.contains(resourceId)) {
+ return;
+ }
+
+ auto& resource = AllocatedResources[resourceId];
+
+ auto now = TInstant::Now();
+
+ for (const auto& worker : resource.Workers) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Free worker " << GetGuidAsString(worker->WorkerId);
+
+ if (failedWorkers.contains(worker->WorkerId)) {
+ Workers.DropWorker(now, worker->WorkerId);
+ } else {
+ Workers.FreeWorker(now, worker);
+ }
+ }
+
+ if (resource.Workers.size() == 1) {
+ DecrLiteralQueries(*resource.Clusters.begin());
+ }
+
+ Workers.UpdateResourceUseTime(now - resource.StartTime, resource.UsedFiles);
+ AllocatedResources.erase(resourceId);
+ }
+
+ void OnFreeWorkers(TEvFreeWorkersNotify::TPtr& ev, const TActorContext&) {
+ auto resourceId = ev->Get()->Record.GetResourceId();
+ if (AllocatedResources.contains(resourceId)) {
+ YQL_LOG_CTX_ROOT_SESSION_SCOPE(ev->Get()->Record.GetTraceId());
+ YQL_CLOG(DEBUG, ProviderDq) << "TEvFreeWorkersNotify " << resourceId;
+ THashSet<TGUID> failedWorkers;
+ for (const auto& workerInfo : ev->Get()->Record.GetFailedWorkerGuid()) {
+ auto guid = GetGuid(workerInfo);
+ YQL_CLOG(DEBUG, ProviderDq) << "Failed worker: " << GetGuidAsString(guid);
+ failedWorkers.insert(guid);
+ }
+ FreeResource(resourceId, failedWorkers);
+ } else {
+ DeadOperations.insert(resourceId);
+ }
+
+ MarkDirty();
+ }
+
+ void OnJobStop(TEvJobStop::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ auto request = ev->Get()->Record.GetRequest();
+ TString requestStr;
+ TStringOutput output1(requestStr);
+ SerializeToTextFormat(request, output1);
+
+ YQL_CLOG(DEBUG, ProviderDq) << "JobStop " << requestStr;
+
+ if (request.GetForce()) {
+ TWorkerStopFilter filter(request);
+ Workers.Visit([&](const TWorkerInfo::TPtr& workerInfo) {
+ if (!workerInfo->IsDead && filter.Match(*workerInfo)) {
+ workerInfo->Stopping = true;
+ YQL_CLOG(DEBUG, ProviderDq) << "Force stop worker on user request " << GetGuidAsString(workerInfo->WorkerId);
+ Send(MakeWorkerManagerActorID(workerInfo->NodeId), new TEvents::TEvPoison);
+ }
+ });
+ } else {
+ StopFilters.emplace_back(request);
+ }
+ }
+
+ void OnClusterStatusStub(TEvClusterStatus::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto response = MakeHolder<TEvClusterStatusResponse>();
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnClusterStatus(TEvClusterStatus::TPtr& ev, const TActorContext& ctx) {
+ YQL_CLOG(DEBUG, ProviderDq) << "TEvClusterStatus ";
+
+ Y_UNUSED(ctx);
+
+ auto response = MakeHolder<TEvClusterStatusResponse>();
+
+ auto* r = response->Record.MutableResponse();
+
+ r->SetRevision(Revision);
+
+ Workers.ClusterStatus(r);
+
+ Scheduler->ForEach([&] (const auto& item) {
+ auto* info = r->AddWaitList();
+ info->SetCount(item.Request.GetCount());
+ info->SetOperationId(item.Request.GetTraceId());
+ info->SetUserName(item.Request.GetUser());
+
+ THashMap<TString, Yql::DqsProto::ClusterStatusResponse::File> files;
+
+ for (const auto& rec : item.Request.GetFiles()) {
+ auto& file = files[rec.GetObjectId()];
+ file.SetObjectId(rec.GetObjectId());
+ file.SetName(rec.GetName());
+ if (item.ResLeft.contains(rec.GetObjectId())) {
+ file.SetLeft(item.ResLeft[rec.GetObjectId()]);
+ }
+ file.SetSize(rec.GetSize());
+ }
+
+ for (const auto& it : item.Request.GetWorkerFilterPerTask()) {
+ for (const auto& rec : it.GetFile()) {
+ auto& file = files[rec.GetObjectId()];
+ file.SetObjectId(rec.GetObjectId());
+ file.SetName(rec.GetName());
+ file.SetCount(file.GetCount()+1);
+ file.SetSize(rec.GetSize());
+ }
+ }
+
+ for (const auto& [_, v] : files) {
+ *info->AddResources() = v;
+ }
+ });
+
+ for (const auto& i : Uploading) {
+ *r->AddUploading() = i.first;
+ }
+
+ for (const auto& [guid, node] : AllCriticalFiles.GetNodes()) {
+ auto* nodeInfo = r->AddServiceNode();
+ nodeInfo->SetRevision(node.Revision);
+ nodeInfo->SetNodeId(node.NodeId);
+ nodeInfo->SetPid(node.Pid);
+ nodeInfo->SetAddress(node.Address);
+ nodeInfo->SetGuid(GetGuidAsString(guid));
+ for (const auto& file : node.OrderedFiles) {
+ *nodeInfo->AddFile() = file;
+ }
+ }
+
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnQueryStatusStub(TEvQueryStatus::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto response = MakeHolder<TEvQueryStatusResponse>();
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnQueryStatus(TEvQueryStatus::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ auto sessionId = ev->Get()->Record.request().GetSession();
+
+ bool queued = false;
+ ui32 workersCount;
+
+ Scheduler->ForEach([&queued, &sessionId, &workersCount] (const NDq::IScheduler::TWaitInfo& item) {
+ if (sessionId == item.Request.GetTraceId()) {
+ queued = true;
+ workersCount = item.Request.GetCount();
+ }
+ });
+
+ auto response = MakeHolder<TEvQueryStatusResponse>();
+
+ auto* r = response->Record.MutableResponse();
+
+ if (queued) {
+ if (workersCount > Workers.FreeSlots()) {
+ r->SetStatus("Awaiting");
+ } else {
+ r->SetStatus("Uploading artifacts");
+ }
+ } else {
+ r->SetStatus("Executing");
+ }
+
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnIsReadyStub(TEvIsReady::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ auto response = MakeHolder<TEvIsReadyResponse>();
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnOperationStopStub(TEvOperationStop::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ auto response = MakeHolder<TEvOperationStopResponse>();
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnOperationStop(TEvOperationStop::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ Scheduler->Process(Workers.Capacity(), Workers.FreeSlots(), [&] (const auto& item) {
+ if (item.Request.GetTraceId() == ev->Get()->Record.GetRequest().GetOperationId()) {
+ Send(item.Sender, new TEvDqFailure(NYql::NDqProto::StatusIds::ABORTED, TIssue("Operation stopped by scheduler").SetCode(TIssuesIds::DQ_GATEWAY_ERROR, TSeverityIds::S_ERROR)));
+ return true;
+ }
+ return false;
+ });
+
+ auto response = MakeHolder<TEvOperationStopResponse>();
+ Send(ev->Sender, response.Release());
+ }
+
+ void FillCriticalFiles(TEvIsReady::TPtr& ev) {
+ if (FollowingMode) {
+ return;
+ }
+ if (ev->Get()->Record.GetIsForwarded()) {
+ return;
+ }
+ LocalCriticalFiles->Clear();
+ for (auto& f : ev->Get()->Record.GetRequest().GetFiles()) {
+ //TODO: Add support types in LocalCriticalFiles
+ //Don't uncomment this
+ //if (f.GetObjectType() == TWorkerInfo::TFileResource::EEXE_FILE) {
+ // LocalCriticalFiles supports only udfs
+ // continue;
+ //}
+ LocalCriticalFiles->AddFile("unused", f.GetObjectId());
+ }
+ TVector<TFileResource> files;
+ LocalCriticalFiles->Walk([&](const TString& objectId) {
+ TFileResource r;
+ r.SetObjectId(objectId);
+ files.push_back(r);
+ });
+ TFileResource r;
+ r.SetObjectId(Revision);
+ files.push_back(r);
+
+ AllCriticalFiles.Add(TGUID(), SelfId().NodeId(), Revision, Address, Pid, files);
+ }
+
+ void OnIsReadyForward(TEvIsReady::TPtr& ev, const TActorContext& ctx) {
+ auto [_, error] = MaybeUpload(ev->Get()->Record.GetIsForwarded(), TVector<TFileResource>(ev->Get()->Record.GetRequest().GetFiles().begin(),ev->Get()->Record.GetRequest().GetFiles().end()), true);
+ if (!error.empty()) {
+ YQL_CLOG(WARN, ProviderDq) << "TEvIsReady error on upload: " << error;
+ }
+
+ if (FollowingMode) {
+ auto response = MakeHolder<TEvIsReadyResponse>();
+ response->Record.SetIsReady(true);
+ Send(ev->Sender, response.Release());
+ } else {
+ FillCriticalFiles(ev);
+ ev->Get()->Record.SetIsForwarded(true);
+ ctx.Send(ev->Forward(MakeWorkerManagerActorID(LeaderId)));
+ }
+ }
+
+ void OnIsReady(TEvIsReady::TPtr& ev, const TActorContext& ctx) {
+ YQL_CLOG(DEBUG, ProviderDq) << "TEvIsReady ";
+ Y_UNUSED(ctx);
+
+ FillCriticalFiles(ev);
+
+ auto [_, error] = MaybeUpload(ev->Get()->Record.GetIsForwarded(), TVector<TFileResource>(ev->Get()->Record.GetRequest().GetFiles().begin(),ev->Get()->Record.GetRequest().GetFiles().end()), true);
+ if (!error.empty()) {
+ YQL_CLOG(WARN, ProviderDq) << "TEvIsReady error on upload: " << error;
+ }
+ THashMap<TString , std::pair<ui32, ui32>> clusterMap;
+ for (const auto& options : ResourceUploaderOptions) {
+ clusterMap.emplace(options.YtBackend.GetClusterName(), std::make_pair(options.YtBackend.GetMaxJobs(), 0));
+ }
+
+ auto resources = ev->Get()->Record.GetRequest().GetFiles();
+ Workers.IsReady(TVector<TFileResource>(resources.begin(), resources.end()), clusterMap);
+
+ // any cluster has at least 50% of workers with actual vanilla job ready
+ bool isReady = false;
+ for (const auto& [cluster,pair] : clusterMap) {
+ YQL_CLOG(DEBUG, ProviderDq) << cluster << " ready: " << pair.second << "/" << pair.first << " workers ready";
+ isReady |= pair.second * 2 >= pair.first;
+ }
+
+ YQL_CLOG(DEBUG, ProviderDq) << "IsReady status : " << isReady;
+
+ auto response = MakeHolder<TEvIsReadyResponse>();
+ response->Record.SetIsReady(isReady);
+
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnGetMasterStub(TEvGetMasterRequest::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto response = MakeHolder<TEvGetMasterResponse>();
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnGetMaster(TEvGetMasterRequest::TPtr& ev, const TActorContext& ctx) {
+ YQL_CLOG(DEBUG, ProviderDq) << "TEvGetMasterRequest ";
+
+ Y_UNUSED(ctx);
+
+ auto response = MakeHolder<TEvGetMasterResponse>();
+
+ auto* r = response->Record.MutableResponse();
+
+ r->SetHost(LeaderHost);
+ r->SetPort(LeaderPort);
+
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnConfigureFailureInjectorStub(TEvConfigureFailureInjectorRequest::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ auto response = MakeHolder<TEvConfigureFailureInjectorResponse>();
+ Send(ev->Sender, response.Release());
+ }
+
+ void OnConfigureFailureInjector(TEvConfigureFailureInjectorRequest::TPtr& ev, const TActorContext& ctx) {
+ YQL_CLOG(DEBUG, ProviderDq) << "TEvConfigureFailureInjectorRequest ";
+
+ Y_UNUSED(ctx);
+
+ // TODO: support resending the event to all workers and gathering the results?
+ auto& request = ev->Get()->Record.GetRequest();
+ // just forward the event to a worker node
+ if (request.GetNodeId() == SelfId().NodeId()) {
+ TFailureInjector::Set(request.GetName(), request.GetSkip(), request.GetCountOfFails());
+
+ auto response = MakeHolder<TEvConfigureFailureInjectorResponse>();
+ auto* r = response->Record.MutableResponse();
+ r->Setsuccess(true);
+ Send(ev->Sender, response.Release());
+ } else {
+ ctx.Send(ev->Forward(MakeWorkerManagerActorID(request.GetNodeId())));
+ }
+ }
+
+ void UpdateMetrics() {
+ if (!WaitListSize) {
+ WaitListSize = Metrics->GetSubgroup("component", "lists")->GetCounter("WaitListSize");
+ }
+ *WaitListSize = Scheduler->UpdateMetrics();
+ Workers.UpdateMetrics();
+ }
+
+private:
+ const ICoordinationHelper::TPtr Coordinator;
+ const std::shared_ptr<TSingleNodeResolver> LeaderResolver;
+ TActorId LeaderPinger;
+ TActorId LockId;
+
+ TSensorsGroupPtr Metrics;
+ TSensorsGroupPtr UploaderMetrics;
+ THistogramPtr LatencyHistogram;
+ THashMap<TString,NMonitoring::TDynamicCounters::TCounterPtr> LiteralQueries;
+ TWorkersStorage Workers;
+
+ struct TResourceInfo {
+ TActorId ActorId;
+ TVector<TWorkerInfo::TPtr> Workers;
+ THashSet<TString> UsedFiles;
+ THashSet<TString> Clusters;
+ TInstant StartTime;
+ };
+ THashMap<ui64, TResourceInfo> AllocatedResources;
+ THashSet<ui64> DeadOperations;
+
+ ui32 LeaderId = static_cast<ui32>(-1);
+ TString LeaderRevision = "";
+ TString LeaderHost = "";
+ ui32 LeaderPort = 0u;
+
+ ui32 LeaderEpoch;
+ TDqResourceId CurrentResourceId;
+
+ const NDq::IScheduler::TPtr Scheduler;
+ const TString Revision;
+
+ THashMap<TActorId, TVector<TString>> UploadProcesses; // actorId -> objects
+
+ struct TUploadingInfo {
+ EFileType ObjectType;
+ TString ObjectName;
+ int Clusters;
+ i64 Size;
+ };
+ THashMap<TString, TUploadingInfo> Uploading; // objectId -> objectName
+
+ TVector<TResourceManagerOptions> ResourceUploaderOptions;
+
+ TList<TWorkerStopFilter> StopFilters;
+
+ TDynamicCounters::TCounterPtr WaitListSize;
+
+ // filled in IsReady
+ IFileCache::TPtr LocalCriticalFiles = MakeIntrusive<TLocalCriticalFiles>();
+ // filled in IsReady for local files
+ // filled in OnRegister for other files
+ TAllCriticalFiles AllCriticalFiles;
+
+ THashMap<TString, TInstant> LastUploadCache;
+
+ // Don't reschedule too frequently to avoid GWM hanging
+ TDuration ScheduleInterval = TDuration::MilliSeconds(100);
+ size_t ScheduleWaitCount = std::numeric_limits<size_t>::max(); // max - no, 0 - any, > 0 count
+ TInstant LastCleanTime;
+ TDuration CleanInterval = TDuration::Seconds(2);
+ bool FollowingMode = false;
+ const TString Address = HostName();
+ const ui32 Pid = GetPID();
+};
+
+NActors::IActor* CreateGlobalWorkerManager(
+ const ICoordinationHelper::TPtr& coordinator,
+ const TVector<TResourceManagerOptions>& resourceUploaderOptions,
+ IMetricsRegistryPtr metricsRegistry,
+ const NProto::TDqConfig::TScheduler& schedulerConfig) {
+ return new TGlobalWorkerManager(coordinator, resourceUploaderOptions, std::move(metricsRegistry), schedulerConfig);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.h b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.h
new file mode 100644
index 0000000000..8173f2c3fe
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <ydb/library/yql/providers/dq/worker_manager/interface/events.h>
+#include <ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h>
+#include <ydb/library/yql/providers/dq/actors/events.h>
+#include "coordination_helper.h"
+#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h>
+#include <ydb/library/yql/providers/common/metrics/metrics_registry.h>
+
+#include <library/cpp/actors/core/events.h>
+
+namespace NYql {
+
+NActors::IActor* CreateGlobalWorkerManager(
+ const ICoordinationHelper::TPtr& coordinator,
+ const TVector<TResourceManagerOptions>& resourceUploaderOptions,
+ IMetricsRegistryPtr metricsRegistry,
+ const NProto::TDqConfig::TScheduler& schedulerConfig);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp
new file mode 100644
index 0000000000..85825306f7
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp
@@ -0,0 +1,439 @@
+#include "global_worker_manager.h"
+#include <ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/actors/testlib/test_runtime.h>
+#include <library/cpp/yson/node/node_io.h>
+#include <yt/cpp/mapreduce/interface/fluent.h>
+#include <ydb/library/yql/providers/common/metrics/metrics_registry.h>
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+#include <ydb/library/yql/providers/dq/common/attrs.h>
+#include <ydb/library/yql/providers/dq/actors/dynamic_nameserver.h>
+#include <ydb/library/yql/providers/dq/actors/resource_allocator.h>
+
+using namespace NYql;
+using namespace NActors;
+using namespace NProto;
+using namespace NDqs;
+
+namespace {
+
+class TMockLock: public TActor<TMockLock> {
+public:
+ TMockLock()
+ : TActor(&TMockLock::Main)
+ {
+ }
+
+ STATEFN(Main) {
+ switch (ev->GetTypeRewrite()) {
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ }
+ }
+};
+
+class TTestCoordinationHelper: public ICoordinationHelper {
+public:
+
+ TTestCoordinationHelper(ui32 nodeId)
+ : NodeId(nodeId)
+ {}
+
+ ui32 GetNodeId() override {
+ return NodeId;
+ }
+
+ ui32 GetNodeId(const TMaybe<ui32> nodeId, const TMaybe<TString>& grpcPort, ui32 minNodeId, ui32 maxNodeId, const THashMap<TString, TString>& attributes) override {
+ Y_UNUSED(nodeId);
+ Y_UNUSED(grpcPort);
+ Y_UNUSED(minNodeId);
+ Y_UNUSED(maxNodeId);
+ Y_UNUSED(attributes);
+ return NodeId;
+ }
+
+ TString GetHostname() override {
+ return "localhost";
+ }
+
+ TString GetIp() override {
+ return "::1";
+ }
+
+ NActors::IActor* CreateLockOnCluster(NActors::TActorId ytWrapper, const TString& prefix, const TString& lockName, bool temporary) override {
+ Y_UNUSED(ytWrapper);
+ Y_UNUSED(prefix);
+ Y_UNUSED(lockName);
+ Y_UNUSED(temporary);
+
+ return new TMockLock();
+ }
+
+ NActors::IActor* CreateLock(const TString& lockName, bool temporary) override {
+ Y_UNUSED(lockName);
+ Y_UNUSED(temporary);
+ return new TMockLock();
+ }
+
+ NActors::IActor* CreateServiceNodePinger(const IServiceNodeResolver::TPtr& ptr, const TResourceManagerOptions& rmOptions, const THashMap<TString, TString>& attributes = {}) override {
+ Y_UNUSED(ptr);
+ Y_UNUSED(rmOptions);
+ Y_UNUSED(attributes);
+ return nullptr;
+ }
+
+ void StartRegistrator(NActors::TActorSystem* actorSystem) override {
+ Y_UNUSED(actorSystem);
+ }
+
+ void StartGlobalWorker(NActors::TActorSystem* actorSystem, const TVector<TResourceManagerOptions>& resourceUploaderOptions, IMetricsRegistryPtr metricsRegistry) override {
+ Y_UNUSED(actorSystem);
+ Y_UNUSED(resourceUploaderOptions);
+ Y_UNUSED(metricsRegistry);
+ }
+
+ void StartCleaner(NActors::TActorSystem* actorSystem, const TMaybe<TString>& role) override {
+ Y_UNUSED(actorSystem);
+ Y_UNUSED(role);
+ }
+
+ NYql::IServiceNodeResolver::TPtr CreateServiceNodeResolver(NActors::TActorSystem* actorSystem, const TVector<TString>& hostPortPairs) override {
+ Y_UNUSED(actorSystem);
+ Y_UNUSED(hostPortPairs);
+
+ return NYql::IServiceNodeResolver::TPtr();
+ }
+
+ const NProto::TDqConfig::TYtCoordinator& GetConfig() override {
+ return YtCoordinator_;
+ }
+
+ const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem) override {
+ Y_UNUSED(actorSystem);
+ return NActors::TActorId();
+ }
+
+ const NActors::TActorId GetWrapper(NActors::TActorSystem* actorSystem, const TString& cluster, const TString& user, const TString& token) override {
+ Y_UNUSED(actorSystem);
+ Y_UNUSED(cluster);
+ Y_UNUSED(user);
+ Y_UNUSED(token);
+ return NActors::TActorId();
+ }
+
+ const NActors::TActorId GetWrapper() override {
+ return NActors::TActorId();
+ }
+
+ TWorkerRuntimeData* GetRuntimeData() override {
+ return nullptr;
+ }
+
+ void Stop(NActors::TActorSystem* actorSystem) override {
+ Y_UNUSED(actorSystem);
+ }
+
+ TString GetRevision() override {
+ return TString();
+ }
+
+private:
+ ui32 NodeId;
+ NProto::TDqConfig::TYtCoordinator YtCoordinator_;
+};
+}
+
+class TGlobalWorkerManagerTest: public TTestBase {
+public:
+ UNIT_TEST_SUITE(TGlobalWorkerManagerTest);
+ UNIT_TEST(TestTasksAllocation1)
+ UNIT_TEST(TestTasksAllocation2)
+ UNIT_TEST(TestTasksAllocation3)
+ UNIT_TEST(TestTasksAllocation4)
+ UNIT_TEST(TestTasksAllocation5)
+ UNIT_TEST_SUITE_END();
+
+ void SetUp() override {
+ // Service node: 0
+ // Worker nodes: 1..nodesNumber
+ const ui32 nodesNumber = 7;
+ ActorRuntime_.Reset(new NActors::TTestActorRuntimeBase(nodesNumber, true));
+
+ // Name server.
+ TIntrusivePtr<TTableNameserverSetup> nameserverTable = new TTableNameserverSetup();
+ THashSet<ui32> staticNodeId;
+
+ for (ui32 i = 0; i < 100; i++) {
+ nameserverTable->StaticNodeTable[i] = std::make_pair(ToString(i), i);
+ }
+ ActorRuntime_->AddLocalService(GetNameserviceActorId(),
+ TActorSetupCmd(NYql::NDqs::CreateDynamicNameserver(nameserverTable), TMailboxType::Simple, 0));
+
+ // GWM
+ TVector<TResourceManagerOptions> uploadResourcesOptions;
+ const TIntrusivePtr<TTestCoordinationHelper> coordPtr = new TTestCoordinationHelper(NodeId());
+ const TIntrusivePtr<TSensorsGroup> sensorsPtr = new TSensorsGroup();
+ const auto gwmActor = CreateGlobalWorkerManager(coordPtr, uploadResourcesOptions, CreateMetricsRegistry(sensorsPtr), NProto::TDqConfig::TScheduler());
+ ActorRuntime_->AddLocalService(MakeWorkerManagerActorID(NodeId()),
+ TActorSetupCmd{gwmActor, TMailboxType::Simple, 0});
+
+ // Local WM.
+ for (ui32 i = 1; i < nodesNumber; i++) {
+ NYql::NDqs::TLocalWorkerManagerOptions lwmOptions;
+ lwmOptions.TaskRunnerInvokerFactory = new NDqs::TTaskRunnerInvokerFactory();
+ lwmOptions.TaskRunnerActorFactory = NYql::NDq::NTaskRunnerActor::CreateTaskRunnerActorFactory(
+ lwmOptions.Factory, lwmOptions.TaskRunnerInvokerFactory);
+ auto localWM = CreateLocalWorkerManager(lwmOptions);
+ ActorRuntime_->AddLocalService(MakeWorkerManagerActorID(NodeId(i)),
+ TActorSetupCmd{localWM, TMailboxType::Simple, 0}, i);
+ ActorRuntime_->AddLocalService(GetNameserviceActorId(),
+ TActorSetupCmd(NYql::NDqs::CreateDynamicNameserver(nameserverTable), TMailboxType::Simple, 0), i);
+ }
+
+ ActorRuntime_->Initialize();
+
+ NActors::TDispatchOptions options;
+ options.FinalEvents.emplace_back(NActors::TEvents::TSystem::Bootstrap, nodesNumber);
+ ActorRuntime_->DispatchEvents(options);
+
+ auto attributes = NYT::BuildYsonNodeFluently()
+ .BeginMap()
+ .Item(NCommonAttrs::ACTOR_NODEID_ATTR).Value(NodeId())
+ .Item(NCommonAttrs::HOSTNAME_ATTR).Value("localhost")
+ .Item(NCommonAttrs::GRPCPORT_ATTR).Value("1")
+ .Item(NCommonAttrs::UPLOAD_EXECUTABLE_ATTR).Value("true")
+ .EndMap();
+
+ // Init GWM.
+ auto leaderEv = MakeHolder<TEvBecomeLeader>(1, "1", NYT::NodeToYsonString(attributes));
+ const auto dummyActor = ActorRuntime_->AllocateEdgeActor();
+ ActorRuntime_->Send(new IEventHandle(MakeWorkerManagerActorID(NodeId()), dummyActor, leaderEv.Release()));
+
+ auto statusEv = MakeHolder<TEvClusterStatus>();
+ const auto statusSender = ActorRuntime_->AllocateEdgeActor();
+ ActorRuntime_->Send(new IEventHandle(MakeWorkerManagerActorID(NodeId()), statusSender, statusEv.Release()));
+ auto resp = ActorRuntime_->GrabEdgeEvent<TEvClusterStatusResponse>(statusSender);
+
+ // Fake file for GWM.
+ TFile outFile("/tmp/test", CreateAlways | ARW);
+ outFile.Close();
+ }
+
+ void TearDown() override {
+ ActorRuntime_.Reset();
+ remove("/tmp/test");
+ }
+
+ void TestTasksAllocation1() {
+ TVector<TVector<TString>> filesPerNode = {
+ {"file0", "file11"},
+ {"file0", "file22"},
+ {"file0", "file33", "file333"},
+ {"file0", "file44"},
+ {"file0", "file55", "file555"},
+ {"file0", "file66", "file6"}
+ };
+
+ TVector<TVector<TString>> filesPerTask = {
+ {"file333"} // Expect mapping on node 3.
+ };
+
+ TVector<ui32> expectedTasksMapping = {3};
+
+ CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping);
+ }
+
+ void TestTasksAllocation2() {
+ TVector<TVector<TString>> filesPerNode = {
+ {"file0", "file11"},
+ {"file0", "file22"},
+ {"file0", "file33", "file333"},
+ {"file0", "file44"},
+ {"file0", "file55", "file555"},
+ {"file0", "file66", "file6"}
+ };
+
+ TVector<TVector<TString>> filesPerTask = {
+ {"file55", "file0"}, // Expect mapping on node 5.
+ {"file0", "file11"} // Expect mapping on node 1.
+ };
+
+ TVector<ui32> expectedTasksMapping = {5, 1};
+
+ CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping);
+ }
+
+ void TestTasksAllocation3() {
+ TVector<TVector<TString>> filesPerNode = {
+ {"file0", "file11"},
+ {"file0", "file22"},
+ {"file0", "file33", "file333"},
+ {"file0", "file44"},
+ {"file0", "file55", "file555"},
+ {"file0", "file66", "file6"}
+ };
+
+ TVector<TVector<TString>> filesPerTask = {
+ {"file0", "file11"}, // Expect mapping on node 1.
+ {"file55", "file0"}, // Expect mapping on node 5.
+ {"file44", "file0"} // Expect mapping on node 4.
+ };
+
+ TVector<ui32> expectedTasksMapping = {1, 5, 4};
+
+ CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping);
+ }
+
+ void TestTasksAllocation4() {
+ TVector<TVector<TString>> filesPerNode = {
+ {"file0", "file11"},
+ {"file0", "file22"},
+ {"file0", "file33", "file333"},
+ {"file0", "file44"},
+ {"file0", "file55", "file555"},
+ {"file0", "file66", "file6"}
+ };
+
+ TVector<TVector<TString>> filesPerTask = {
+ {"file11"}, // Expect mapping on node 1.
+ {"file0", "file333", "file33"}, // Expect mapping on node 3.
+ {"file0", "file22"}, // Expect mapping on node 2.
+ {"file0", "file555"} // Expect mapping on node 5.
+ };
+
+ TVector<ui32> expectedTasksMapping = {1, 3, 2, 5};
+
+ CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping);
+ }
+
+ void TestTasksAllocation5() {
+ TVector<TVector<TString>> filesPerNode = {
+ {"file0", "file11"},
+ {"file0", "file22"},
+ {"file0", "file33", "file333"},
+ {"file0", "file44"},
+ {"file0", "file55", "file555"},
+ {"file0", "file66", "file5"}
+ };
+
+ TVector<TVector<TString>> filesPerTask = {
+ {"file0", "file5", "file66"}, // Expect mapping on node 6.
+ {"file0", "file333"}, // Expect mapping on node 3.
+ {"file0", "file11"}, // Expect mapping on node 1.
+ {"file0", "file44"}, // Expect mapping on node 4.
+ {"file0", "file55", "file555"} // Expect mapping on node 5.
+ };
+
+ TVector<ui32> expectedTasksMapping = {6, 3, 1, 4, 5};
+
+ CheckTasksAllocation(filesPerNode, filesPerTask, expectedTasksMapping);
+ }
+
+ void CheckTasksAllocation(
+ TVector<TVector<TString>>& filesPerNode,
+ TVector<TVector<TString>>& filesPerTask,
+ const TVector<ui32>& expectedTasksMapping
+ ) {
+ Y_ASSERT(expectedTasksMapping.size() == filesPerTask.size());
+ Y_ASSERT(filesPerTask.size() <= filesPerNode.size());
+
+ SetupWorkerNodes(filesPerNode);
+
+ const auto execActor = ActorRuntime_->AllocateEdgeActor();
+ const auto allocatorActor = RegisterResourceAllocator(filesPerTask.size(), execActor);
+
+ auto allocateRequest = MakeAllocationRequest(filesPerTask);
+
+ ActorRuntime_->Send(new IEventHandle(MakeWorkerManagerActorID(NodeId()), allocatorActor, allocateRequest.Release()));
+
+ auto resp = ActorRuntime_->GrabEdgeEvent<TEvAllocateWorkersResponse>(execActor);
+
+ auto allocatedWorkers = resp->Get()->Record.GetWorkers();
+
+ UNIT_ASSERT_VALUES_EQUAL(filesPerTask.size(), allocatedWorkers.WorkerSize());
+ for (ui32 task = 0; task < expectedTasksMapping.size(); ++task) {
+ ui32 expectedNode = expectedTasksMapping[task];
+ UNIT_ASSERT_VALUES_EQUAL(NodeId(expectedNode), allocatedWorkers.worker(task).GetNodeId());
+ }
+ }
+
+ TActorId RegisterResourceAllocator(const ui32 workersCount, const TActorId& execActor) const {
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+ auto gwmActor = MakeWorkerManagerActorID(NodeId());
+ auto allocator = CreateResourceAllocator(gwmActor, execActor, execActor, workersCount, "TraceId", new TDqConfiguration(), counters);
+ const auto allocatorId = ActorRuntime_->Register(allocator);
+ return allocatorId;
+ }
+
+ void SetupWorkerNodes(TVector<TVector<TString>>& filesOnNodes) const {
+ ui32 nodeId = 1;
+ for (const auto& nodeFiles : filesOnNodes) {
+ RegisterNodeInGwm(NodeId(nodeId++), nodeFiles);
+ }
+ }
+
+ THolder<TEvAllocateWorkersRequest> MakeAllocationRequest(TVector<TVector<TString>>& filesPerTask) const {
+ auto allocateRequest = MakeHolder<TEvAllocateWorkersRequest>(filesPerTask.size(), "Username");
+ allocateRequest->Record.SetTraceId("TraceId");
+
+ THashSet<TString> allFiles;
+
+ for (const auto& tf : filesPerTask) {
+ Yql::DqsProto::TWorkerFilter taskFiles;
+ for (const auto& f : tf) {
+ Yql::DqsProto::TFile file;
+ file.SetObjectId(f + "_id");
+ file.SetLocalPath("/tmp/test");
+ *taskFiles.AddFile() = file;
+ allFiles.emplace(f);
+ }
+ *allocateRequest->Record.AddWorkerFilterPerTask() = taskFiles;
+ }
+
+ for (const auto& f : allFiles) {
+ Yql::DqsProto::TFile file;
+ file.SetObjectId(f + "_id");
+ file.SetLocalPath("/tmp/test");
+ *allocateRequest->Record.AddFiles() = file;
+ }
+
+ return allocateRequest;
+ }
+
+ ui32 NodeId(ui32 id = 0) const {
+ return ActorRuntime_->GetNodeId(id);
+ }
+
+ void RegisterNodeInGwm(const ui32 nodeId, const TVector<TString>& files) const {
+ Yql::DqsProto::RegisterNodeRequest req;
+ req.SetCapabilities(Yql::DqsProto::RegisterNodeRequest::ECAP_RUNEXE);
+ req.SetNodeId(nodeId);
+ req.SetPort(1);
+ req.SetRole("worker_node");
+ req.SetAddress("Address");
+ req.SetRevision(ToString(GetProgramCommitId()));
+ req.SetRunningWorkers(1);
+ req.SetClusterName("plato");
+ req.MutableKnownNodes()->Add(NodeId());
+ req.MutableKnownNodes()->Add(nodeId);
+ req.MutableGuid()->SetDw0(1);
+ req.MutableGuid()->SetDw1(1);
+ req.MutableGuid()->SetDw2(1);
+ req.MutableGuid()->SetDw3(nodeId);
+
+ for (const auto& file : files) {
+ auto fileOnNode = req.AddFilesOnNode();
+ fileOnNode->SetObjectId(file + "_id");
+ }
+
+ auto ev = MakeHolder<TEvRegisterNode>(req);
+ const auto sender = ActorRuntime_->AllocateEdgeActor();
+ ActorRuntime_->Send(new IEventHandle(MakeWorkerManagerActorID(NodeId()), sender, ev.Release()));
+
+ ActorRuntime_->GrabEdgeEvent<TEvRegisterNodeResponse>(sender);
+ }
+
+ THolder<NActors::TTestActorRuntimeBase> ActorRuntime_;
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TGlobalWorkerManagerTest)
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp b/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp
new file mode 100644
index 0000000000..7a2d192516
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.cpp
@@ -0,0 +1,398 @@
+#include "service_node_pinger.h"
+#include <ydb/library/yql/providers/dq/worker_manager/interface/events.h>
+
+#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <library/cpp/actors/interconnect/interconnect.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+
+#include <util/generic/guid.h>
+#include <util/system/getpid.h>
+#include <util/system/fs.h>
+
+#include <ydb/library/yql/providers/dq/api/grpc/api.grpc.pb.h>
+#include <ydb/library/yql/providers/dq/runtime/runtime_data.h>
+
+#include <ydb/library/yql/providers/dq/actors/events/events.h>
+#include <ydb/library/yql/providers/dq/actors/yt/resource_manager.h>
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/execution_helpers.h>
+#include <ydb/library/yql/providers/dq/task_runner/file_cache.h>
+
+namespace NYql {
+
+using namespace NActors;
+
+struct TEvRegisterNodeResponse
+ : NActors::TEventLocal<TEvRegisterNodeResponse, TDqEvents::ES_OTHER1> {
+ TEvRegisterNodeResponse()
+ : Error(true)
+ { }
+
+ TEvRegisterNodeResponse(const Yql::DqsProto::RegisterNodeResponse& res)
+ : Error(false)
+ , Response(res)
+ { }
+
+ bool Error;
+ const Yql::DqsProto::RegisterNodeResponse Response;
+};
+
+class TServiceNodePinger: public TActor<TServiceNodePinger> {
+public:
+ static constexpr char ActorName[] = "PINGER";
+
+ TServiceNodePinger(
+ ui32 nodeId,
+ const TString& address,
+ ui16 port,
+ const TString& role,
+ const THashMap<TString, TString>& attributes,
+ const IServiceNodeResolver::TPtr& resolver,
+ const ICoordinationHelper::TPtr& coordinator,
+ const TResourceManagerOptions& options)
+ : TActor<TServiceNodePinger>(&TServiceNodePinger::Handler)
+ , NodeId(nodeId)
+ , Address(address)
+ , Port(port)
+ , Role(role)
+ , Attributes(attributes)
+ , Resolver(resolver)
+ , Revision(coordinator->GetRevision())
+ , Options(options)
+ , RuntimeData(coordinator->GetRuntimeData())
+ , Coordinator(coordinator)
+ {
+ CreateGuid(&Guid);
+
+ RuntimeData->WorkerId = Guid;
+ if (Options.AnnounceClusterName) {
+ RuntimeData->ClusterName = *Options.AnnounceClusterName;
+ } else {
+ RuntimeData->ClusterName = Options.YtBackend.GetClusterName();
+ }
+
+ if (coordinator->GetConfig().HasHeartbeatPeriodMs()) {
+ HeartbeatPeriod = TDuration::MilliSeconds(coordinator->GetConfig().GetHeartbeatPeriodMs());
+ }
+
+ YQL_CLOG(DEBUG, ProviderDq) << "Node started nodeId|role|guid " << NodeId << "|" << Role << "|" << GetGuidAsString(Guid);
+ }
+
+ ~TServiceNodePinger() {
+ Resolver->Stop();
+ }
+
+ STRICT_STFUNC(Handler, {
+ CFunc(TEvents::TEvBootstrap::EventType, Ping)
+ HFunc(TEvInterconnect::TEvNodesInfo, OnNodesInfo)
+ HFunc(TEvDownloadComplete, OnDownloadComplete)
+ HFunc(TEvRegisterNodeResponse, OnRegisterNodeResponse)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ });
+
+private:
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0);
+ }
+
+ void OnDownloadComplete(TEvDownloadComplete::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ auto id = ev->Sender;
+ auto it = DownloadProcess.find(id);
+ if (it == DownloadProcess.end()) {
+ return;
+ }
+
+ for (auto file : it->second) {
+ Downloading.erase(/*objectId = */ file.LocalFileName);
+ }
+
+ DownloadProcess.erase(it);
+ }
+
+ void OnNodesInfo(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ KnownNodes.clear();
+ KnownNodes.reserve(ev->Get()->Nodes.size());
+ for (const auto& node: ev->Get()->Nodes) {
+ KnownNodes.push_back(node.NodeId);
+ }
+ }
+
+ void SchedulePing() {
+ auto now = TInstant::Now();
+ auto delta = now - LastPingTime;
+ if (delta > HeartbeatPeriod) {
+// YQL_CLOG(DEBUG, ProviderDq) << "Ping Now";
+ Send(SelfId(), new TEvents::TEvBootstrap);
+ } else {
+ // YQL_CLOG(DEBUG, ProviderDq) << "Ping After " << (HeartbeatPeriod - delta).MilliSeconds();
+ Schedule(HeartbeatPeriod - delta, new TEvents::TEvBootstrap);
+ }
+ }
+
+ void OnRegisterNodeResponse(TEvRegisterNodeResponse::TPtr& ev, const TActorContext& ctx) {
+// YQL_CLOG(DEBUG, ProviderDq) << "Pong";
+
+ if (ev->Get()->Error) {
+ Errors += 1;
+ Oks = 0;
+ if (Options.ExitOnPingFail && Errors > 3) {
+ YQL_CLOG(DEBUG, ProviderDq) << "ExitOnPingFail";
+ _exit(-1);
+ }
+ SchedulePing();
+ return;
+ }
+
+ Oks += 1;
+ Errors = 0;
+
+ auto& resp = ev->Get()->Response;
+
+ THolder<TEvInterconnect::TEvNodesInfo>
+ reply(new TEvInterconnect::TEvNodesInfo());
+ reply->Nodes.reserve(resp.GetNodes().size());
+ if (resp.GetEpoch()) {
+ RuntimeData->Epoch = resp.GetEpoch();
+ }
+
+ for (auto& node : resp.GetNodes()) {
+ reply->Nodes.emplace_back(
+ node.GetNodeId(),
+ node.GetAddress(),
+ node.GetAddress(),
+ node.GetAddress(),
+ node.GetPort(),
+ NActors::TNodeLocation());
+ }
+
+ TVector<TResourceFile> downloadList;
+ for (auto& file : resp.GetDownloadList()) {
+ if (Downloading.contains(file.GetObjectId()) || (Options.FileCache && Options.FileCache->Contains(file.GetObjectId())))
+ {
+ continue;
+ }
+ TResourceFile resource;
+
+ switch (file.GetObjectType()) {
+ case Yql::DqsProto::TFile::EEXE_FILE:
+ resource.RemoteFileName = "bin/" + file.GetObjectId() + "/" + file.GetName();
+ break;
+ default:
+ resource.RemoteFileName = "udfs/" + file.GetObjectId();
+ break;
+ }
+
+ resource.LocalFileName = file.GetObjectId();
+
+ downloadList.push_back(resource);
+
+ YQL_CLOG(DEBUG, ProviderDq) << "Start downloading: " << file.GetObjectId();
+
+ Downloading.insert(file.GetObjectId());
+ }
+
+ if (!downloadList.empty() && Options.FileCache) {
+ auto rmOptions = Options;
+ rmOptions.Files = downloadList;
+ rmOptions.UploadPrefix = rmOptions.YtBackend.GetUploadPrefix();
+
+ YQL_CLOG(DEBUG, ProviderDq) << "Start downloading from " << rmOptions.YtBackend.GetClusterName();
+ DownloadProcess.emplace(ctx.Register(CreateResourceDownloader(rmOptions, Coordinator)), downloadList);
+ }
+
+ Send(NActors::GetNameserviceActorId(), reply.Release());
+ Send(NActors::GetNameserviceActorId(), new TEvInterconnect::TEvListNodes);
+ SchedulePing();
+ }
+
+ void Ping(const TActorContext& ctx) {
+ Yql::DqsProto::RegisterNodeRequest req;
+ req.SetCapabilities(Yql::DqsProto::RegisterNodeRequest::ECAP_RUNEXE | Options.Capabilities);
+ req.SetNodeId(NodeId);
+ req.SetPort(Port);
+ req.SetRole(Role);
+ req.SetAddress(Address);
+ req.SetRevision(Revision);
+ req.SetPid(Pid);
+ auto runningWorkers = RuntimeData->GetRunningWorkers();
+ int sum = 0;
+ for (const auto& [id, count] : runningWorkers) {
+ req.AddRunningOperation(id);
+ sum += count;
+ }
+ req.SetRunningWorkers(sum);
+ req.SetClusterName(RuntimeData->ClusterName);
+ req.SetStartTime(StartTime);
+ if (RuntimeData->Epoch) {
+ req.SetEpoch(RuntimeData->Epoch);
+ }
+ auto rusageFull = RuntimeData->GetRusage();
+ req.MutableRusage()->SetStime(rusageFull.Stime.MicroSeconds());
+ req.MutableRusage()->SetUtime(rusageFull.Utime.MicroSeconds());
+ req.MutableRusage()->SetMajorPageFaults(rusageFull.MajorPageFaults);
+
+ //if (Options.MetricsRegistry) {
+ // auto* metrics = req.MutableMetricsRegistry();
+ // metrics->SetDontIncrement(true); // don't invalidate metrics
+ // Options.MetricsRegistry->TakeSnapshot(metrics);
+ //}
+
+ if (Options.FileCache) {
+ req.SetUsedDiskSize(Options.FileCache->UsedDiskSize());
+ req.SetFreeDiskSize(Options.FileCache->FreeDiskSize());
+ }
+ for (const auto& [k, v] : Attributes) {
+ auto* attr = req.AddAttribute();
+ attr->SetKey(k);
+ attr->SetValue(v);
+ }
+
+ NDqs::NExecutionHelpers::GuidToProto(*req.MutableGuid(), Guid);
+ req.SetCapacity(Options.YtBackend.GetWorkerCapacity());
+
+ if (Options.FileCache) {
+ Options.FileCache->Walk([&] (const TString& objectId) {
+ req.AddFilesOnNode()->SetObjectId(objectId);
+ });
+ }
+
+ // self exe must be at the end of list
+ req.AddFilesOnNode()->SetObjectId(Revision);
+
+ for (auto node : KnownNodes) {
+ req.AddKnownNodes(node);
+ }
+
+ auto* actorSystem = ctx.ExecutorThread.ActorSystem;
+ auto selfId = SelfId();
+
+ Resolver->GetConnection()
+ .Apply([actorSystem, selfId, req, maybeResolver=std::weak_ptr<IServiceNodeResolver>(Resolver), timeout=HeartbeatPeriod, lastPingTime=LastPingTime, errors=Errors, oks=Oks] (const NThreading::TFuture<IServiceNodeResolver::TConnectionResult>& resultFuture) {
+ const auto& result = resultFuture.GetValueSync();
+ if (!result.Success()) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Cannot resolve service node";
+ actorSystem->Send(selfId, new TEvRegisterNodeResponse());
+ return;
+ }
+
+ if (!maybeResolver.lock()) {
+ return;
+ }
+
+ if (!lastPingTime || errors > 1 || oks < 2 || !result.NodeId) {
+ // GRPC ping
+ NGrpc::TCallMeta meta;
+ meta.Timeout = timeout;
+ result.Connection->DoRequest<Yql::DqsProto::RegisterNodeRequest, Yql::DqsProto::RegisterNodeResponse>(
+ req, [=] (NGrpc::TGrpcStatus&& status, Yql::DqsProto::RegisterNodeResponse&& resp) {
+ if (!status.Ok()) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Error on service node ping " << status.Msg;
+ if (auto resolver = maybeResolver.lock()) {
+ resolver->InvalidateCache();
+ }
+ Y_ABORT_UNLESS(status.GRpcStatusCode != grpc::INVALID_ARGUMENT);
+ actorSystem->Send(selfId, new TEvRegisterNodeResponse());
+ return;
+ }
+
+ actorSystem->Send(selfId, new TEvRegisterNodeResponse(resp));
+ }, &Yql::DqsProto::DqService::Stub::AsyncRegisterNode, meta, result.GRpcContext.get());
+ } else {
+ // IC Ping
+ YQL_CLOG(DEBUG, ProviderDq) << "IC Ping " << result.NodeId;
+ auto ev = MakeHolder<NDqs::TEvRegisterNode>(req);
+ using ResultEv = NDqs::TEvRegisterNodeResponse;
+
+ // Handle errors and timeouts
+ auto callback = MakeHolder<TRichActorFutureCallback<ResultEv>>(
+ [actorSystem, selfId] (TAutoPtr<TEventHandle<ResultEv>>& event) mutable {
+ actorSystem->Send(selfId, new TEvRegisterNodeResponse(event->Get()->Record.GetResponse()));
+ },
+ [actorSystem, selfId, maybeResolver] () mutable {
+ YQL_CLOG(DEBUG, ProviderDq) << "Error on service node ping";
+ if (auto resolver = maybeResolver.lock()) {
+ resolver->InvalidateCache();
+ }
+ actorSystem->Send(selfId, new TEvRegisterNodeResponse());
+ },
+ timeout);
+
+ TActorId callbackId = actorSystem->Register(callback.Release());
+
+ actorSystem->Send(new IEventHandle(
+ NDqs::MakeWorkerManagerActorID(result.NodeId),
+ callbackId,
+ ev.Release(),
+ IEventHandle::FlagTrackDelivery));
+ }
+ });
+
+ LastPingTime = TInstant::Now();
+
+ CheckFs();
+ }
+
+ void CheckFs() {
+ if (Options.DieOnFileAbsence.empty()) {
+ return;
+ }
+
+ auto now = TInstant::Now();
+
+ if (now - LastCheckTime < TDuration::Seconds(10)) {
+ return;
+ }
+
+ Y_ABORT_UNLESS(NFs::Exists(Options.DieOnFileAbsence));
+
+ LastCheckTime = now;
+ }
+
+private:
+ const ui32 NodeId;
+ const TString Address;
+ const ui16 Port;
+ const TString Role;
+ const THashMap<TString, TString> Attributes;
+ const IServiceNodeResolver::TPtr Resolver;
+ TVector<ui32> KnownNodes;
+ const TString Revision;
+ const TString StartTime = ToString(TInstant::Now());
+ const TProcessId Pid = GetPID();
+
+ TResourceManagerOptions Options;
+ TWorkerRuntimeData* RuntimeData;
+ const ICoordinationHelper::TPtr Coordinator;
+
+ THashSet<TString> Downloading;
+ THashMap<TActorId, TVector<TResourceFile>> DownloadProcess;
+
+ TDuration HeartbeatPeriod = TDuration::Seconds(5);
+
+ TGUID Guid;
+ TInstant LastPingTime;
+ TInstant LastCheckTime;
+ int Errors = 0;
+ int Oks = 0;
+};
+
+IActor* CreateServiceNodePinger(
+ ui32 nodeId,
+ const TString& address,
+ ui16 port,
+ const TString& role,
+ const THashMap<TString, TString>& attributes,
+ const IServiceNodeResolver::TPtr& ptr,
+ const ICoordinationHelper::TPtr& coordinator,
+ const TResourceManagerOptions& rmOptions)
+{
+ return new TServiceNodePinger(nodeId, address, port, role, attributes, ptr, coordinator, rmOptions);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.h b/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.h
new file mode 100644
index 0000000000..4a7efd1209
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_pinger.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include <library/cpp/actors/core/actor.h>
+
+#include "service_node_resolver.h"
+#include "coordination_helper.h"
+
+namespace NYql {
+
+struct TWorkerRuntimeData;
+struct TResourceManagerOptions;
+
+NActors::IActor* CreateServiceNodePinger(
+ ui32 nodeId,
+ const TString& address,
+ ui16 port,
+ const TString& role,
+ const THashMap<TString, TString>& attributes,
+ const IServiceNodeResolver::TPtr& ptr,
+ const ICoordinationHelper::TPtr& coordinator,
+ const TResourceManagerOptions& rmOptions);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp
new file mode 100644
index 0000000000..744abcdf68
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp
@@ -0,0 +1,303 @@
+#include "service_node_resolver.h"
+
+#include <library/cpp/actors/core/actorsystem.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/yson/node/node_io.h>
+
+#include <ydb/library/yql/utils/log/log.h>
+#include <ydb/library/yql/utils/yql_panic.h>
+
+#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
+#include <ydb/library/yql/providers/dq/actors/yt/yt_wrapper.h>
+#include <ydb/library/yql/providers/dq/actors/yt/nodeid_assigner.h>
+
+#include <util/system/mutex.h>
+#include <util/random/random.h>
+
+namespace NYql {
+
+using namespace NThreading;
+using namespace NActors;
+
+struct TNodeInfo {
+ NGrpc::TGRpcClientConfig ClientConfig;
+ ui32 NodeId;
+};
+
+class TNodesHolder {
+public:
+ TMaybe<TNodeInfo> GetNode() {
+ TGuard<TMutex> guard(Mutex);
+ if (Nodes.empty()) {
+ return {};
+ } else {
+ auto index = RandomNumber(Min<ui32>(3, Nodes.size()));
+ return Nodes[index];
+ }
+ }
+
+ void Swap(TVector<TNodeInfo>& nodes)
+ {
+ TGuard<TMutex> guard(Mutex);
+ Nodes.swap(nodes);
+ }
+
+private:
+ TMutex Mutex;
+ TVector<TNodeInfo> Nodes;
+};
+
+class TAbstractNodeResolver: public IServiceNodeResolver {
+public:
+ TAbstractNodeResolver()
+ : ClientLow(1)
+ , ChannelPool(NGrpc::TTcpKeepAliveSettings{true, 30, 5, 10})
+ { }
+
+ ~TAbstractNodeResolver()
+ {
+ Stop();
+ }
+
+ virtual TMaybe<TNodeInfo> GetNode() = 0;
+
+ TFuture<TConnectionResult> GetConnection() override {
+ auto nodeInfo = GetNode();
+
+ if (!nodeInfo) {
+ return MakeFuture(NCommon::ResultFromError<TConnectionResult>("Unable to get node"));
+ }
+
+ auto context = ClientLow.CreateContext();
+ if (!context) {
+ return MakeFuture(NCommon::ResultFromError<TConnectionResult>("Unable to create grpc request context"));
+ }
+
+ std::unique_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>> conn;
+ ChannelPool.GetStubsHolderLocked(
+ nodeInfo->ClientConfig.Locator, nodeInfo->ClientConfig, [&conn, this](NGrpc::TStubsHolder& holder) mutable {
+ conn.reset(ClientLow.CreateGRpcServiceConnection<Yql::DqsProto::DqService>(holder).release());
+ });
+
+ return MakeFuture(TConnectionResult(std::move(conn), std::move(context), nodeInfo->NodeId, nodeInfo->ClientConfig.Locator));
+ }
+
+ void Stop() override {
+ ClientLow.Stop(true);
+ }
+
+protected:
+ NGrpc::TGRpcClientLow ClientLow;
+ NGrpc::TChannelPool ChannelPool;
+};
+
+class TNodeUpdater: public TActor<TNodeUpdater> {
+public:
+ static constexpr char ActorName[] = "NODE_UPDATER";
+
+ TNodeUpdater(const std::shared_ptr<TNodesHolder>& holder, const TDynamicResolverOptions& options)
+ : TActor(&TNodeUpdater::Handler)
+ , Holder(holder)
+ , Options(options)
+ , LastUpdateTime(TInstant::Now())
+ { }
+
+ TEvListNode* ListNodeCommand() {
+ NYT::NApi::TListNodeOptions options;
+ options.Attributes = {
+ NCommonAttrs::ACTOR_NODEID_ATTR,
+ NCommonAttrs::ROLE_ATTR,
+ NCommonAttrs::HOSTNAME_ATTR,
+ NCommonAttrs::GRPCPORT_ATTR,
+ "modification_time"
+ };
+ if (InvalidateCache) {
+ InvalidateCache = false;
+ } else {
+ options.ReadFrom = NYT::NApi::EMasterChannelKind::Cache;
+ }
+ return new TEvListNode(Options.Prefix, options);
+ }
+
+ TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override {
+ Y_UNUSED(parentId);
+ return new IEventHandle(Options.YtWrapper, self, ListNodeCommand(), 0);
+ }
+
+ STRICT_STFUNC(Handler, {
+ HFunc(TEvListNodeResponse, OnListNodeResponse)
+ cFunc(NActors::TEvents::TEvPoison::EventType, PassAway)
+ cFunc(NActors::TEvents::TEvWakeup::EventType, [&] () {
+ InvalidateCache = true;
+ })
+ });
+
+ void ProcessResult(const TVector<NYT::TNode>& nodes)
+ {
+ TVector<std::tuple<TInstant, TString, ui32>> hostPort;
+ hostPort.reserve(nodes.size());
+ for (const auto& node : nodes) {
+ const auto& attributes = node.GetAttributes().AsMap();
+ auto maybeRole = attributes.find(NCommonAttrs::ROLE_ATTR);
+ auto maybeHostname = attributes.find(NCommonAttrs::HOSTNAME_ATTR);
+ auto maybeGrpcPort = attributes.find(NCommonAttrs::GRPCPORT_ATTR);
+ auto maybeNodeId = attributes.find(NCommonAttrs::ACTOR_NODEID_ATTR);
+
+ auto maybeModificationTime = attributes.find("modification_time");
+ YQL_ENSURE(maybeModificationTime != attributes.end());
+
+ if (maybeRole == attributes.end()
+ || maybeHostname == attributes.end()
+ || maybeGrpcPort == attributes.end()
+ || maybeNodeId == attributes.end())
+ {
+ continue;
+ }
+
+ if (maybeRole->second != "service_node") {
+ continue;
+ }
+
+ ui32 nodeId = maybeNodeId->second.AsUint64();
+
+ auto modificationTime = TInstant::ParseIso8601(maybeModificationTime->second.AsString());
+
+ hostPort.emplace_back(modificationTime, maybeHostname->second.AsString() + ":" + maybeGrpcPort->second.AsString(), nodeId);
+ }
+
+ std::sort(hostPort.begin(), hostPort.end());
+
+ TVector<TNodeInfo> locations;
+ locations.reserve(hostPort.size());
+ for (auto it = hostPort.rbegin(); it != hostPort.rend(); ++it) {
+ locations.push_back({NGrpc::TGRpcClientConfig(std::get<1>(*it), TDuration::Seconds(15)), std::get<2>(*it)});
+ }
+
+ if (locations.empty()) {
+ YQL_CLOG(WARN, ProviderDq) << "Empty locations";
+ }
+
+ Holder->Swap(locations);
+ }
+
+ void OnListNodeResponse(TEvListNodeResponse::TPtr& ev, const NActors::TActorContext& ctx) {
+ Y_UNUSED(ctx);
+
+ auto now = TInstant::Now();
+
+ auto result = std::get<0>(*ev->Get());
+
+ try {
+ ProcessResult(NYT::NodeFromYsonString(result.ValueOrThrow()).AsList());
+ } catch (...) {
+ YQL_CLOG(ERROR, ProviderDq) << "Error on list node '" << ToString(result) << "' " << CurrentExceptionMessage();
+ }
+
+ TActivationContext::Schedule(Options.UpdatePeriod, new IEventHandle(Options.YtWrapper, SelfId(), ListNodeCommand(), 0));
+
+ LastUpdateTime = now;
+ }
+
+private:
+ std::shared_ptr<TNodesHolder> Holder;
+ TDynamicResolverOptions Options;
+ TInstant LastUpdateTime;
+ bool InvalidateCache = false;
+};
+
+class TDynamicResolver: public TAbstractNodeResolver {
+public:
+ TDynamicResolver(NActors::TActorSystem* actorSystem, const TDynamicResolverOptions& options)
+ : Nodes(new TNodesHolder)
+ , ActorSystem(actorSystem)
+ , Options(options)
+ , NodeUpdater(ActorSystem->Register(new TNodeUpdater(Nodes, Options)))
+ { }
+
+ ~TDynamicResolver()
+ { }
+
+ TMaybe<TNodeInfo> GetNode() override {
+ return Nodes->GetNode();
+ }
+
+ void InvalidateCache() override {
+ ActorSystem->Send(NodeUpdater, new TEvents::TEvWakeup);
+ }
+
+private:
+ std::shared_ptr<TNodesHolder> Nodes;
+
+ NActors::TActorSystem* ActorSystem;
+ TDynamicResolverOptions Options;
+ NActors::TActorId NodeUpdater;
+};
+
+class TStaticResolver: public TAbstractNodeResolver {
+public:
+ TStaticResolver(const TVector<TString>& hostPortPairs)
+ {
+ Nodes.reserve(hostPortPairs.size());
+ for (const auto& hostPort : hostPortPairs) {
+ Nodes.emplace_back(hostPort, TDuration::Seconds(1));
+ }
+ }
+
+ TMaybe<TNodeInfo> GetNode() override {
+ auto clientConfig = NGrpc::TGRpcClientConfig{Nodes[CurrentNodeIndex]};
+ CurrentNodeIndex = (CurrentNodeIndex + 1) % Nodes.size();
+
+ return TNodeInfo{clientConfig, 0};
+ }
+
+ void InvalidateCache() override {
+ }
+
+private:
+ TVector<NGrpc::TGRpcClientConfig> Nodes;
+ int CurrentNodeIndex = 0;
+};
+
+TSingleNodeResolver::TSingleNodeResolver()
+ : ClientLow(1)
+ , ChannelPool(NGrpc::TTcpKeepAliveSettings{true, 30, 5, 10})
+{ }
+
+TSingleNodeResolver::~TSingleNodeResolver()
+{
+ ClientLow.Stop(true);
+}
+
+TFuture<IServiceNodeResolver::TConnectionResult> TSingleNodeResolver::GetConnection() {
+ auto clientConfig = NGrpc::TGRpcClientConfig(LeaderHostPort);
+
+ auto context = ClientLow.CreateContext();
+ if (!context) {
+ return MakeFuture(NCommon::ResultFromError<IServiceNodeResolver::TConnectionResult>("Unable to create grpc request context"));
+ }
+
+ std::unique_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>> conn;
+ ChannelPool.GetStubsHolderLocked(
+ clientConfig.Locator, clientConfig, [&conn, this](NGrpc::TStubsHolder& holder) mutable {
+ conn.reset(ClientLow.CreateGRpcServiceConnection<Yql::DqsProto::DqService>(holder).release());
+ });
+
+ return MakeFuture(IServiceNodeResolver::TConnectionResult(std::move(conn), std::move(context)));
+}
+
+void TSingleNodeResolver::SetLeaderHostPort(const TString& leaderHostPort) {
+ LeaderHostPort = leaderHostPort;
+}
+
+IServiceNodeResolver::TPtr CreateStaticResolver(const TVector<TString>& hostPortPairs) {
+ return std::make_shared<TStaticResolver>(hostPortPairs);
+}
+
+IServiceNodeResolver::TPtr CreateDynamicResolver(NActors::TActorSystem* actorSystem, const TDynamicResolverOptions& options) {
+ Y_ABORT_UNLESS(options.YtWrapper);
+ Y_ABORT_UNLESS(!options.Prefix.empty());
+
+ return std::make_shared<TDynamicResolver>(actorSystem, options);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h
new file mode 100644
index 0000000000..c2647904b1
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h
@@ -0,0 +1,80 @@
+#pragma once
+
+#include <ydb/library/yql/providers/common/gateway/yql_provider_gateway.h>
+#include <ydb/library/yql/providers/dq/api/grpc/api.grpc.pb.h>
+
+#include <util/generic/string.h>
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/threading/future/future.h>
+#include <library/cpp/grpc/client/grpc_client_low.h>
+
+namespace NYql {
+
+class IServiceNodeResolver {
+public:
+ using TPtr = std::shared_ptr<IServiceNodeResolver>;
+
+ struct TConnectionResult : public NCommon::TOperationResult {
+
+ TConnectionResult() {}
+
+ TConnectionResult(
+ std::unique_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>>&& connection,
+ std::shared_ptr<NGrpc::IQueueClientContext>&& ctx = nullptr,
+ ui32 nodeId = 0,
+ const TString& location = "")
+ : Connection(connection.release())
+ , GRpcContext(std::move(ctx))
+ , NodeId(nodeId)
+ , Location(location)
+ {
+ if (GRpcContext) {
+ SetSuccess();
+ }
+ }
+
+ std::shared_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>> Connection;
+ std::shared_ptr<NGrpc::IQueueClientContext> GRpcContext;
+ ui32 NodeId;
+ TString Location;
+ };
+
+ virtual ~IServiceNodeResolver() = default;
+ virtual NThreading::TFuture<TConnectionResult> GetConnection() = 0;
+ virtual void InvalidateCache() = 0;
+ virtual void Stop() = 0;
+};
+
+struct TDynamicResolverOptions {
+ NActors::TActorId YtWrapper;
+ TString Prefix;
+ TDuration UpdatePeriod = TDuration::Seconds(5);
+ TDuration RetryPeriod = TDuration::Seconds(10);
+};
+
+class TSingleNodeResolver: public IServiceNodeResolver {
+public:
+ TSingleNodeResolver();
+
+ ~TSingleNodeResolver();
+
+ NThreading::TFuture<IServiceNodeResolver::TConnectionResult> GetConnection() override;
+
+ void SetLeaderHostPort(const TString& leaderHostPort);
+
+ void InvalidateCache() override { }
+
+ void Stop() override { }
+
+private:
+ NGrpc::TGRpcClientLow ClientLow;
+ NGrpc::TChannelPool ChannelPool;
+
+ TString LeaderHostPort;
+};
+
+IServiceNodeResolver::TPtr CreateStaticResolver(const TVector<TString>& hostPortPairs);
+IServiceNodeResolver::TPtr CreateDynamicResolver(NActors::TActorSystem* actorSystem, const TDynamicResolverOptions& options);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..d2252b933a
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,77 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(yql-providers-dq-global_worker_manager-ut)
+target_compile_options(yql-providers-dq-global_worker_manager-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(yql-providers-dq-global_worker_manager-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager
+)
+target_link_libraries(yql-providers-dq-global_worker_manager-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ providers-dq-global_worker_manager
+ cpp-actors-testlib
+ udf-service-stub
+ yql-sql-pg_dummy
+ dq-actors-yt
+ providers-dq-actors
+ dq-actors-compute
+)
+target_link_options(yql-providers-dq-global_worker_manager-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(yql-providers-dq-global_worker_manager-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp
+)
+set_property(
+ TARGET
+ yql-providers-dq-global_worker_manager-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ yql-providers-dq-global_worker_manager-ut
+ TEST_TARGET
+ yql-providers-dq-global_worker_manager-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ yql-providers-dq-global_worker_manager-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ yql-providers-dq-global_worker_manager-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(yql-providers-dq-global_worker_manager-ut
+ system_allocator
+)
+vcs_info(yql-providers-dq-global_worker_manager-ut)
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..5914005f21
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,81 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(yql-providers-dq-global_worker_manager-ut)
+target_compile_options(yql-providers-dq-global_worker_manager-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(yql-providers-dq-global_worker_manager-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager
+)
+target_link_libraries(yql-providers-dq-global_worker_manager-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ providers-dq-global_worker_manager
+ cpp-actors-testlib
+ udf-service-stub
+ yql-sql-pg_dummy
+ dq-actors-yt
+ providers-dq-actors
+ dq-actors-compute
+)
+target_link_options(yql-providers-dq-global_worker_manager-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+ -lutil
+)
+target_sources(yql-providers-dq-global_worker_manager-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp
+)
+set_property(
+ TARGET
+ yql-providers-dq-global_worker_manager-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ yql-providers-dq-global_worker_manager-ut
+ TEST_TARGET
+ yql-providers-dq-global_worker_manager-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ yql-providers-dq-global_worker_manager-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ yql-providers-dq-global_worker_manager-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(yql-providers-dq-global_worker_manager-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(yql-providers-dq-global_worker_manager-ut)
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..7a7c261c19
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,83 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(yql-providers-dq-global_worker_manager-ut)
+target_compile_options(yql-providers-dq-global_worker_manager-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(yql-providers-dq-global_worker_manager-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager
+)
+target_link_libraries(yql-providers-dq-global_worker_manager-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ providers-dq-global_worker_manager
+ cpp-actors-testlib
+ udf-service-stub
+ yql-sql-pg_dummy
+ dq-actors-yt
+ providers-dq-actors
+ dq-actors-compute
+)
+target_link_options(yql-providers-dq-global_worker_manager-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+ -lutil
+)
+target_sources(yql-providers-dq-global_worker_manager-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp
+)
+set_property(
+ TARGET
+ yql-providers-dq-global_worker_manager-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ yql-providers-dq-global_worker_manager-ut
+ TEST_TARGET
+ yql-providers-dq-global_worker_manager-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ yql-providers-dq-global_worker_manager-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ yql-providers-dq-global_worker_manager-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(yql-providers-dq-global_worker_manager-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(yql-providers-dq-global_worker_manager-ut)
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..a2c865ae35
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,14 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(dq-global_worker_manager-ut INTERFACE)
+target_link_libraries(dq-global_worker_manager-ut INTERFACE
+ contrib-libs-cxxsupp
+ yutil
+)
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ut/ya.make b/ydb/library/yql/providers/dq/global_worker_manager/ut/ya.make
new file mode 100644
index 0000000000..b67eba48b8
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/ut/ya.make
@@ -0,0 +1,29 @@
+IF (NOT OS_WINDOWS)
+ UNITTEST_FOR(ydb/library/yql/providers/dq/global_worker_manager)
+
+ SIZE(SMALL)
+
+ PEERDIR(
+ library/cpp/actors/testlib
+ ydb/library/yql/public/udf/service/stub
+ ydb/library/yql/sql/pg_dummy
+ ydb/library/yql/providers/dq/actors/yt
+ ydb/library/yql/providers/dq/actors
+ ydb/library/yql/dq/actors/compute
+ )
+
+ SRCS(
+ global_worker_manager_ut.cpp
+ workers_storage_ut.cpp
+ )
+
+ INCLUDE(${ARCADIA_ROOT}/ydb/tests/supp/ubsan_supp.inc)
+
+ YQL_LAST_ABI_VERSION()
+
+ END()
+ELSE()
+ LIBRARY()
+
+ END()
+ENDIF()
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp b/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp
new file mode 100644
index 0000000000..8b5f443c63
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.cpp
@@ -0,0 +1,70 @@
+#include "worker_filter.h"
+
+namespace NYql {
+
+using namespace NDqs;
+
+TWorkerFilter::TWorkerFilter(const Yql::DqsProto::TWorkerFilter& filter)
+ : Filter(filter)
+ , FullMatch(
+ Filter.GetClusterName()
+ || (Filter.GetAddress().size() > 0)
+ || (Filter.GetNodeId().size() > 0)
+ || Filter.GetRevision())
+{
+ for (const auto& address : Filter.GetAddress()) {
+ Addresses.insert(address);
+ }
+ for (const auto& nodeId : Filter.GetNodeId()) {
+ NodeIds.insert(nodeId);
+ }
+ for (const auto& nodeId : Filter.GetNodeIdHint()) {
+ NodeIdHints.insert(nodeId);
+ }
+}
+
+TWorkerFilter::EMatchStatus TWorkerFilter::Match(const TWorkerInfo::TPtr& workerInfo, int taskId, TStats* stats) const {
+ bool allExists = true;
+ bool partial = false;
+ if (FullMatch) {
+ if (Filter.GetClusterName() && workerInfo->ClusterName != Filter.GetClusterName()) {
+ return EFAIL;
+ }
+ if (!Addresses.empty() && Addresses.find(workerInfo->Address) == Addresses.end()) {
+ return EFAIL;
+ }
+ if (!NodeIds.empty() && NodeIds.find(workerInfo->NodeId) == NodeIds.end()) {
+ return EFAIL;
+ }
+ }
+ if (Filter.GetClusterNameHint() && workerInfo->ClusterName != Filter.GetClusterNameHint()) {
+ partial = true;
+ }
+ if (!NodeIdHints.empty() && NodeIdHints.find(workerInfo->NodeId) == NodeIdHints.end()) {
+ partial = true;
+ }
+ for (const auto& file : Filter.GetFile()) {
+ const auto& id = file.GetObjectId();
+ auto flag = workerInfo->GetResources().contains(id);
+ allExists &= flag;
+ if (stats) {
+ if (flag) {
+ (*stats->WaitingResources)[id].insert(taskId);
+ } else {
+ (*stats->WaitingResources)[id].erase(taskId);
+ stats->Uploaded->find(id)->second.TryCount ++;
+ }
+ }
+ }
+ return allExists
+ ? (partial?EPARTIAL:EOK)
+ : EFAIL;
+}
+
+void TWorkerFilter::Visit(const std::function<void(const Yql::DqsProto::TFile&)>& visitor) const {
+ for (const auto& file : Filter.GetFile()) {
+ visitor(file);
+ }
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.h b/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.h
new file mode 100644
index 0000000000..8e938e72d4
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/worker_filter.h
@@ -0,0 +1,54 @@
+#pragma once
+
+#include <ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h>
+
+namespace NYql {
+
+struct TResourceStat {
+ TString Id;
+ Yql::DqsProto::TFile::EFileType ObjectType;
+ TString Name;
+ i64 Size;
+ TDuration UseTime;
+ TDuration WaitTime;
+ i64 UseCount = 0;
+ i64 TryCount = 0;
+ bool Uploaded = false;
+ TInstant LastSeenTime = TInstant::Now();
+
+ TResourceStat(const TString& id, Yql::DqsProto::TFile::EFileType type, const TString& name, i64 size)
+ : Id(id)
+ , ObjectType(type)
+ , Name(name)
+ , Size(size)
+ { }
+};
+
+class TWorkerFilter {
+public:
+ struct TStats {
+ THashMap<TString, THashSet<int>>* WaitingResources;
+ THashMap<TString, TResourceStat>* Uploaded;
+ };
+
+ enum EMatchStatus {
+ EFAIL = 0,
+ EOK = 1,
+ EPARTIAL = 2
+ };
+
+ TWorkerFilter(const Yql::DqsProto::TWorkerFilter& filter);
+
+ EMatchStatus Match(const NDqs::TWorkerInfo::TPtr& workerInfo, int taskId, TStats* stats) const;
+
+ void Visit(const std::function<void(const Yql::DqsProto::TFile&)>& visitor) const;
+
+private:
+ Yql::DqsProto::TWorkerFilter Filter;
+ bool FullMatch;
+ THashSet<TString> Addresses;
+ THashSet<ui64> NodeIds;
+ THashSet<ui64> NodeIdHints;
+};
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp
new file mode 100644
index 0000000000..f2f0206b93
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.cpp
@@ -0,0 +1,580 @@
+#include "workers_storage.h"
+
+#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <util/generic/scope.h>
+
+namespace NYql {
+
+using namespace NDqs;
+using namespace NMonitoring;
+
+using EFileType = Yql::DqsProto::TFile::EFileType;
+using TFileResource = Yql::DqsProto::TFile;
+
+TWorkersStorage::TWorkersStorage(ui32 nodeId, TSensorsGroupPtr metrics, TSensorsGroupPtr workerMetrics)
+ : NodeId(nodeId)
+ , Metrics(std::move(metrics))
+ , WorkerMetrics(std::move(workerMetrics))
+ , ResourceCounters(Metrics->GetSubgroup("component", "ResourceCounters"))
+ , ResourceDownloadCounters(Metrics->GetSubgroup("component", "ResourceDownloadCounters"))
+ , ResourceWaitTime(Metrics
+ ->GetSubgroup("component", "Resource")
+ ->GetHistogram("WaitTime", ExponentialHistogram(10, 2, 1000)))
+ , WorkersSize(nullptr)
+ , FreeListSize(nullptr)
+ , GlobalResources(Metrics)
+{
+ NodeIds.resize(65536);
+}
+
+void TWorkersStorage::Clear()
+{
+ Workers.clear();
+ FreeList.clear();
+ WorkersSize = nullptr;
+ FreeListSize = nullptr;
+ Metrics->RemoveSubgroup("component", "lists");
+}
+
+TList<TWorkerInfo::TPtr> TWorkersStorage::GetList() {
+ TList<TWorkerInfo::TPtr> workers;
+
+ for (const auto& [_, workerInfo] : Workers) {
+ workers.push_back(workerInfo);
+ }
+
+ return workers;
+}
+
+std::tuple<TWorkerInfo::TPtr, bool> TWorkersStorage::CreateOrUpdate(ui32 nodeId, TGUID workerId, Yql::DqsProto::RegisterNodeRequest& request) {
+ auto& knownWorkers = request.GetKnownNodes();
+ bool needResume = false;
+ auto maybeWorker = Workers.find(workerId);
+ TWorkerInfo::TPtr workerInfo;
+
+ CheckZombie(nodeId, workerId, request);
+
+ if (maybeWorker == Workers.end()) {
+ for (auto node : knownWorkers) {
+ if (node == NodeId) {
+ YQL_CLOG(DEBUG, ProviderDq)
+ << "Resume " << GetGuidAsString(workerId)
+ << " " << request.GetRevision()
+ << " " << request.GetRunningWorkers()
+ << " " << request.GetEpoch();
+
+ TInflightLimiter::TPtr limiter = InflightLimiters[request.GetClusterName()];
+ if (!limiter) {
+ limiter = new NDqs::TInflightLimiter(MaxDownloadsPerFile);
+ InflightLimiters[request.GetClusterName()] = limiter;
+ }
+
+ workerInfo = MakeIntrusive<TWorkerInfo>(StartTime, nodeId, workerId, request, WorkerMetrics, limiter, GlobalResources);
+ Workers[workerId] = workerInfo;
+ FreeList.insert(workerInfo);
+ needResume = true;
+ break;
+ }
+ }
+ } else {
+ workerInfo = maybeWorker->second;
+ FreeList.erase(workerInfo);
+ needResume = workerInfo->Update(request);
+ if (workerInfo->RunningRequests < workerInfo->Capacity) {
+ FreeList.insert(workerInfo);
+ }
+ }
+
+ return std::make_tuple(workerInfo, needResume);
+}
+
+void TWorkersStorage::CheckZombie(ui32 nodeId, TGUID workerId, Yql::DqsProto::RegisterNodeRequest& request) {
+ Y_ABORT_UNLESS(nodeId < NodeIds.size());
+
+ if (request.GetStartTime().empty()) {
+ request.SetStartTime(StartTime);
+ }
+
+ if (!NodeIds[nodeId].first) {
+ NodeIds[nodeId] = std::make_pair(workerId, request.GetStartTime());
+ return;
+ }
+ if (NodeIds[nodeId].first != workerId) {
+ auto curStartTime = TInstant::ParseIso8601(request.GetStartTime());
+ auto prevStartTime = TInstant::ParseIso8601(NodeIds[nodeId].second);
+
+ if (prevStartTime < curStartTime) {
+ // replace old nodeId
+ YQL_CLOG(DEBUG, ProviderDq) << "Zombie worker " << GetGuidAsString(NodeIds[nodeId].first) << " " << nodeId;
+ NodeIds[nodeId] = std::make_pair(workerId, request.GetStartTime());
+ } else {
+ YQL_CLOG(DEBUG, ProviderDq) << "Zombie worker " << GetGuidAsString(workerId) << " " << nodeId;
+ request.SetZombie(true);
+ }
+ }
+}
+
+void TWorkersStorage::CleanUp(TInstant now, TDuration duration) {
+ TVector<TGUID> deadWorkers;
+ THashMap<TString, TWorkerInfo::TFileResource> downloadList;
+ for (const auto& [k, v] : Workers) {
+ if (now - v->LastPingTime > duration) {
+ FreeList.erase(v);
+
+ deadWorkers.push_back(k);
+ downloadList.insert(v->GetDownloadList().begin(), v->GetDownloadList().end());
+ v->OnDead();
+ }
+ }
+
+ for (auto workerId : deadWorkers) {
+ YQL_CLOG(WARN, ProviderDq) << "Remove dead worker " << GetGuidAsString(workerId);
+ Workers.erase(workerId);
+ }
+
+ // pass dead download list to live workers
+ auto it = Workers.begin();
+ for (ui32 i = 0; i < deadWorkers.size() && it != Workers.end(); ++i, ++it) {
+ it->second->AddToDownloadList(downloadList);
+ }
+
+ THashSet<TString> dropOldUploaded;
+ for (const auto& [k, v] : Uploaded) {
+ if (now - v.LastSeenTime > TDuration::Hours(1)) {
+ dropOldUploaded.insert(k);
+ }
+ }
+ for (const auto& k : dropOldUploaded) {
+ Uploaded.erase(k);
+ }
+}
+
+void TWorkersStorage::DropWorker(TInstant now, TGUID workerId) {
+ Y_UNUSED(now);
+ auto it = Workers.find(workerId);
+ if (it != Workers.end()) {
+ FreeList.erase(it->second);
+ it->second->OnDead();
+ Workers.erase(it);
+ }
+}
+
+void TWorkersStorage::FreeWorker(TInstant now, const TWorkerInfo::TPtr& worker) {
+ Y_UNUSED(now);
+
+ FreeList.erase(worker);
+ if (worker->Release()) {
+ FreeList.insert(worker);
+ }
+}
+
+TWorkersStorage::TFreeList::iterator TWorkersStorage::ProcessMatched(
+ TWorkersStorage::TFreeList::iterator it,
+ TVector<NDqs::TWorkerInfo::TPtr>& result,
+ TFreeList* freeList,
+ THashMap<TWorkerInfo::TPtr, int> tasksPerWorker,
+ THashMap<i32, TWorkerFilter>& tasksToAllocate,
+ int taskId)
+{
+ auto workerInfo = *it;
+
+ it = freeList->erase(it);
+ if (!workerInfo->Acquire()) {
+ freeList->insert(workerInfo);
+ it = freeList->begin();
+ }
+
+ tasksPerWorker[workerInfo] ++;
+
+ Y_ASSERT(result[taskId] == nullptr);
+ result[taskId] = workerInfo;
+ tasksToAllocate.erase(taskId);
+
+ return it;
+}
+
+void TWorkersStorage::SearchFreeList(
+ TVector<TWorkerInfo::TPtr>& result,
+ TFreeList* freeList,
+ const i32 maxTasksPerWorker,
+ THashMap<TWorkerInfo::TPtr, int>& tasksPerWorker,
+ THashMap<i32, TWorkerFilter>& tasksToAllocate,
+ THashMap<TString, THashSet<int>>& waitingResources,
+ TWorkerFilter::EMatchStatus matchMode)
+{
+ TWorkerFilter::TStats stats;
+ stats.WaitingResources = &waitingResources;
+ stats.Uploaded = &Uploaded;
+
+ // fresh nodes must be first
+ for (auto it = freeList->begin(); it != freeList->end(); ) {
+ // must exists
+ auto workerInfo = *it;
+ if (workerInfo->IsDead) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Remove dead worker from free list "
+ << GetGuidAsString(workerInfo->WorkerId);
+ it = freeList->erase(it);
+ continue;
+ }
+ if (workerInfo->Capacity <= workerInfo->RunningRequests) {
+ it = freeList->erase(it);
+ continue;
+ }
+
+ if (maxTasksPerWorker > 0 && tasksPerWorker[workerInfo] >= maxTasksPerWorker) {
+ ++it;
+ continue;
+ }
+
+ if (workerInfo->Stopping) {
+ ++it;
+ continue;
+ }
+
+ i32 taskId;
+ bool ok = false;
+ for (const auto& [taskId_, filter] : tasksToAllocate) {
+ taskId = taskId_;
+ if ((matchMode == filter.Match(workerInfo, taskId, &stats))) {
+ ok = true;
+ break;
+ }
+ }
+
+ if (ok) {
+ it = ProcessMatched(
+ it, result, freeList, tasksPerWorker, tasksToAllocate, taskId);
+ } else {
+ ++it;
+ }
+
+ if (tasksToAllocate.empty()) {
+ break;
+ }
+ }
+}
+
+TVector<TWorkerInfo::TPtr> TWorkersStorage::TryAllocate(const NDq::IScheduler::TWaitInfo& waitInfo) noexcept
+{
+ auto& request = waitInfo.Request;
+ auto count = request.GetCount();
+ TVector<TWorkerInfo::TPtr> result(count, nullptr);
+ auto& filterPerTask = request.GetWorkerFilterPerTask();
+ THashMap<i32, TWorkerFilter> tasksToAllocate;
+
+ YQL_LOG_CTX_ROOT_SESSION_SCOPE(waitInfo.Request.GetTraceId());
+ YQL_CLOG(DEBUG, ProviderDq) << "TryAllocate, workers count: " << count << " files per task: " << filterPerTask.size();
+
+ if (filterPerTask.empty()) {
+ // COMPAT demand all files on all workers
+ Yql::DqsProto::TWorkerFilter commonFilter;
+ for (const auto& file : request.GetFiles()) {
+ *commonFilter.AddFile() = file;
+ }
+ for (ui32 i = 0; i < count; ++i) {
+ tasksToAllocate.emplace(i, commonFilter);
+ }
+ } else {
+ for (i32 i = 0; i < filterPerTask.size(); ++i) {
+ tasksToAllocate.emplace(i, filterPerTask.Get(i));
+ }
+ }
+
+ Y_ASSERT(tasksToAllocate.size() == count);
+
+ auto now = TInstant::Now();
+
+ for (const auto& file : request.GetFiles()) {
+ auto id = file.GetObjectId();
+ auto maybeUploaded = Uploaded.find(id);
+ if (maybeUploaded == Uploaded.end()) {
+ Uploaded.emplace(id, TResourceStat{id, file.GetObjectType(), file.GetName(), file.GetSize()});
+ } else {
+ if (maybeUploaded->second.Name.empty()) {
+ maybeUploaded->second = TResourceStat{id, file.GetObjectType(), file.GetName(), file.GetSize()};
+ } else {
+ maybeUploaded->second.LastSeenTime = now;
+ }
+ }
+ }
+
+ THashMap<TString, THashSet<int>> waitingResources; // resourceId -> taskId
+
+ TFreeList boundedList;
+ auto* freeList = &FreeList;
+
+ Y_SCOPE_EXIT(&) {
+ for (const auto& workerInfo : boundedList) {
+ FreeList.insert(workerInfo);
+ }
+ };
+
+ i32 maxTasksPerWorker = 0;
+ if (request.GetWorkersCount()) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Bounded mode";
+ maxTasksPerWorker = (count + request.GetWorkersCount() - 1) / request.GetWorkersCount();
+ freeList = &boundedList;
+ auto slots = count; slots = 0;
+
+ for (auto it = FreeList.begin(); it != FreeList.end() && freeList->size() < request.GetWorkersCount(); ) {
+ auto workerInfo = *it;
+ if (workerInfo->IsDead || workerInfo->Capacity <= workerInfo->RunningRequests) {
+ it = FreeList.erase(it);
+ continue;
+ }
+ auto freeSlots = workerInfo->Capacity - workerInfo->RunningRequests;
+ if (freeSlots >= maxTasksPerWorker) {
+ freeList->insert(workerInfo);
+ slots += freeSlots;
+ it = FreeList.erase(it);
+ } else {
+ ++it;
+ }
+ }
+
+ if (slots < count) {
+ YQL_CLOG(DEBUG, ProviderDq) << "Not enough slots";
+ return { };
+ }
+ }
+
+ THashMap<TWorkerInfo::TPtr, int> tasksPerWorker;
+ SearchFreeList(result, freeList, maxTasksPerWorker, tasksPerWorker, tasksToAllocate, waitingResources, TWorkerFilter::EOK);
+ if (!tasksToAllocate.empty()) {
+ SearchFreeList(result, freeList, maxTasksPerWorker, tasksPerWorker, tasksToAllocate, waitingResources, TWorkerFilter::EPARTIAL);
+ }
+
+ for (const auto& file : request.GetFiles()) {
+ const auto& id = file.GetObjectId();
+
+ if (waitingResources.contains(id)) {
+ waitInfo.Stat.StartCounter(id, now);
+ } else {
+ waitInfo.Stat.FlushCounter(id);
+ }
+ }
+
+ waitInfo.ResLeft.clear();
+ if (tasksToAllocate.empty()) {
+ auto now = TInstant::Now();
+ for (const auto& workerInfo : result) {
+ Y_ASSERT(workerInfo != nullptr);
+ workerInfo->UseCount ++;
+ workerInfo->RequestStartTime = now;
+ }
+ for (const auto& file : request.GetFiles()) {
+ auto id = file.GetObjectId();
+ auto it = Uploaded.find(id);
+ auto delta = waitInfo.Stat.Get().contains(id)
+ ? TDuration::MilliSeconds(waitInfo.Stat.Get().find(id)->second.Sum)
+ : TDuration();
+
+ waitInfo.Stat.Clear();
+
+ it->second.UseCount += count;
+ it->second.WaitTime += delta;
+
+ TString resourceName;
+
+ switch (file.GetObjectType()) {
+ case Yql::DqsProto::TFile::EEXE_FILE:
+ resourceName = "ResourceExe";
+ waitInfo.Stat.AddCounter(resourceName, delta);
+ break;
+ case Yql::DqsProto::TFile::EUDF_FILE:
+ if (!file.GetName().empty()) {
+ resourceName = "ResourceUdf" + file.GetName();
+ waitInfo.Stat.AddCounter(resourceName, delta);
+ }
+ waitInfo.Stat.AddCounter("ResourceUdf", delta);
+ break;
+ default:
+ resourceName = "ResourceFile";
+ waitInfo.Stat.AddCounter(resourceName, delta);
+ break;
+ }
+
+ ResourceWaitTime->Collect(delta.MilliSeconds());
+ *ResourceCounters->GetCounter(resourceName, /*derivative=*/ true) += count;
+ }
+
+ return result;
+ } else {
+ YQL_CLOG(DEBUG, ProviderDq) << "Tasks left " << tasksToAllocate.size();
+ // schedule downloads
+ for (const auto& workerInfo : result) {
+ if (workerInfo == nullptr) {
+ continue;
+ }
+ freeList->erase(workerInfo);
+ if (workerInfo->Release()) {
+ freeList->insert(workerInfo);
+ }
+ }
+ for (const auto& [_, filter] : tasksToAllocate) {
+ filter.Visit([&](const auto& file) {
+ waitInfo.ResLeft[file.GetObjectId()]++;
+ });
+ }
+
+ auto it = freeList->begin();
+ for (const auto& [_, filter] : tasksToAllocate) {
+ if (it == freeList->end()) {
+ it = freeList->begin();
+ }
+ if (it == freeList->end()) {
+ // begin == end => no free workers
+ break;
+ }
+ const auto workerInfo = *it++;
+ if (workerInfo->Stopping) {
+ continue;
+ }
+ filter.Visit([&](const auto& file) {
+ if (workerInfo->AddToDownloadList(file.GetObjectId(), file)) {
+ YQL_CLOG(TRACE, ProviderDq) << "Added " << file.GetName() << "|" << file.GetObjectId() << " to worker's " << GetGuidAsString(workerInfo->WorkerId) << " download list" ;
+ TStringBuilder resourceDownloadName;
+ resourceDownloadName << "ResourceDownload";
+ switch (file.GetObjectType()) {
+ case Yql::DqsProto::TFile::EEXE_FILE:
+ resourceDownloadName << "Exe";
+ break;
+ case Yql::DqsProto::TFile::EUDF_FILE:
+ resourceDownloadName << "Udf" << file.GetName();
+ break;
+ default:
+ resourceDownloadName << "File";
+ break;
+ }
+ *ResourceDownloadCounters->GetCounter(resourceDownloadName, /*derivative=*/ true) += 1;
+ }
+ });
+ }
+
+ return { };
+ }
+}
+
+void TWorkersStorage::IsReady(const TVector<TFileResource>& resources, THashMap<TString, std::pair<ui32, ui32>>& clusterMap) {
+ for (const auto& [_,workerInfo] : Workers) {
+ auto count = 0;
+ for (const auto& r : resources){
+ if (workerInfo->GetResources().contains(r.GetObjectId())) {
+ count++;
+ }
+ }
+ YQL_CLOG(TRACE, ProviderDq) << workerInfo->ClusterName << " worker " << GetGuidAsString(workerInfo->WorkerId) << " has: " << count << "/" << int(resources.size()) << " resources";
+ clusterMap[workerInfo->ClusterName].second += count == int(resources.size());
+ }
+};
+
+
+void TWorkersStorage::ClusterStatus(Yql::DqsProto::ClusterStatusResponse* r) const {
+ for (const auto& [_, workerInfo] : Workers) {
+ auto* node = r->AddWorker();
+ node->SetGuid(GetGuidAsString(workerInfo->WorkerId));
+ node->SetNodeId(workerInfo->NodeId);
+ node->SetLastPingTime(workerInfo->LastPingTime.ToString());
+ node->SetRevision(workerInfo->Revision);
+ node->SetClusterName(workerInfo->ClusterName);
+ node->SetAddress(workerInfo->Address);
+ node->SetPort(workerInfo->Port);
+ node->SetEpoch(workerInfo->Epoch);
+ node->SetStopping(workerInfo->Stopping);
+ node->SetStartTime(workerInfo->StartTime);
+
+ for (const auto& f : workerInfo->GetResourcesOrdered()) {
+ *node->AddResource() = f.GetObjectId();
+ }
+
+ for (const auto& [k, _] : workerInfo->GetDownloadList()) {
+ node->AddDownloadList()->SetObjectId(k);
+ }
+
+ for (const auto& [k, _] : workerInfo->GetActiveDownloads()) {
+ node->AddActiveDownload()->SetObjectId(k);
+ }
+ for (const auto& op : workerInfo->Operations){
+ node->AddOperation(op);
+ }
+ node->SetFreeDiskSize(workerInfo->FreeDiskSize);
+ node->SetUsedDiskSize(workerInfo->UsedDiskSize);
+ node->SetCapacity(workerInfo->Capacity);
+ node->SetRunningWorkerActors(workerInfo->RunningWorkerActors);
+ node->SetRunningRequests(workerInfo->RunningRequests);
+ node->SetDead(workerInfo->IsDead);
+
+ node->MutableRusage()->SetStime(workerInfo->Stime.MicroSeconds());
+ node->MutableRusage()->SetUtime(workerInfo->Utime.MicroSeconds());
+ node->MutableRusage()->SetMajorPageFaults(workerInfo->MajorPageFaults);
+ node->MutableRusage()->SetCpuSystem(workerInfo->CpuSystem);
+ node->MutableRusage()->SetCpuUser(workerInfo->CpuUser);
+ node->MutableRusage()->SetCpuTotal(workerInfo->CpuTotal);
+
+ for (const auto& [k, v] : workerInfo->Attributes) {
+ auto* attr = node->AddAttribute();
+ attr->SetKey(k);
+ attr->SetValue(v);
+ }
+
+ node->SetUseCount(workerInfo->UseCount);
+ node->SetUseTime(workerInfo->UseTime.MilliSeconds());
+ }
+
+ for (const auto& [_, i] : Uploaded) {
+ auto* rr = r->AddResource();
+ rr->SetId(i.Id);
+ rr->SetName(i.Name);
+ rr->SetUseTime(i.UseTime.MilliSeconds());
+ rr->SetWaitTime(i.WaitTime.MilliSeconds());
+ rr->SetUseCount(i.UseCount);
+ rr->SetTryCount(i.TryCount);
+ rr->SetSize(i.Size);
+ }
+
+ r->SetFreeListSize(FreeList.size());
+ r->SetCapacity(GlobalResources.GetCapacity());
+ r->SetRunningRequests(GlobalResources.GetRunningRequests());
+}
+
+void TWorkersStorage::UpdateResourceUseTime(TDuration duration, const THashSet<TString>& ids) {
+ for (const auto& id : ids) {
+ auto it = Uploaded.find(id);
+ if (it != Uploaded.end()) {
+ it->second.UseTime += duration;
+ }
+ }
+}
+
+bool TWorkersStorage::HasResource(const TString& id) const {
+ auto it = Uploaded.find(id);
+ return (it != Uploaded.end()) && it->second.Uploaded;
+}
+
+void TWorkersStorage::AddResource(const TString& id, EFileType type, const TString& name, i64 size) {
+ if (!Uploaded.contains(id)) {
+ Uploaded.emplace(id, TResourceStat{id, type, name, size});
+ }
+ Uploaded.find(id)->second.Uploaded = true;
+}
+
+void TWorkersStorage::UpdateMetrics() {
+ if (!WorkersSize) {
+ WorkersSize = Metrics->GetSubgroup("component", "lists")->GetCounter("WorkersSize");
+ }
+ if (!FreeListSize) {
+ FreeListSize = Metrics->GetSubgroup("component", "lists")->GetCounter("FreeListSize");
+ }
+ *WorkersSize = Workers.size();
+ *FreeListSize = FreeList.size();
+}
+
+void TWorkersStorage::Visit(const std::function<void(const TWorkerInfo::TPtr& workerInfo)>& f) {
+ for (const auto& [_, workerInfo] : Workers) {
+ f(workerInfo);
+ }
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.h b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.h
new file mode 100644
index 0000000000..d40366596d
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage.h
@@ -0,0 +1,90 @@
+#pragma once
+
+#include "worker_filter.h"
+
+#include <ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h>
+
+#include <ydb/library/yql/providers/dq/scheduler/dq_scheduler.h>
+
+namespace NYql {
+
+class TWorkersStorage {
+public:
+ TWorkersStorage(ui32 nodeId, TSensorsGroupPtr metrics, TSensorsGroupPtr workerMetrics);
+
+ void Clear();
+
+ TList<NDqs::TWorkerInfo::TPtr> GetList();
+
+ size_t Capacity() const { return GlobalResources.GetCapacity(); }
+ size_t FreeSlots() const { return Max<i64>(GlobalResources.GetCapacity() - GlobalResources.GetRunningRequests(), 0); }
+
+ std::tuple<NDqs::TWorkerInfo::TPtr, bool> CreateOrUpdate(ui32 nodeId, TGUID workerId, Yql::DqsProto::RegisterNodeRequest& request);
+
+ void CheckZombie(ui32 nodeId, TGUID workerId, Yql::DqsProto::RegisterNodeRequest& request);
+
+ void CleanUp(TInstant now, TDuration duration);
+
+ void DropWorker(TInstant now, TGUID workerId);
+
+ void FreeWorker(TInstant now, const NDqs::TWorkerInfo::TPtr& worker);
+
+ TVector<NDqs::TWorkerInfo::TPtr> TryAllocate(const NDq::IScheduler::TWaitInfo& waitInfo) noexcept;
+
+ void IsReady(const TVector<NDqs::TWorkerInfo::TFileResource>& resources, THashMap<TString, std::pair<ui32, ui32>>& clusterMap);
+
+ void ClusterStatus(Yql::DqsProto::ClusterStatusResponse* r) const;
+
+ void UpdateResourceUseTime(TDuration duration, const THashSet<TString>& ids);
+
+ bool HasResource(const TString& id) const;
+
+ void AddResource(const TString& id, Yql::DqsProto::TFile::EFileType type, const TString& name, i64 size);
+
+ void UpdateMetrics();
+
+ void Visit(const std::function<void(const NDqs::TWorkerInfo::TPtr& workerInfo)>& f);
+
+private:
+
+ const i32 MaxDownloadsPerFile = 20;
+ ui32 NodeId;
+ THashMap<TString, NDqs::TInflightLimiter::TPtr> InflightLimiters;
+ THashMap<TGUID, NDqs::TWorkerInfo::TPtr> Workers;
+
+ using TFreeList = std::set<NDqs::TWorkerInfo::TPtr, NDqs::TWorkerInfoPtrComparator>;
+ TFreeList FreeList;
+
+ TSensorsGroupPtr Metrics;
+ TSensorsGroupPtr WorkerMetrics;
+ TSensorsGroupPtr ResourceCounters;
+ TSensorsGroupPtr ResourceDownloadCounters;
+ NMonitoring::THistogramPtr ResourceWaitTime;
+ NMonitoring::TDynamicCounters::TCounterPtr WorkersSize;
+ NMonitoring::TDynamicCounters::TCounterPtr FreeListSize;
+ TString StartTime = ToString(TInstant::Now());
+ TVector<std::pair<TGUID, TString>> NodeIds;
+
+ THashMap<TString, TResourceStat> Uploaded;
+
+ NDqs::TGlobalResources GlobalResources;
+
+ TFreeList::iterator ProcessMatched(
+ TWorkersStorage::TFreeList::iterator it,
+ TVector<NDqs::TWorkerInfo::TPtr>& result,
+ TFreeList* freeList,
+ THashMap<NDqs::TWorkerInfo::TPtr, int> tasksPerWorker,
+ THashMap<i32, TWorkerFilter>& tasksToAllocate,
+ int taskId);
+
+ void SearchFreeList(
+ TVector<NDqs::TWorkerInfo::TPtr>& result,
+ TFreeList* freeList,
+ const i32 maxTasksPerWorker,
+ THashMap<NDqs::TWorkerInfo::TPtr, int>& tasksPerWorker,
+ THashMap<i32, TWorkerFilter>& tasksToAllocate,
+ THashMap<TString, THashSet<int>>& waitingResources,
+ TWorkerFilter::EMatchStatus matchMode);
+};
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp
new file mode 100644
index 0000000000..efa23c8a32
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/workers_storage_ut.cpp
@@ -0,0 +1,82 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <ydb/library/yql/providers/dq/global_worker_manager/workers_storage.h>
+
+using namespace NYql;
+using namespace NYql::NDq;
+
+Y_UNIT_TEST_SUITE(WorkersBenchmark) {
+ Y_UNIT_TEST(Basic) {
+ int workers = 1000;
+ int i;
+ TWorkersStorage storage(1, new TSensorsGroup, new TSensorsGroup);
+ storage.Clear();
+ for (i = 0; i < workers; i++) {
+ TGUID guid;
+ Yql::DqsProto::RegisterNodeRequest request;
+ request.SetCapacity(100);
+ request.AddKnownNodes(1);
+ CreateGuid(&guid);
+ storage.CreateOrUpdate(i+100, guid, request);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(workers*100, storage.FreeSlots());
+
+ TVector<NDqs::TWorkerInfo::TPtr> all;
+ TInstant now = TInstant::Now();
+ while (1) {
+ NYql::NDqProto::TAllocateWorkersRequest request;
+ request.SetCount(10);
+ IScheduler::TWaitInfo waitInfo(request, NActors::TActorId());
+ auto result = storage.TryAllocate(waitInfo);
+ if (result.empty()) {
+ break;
+ }
+ all.insert(all.end(), result.begin(), result.end());
+ }
+
+ Cerr << (TInstant::Now() - now) << Endl;
+
+ UNIT_ASSERT_VALUES_EQUAL(0, storage.FreeSlots());
+
+
+ now = TInstant::Now();
+ for (auto& node : all) {
+ storage.FreeWorker(now, node);
+ }
+ Cerr << (TInstant::Now() - now) << Endl;
+
+ UNIT_ASSERT_VALUES_EQUAL(workers*100, storage.FreeSlots());
+ }
+
+ Y_UNIT_TEST(AcquireAll) {
+ int workers = 1;
+ int i;
+ TWorkersStorage storage(1, new TSensorsGroup, new TSensorsGroup);
+ storage.Clear();
+ for (i = 0; i < workers; i++) {
+ TGUID guid;
+ Yql::DqsProto::RegisterNodeRequest request;
+ request.SetCapacity(100);
+ request.AddKnownNodes(1);
+ CreateGuid(&guid);
+ storage.CreateOrUpdate(i+100, guid, request);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(workers*100, storage.FreeSlots());
+
+ TVector<NDqs::TWorkerInfo::TPtr> all;
+ while (1) {
+ NYql::NDqProto::TAllocateWorkersRequest request;
+ request.SetCount(10);
+ IScheduler::TWaitInfo waitInfo(request, NActors::TActorId());
+ auto result = storage.TryAllocate(waitInfo);
+ if (result.empty()) {
+ break;
+ }
+ all.insert(all.end(), result.begin(), result.end());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(all.size(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(0, storage.FreeSlots());
+ }
+}
diff --git a/ydb/library/yql/providers/dq/global_worker_manager/ya.make b/ydb/library/yql/providers/dq/global_worker_manager/ya.make
new file mode 100644
index 0000000000..f2ce78ffd4
--- /dev/null
+++ b/ydb/library/yql/providers/dq/global_worker_manager/ya.make
@@ -0,0 +1,53 @@
+LIBRARY()
+
+PEERDIR(
+ ydb/library/yql/utils/failure_injector
+ ydb/library/yql/providers/common/config
+ ydb/library/yql/providers/common/gateway
+ ydb/library/yql/providers/common/metrics
+ ydb/library/yql/providers/dq/actors
+ ydb/library/yql/providers/dq/api/grpc
+ ydb/library/yql/providers/dq/api/protos
+ ydb/library/yql/providers/dq/config
+ ydb/library/yql/providers/dq/counters
+ ydb/library/yql/providers/dq/runtime
+ ydb/library/yql/providers/dq/task_runner
+ ydb/library/yql/providers/dq/actors/yt
+ ydb/library/yql/providers/dq/scheduler
+)
+
+YQL_LAST_ABI_VERSION()
+
+SET(
+ SOURCE
+ benchmark.cpp
+ global_worker_manager.cpp
+ service_node_pinger.cpp
+ workers_storage.cpp
+ worker_filter.cpp
+)
+
+IF (NOT OS_WINDOWS)
+ SET(
+ SOURCE
+ ${SOURCE}
+ service_node_resolver.cpp
+ coordination_helper.cpp
+ )
+ELSE()
+ SET(
+ SOURCE
+ ${SOURCE}
+ coordination_helper_win.cpp
+ )
+ENDIF()
+
+SRCS(
+ ${SOURCE}
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..450d2ffafa
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,29 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(providers-dq-scheduler)
+target_compile_options(providers-dq-scheduler PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(providers-dq-scheduler PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yql-dq-common
+ yql-dq-proto
+ providers-dq-config
+ providers-common-proto
+ dq-api-protos
+ providers-dq-common
+ providers-dq-counters
+ cpp-actors-protos
+)
+target_sources(providers-dq-scheduler PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp
+)
diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..2479c98297
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,30 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(providers-dq-scheduler)
+target_compile_options(providers-dq-scheduler PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(providers-dq-scheduler PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yql-dq-common
+ yql-dq-proto
+ providers-dq-config
+ providers-common-proto
+ dq-api-protos
+ providers-dq-common
+ providers-dq-counters
+ cpp-actors-protos
+)
+target_sources(providers-dq-scheduler PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp
+)
diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..2479c98297
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,30 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(providers-dq-scheduler)
+target_compile_options(providers-dq-scheduler PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(providers-dq-scheduler PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yql-dq-common
+ yql-dq-proto
+ providers-dq-config
+ providers-common-proto
+ dq-api-protos
+ providers-dq-common
+ providers-dq-counters
+ cpp-actors-protos
+)
+target_sources(providers-dq-scheduler PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp
+)
diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/dq/scheduler/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..450d2ffafa
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,29 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(providers-dq-scheduler)
+target_compile_options(providers-dq-scheduler PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(providers-dq-scheduler PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yql-dq-common
+ yql-dq-proto
+ providers-dq-config
+ providers-common-proto
+ dq-api-protos
+ providers-dq-common
+ providers-dq-counters
+ cpp-actors-protos
+)
+target_sources(providers-dq-scheduler PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp
+)
diff --git a/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp b/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp
new file mode 100644
index 0000000000..1291cc949f
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/dq_scheduler.cpp
@@ -0,0 +1,240 @@
+#include "dq_scheduler.h"
+
+#include <queue>
+#include <list>
+#include <unordered_map>
+
+#include <ydb/library/yql/utils/log/log.h>
+
+namespace NYql::NDq {
+
+IScheduler::TWaitInfo::TWaitInfo(const NYql::NDqProto::TAllocateWorkersRequest& record, const NActors::TActorId& sender)
+ : Request(record), Sender(sender), StartTime(TInstant::Now())
+{ }
+
+namespace {
+
+struct TMyCounters {
+ using TPtr = std::unique_ptr<TMyCounters>;
+
+ const TSensorsGroupPtr Group;
+ const NMonitoring::TDynamicCounters::TCounterPtr QueueSizeForSmall;
+ const NMonitoring::TDynamicCounters::TCounterPtr QueueSizeForLarge;
+ const NMonitoring::TDynamicCounters::TCounterPtr IntegralQueueSizeForLarge;
+ const NMonitoring::TDynamicCounters::TCounterPtr AllocatedTotal;
+ const NMonitoring::TDynamicCounters::TCounterPtr KnownUsers;
+ const NMonitoring::TDynamicCounters::TCounterPtr UserLimited;
+
+ TMyCounters(TSensorsGroupPtr&& group)
+ : Group(std::move(group))
+ , QueueSizeForSmall(Group->GetCounter("QueueSizeForSmall"))
+ , QueueSizeForLarge(Group->GetCounter("QueueSizeForLarge"))
+ , IntegralQueueSizeForLarge(Group->GetCounter("IntegralQueueSizeForLarge"))
+ , AllocatedTotal(Group->GetCounter("AllocatedTotal"))
+ , KnownUsers(Group->GetCounter("KnownUsers"))
+ , UserLimited(Group->GetCounter("UserLimited"))
+ {}
+
+ static TPtr Make(IMetricsRegistryPtr metricsRegistry) {
+ return metricsRegistry ? std::make_unique<TMyCounters>(metricsRegistry->GetSensors()->GetSubgroup("component", "scheduler")) : nullptr;
+ }
+};
+
+class TScheduler : public IScheduler {
+public:
+ TScheduler(IMetricsRegistryPtr metricsRegistry, const NProto::TDqConfig::TScheduler& config)
+ : KeepReserveForLiteralRequests(config.GetKeepReserveForLiteralRequests())
+ , HistoryKeepingTime(TDuration::Minutes(config.GetHistoryKeepingTime()))
+ , MaxOperations(config.GetMaxOperations())
+ , MaxOperationsPerUser(config.GetMaxOperationsPerUser())
+ , Counters(TMyCounters::Make(metricsRegistry))
+ , EnableLimiter(config.GetLimitTasksPerWindow())
+ , LimiterNumerator(config.GetLimiterNumerator())
+ , LimiterDenumerator(config.GetLimiterDenumerator())
+ {}
+
+private:
+ struct TUserInfo {
+ ui64 Await = 0ULL;
+ ui64 Allocated = 0ULL;
+ ui64 AwaitOperations = 0LL;
+ std::queue<std::pair<TInstant, ui32>> History;
+ struct {
+ NMonitoring::TDynamicCounters::TCounterPtr Await, AwaitOperations, Allocated;
+ } Counters;
+ };
+
+ using THistoryMap = std::unordered_map<TString, TUserInfo>;
+
+ struct TFullWaitInfo : public TWaitInfo {
+ TFullWaitInfo(TWaitInfo&& info, THistoryMap::value_type* userInfo)
+ : TWaitInfo(std::move(info)), UserInfo(userInfo)
+ {}
+
+ THistoryMap::value_type* const UserInfo;
+ };
+
+ bool Suspend(TWaitInfo&& info) final {
+ const auto ins = AllocationsHistory.emplace(info.Request.GetUser(), TUserInfo());
+ auto& userInfo = *ins.first;
+
+ if (info.Request.GetCount() > 1U) {
+ if (userInfo.second.AwaitOperations >= MaxOperationsPerUser) {
+ return false;
+ }
+ if (LargeWaitList.size() >= MaxOperations) {
+ return false;
+ }
+ }
+
+ userInfo.second.Await += info.Request.GetCount();
+ userInfo.second.AwaitOperations += 1;
+ if (ins.second && Counters) {
+ const auto group = Counters->Group->GetSubgroup("user", info.Request.GetUser());
+ userInfo.second.Counters.Await = group->GetCounter("Await");
+ userInfo.second.Counters.AwaitOperations = group->GetCounter("AwaitOperations");
+ userInfo.second.Counters.Allocated = group->GetCounter("Allocated");
+ }
+ (info.Request.GetCount() > 1U ? LargeWaitList : SmallWaitList).emplace_back(std::move(info), &userInfo);
+ return true;
+ }
+
+ std::vector<NActors::TActorId> Cleanup() final {
+ std::vector<NActors::TActorId> senders;
+ senders.reserve(SmallWaitList.size() + LargeWaitList.size());
+ std::transform(SmallWaitList.cbegin(), SmallWaitList.cend(), std::back_inserter(senders), [](const TWaitInfo& info) { return info.Sender; });
+ std::transform(LargeWaitList.cbegin(), LargeWaitList.cend(), std::back_inserter(senders), [](const TWaitInfo& info) { return info.Sender; });
+ SmallWaitList.clear();
+ LargeWaitList.clear();
+
+ if (Counters) {
+ for (auto it = AllocationsHistory.cbegin(); AllocationsHistory.cend() != it; ++it) {
+ *it->second.Counters.Await = 0;
+ *it->second.Counters.AwaitOperations = 0;
+ *it->second.Counters.Allocated = 0;
+ }
+ }
+
+ AllocationsHistory.clear();
+ return senders;
+ }
+
+ size_t UpdateMetrics() final {
+ if (Counters) {
+ auto allocated = 0ULL;
+ for (auto it = AllocationsHistory.cbegin(); AllocationsHistory.cend() != it; ++it) {
+ *it->second.Counters.Await = it->second.Await;
+ *it->second.Counters.AwaitOperations = it->second.AwaitOperations;
+ *it->second.Counters.Allocated = it->second.Allocated;
+ allocated += it->second.Allocated;
+ }
+
+ *Counters->KnownUsers = AllocationsHistory.size();
+ *Counters->AllocatedTotal = allocated;
+ *Counters->QueueSizeForSmall = SmallWaitList.size();
+ *Counters->QueueSizeForLarge = LargeWaitList.size();
+ *Counters->IntegralQueueSizeForLarge = std::accumulate(LargeWaitList.cbegin(), LargeWaitList.cend(), 0ULL,
+ [] (ui64 c, const TFullWaitInfo& info) { return c += info.Request.GetCount(); }
+ );
+ }
+
+ return SmallWaitList.size() + LargeWaitList.size();
+ }
+
+ void Process(size_t total, size_t count, const TProcessor& processor, const TInstant& now) final {
+ const auto from = now - HistoryKeepingTime;
+ for (auto& info : AllocationsHistory) {
+ for (auto& history = info.second.History; !history.empty() && history.front().first <= from; history.pop())
+ info.second.Allocated -= history.front().second;
+ };
+
+ const auto sort = [](const TFullWaitInfo& lhs, const TFullWaitInfo& rhs) {
+ return lhs.UserInfo->second.Allocated < rhs.UserInfo->second.Allocated;
+ };
+
+ SmallWaitList.sort(sort);
+ LargeWaitList.sort(sort);
+
+ const auto work = [&processor, &now, &total, this] (size_t& quota, std::list<TFullWaitInfo>& list) {
+ list.remove_if([&](const TFullWaitInfo& info) {
+ const auto count = info.Request.GetCount();
+ if (quota < count)
+ return false;
+
+ if (EnableLimiter && (info.UserInfo->second.Allocated+count) > LimiterNumerator * total / LimiterDenumerator) {
+ if (Counters) {
+ *Counters->UserLimited += 1;
+ }
+ return false;
+ }
+
+ if (processor(info)) {
+ info.UserInfo->second.Await -= count;
+ info.UserInfo->second.AwaitOperations -= 1;
+ info.UserInfo->second.Allocated += count;
+ info.UserInfo->second.History.emplace(now, count);
+ quota -= count;
+ return true;
+ }
+ return false;
+ });
+ };
+
+ const auto full = count;
+ const auto half = full >> 1U;
+
+ if (count -= half)
+ work(count, SmallWaitList);
+
+ if (KeepReserveForLiteralRequests)
+ count = SmallWaitList.empty() && count + half >= total >> 2U ? std::min(count + half, full - (half >> 1U)) : half;
+ else
+ count += half;
+
+ if (count > 1U)
+ work(count, LargeWaitList);
+
+ if (count && LargeWaitList.empty())
+ work(count, SmallWaitList);
+ }
+
+ void ProcessAll(const TProcessor& processor) final {
+ const auto proc = [&processor](const TFullWaitInfo& info) {
+ if (processor(info)) {
+ info.UserInfo->second.Await -= info.Request.GetCount();
+ return true;
+ }
+ return false;
+ };
+
+ SmallWaitList.remove_if(proc);
+ LargeWaitList.remove_if(proc);
+ }
+
+ void ForEach(const std::function<void(const TWaitInfo& info)>& processor) final {
+ std::for_each(SmallWaitList.cbegin(), SmallWaitList.cend(), processor);
+ std::for_each(LargeWaitList.cbegin(), LargeWaitList.cend(), processor);
+ }
+
+ const bool KeepReserveForLiteralRequests;
+ const TDuration HistoryKeepingTime;
+ const size_t MaxOperations;
+ const size_t MaxOperationsPerUser;
+
+ const TMyCounters::TPtr Counters;
+
+ THistoryMap AllocationsHistory;
+ std::list<TFullWaitInfo> SmallWaitList, LargeWaitList;
+
+ bool EnableLimiter;
+ ui32 LimiterNumerator;
+ ui32 LimiterDenumerator;
+};
+
+}
+
+IScheduler::TPtr IScheduler::Make(const NProto::TDqConfig::TScheduler& config, IMetricsRegistryPtr metricsRegistry) {
+ return std::make_unique<TScheduler>(metricsRegistry, config);
+}
+
+}
diff --git a/ydb/library/yql/providers/dq/scheduler/dq_scheduler.h b/ydb/library/yql/providers/dq/scheduler/dq_scheduler.h
new file mode 100644
index 0000000000..d6a976cd5e
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/dq_scheduler.h
@@ -0,0 +1,47 @@
+#pragma once
+
+#include <library/cpp/actors/core/events.h>
+
+#include <ydb/library/yql/providers/dq/counters/counters.h>
+#include <ydb/library/yql/providers/dq/api/protos/dqs.pb.h>
+#include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
+#include <ydb/library/yql/providers/dq/config/config.pb.h>
+#include <ydb/library/yql/providers/common/metrics/metrics_registry.h>
+
+namespace NYql::NDq {
+
+class IScheduler {
+public:
+ using TPtr = std::unique_ptr<IScheduler>;
+
+ static TPtr Make(const NProto::TDqConfig::TScheduler& schedulerConfig = {}, IMetricsRegistryPtr metricsRegistry = {});
+
+ virtual ~IScheduler() = default;
+
+ struct TWaitInfo {
+ const NYql::NDqProto::TAllocateWorkersRequest Request;
+ const NActors::TActorId Sender;
+ const TInstant StartTime;
+
+ TCounters Stat;
+ mutable THashMap<TString, int> ResLeft;
+
+ TWaitInfo(const NYql::NDqProto::TAllocateWorkersRequest& record, const NActors::TActorId& sender);
+ };
+
+ virtual bool Suspend(TWaitInfo&& info) = 0;
+
+ virtual std::vector<NActors::TActorId> Cleanup() = 0;
+
+ virtual size_t UpdateMetrics() = 0;
+
+ using TProcessor = std::function<bool(const TWaitInfo& info)>;
+
+ virtual void Process(size_t totalWorkersCount, size_t freeWorkersCount, const TProcessor& processor, const TInstant& timestamp = TInstant::Now()) = 0;
+
+ virtual void ProcessAll(const TProcessor& processor) = 0;
+
+ virtual void ForEach(const std::function<void(const TWaitInfo& info)>& processor) = 0;
+};
+
+}
diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..aa754d167c
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,72 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-providers-dq-scheduler-ut)
+target_compile_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler
+)
+target_link_libraries(ydb-library-yql-providers-dq-scheduler-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ providers-dq-scheduler
+ udf-service-stub
+ yql-utils-log
+)
+target_link_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-providers-dq-scheduler-ut
+ TEST_TARGET
+ ydb-library-yql-providers-dq-scheduler-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-providers-dq-scheduler-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-providers-dq-scheduler-ut)
diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..cdeda2cc91
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,75 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-providers-dq-scheduler-ut)
+target_compile_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler
+)
+target_link_libraries(ydb-library-yql-providers-dq-scheduler-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ providers-dq-scheduler
+ udf-service-stub
+ yql-utils-log
+)
+target_link_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-providers-dq-scheduler-ut
+ TEST_TARGET
+ ydb-library-yql-providers-dq-scheduler-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-providers-dq-scheduler-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-providers-dq-scheduler-ut)
diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..3ee08ce188
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,77 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-providers-dq-scheduler-ut)
+target_compile_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler
+)
+target_link_libraries(ydb-library-yql-providers-dq-scheduler-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ providers-dq-scheduler
+ udf-service-stub
+ yql-utils-log
+)
+target_link_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-providers-dq-scheduler-ut
+ TEST_TARGET
+ ydb-library-yql-providers-dq-scheduler-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-providers-dq-scheduler-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-providers-dq-scheduler-ut)
diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..e5ba4657a5
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,65 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-providers-dq-scheduler-ut)
+target_compile_options(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler
+)
+target_link_libraries(ydb-library-yql-providers-dq-scheduler-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ providers-dq-scheduler
+ udf-service-stub
+ yql-utils-log
+)
+target_sources(ydb-library-yql-providers-dq-scheduler-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-providers-dq-scheduler-ut
+ TEST_TARGET
+ ydb-library-yql-providers-dq-scheduler-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-dq-scheduler-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-providers-dq-scheduler-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-providers-dq-scheduler-ut)
diff --git a/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp b/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp
new file mode 100644
index 0000000000..1aea552250
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/ut/dq_scheduler_ut.cpp
@@ -0,0 +1,311 @@
+#include "../dq_scheduler.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NYql::NDq;
+
+namespace {
+NYql::NDqProto::TAllocateWorkersRequest MakeRequest(ui32 count, const TString& user) {
+ NYql::NDqProto::TAllocateWorkersRequest request;
+ request.SetCount(count);
+ request.SetUser(user);
+ return request;
+}
+
+}
+
+Y_UNIT_TEST_SUITE(TSchedulerTest) {
+ Y_UNIT_TEST(SimpleFifo) {
+ const auto scheduler = IScheduler::Make();
+ UNIT_ASSERT(scheduler);
+
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(7U, "user2"), {}});
+ scheduler->Suspend({MakeRequest(5U, "user3"), {}});
+
+ size_t workers = 30U;
+ std::vector<size_t> counts;
+ const auto processor = [&](const IScheduler::TWaitInfo& wait) {
+ counts.emplace_back(wait.Request.GetCount());
+ return true;
+ };
+
+ scheduler->Process(workers, workers, processor);
+
+ const std::vector<size_t> expected = {3U, 7U, 5U};
+ UNIT_ASSERT_VALUES_EQUAL(counts, expected);
+ }
+
+ Y_UNIT_TEST(ReserveForSmall) {
+ const auto scheduler = IScheduler::Make();
+ UNIT_ASSERT(scheduler);
+
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(7U, "user2"), {}});
+ scheduler->Suspend({MakeRequest(5U, "user3"), {}});
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 3U);
+
+ size_t workers = 5U;
+ std::vector<size_t> counts;
+ const auto processor = [&](const IScheduler::TWaitInfo& wait) {
+ counts.emplace_back(wait.Request.GetCount());
+ return true;
+ };
+
+ scheduler->Process(15U, workers, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 2U);
+
+ scheduler->Process(15U, workers = 8U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U);
+
+ scheduler->Process(15U, workers = 8U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U);
+
+ const std::vector<size_t> expected = {3U, 5U};
+ UNIT_ASSERT_VALUES_EQUAL(counts, expected);
+ }
+
+ Y_UNIT_TEST(OneUserForCluster) {
+ NYql::NProto::TDqConfig::TScheduler cfg;
+ cfg.SetHistoryKeepingTime(1);
+ cfg.SetLimitTasksPerWindow(true);
+
+ const auto scheduler = IScheduler::Make(cfg);
+ UNIT_ASSERT(scheduler);
+
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 2U);
+
+ std::vector<size_t> counts;
+ const auto processor = [&](const IScheduler::TWaitInfo& wait) {
+ counts.emplace_back(wait.Request.GetCount());
+ return true;
+ };
+
+ scheduler->Process(11U, 7U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U);
+
+ const std::vector<size_t> expected = {3U};
+ UNIT_ASSERT_VALUES_EQUAL(counts, expected);
+ }
+
+ Y_UNIT_TEST(DoNotReserveForSmall) {
+ NYql::NProto::TDqConfig::TScheduler cfg;
+ cfg.SetKeepReserveForLiteralRequests(false);
+
+ const auto scheduler = IScheduler::Make(cfg);
+ UNIT_ASSERT(scheduler);
+
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(7U, "user2"), {}});
+ scheduler->Suspend({MakeRequest(5U, "user3"), {}});
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 3U);
+
+ size_t workers = 5U;
+ std::vector<size_t> counts;
+ const auto processor = [&](const IScheduler::TWaitInfo& wait) {
+ counts.emplace_back(wait.Request.GetCount());
+ return true;
+ };
+
+ scheduler->Process(30U, workers, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 2U);
+
+ scheduler->Process(30U, workers = 8U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U);
+
+ scheduler->Process(30U, workers = 8U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 0U);
+
+ const std::vector<size_t> expected = {3U, 7U, 5U};
+ UNIT_ASSERT_VALUES_EQUAL(counts, expected);
+ }
+
+ Y_UNIT_TEST(NewbieFirst) {
+ const auto scheduler = IScheduler::Make();
+ UNIT_ASSERT(scheduler);
+
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(7U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(5U, "user1"), {}});
+
+ size_t workers = 15U;
+ std::vector<size_t> counts;
+ const auto processor = [&](const IScheduler::TWaitInfo& wait) {
+ counts.emplace_back(wait.Request.GetCount());
+ return true;
+ };
+
+ scheduler->Process(33U, workers, processor);
+
+ scheduler->Suspend({MakeRequest(3U, "user2"), {}});
+ scheduler->Suspend({MakeRequest(7U, "user2"), {}});
+ scheduler->Suspend({MakeRequest(5U, "user2"), {}});
+
+ workers += 10U;
+
+ scheduler->Process(33U, workers, processor);
+ workers += 10U;
+
+ scheduler->Process(33U, workers, processor);
+
+ const std::vector<size_t> expected = {3U, 7U, 3U, 7U, 5U, 5U};
+ UNIT_ASSERT_VALUES_EQUAL(counts, expected);
+ }
+
+ Y_UNIT_TEST(FifoAfterOneHour) {
+ const auto scheduler = IScheduler::Make();
+ UNIT_ASSERT(scheduler);
+
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(7U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(5U, "user1"), {}});
+
+ size_t workers = 15U;
+ std::vector<size_t> counts;
+ const auto processor = [&](const IScheduler::TWaitInfo& wait) {
+ counts.emplace_back(wait.Request.GetCount());
+ return true;
+ };
+
+ scheduler->Process(40U, workers, processor);
+
+ scheduler->Suspend({MakeRequest(3U, "user2"), {}});
+ scheduler->Suspend({MakeRequest(7U, "user2"), {}});
+ scheduler->Suspend({MakeRequest(5U, "user2"), {}});
+
+ workers += 10U;
+
+ const auto skipHour = TInstant::Now() + TDuration::Hours(1U);
+
+ scheduler->Process(40U, workers, processor, skipHour);
+ workers += 10U;
+
+ scheduler->Process(40U, workers, processor, skipHour);
+
+ const std::vector<size_t> expected = {3U, 7U, 5U, 3U, 7U, 5U};
+ UNIT_ASSERT_VALUES_EQUAL(counts, expected);
+ }
+
+ Y_UNIT_TEST(HalfWorkersForSmall) {
+ const auto scheduler = IScheduler::Make();
+ UNIT_ASSERT(scheduler);
+
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(7U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(5U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(1U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(1U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(1U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(1U, "user1"), {}});
+
+ size_t workers = 6U;
+ std::vector<size_t> counts;
+ const auto processor = [&](const IScheduler::TWaitInfo& wait) {
+ counts.emplace_back(wait.Request.GetCount());
+ return true;
+ };
+
+ scheduler->Process(30U, workers, processor);
+
+ workers += 15U;
+
+ scheduler->Process(30U, workers, processor);
+
+ const std::vector<size_t> expected = {1U, 1U, 1U, 3U, 1U, 7U, 5U};
+ UNIT_ASSERT_VALUES_EQUAL(counts, expected);
+ }
+
+ Y_UNIT_TEST(Use75PercentForLargeInNonOverload) {
+ const auto scheduler = IScheduler::Make();
+ UNIT_ASSERT(scheduler);
+
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user2"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user3"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user4"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user5"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user6"), {}});
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 6U);
+
+ size_t workers = 4U;
+ std::vector<size_t> counts;
+ const auto processor = [&](const IScheduler::TWaitInfo& wait) {
+ counts.emplace_back(wait.Request.GetCount());
+ return true;
+ };
+
+ scheduler->Process(16U, workers, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 5U);
+
+ scheduler->Process(16U, workers = 4U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 4U);
+
+ scheduler->Process(16U, workers = 4U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 3U);
+
+ scheduler->Process(16U, workers = 4U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 2U);
+
+ scheduler->Process(16U, workers = 4U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 1U);
+
+ scheduler->Process(16U, workers = 4U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 0U);
+ }
+
+ Y_UNIT_TEST(UseOnlyHalfForLargeInOverload) {
+ const auto scheduler = IScheduler::Make();
+ UNIT_ASSERT(scheduler);
+
+ scheduler->Suspend({MakeRequest(3U, "user1"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user2"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user3"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user4"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user5"), {}});
+ scheduler->Suspend({MakeRequest(3U, "user6"), {}});
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 6U);
+
+ size_t workers = 4U;
+ std::vector<size_t> counts;
+ const auto processor = [&](const IScheduler::TWaitInfo& wait) {
+ counts.emplace_back(wait.Request.GetCount());
+ return true;
+ };
+
+ scheduler->Process(20U, workers, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 6U);
+
+ scheduler->Process(20U, workers = 5U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 5U);
+
+ scheduler->Process(20U, workers = 4U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 5U);
+
+ scheduler->Process(20U, workers = 5U, processor);
+
+ UNIT_ASSERT_VALUES_EQUAL(scheduler->UpdateMetrics(), 4U);
+ }
+}
diff --git a/ydb/library/yql/providers/dq/scheduler/ut/ya.make b/ydb/library/yql/providers/dq/scheduler/ut/ya.make
new file mode 100644
index 0000000000..069195b8b4
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/ut/ya.make
@@ -0,0 +1,16 @@
+UNITTEST_FOR(ydb/library/yql/providers/dq/scheduler)
+
+SIZE(SMALL)
+
+SRCS(
+ dq_scheduler_ut.cpp
+)
+
+PEERDIR(
+ ydb/library/yql/public/udf/service/stub
+ ydb/library/yql/utils/log
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/providers/dq/scheduler/ya.make b/ydb/library/yql/providers/dq/scheduler/ya.make
new file mode 100644
index 0000000000..ed9eca8c37
--- /dev/null
+++ b/ydb/library/yql/providers/dq/scheduler/ya.make
@@ -0,0 +1,24 @@
+LIBRARY()
+
+PEERDIR(
+ ydb/library/yql/dq/common
+ ydb/library/yql/dq/proto
+ ydb/library/yql/providers/dq/config
+ ydb/library/yql/providers/common/proto
+ ydb/library/yql/providers/dq/api/protos
+ ydb/library/yql/providers/dq/common
+ ydb/library/yql/providers/dq/counters
+ library/cpp/actors/protos
+)
+
+SRCS(
+ dq_scheduler.cpp
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/providers/dq/ya.make b/ydb/library/yql/providers/dq/ya.make
index e354a8a22f..b134a0eeef 100644
--- a/ydb/library/yql/providers/dq/ya.make
+++ b/ydb/library/yql/providers/dq/ya.make
@@ -5,6 +5,8 @@ RECURSE(
config
counters
expr_nodes
+ scheduler
+ global_worker_manager
interface
mkql
opt
diff --git a/yt/yt/CMakeLists.darwin-x86_64.txt b/yt/yt/CMakeLists.darwin-x86_64.txt
index b0414f0741..91d55b509b 100644
--- a/yt/yt/CMakeLists.darwin-x86_64.txt
+++ b/yt/yt/CMakeLists.darwin-x86_64.txt
@@ -7,5 +7,6 @@
add_subdirectory(build)
+add_subdirectory(client)
add_subdirectory(core)
add_subdirectory(library)
diff --git a/yt/yt/client/CMakeLists.darwin-x86_64.txt b/yt/yt/client/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..3536fc456a
--- /dev/null
+++ b/yt/yt/client/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,199 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(query_tracker_client)
+set(
+ YT_RPC_MODIFY_ROWS_STRONG_LOCKS_VERSION
+ 2
+)
+set(
+ YT_RPC_PROXY_CLIENT_PROTOCOL_VERSION_MINOR
+ 1
+)
+set(
+ YT_RPC_PROXY_PROTOCOL_VERSION_MAJOR
+ 1
+)
+set(
+ YT_RPC_PROXY_SERVER_PROTOCOL_VERSION_MINOR
+ 2
+)
+
+add_library(yt-yt-client)
+target_compile_options(yt-yt-client PRIVATE
+ -Wdeprecated-this-capture
+)
+target_include_directories(yt-yt-client PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+ ${CMAKE_BINARY_DIR}/yt/yt/client/_/api/rpc_proxy
+)
+target_link_libraries(yt-yt-client PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-client-query_tracker_client
+ yt-yt-core
+ yt-core-http
+ yt-core-https
+ yt-library-auth
+ yt-library-decimal
+ yt-library-re2
+ yt-library-erasure
+ yt-library-numeric
+ yt-library-quantile_digest
+ yt_proto-yt-client
+ library-cpp-json
+ contrib-libs-pfr
+)
+target_sources(yt-yt-client PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/client_common.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/client_cache.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/delegating_client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/etc_client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/journal_client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/operation_client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/security_client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/table_client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/query_tracker_client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/internal_client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/operation_archive_schema.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rowset.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/skynet.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/transaction.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/persistent_queue.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/sticky_transaction_pool.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/address_helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/client_impl.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/client_base.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/connection.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/connection_impl.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/file_reader.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/file_writer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/journal_reader.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/journal_writer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/table_mount_cache.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/table_reader.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/table_writer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/timestamp_provider.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/transaction.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/transaction_impl.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/row_stream.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/wire_row_stream.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/election/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/hive/timestamp_map.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/hydra/version.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chaos_client/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chaos_client/replication_card.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chaos_client/replication_card_cache.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chaos_client/replication_card_serialization.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/chunk_replica.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/data_statistics.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/read_limit.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/chunk_client/ready_event_reader_base.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/file_client/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/journal_client/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/journal_client/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/cypress_client/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/node_tracker_client/node_directory.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/node_tracker_client/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/node_tracker_client/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/object_client/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/object_client/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/scheduler/operation_id_or_alias.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/scheduler/operation_cache.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/security_client/acl.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/security_client/access_control.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/security_client/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/security_client/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/adapters.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/table_output.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/blob_reader.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/check_schema_compatibility.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/chunk_stripe_statistics.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/column_rename_descriptor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/column_sort_schema.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/comparator.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/key.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/key_bound.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/key_bound_compressor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/pipe.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/versioned_row.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/unversioned_row.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/unversioned_value.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/versioned_reader.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/row_base.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/row_batch.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/row_buffer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/schema.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/schema_serialization_helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/serialize.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/logical_type.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/name_table.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/wire_protocol.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/columnar_statistics.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/value_consumer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/table_consumer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/schemaless_row_reorderer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/unordered_schemaful_reader.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/validate_logical_type.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/composite_compare.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/columnar.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/record_codegen_cpp.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/table_client/record_helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/table_mount_cache_detail.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/table_mount_cache.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/tablet_client/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/common.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/consumer_client.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/partition_reader.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/queue_client/queue_rowset.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/ypath/rich.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/ypath/parser_detail.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/batching_timestamp_provider.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/noop_timestamp_provider.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/remote_timestamp_provider.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/transaction_client/timestamp_provider_base.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/misc/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/misc/io_tags.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/misc/method_helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/misc/workload.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/job_tracker_client/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/job_tracker_client/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/query_client/query_builder.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/query_client/query_statistics.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/check_yson_token.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/check_type_compatibility.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/infinite_entity.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/yson_format_conversion.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/uuid_text.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/complex_types/time_text.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/zookeeper/packet.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/zookeeper/protocol.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/zookeeper/requests.cpp
+)
+configure_file(
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/api/rpc_proxy/protocol_version_variables.h.in
+ ${CMAKE_BINARY_DIR}/yt/yt/client/_/api/rpc_proxy/protocol_version_variables.h
+)
diff --git a/yt/yt/client/CMakeLists.txt b/yt/yt/client/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/yt/yt/client/CMakeLists.txt
+++ b/yt/yt/client/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()
diff --git a/yt/yt/client/query_tracker_client/CMakeLists.darwin-x86_64.txt b/yt/yt/client/query_tracker_client/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..3d1ea8d25a
--- /dev/null
+++ b/yt/yt/client/query_tracker_client/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,21 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-client-query_tracker_client)
+target_compile_options(yt-client-query_tracker_client PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-client-query_tracker_client PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+)
+target_sources(yt-client-query_tracker_client PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/client/query_tracker_client/public.cpp
+)
diff --git a/yt/yt/client/query_tracker_client/CMakeLists.txt b/yt/yt/client/query_tracker_client/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/yt/yt/client/query_tracker_client/CMakeLists.txt
+++ b/yt/yt/client/query_tracker_client/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()
diff --git a/yt/yt/library/CMakeLists.darwin-x86_64.txt b/yt/yt/library/CMakeLists.darwin-x86_64.txt
index 20f7fe76fa..d26f3bae28 100644
--- a/yt/yt/library/CMakeLists.darwin-x86_64.txt
+++ b/yt/yt/library/CMakeLists.darwin-x86_64.txt
@@ -6,7 +6,13 @@
# original buildsystem will not be accepted.
+add_subdirectory(auth)
+add_subdirectory(decimal)
+add_subdirectory(erasure)
+add_subdirectory(numeric)
add_subdirectory(profiling)
+add_subdirectory(quantile_digest)
+add_subdirectory(re2)
add_subdirectory(syncmap)
add_subdirectory(tracing)
add_subdirectory(tvm)
diff --git a/yt/yt/library/auth/CMakeLists.darwin-x86_64.txt b/yt/yt/library/auth/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..b026c98348
--- /dev/null
+++ b/yt/yt/library/auth/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,24 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-auth)
+target_compile_options(yt-library-auth PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-auth PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ yt-library-tvm
+)
+target_sources(yt-library-auth PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/auth/auth.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/auth/authentication_options.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/auth/credentials_injecting_channel.cpp
+)
diff --git a/yt/yt/library/auth/CMakeLists.txt b/yt/yt/library/auth/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/yt/yt/library/auth/CMakeLists.txt
+++ b/yt/yt/library/auth/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()
diff --git a/yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt b/yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..2b82fc6b48
--- /dev/null
+++ b/yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-decimal)
+target_compile_options(yt-library-decimal PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-decimal PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ library-cpp-int128
+)
+target_sources(yt-library-decimal PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/decimal/decimal.cpp
+)
diff --git a/yt/yt/library/decimal/CMakeLists.txt b/yt/yt/library/decimal/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/yt/yt/library/decimal/CMakeLists.txt
+++ b/yt/yt/library/decimal/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()
diff --git a/yt/yt/library/erasure/CMakeLists.darwin-x86_64.txt b/yt/yt/library/erasure/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..232e2fa1c0
--- /dev/null
+++ b/yt/yt/library/erasure/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,21 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-erasure)
+target_compile_options(yt-library-erasure PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-erasure PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+)
+target_sources(yt-library-erasure PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/erasure/public.cpp
+)
diff --git a/yt/yt/library/erasure/CMakeLists.txt b/yt/yt/library/erasure/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/yt/yt/library/erasure/CMakeLists.txt
+++ b/yt/yt/library/erasure/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()
diff --git a/yt/yt/library/numeric/CMakeLists.darwin-x86_64.txt b/yt/yt/library/numeric/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..9f1b4b1c1e
--- /dev/null
+++ b/yt/yt/library/numeric/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,21 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-numeric)
+target_compile_options(yt-library-numeric PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-numeric PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-yt-small_containers
+)
+target_sources(yt-library-numeric PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/numeric/piecewise_linear_function.cpp
+)
diff --git a/yt/yt/library/numeric/CMakeLists.txt b/yt/yt/library/numeric/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/yt/yt/library/numeric/CMakeLists.txt
+++ b/yt/yt/library/numeric/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()
diff --git a/yt/yt/library/quantile_digest/CMakeLists.darwin-x86_64.txt b/yt/yt/library/quantile_digest/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..ceac4e685a
--- /dev/null
+++ b/yt/yt/library/quantile_digest/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,54 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(yt-library-quantile_digest)
+target_compile_options(yt-library-quantile_digest PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-quantile_digest PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-yt-memory
+ library-cpp-tdigest
+ yt-yt-core
+ contrib-libs-protobuf
+)
+target_proto_messages(yt-library-quantile_digest PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/quantile_digest/proto/quantile_digest.proto
+)
+target_sources(yt-library-quantile_digest PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/quantile_digest/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/quantile_digest/quantile_digest.cpp
+)
+target_proto_addincls(yt-library-quantile_digest
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(yt-library-quantile_digest
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/yt/yt/library/quantile_digest/CMakeLists.txt b/yt/yt/library/quantile_digest/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/yt/yt/library/quantile_digest/CMakeLists.txt
+++ b/yt/yt/library/quantile_digest/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()
diff --git a/yt/yt/library/re2/CMakeLists.darwin-x86_64.txt b/yt/yt/library/re2/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..7310b7ab28
--- /dev/null
+++ b/yt/yt/library/re2/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-re2)
+target_compile_options(yt-library-re2 PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-re2 PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ contrib-libs-re2
+)
+target_sources(yt-library-re2 PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/re2/re2.cpp
+)
diff --git a/yt/yt/library/re2/CMakeLists.txt b/yt/yt/library/re2/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/yt/yt/library/re2/CMakeLists.txt
+++ b/yt/yt/library/re2/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()
diff --git a/yt/yt_proto/yt/CMakeLists.darwin-x86_64.txt b/yt/yt_proto/yt/CMakeLists.darwin-x86_64.txt
index d8f6bf9771..aa7a2b73aa 100644
--- a/yt/yt_proto/yt/CMakeLists.darwin-x86_64.txt
+++ b/yt/yt_proto/yt/CMakeLists.darwin-x86_64.txt
@@ -6,5 +6,6 @@
# original buildsystem will not be accepted.
+add_subdirectory(client)
add_subdirectory(core)
add_subdirectory(formats)
diff --git a/yt/yt_proto/yt/client/CMakeLists.darwin-x86_64.txt b/yt/yt_proto/yt/client/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..bde6ffe9e0
--- /dev/null
+++ b/yt/yt_proto/yt/client/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,296 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(yt_proto-yt-client)
+target_include_directories(yt_proto-yt-client PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+)
+target_link_libraries(yt_proto-yt-client PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt_proto-yt-core
+ contrib-libs-protobuf
+)
+target_proto_messages(yt_proto-yt-client PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/api/rpc_proxy/proto/discovery_service.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/cell_master/proto/cell_directory.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chaos_client/proto/replication_card.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/data_statistics.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/chunk_meta.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/read_limit.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/chunk_spec.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/chunk_client/proto/confirm_chunk_replica_info.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/discovery_client/proto/discovery_client_service.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/hive/proto/timestamp_map.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/hive/proto/cluster_directory.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/node_tracker_client/proto/node.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/node_tracker_client/proto/node_directory.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/table_chunk_format/proto/chunk_meta.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/table_chunk_format/proto/column_meta.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/table_chunk_format/proto/wire_protocol.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/transaction_client/proto/timestamp_service.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/query_client/proto/query_statistics.proto
+ ${CMAKE_SOURCE_DIR}/yt/yt_proto/yt/client/misc/proto/workload.proto
+)
+target_proto_addincls(yt_proto-yt-client
+ ./yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(yt_proto-yt-client
+ --cpp_out=${CMAKE_BINARY_DIR}/yt
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt
+)
diff --git a/yt/yt_proto/yt/client/CMakeLists.txt b/yt/yt_proto/yt/client/CMakeLists.txt
index 4d48dcdee6..606ff46b4b 100644
--- a/yt/yt_proto/yt/client/CMakeLists.txt
+++ b/yt/yt_proto/yt/client/CMakeLists.txt
@@ -8,6 +8,8 @@
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
endif()