diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2023-11-21 10:06:38 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2023-11-21 10:28:10 +0300 |
commit | fc6e7dcd6be0eee17a68aabecc15da6bbc063cb7 (patch) | |
tree | 0554971ad955e3c9bdc985ab6e364e0a85c46110 | |
parent | 7c95ccc901f91884d3be99629c396b1772dd8835 (diff) | |
download | ydb-fc6e7dcd6be0eee17a68aabecc15da6bbc063cb7.tar.gz |
Intermediate changes
29 files changed, 618 insertions, 7 deletions
diff --git a/.mapping.json b/.mapping.json index 8ff830aa3e..269bba78ff 100644 --- a/.mapping.json +++ b/.mapping.json @@ -12082,6 +12082,12 @@ "yt/yt/library/tvm/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/tvm/CMakeLists.txt":"", "yt/yt/library/tvm/CMakeLists.windows-x86_64.txt":"", + "yt/yt/library/tvm/service/CMakeLists.darwin-arm64.txt":"", + "yt/yt/library/tvm/service/CMakeLists.darwin-x86_64.txt":"", + "yt/yt/library/tvm/service/CMakeLists.linux-aarch64.txt":"", + "yt/yt/library/tvm/service/CMakeLists.linux-x86_64.txt":"", + "yt/yt/library/tvm/service/CMakeLists.txt":"", + "yt/yt/library/tvm/service/CMakeLists.windows-x86_64.txt":"", "yt/yt/library/undumpable/CMakeLists.darwin-arm64.txt":"", "yt/yt/library/undumpable/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/undumpable/CMakeLists.linux-aarch64.txt":"", diff --git a/yt/yt/library/tracing/example/main.cpp b/yt/yt/library/tracing/example/main.cpp index 2dc5089e6f..02baaba24e 100644 --- a/yt/yt/library/tracing/example/main.cpp +++ b/yt/yt/library/tracing/example/main.cpp @@ -6,6 +6,8 @@ #include <yt/yt/library/tracing/jaeger/tracer.h> +#include <util/system/env.h> + using namespace NYT; using namespace NYT::NTracing; @@ -53,21 +55,52 @@ void DelayedSamplingExample(std::optional<TString> endpoint) traceContext->Finish(); } +NAuth::TTvmServiceConfigPtr GetTvmMockConfig() { + auto config = New<NAuth::TTvmServiceConfig>(); + config->EnableMock = true; + config->ClientSelfSecret = "TestSecret-0"; + config->ClientDstMap["tracing"] = 10; + config->ClientEnableServiceTicketFetching = true; + + return config; +} + +NAuth::TTvmServiceConfigPtr GetTvmConfig() { + auto config = New<NAuth::TTvmServiceConfig>(); + config->ClientSelfId = FromString<NAuth::TTvmId>(GetEnv("TVM_ID")); + config->ClientSelfSecretEnv = "TVM_SECRET"; + config->ClientDstMap["tracing"] = FromString<NAuth::TTvmId>(GetEnv("TRACING_TVM_ID")); + config->ClientEnableServiceTicketFetching = true; + + return config; +} + int main(int argc, char* argv[]) { try { + + bool test = false; + auto usage = Format("usage: %v [--test] COLLECTOR_ENDPOINTS", argv[0]); + + if (argc >= 2 && argv[1] == TString("--test")) { + test = true; + argv++; + argc--; + } + if (argc < 2) { - throw yexception() << "usage: " << argv[0] << " COLLECTOR_ENDPOINTS"; + throw yexception() << usage; } auto config = New<NTracing::TJaegerTracerConfig>(); config->CollectorChannelConfig = New<NRpc::NGrpc::TChannelConfig>(); config->CollectorChannelConfig->Address = argv[1]; - config->FlushPeriod = TDuration::MilliSeconds(100); + config->FlushPeriod = TDuration::MilliSeconds(test ? 100 : 1000); config->ServiceName = "example"; config->ProcessTags["host"] = "prime-dev.qyp.yandex-team.ru"; + config->TvmService = test ? GetTvmMockConfig() : GetTvmConfig(); auto jaeger = New<NTracing::TJaegerTracer>(config); SetGlobalTracer(jaeger); diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.darwin-arm64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.darwin-arm64.txt index ebc07421e3..7604b7dc8d 100644 --- a/yt/yt/library/tracing/jaeger/CMakeLists.darwin-arm64.txt +++ b/yt/yt/library/tracing/jaeger/CMakeLists.darwin-arm64.txt @@ -27,6 +27,7 @@ target_link_libraries(library-tracing-jaeger PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) @@ -59,6 +60,7 @@ target_link_libraries(library-tracing-jaeger.global PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt index ebc07421e3..7604b7dc8d 100644 --- a/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt @@ -27,6 +27,7 @@ target_link_libraries(library-tracing-jaeger PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) @@ -59,6 +60,7 @@ target_link_libraries(library-tracing-jaeger.global PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt index 0946a55b6c..2dfa2a2dc2 100644 --- a/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt +++ b/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt @@ -28,6 +28,7 @@ target_link_libraries(library-tracing-jaeger PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) @@ -61,6 +62,7 @@ target_link_libraries(library-tracing-jaeger.global PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt index 0946a55b6c..2dfa2a2dc2 100644 --- a/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt +++ b/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt @@ -28,6 +28,7 @@ target_link_libraries(library-tracing-jaeger PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) @@ -61,6 +62,7 @@ target_link_libraries(library-tracing-jaeger.global PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt index e6ee22e85b..8cf111cc89 100644 --- a/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt +++ b/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt @@ -24,6 +24,7 @@ target_link_libraries(library-tracing-jaeger PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) @@ -53,6 +54,7 @@ target_link_libraries(library-tracing-jaeger.global PUBLIC contrib-libs-cxxsupp yutil yt-library-tracing + library-tvm-service core-rpc-grpc contrib-libs-protobuf ) diff --git a/yt/yt/library/tracing/jaeger/tracer.cpp b/yt/yt/library/tracing/jaeger/tracer.cpp index 25dc484e1f..42a6df2c38 100644 --- a/yt/yt/library/tracing/jaeger/tracer.cpp +++ b/yt/yt/library/tracing/jaeger/tracer.cpp @@ -4,6 +4,8 @@ #include <yt/yt/library/profiling/sensor.h> +#include <yt/yt/library/tvm/service/tvm_service.h> + #include <yt/yt/core/rpc/grpc/channel.h> #include <yt/yt/core/concurrency/action_queue.h> @@ -27,11 +29,14 @@ using namespace NRpc; using namespace NConcurrency; using namespace NProfiling; using namespace NYTree; +using namespace NAuth; //////////////////////////////////////////////////////////////////////////////// static const NLogging::TLogger Logger{"Jaeger"}; static const NProfiling::TProfiler Profiler{"/tracing"}; +static const TString ServiceTicketMetadataName = "x-ya-service-ticket"; +static const TString TracingServiceAlias = "tracing"; //////////////////////////////////////////////////////////////////////////////// @@ -85,6 +90,9 @@ void TJaegerTracerConfig::Register(TRegistrar registrar) .Default(); registrar.Parameter("enable_pid_tag", &TThis::EnablePidTag) .Default(false); + + registrar.Parameter("tvm_service", &TThis::TvmService) + .Optional(); } TJaegerTracerConfigPtr TJaegerTracerConfig::ApplyDynamic(const TJaegerTracerDynamicConfigPtr& dynamicConfig) const @@ -108,6 +116,7 @@ TJaegerTracerConfigPtr TJaegerTracerConfig::ApplyDynamic(const TJaegerTracerDyna config->ServiceName = ServiceName; config->ProcessTags = ProcessTags; config->EnablePidTag = EnablePidTag; + config->TvmService = TvmService; config->Postprocess(); return config; @@ -287,8 +296,12 @@ TJaegerChannelManager::TJaegerChannelManager() , RpcTimeout_() { } -TJaegerChannelManager::TJaegerChannelManager(const TIntrusivePtr<TJaegerTracerConfig>& config, const TString& endpoint) - : Endpoint_(endpoint) +TJaegerChannelManager::TJaegerChannelManager( + const TIntrusivePtr<TJaegerTracerConfig>& config, + const TString& endpoint, + const ITvmServicePtr& tvmService) + : TvmService_(tvmService) + , Endpoint_(endpoint) , ReopenTime_(TInstant::Now() + config->ReconnectPeriod + RandomDuration(config->ReconnectPeriod)) , RpcTimeout_(config->RpcTimeout) , PushedBytes_(Profiler.WithTag("endpoint", endpoint).Counter("/pushed_bytes")) @@ -312,6 +325,12 @@ bool TJaegerChannelManager::Push(const std::vector<TSharedRef>& batches, int spa req->SetEnableLegacyRpcCodecs(false); req->set_batch(MergeRefsToString(batches)); + if (TvmService_) { + auto* ticketExt = req->Header().MutableExtension(NRpc::NProto::TCustomMetadataExt::custom_metadata_ext); + ticketExt->mutable_entries()->insert( + {ServiceTicketMetadataName, TvmService_->GetServiceTicket(TracingServiceAlias)}); + } + YT_LOG_DEBUG("Sending spans (SpanCount: %v, PayloadSize: %v, Endpoint: %v)", spanCount, req->batch().size(), @@ -359,6 +378,7 @@ TJaegerTracer::TJaegerTracer( BIND(&TJaegerTracer::Flush, MakeStrong(this)), config->FlushPeriod)) , Config_(config) + , TvmService_(config->TvmService ? CreateTvmService(config->TvmService) : nullptr) { Profiler.AddFuncGauge("/enabled", MakeStrong(this), [this] { return Config_.Acquire()->IsEnabled(); @@ -563,13 +583,13 @@ void TJaegerTracer::Flush() auto it = CollectorChannels_.find(endpoint); if (it == CollectorChannels_.end()) { - it = CollectorChannels_.insert({endpoint, TJaegerChannelManager(config, endpoint)}).first; + it = CollectorChannels_.emplace(endpoint, TJaegerChannelManager(config, endpoint, TvmService_)).first; } auto& channel = it->second; if (channel.NeedsReopen(flushStartTime)) { - channel = TJaegerChannelManager(config, endpoint); + channel = TJaegerChannelManager(config, endpoint, TvmService_); } if (channel.Push(batches, spanCount)) { diff --git a/yt/yt/library/tracing/jaeger/tracer.h b/yt/yt/library/tracing/jaeger/tracer.h index fadc9a1703..2dc2ae6ba6 100644 --- a/yt/yt/library/tracing/jaeger/tracer.h +++ b/yt/yt/library/tracing/jaeger/tracer.h @@ -5,6 +5,8 @@ #include <yt/yt/library/tracing/tracer.h> #include <yt/yt/library/profiling/sensor.h> +#include <yt/yt/library/tvm/service/config.h> +#include <yt/yt/library/tvm/service/public.h> #include <yt/yt/core/misc/mpsc_stack.h> #include <yt/yt/core/misc/atomic_object.h> @@ -77,6 +79,8 @@ public: bool EnablePidTag; + NAuth::TTvmServiceConfigPtr TvmService; + TJaegerTracerConfigPtr ApplyDynamic(const TJaegerTracerDynamicConfigPtr& dynamicConfig) const; bool IsEnabled() const; @@ -120,7 +124,10 @@ class TJaegerChannelManager { public: TJaegerChannelManager(); - TJaegerChannelManager(const TIntrusivePtr<TJaegerTracerConfig>& config, const TString& endpoint); + TJaegerChannelManager( + const TIntrusivePtr<TJaegerTracerConfig>& config, + const TString& endpoint, + const NAuth::ITvmServicePtr& tvmService); bool Push(const std::vector<TSharedRef>& batches, int spanCount); bool NeedsReopen(TInstant currentTime); @@ -130,6 +137,7 @@ public: private: NRpc::IChannelPtr Channel_; + NAuth::ITvmServicePtr TvmService_; TString Endpoint_; @@ -175,6 +183,8 @@ private: THashMap<TString, TJaegerChannelManager> CollectorChannels_; NRpc::NGrpc::TChannelConfigPtr OpenChannelConfig_; + NAuth::ITvmServicePtr TvmService_; + void Flush(); void DequeueAll(const TJaegerTracerConfigPtr& config); void NotifyEmptyQueue(); diff --git a/yt/yt/library/tracing/jaeger/ya.make b/yt/yt/library/tracing/jaeger/ya.make index 2276d20db2..b85e518305 100644 --- a/yt/yt/library/tracing/jaeger/ya.make +++ b/yt/yt/library/tracing/jaeger/ya.make @@ -4,6 +4,7 @@ INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) PEERDIR( yt/yt/library/tracing + yt/yt/library/tvm/service yt/yt/core/rpc/grpc ) diff --git a/yt/yt/library/tvm/CMakeLists.darwin-arm64.txt b/yt/yt/library/tvm/CMakeLists.darwin-arm64.txt index f92c150fd3..5f6d73b807 100644 --- a/yt/yt/library/tvm/CMakeLists.darwin-arm64.txt +++ b/yt/yt/library/tvm/CMakeLists.darwin-arm64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(service) add_library(yt-library-tvm) target_compile_options(yt-library-tvm PRIVATE diff --git a/yt/yt/library/tvm/CMakeLists.darwin-x86_64.txt b/yt/yt/library/tvm/CMakeLists.darwin-x86_64.txt index f92c150fd3..5f6d73b807 100644 --- a/yt/yt/library/tvm/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/library/tvm/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(service) add_library(yt-library-tvm) target_compile_options(yt-library-tvm PRIVATE diff --git a/yt/yt/library/tvm/CMakeLists.linux-aarch64.txt b/yt/yt/library/tvm/CMakeLists.linux-aarch64.txt index 3a22559a59..788d405e94 100644 --- a/yt/yt/library/tvm/CMakeLists.linux-aarch64.txt +++ b/yt/yt/library/tvm/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(service) add_library(yt-library-tvm) target_compile_options(yt-library-tvm PRIVATE diff --git a/yt/yt/library/tvm/CMakeLists.linux-x86_64.txt b/yt/yt/library/tvm/CMakeLists.linux-x86_64.txt index 3a22559a59..788d405e94 100644 --- a/yt/yt/library/tvm/CMakeLists.linux-x86_64.txt +++ b/yt/yt/library/tvm/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(service) add_library(yt-library-tvm) target_compile_options(yt-library-tvm PRIVATE diff --git a/yt/yt/library/tvm/CMakeLists.windows-x86_64.txt b/yt/yt/library/tvm/CMakeLists.windows-x86_64.txt index 5b92f509ce..6f1972e2a8 100644 --- a/yt/yt/library/tvm/CMakeLists.windows-x86_64.txt +++ b/yt/yt/library/tvm/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(service) add_library(yt-library-tvm) target_link_libraries(yt-library-tvm PUBLIC diff --git a/yt/yt/library/tvm/service/CMakeLists.darwin-arm64.txt b/yt/yt/library/tvm/service/CMakeLists.darwin-arm64.txt new file mode 100644 index 0000000000..44caa31066 --- /dev/null +++ b/yt/yt/library/tvm/service/CMakeLists.darwin-arm64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-tvm-service) +target_compile_options(library-tvm-service PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tvm-service PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-memory + cpp-yt-logging + yt-yt-core +) +target_sources(library-tvm-service PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_common.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_dummy.cpp +) diff --git a/yt/yt/library/tvm/service/CMakeLists.darwin-x86_64.txt b/yt/yt/library/tvm/service/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..44caa31066 --- /dev/null +++ b/yt/yt/library/tvm/service/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-tvm-service) +target_compile_options(library-tvm-service PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tvm-service PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-memory + cpp-yt-logging + yt-yt-core +) +target_sources(library-tvm-service PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_common.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_dummy.cpp +) diff --git a/yt/yt/library/tvm/service/CMakeLists.linux-aarch64.txt b/yt/yt/library/tvm/service/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..9ec5940c8b --- /dev/null +++ b/yt/yt/library/tvm/service/CMakeLists.linux-aarch64.txt @@ -0,0 +1,26 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-tvm-service) +target_compile_options(library-tvm-service PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tvm-service PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-memory + cpp-yt-logging + yt-yt-core +) +target_sources(library-tvm-service PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_common.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_dummy.cpp +) diff --git a/yt/yt/library/tvm/service/CMakeLists.linux-x86_64.txt b/yt/yt/library/tvm/service/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..9ec5940c8b --- /dev/null +++ b/yt/yt/library/tvm/service/CMakeLists.linux-x86_64.txt @@ -0,0 +1,26 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-tvm-service) +target_compile_options(library-tvm-service PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tvm-service PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-memory + cpp-yt-logging + yt-yt-core +) +target_sources(library-tvm-service PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_common.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_dummy.cpp +) diff --git a/yt/yt/library/tvm/service/CMakeLists.txt b/yt/yt/library/tvm/service/CMakeLists.txt new file mode 100644 index 0000000000..2dce3a77fe --- /dev/null +++ b/yt/yt/library/tvm/service/CMakeLists.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") + include(CMakeLists.darwin-arm64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/tvm/service/CMakeLists.windows-x86_64.txt b/yt/yt/library/tvm/service/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..0b5bd76ab4 --- /dev/null +++ b/yt/yt/library/tvm/service/CMakeLists.windows-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-tvm-service) +target_link_libraries(library-tvm-service PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-memory + cpp-yt-logging + yt-yt-core +) +target_sources(library-tvm-service PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_common.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/tvm/service/tvm_service_dummy.cpp +) diff --git a/yt/yt/library/tvm/service/config.cpp b/yt/yt/library/tvm/service/config.cpp new file mode 100644 index 0000000000..f1671e8650 --- /dev/null +++ b/yt/yt/library/tvm/service/config.cpp @@ -0,0 +1,63 @@ +#include "config.h" + +namespace NYT::NAuth { + +//////////////////////////////////////////////////////////////////////////////// + +void TTvmServiceConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("use_tvm_tool", &TThis::UseTvmTool) + .Default(false); + registrar.Parameter("client_self_id", &TThis::ClientSelfId) + .Default(0); + registrar.Parameter("client_disk_cache_dir", &TThis::ClientDiskCacheDir) + .Optional(); + registrar.Parameter("tvm_host", &TThis::TvmHost) + .Optional(); + registrar.Parameter("tvm_port", &TThis::TvmPort) + .Optional(); + registrar.Parameter("client_enable_user_ticket_checking", &TThis::ClientEnableUserTicketChecking) + .Default(false); + registrar.Parameter("client_blackbox_env", &TThis::ClientBlackboxEnv) + .Default("ProdYateam"); + registrar.Parameter("client_enable_service_ticket_fetching", &TThis::ClientEnableServiceTicketFetching) + .Default(false); + registrar.Parameter("client_self_secret", &TThis::ClientSelfSecret) + .Optional(); + registrar.Parameter("client_self_secret_path", &TThis::ClientSelfSecretPath) + .Optional(); + registrar.Parameter("client_self_secret_env", &TThis::ClientSelfSecretEnv) + .Optional(); + registrar.Parameter("client_dst_map", &TThis::ClientDstMap) + .Optional(); + registrar.Parameter("client_enable_service_ticket_checking", &TThis::ClientEnableServiceTicketChecking) + .Default(false); + + registrar.Parameter("enable_ticket_parse_cache", &TThis::EnableTicketParseCache) + .Default(false); + registrar.Parameter("ticket_checking_cache_timeout", &TThis::TicketCheckingCacheTimeout) + .Default(TDuration::Minutes(1)); + + registrar.Parameter("tvm_tool_self_alias", &TThis::TvmToolSelfAlias) + .Optional(); + registrar.Parameter("tvm_tool_port", &TThis::TvmToolPort) + .Optional(); + registrar.Parameter("tvm_tool_auth_token", &TThis::TvmToolAuthToken) + .Optional(); + + registrar.Parameter("enable_mock", &TThis::EnableMock) + .Default(false); + registrar.Parameter("require_mock_secret", &TThis::RequireMockSecret) + .Default(true); + + registrar.Postprocessor([] (TThis* config) { + if (config->ClientSelfSecretEnv && config->ClientSelfSecretPath) { + THROW_ERROR_EXCEPTION("Options \"client_self_secret_env\", \"client_self_secret_path\" " + "cannot be used together"); + } + }); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NAuth diff --git a/yt/yt/library/tvm/service/config.h b/yt/yt/library/tvm/service/config.h new file mode 100644 index 0000000000..9f3c8b5af6 --- /dev/null +++ b/yt/yt/library/tvm/service/config.h @@ -0,0 +1,69 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/ytree/yson_struct.h> + +namespace NYT::NAuth { + +//////////////////////////////////////////////////////////////////////////////// + +class TTvmServiceConfig + : public virtual NYTree::TYsonStruct +{ +public: + bool UseTvmTool; + + // TvmClient settings + TTvmId ClientSelfId = 0; + std::optional<TString> ClientDiskCacheDir; + + std::optional<TString> TvmHost; + std::optional<ui16> TvmPort; + + bool ClientEnableUserTicketChecking = false; + TString ClientBlackboxEnv; + + bool ClientEnableServiceTicketFetching = false; + + //! Do not use this option as the plaintext value of secret may be exposed via service orchid or somehow else. + std::optional<TString> ClientSelfSecret; + + //! Name of env variable with TVM secret. Used if ClientSelfSecret is unset. + std::optional<TString> ClientSelfSecretEnv; + + //! Path to TVM secret. Used if ClientSelfSecret is unset. + std::optional<TString> ClientSelfSecretPath; + + THashMap<TString, ui32> ClientDstMap; + + bool ClientEnableServiceTicketChecking = false; + + //! If true, then checked tickets are cached, allowing us to speed up checking. + bool EnableTicketParseCache = false; + TDuration TicketCheckingCacheTimeout; + + TString TvmToolSelfAlias; + //! If not specified, get port from env variable `DEPLOY_TVM_TOOL_URL`. + int TvmToolPort = 0; + //! Do not use this option in production. + //! If not specified, get token from env variable `TVMTOOL_LOCAL_AUTHTOKEN`. + std::optional<TString> TvmToolAuthToken; + + //! For testing only. If enabled, then a mock instead of a real TVM service will be used. + bool EnableMock = false; + + //! If EnableMock and RequireMockSecret is true, then ensures that ClientSelfSecret is equal to + //! "SecretPrefix-" + ToString(ClientSelfId). + bool RequireMockSecret = true; + + REGISTER_YSON_STRUCT(TTvmServiceConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TTvmServiceConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NAuth diff --git a/yt/yt/library/tvm/service/public.h b/yt/yt/library/tvm/service/public.h new file mode 100644 index 0000000000..89b3c4e60d --- /dev/null +++ b/yt/yt/library/tvm/service/public.h @@ -0,0 +1,33 @@ +#pragma once + +#include <library/cpp/yt/memory/ref_counted.h> + +#include <util/generic/hash_set.h> +#include <util/generic/string.h> + +namespace NYT::NAuth { + +//////////////////////////////////////////////////////////////////////////////// + +DECLARE_REFCOUNTED_CLASS(TTvmServiceConfig) +DECLARE_REFCOUNTED_STRUCT(ITvmService) +DECLARE_REFCOUNTED_STRUCT(IDynamicTvmService) + +//////////////////////////////////////////////////////////////////////////////// + +struct TParsedTicket +{ + ui64 DefaultUid; + THashSet<TString> Scopes; +}; + +using TTvmId = ui64; + +struct TParsedServiceTicket +{ + TTvmId TvmId; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NAuth diff --git a/yt/yt/library/tvm/service/tvm_service.h b/yt/yt/library/tvm/service/tvm_service.h new file mode 100644 index 0000000000..7302053e62 --- /dev/null +++ b/yt/yt/library/tvm/service/tvm_service.h @@ -0,0 +1,77 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/library/tvm/public.h> + +#include <yt/yt/library/profiling/sensor.h> + +namespace NYT::NAuth { + +//////////////////////////////////////////////////////////////////////////////// + +struct ITvmService + : public virtual TRefCounted +{ + //! Our TVM id. + virtual TTvmId GetSelfTvmId() = 0; + + //! Get TVM service ticket from us to serviceAlias. Service mapping must be in config. + //! Throws on failure. + virtual TString GetServiceTicket(const TString& serviceAlias) = 0; + + //! Get TVM service ticket from us to serviceId. Service ID must be known (either during + //! construction or explicitly added in dynamic service). + //! Throws on failure. + virtual TString GetServiceTicket(TTvmId serviceId) = 0; + + //! Decode user ticket contents. Throws on failure. + virtual TParsedTicket ParseUserTicket(const TString& ticket) = 0; + + //! Decode service ticket contents. Throws on failure. + virtual TParsedServiceTicket ParseServiceTicket(const TString& ticket) = 0; +}; + +struct IDynamicTvmService + : public virtual ITvmService +{ +public: + //! Add destination service IDs to fetch. It is possible to add the same ID multiple + //! times, though it will be added only once really. + virtual void AddDestinationServiceIds(const std::vector<TTvmId>& serviceIds) = 0; +}; + +DEFINE_REFCOUNTED_TYPE(ITvmService) +DEFINE_REFCOUNTED_TYPE(IDynamicTvmService) + +//////////////////////////////////////////////////////////////////////////////// + +ITvmServicePtr CreateTvmService( + TTvmServiceConfigPtr config, + NProfiling::TProfiler profiler = {}); + +IDynamicTvmServicePtr CreateDynamicTvmService( + TTvmServiceConfigPtr config, + NProfiling::TProfiler profiler = {}); + +//////////////////////////////////////////////////////////////////////////////// + +IServiceTicketAuthPtr CreateServiceTicketAuth( + ITvmServicePtr tvmService, + TTvmId dstServiceId); + +IServiceTicketAuthPtr CreateServiceTicketAuth( + ITvmServicePtr tvmService, + TString dstServiceAlias); + +//////////////////////////////////////////////////////////////////////////////// + +TStringBuf RemoveTicketSignature(TStringBuf ticketBody); + +//////////////////////////////////////////////////////////////////////////////// + +bool IsDummyTvmServiceImplementation(); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NAuth diff --git a/yt/yt/library/tvm/service/tvm_service_common.cpp b/yt/yt/library/tvm/service/tvm_service_common.cpp new file mode 100644 index 0000000000..f031da6bfb --- /dev/null +++ b/yt/yt/library/tvm/service/tvm_service_common.cpp @@ -0,0 +1,55 @@ +#include "tvm_service.h" + +#include <library/cpp/yt/memory/new.h> + +#include <yt/yt/library/tvm/tvm_base.h> + +namespace NYT::NAuth { + +//////////////////////////////////////////////////////////////////////////////// + +template <typename TId> +class TServiceTicketAuth + : public IServiceTicketAuth +{ +public: + TServiceTicketAuth( + ITvmServicePtr tvmService, + TId destServiceId) + : TvmService_(std::move(tvmService)) + , DstServiceId_(std::move(destServiceId)) + { } + + TString IssueServiceTicket() override + { + return TvmService_->GetServiceTicket(DstServiceId_); + } + +private: + const ITvmServicePtr TvmService_; + const TId DstServiceId_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +IServiceTicketAuthPtr CreateServiceTicketAuth( + ITvmServicePtr tvmService, + TTvmId dstServiceId) +{ + YT_VERIFY(tvmService); + + return New<TServiceTicketAuth<TTvmId>>(std::move(tvmService), dstServiceId); +} + +IServiceTicketAuthPtr CreateServiceTicketAuth( + ITvmServicePtr tvmService, + TString dstServiceAlias) +{ + YT_VERIFY(tvmService); + + return New<TServiceTicketAuth<TString>>(std::move(tvmService), std::move(dstServiceAlias)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NAuth diff --git a/yt/yt/library/tvm/service/tvm_service_dummy.cpp b/yt/yt/library/tvm/service/tvm_service_dummy.cpp new file mode 100644 index 0000000000..f592d3a198 --- /dev/null +++ b/yt/yt/library/tvm/service/tvm_service_dummy.cpp @@ -0,0 +1,40 @@ +#include "tvm_service.h" +#include "config.h" + +namespace NYT::NAuth { + +using namespace NProfiling; + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK ITvmServicePtr CreateTvmService( + TTvmServiceConfigPtr /*config*/, + TProfiler /*profiler*/) +{ + THROW_ERROR_EXCEPTION("Not implemented"); +} + +Y_WEAK IDynamicTvmServicePtr CreateDynamicTvmService( + TTvmServiceConfigPtr /*config*/, + TProfiler /*profiler*/) +{ + THROW_ERROR_EXCEPTION("Not implemented"); +} + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK TStringBuf RemoveTicketSignature(TStringBuf /*ticketBody*/) +{ + THROW_ERROR_EXCEPTION("Not implemented"); +} + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK bool IsDummyTvmServiceImplementation() +{ + return true; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NAuth diff --git a/yt/yt/library/tvm/service/unittests/ya.make b/yt/yt/library/tvm/service/unittests/ya.make new file mode 100644 index 0000000000..84baf8c216 --- /dev/null +++ b/yt/yt/library/tvm/service/unittests/ya.make @@ -0,0 +1,21 @@ +GTEST(unittester-library-auth_tvm) + +ALLOCATOR(YT) + +INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc) + +PEERDIR( + yt/yt/build + + yt/yt/core/test_framework + + yt/yt/library/tvm/service +) + +EXPLICIT_DATA() + +IF(NOT OPENSOURCE) + INCLUDE(ya_non_opensource.inc) +ENDIF() + +END() diff --git a/yt/yt/library/tvm/service/ya.make b/yt/yt/library/tvm/service/ya.make new file mode 100644 index 0000000000..7a6d913a42 --- /dev/null +++ b/yt/yt/library/tvm/service/ya.make @@ -0,0 +1,25 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + config.cpp + tvm_service_common.cpp + tvm_service_dummy.cpp +) + +PEERDIR( + library/cpp/yt/memory + library/cpp/yt/logging + yt/yt/core +) + +IF(NOT OPENSOURCE) + INCLUDE(ya_non_opensource.inc) +ENDIF() + +END() + +RECURSE_FOR_TESTS( + unittests +) |