Yql_logging_refactoring

[]

10 files changed, 326 insertions, 15 deletions

diff --git a/ydb/library/yql/utils/log/CMakeLists.darwin.txt b/ydb/library/yql/utils/log/CMakeLists.darwin.txt
index a50046760f4..092b94e027d 100644
--- a/ydb/library/yql/utils/log/CMakeLists.darwin.txt
+++ b/ydb/library/yql/utils/log/CMakeLists.darwin.txt
@@ -6,6 +6,7 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(proto)
 add_subdirectory(ut)
 
 add_library(yql-utils-log)
@@ -15,6 +16,7 @@ target_link_libraries(yql-utils-log PUBLIC
   library-cpp-logger
   cpp-logger-global
   cpp-deprecated-atomic
+  utils-log-proto
 )
 target_sources(yql-utils-log PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/utils/log/context.cpp
diff --git a/ydb/library/yql/utils/log/CMakeLists.linux-aarch64.txt b/ydb/library/yql/utils/log/CMakeLists.linux-aarch64.txt
index 424601bfaf3..d836e881843 100644
--- a/ydb/library/yql/utils/log/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/utils/log/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(proto)
 add_subdirectory(ut)
 
 add_library(yql-utils-log)
@@ -16,6 +17,7 @@ target_link_libraries(yql-utils-log PUBLIC
   library-cpp-logger
   cpp-logger-global
   cpp-deprecated-atomic
+  utils-log-proto
 )
 target_sources(yql-utils-log PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/utils/log/context.cpp
diff --git a/ydb/library/yql/utils/log/CMakeLists.linux.txt b/ydb/library/yql/utils/log/CMakeLists.linux.txt
index 424601bfaf3..d836e881843 100644
--- a/ydb/library/yql/utils/log/CMakeLists.linux.txt
+++ b/ydb/library/yql/utils/log/CMakeLists.linux.txt
@@ -6,6 +6,7 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(proto)
 add_subdirectory(ut)
 
 add_library(yql-utils-log)
@@ -16,6 +17,7 @@ target_link_libraries(yql-utils-log PUBLIC
   library-cpp-logger
   cpp-logger-global
   cpp-deprecated-atomic
+  utils-log-proto
 )
 target_sources(yql-utils-log PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/utils/log/context.cpp
diff --git a/ydb/library/yql/utils/log/log.cpp b/ydb/library/yql/utils/log/log.cpp
index b62bfeb907a..978915e543b 100644
--- a/ydb/library/yql/utils/log/log.cpp
+++ b/ydb/library/yql/utils/log/log.cpp
@@ -1,7 +1,10 @@
 #include "log.h"
 
+#include <ydb/library/yql/utils/log/proto/logger_config.pb.h>
+
 #include <library/cpp/logger/stream.h>
 #include <library/cpp/logger/system.h>
+#include <library/cpp/logger/composite.h>
 
 #include <util/datetime/systime.h>
 #include <util/generic/strbuf.h>
@@ -67,6 +70,85 @@ private:
     TAtomic Limit;
 };
 
+// Conversions between NYql::NProto::TLoggingConfig enums and NYql::NLog enums
+
+NYql::NLog::ELevel ConvertLevel(NYql::NProto::TLoggingConfig::ELevel level) {
+    using namespace NYql::NProto;
+    using namespace NYql::NLog;
+    switch (level) {
+    case TLoggingConfig::FATAL: return ELevel::FATAL;
+    case TLoggingConfig::ERROR: return ELevel::ERROR;
+    case TLoggingConfig::WARN: return ELevel::WARN;
+    case TLoggingConfig::INFO: return ELevel::INFO;
+    case TLoggingConfig::DEBUG: return ELevel::DEBUG;
+    case TLoggingConfig::TRACE: return ELevel::TRACE;
+    }
+
+    ythrow yexception() << "unknown log level: "
+            << TLoggingConfig::ELevel_Name(level);
+}
+
+NYql::NLog::EComponent ConvertComponent(NYql::NProto::TLoggingConfig::EComponent c) {
+    using namespace NYql::NProto;
+    using namespace NYql::NLog;
+    switch (c) {
+    case TLoggingConfig::DEFAULT: return EComponent::Default;
+    case TLoggingConfig::CORE: return EComponent::Core;
+    case TLoggingConfig::CORE_EVAL: return EComponent::CoreEval;
+    case TLoggingConfig::CORE_PEEPHOLE: return EComponent::CorePeepHole;
+    case TLoggingConfig::CORE_EXECUTION: return EComponent::CoreExecution;
+    case TLoggingConfig::SQL: return EComponent::Sql;
+    case TLoggingConfig::PROVIDER_COMMON: return EComponent::ProviderCommon;
+    case TLoggingConfig::PROVIDER_CONFIG: return EComponent::ProviderConfig;
+    case TLoggingConfig::PROVIDER_RESULT: return EComponent::ProviderResult;
+    case TLoggingConfig::PROVIDER_YT: return EComponent::ProviderYt;
+    case TLoggingConfig::PROVIDER_KIKIMR: return EComponent::ProviderKikimr;
+    case TLoggingConfig::PROVIDER_KQP: return EComponent::ProviderKqp;
+    case TLoggingConfig::PROVIDER_RTMR: return EComponent::ProviderRtmr;
+    case TLoggingConfig::PERFORMANCE: return EComponent::Perf;
+    case TLoggingConfig::NET: return EComponent::Net;
+    case TLoggingConfig::PROVIDER_STAT: return EComponent::ProviderStat;
+    case TLoggingConfig::PROVIDER_SOLOMON: return EComponent::ProviderSolomon;
+    case TLoggingConfig::PROVIDER_DQ: return EComponent::ProviderDq;
+    case TLoggingConfig::PROVIDER_CLICKHOUSE: return EComponent::ProviderClickHouse;
+    case TLoggingConfig::PROVIDER_YDB: return EComponent::ProviderYdb;
+    case TLoggingConfig::PROVIDER_PQ: return EComponent::ProviderPq;
+    case TLoggingConfig::PROVIDER_S3: return EComponent::ProviderS3;
+    case TLoggingConfig::CORE_DQ: return EComponent::CoreDq;
+    case TLoggingConfig::HTTP_GATEWAY: return EComponent::HttpGateway;
+    }
+
+    ythrow yexception() << "unknown log component: "
+            << TLoggingConfig::EComponent_Name(c);
+}
+
+TString ConvertDestinationType(NYql::NProto::TLoggingConfig::ELogTo c) {
+    switch (c) {
+    case NYql::NProto::TLoggingConfig::STDOUT: return "stdout";
+    case NYql::NProto::TLoggingConfig::STDERR: return "stderr";
+    case NYql::NProto::TLoggingConfig::CONSOLE: return "console";
+    default : {
+        ythrow yexception() << "unsupported ELogTo destination in Convert";
+    }
+    }
+
+    ythrow yexception() << "unknown ELogTo destination";
+}
+
+NYql::NProto::TLoggingConfig::TLogDestination CreateLogDestination(const TString& c) {
+    NYql::NProto::TLoggingConfig::TLogDestination destination;
+    if (c == "stdout") {
+        destination.SetType(NYql::NProto::TLoggingConfig::STDOUT);
+    } else if (c == "stderr") {
+        destination.SetType(NYql::NProto::TLoggingConfig::STDERR);
+    } else if (c == "console") {
+        destination.SetType(NYql::NProto::TLoggingConfig::CONSOLE);
+    } else {
+        destination.SetType(NYql::NProto::TLoggingConfig::FILE);
+        destination.SetTarget(c);
+    }
+    return destination;
+}
 
 } // namspace
 
@@ -164,6 +246,13 @@ void TYqlLog::SetMaxLogLimit(ui64 limit) {
 }
 
 void InitLogger(const TString& logType, bool startAsDaemon) {
+    NProto::TLoggingConfig config;
+    *config.AddLogDest() = CreateLogDestination(logType);
+
+    InitLogger(config, startAsDaemon);
+}
+
+void InitLogger(const NProto::TLoggingConfig& config, bool startAsDaemon) {
     with_lock(g_InitLoggerMutex) {
         ++g_LoggerInitialized;
         if (g_LoggerInitialized > 1) {
@@ -171,24 +260,58 @@ void InitLogger(const TString& logType, bool startAsDaemon) {
         }
 
         TComponentLevels levels;
-        levels.fill(ELevel::INFO);
+        if (config.HasAllComponentsLevel()) {
+            levels.fill(ConvertLevel(config.GetAllComponentsLevel()));
+        } else {
+            levels.fill(ELevel::INFO);
+        }
 
-        if (startAsDaemon && (
-                TStringBuf("console") == logType ||
-                TStringBuf("cout") == logType ||
-                TStringBuf("cerr") == logType))
-        {
-            TLoggerOperator<TYqlLog>::Set(new TYqlLog("null", levels));
-            return;
+        for (const auto& cmpLevel: config.GetLevels()) {
+            auto component = ConvertComponent(cmpLevel.GetC());
+            auto level = ConvertLevel(cmpLevel.GetL());
+            levels[EComponentHelpers::ToInt(component)] = level;
         }
+        TLoggerOperator<TYqlLog>::Set(new TYqlLog("null", levels));
 
-        if (TStringBuf("syslog") == logType) {
-            auto backend = MakeHolder<TSysLogBackend>(
-                        GetProgramName().data(), TSysLogBackend::TSYSLOG_LOCAL1);
-            auto& logger = TLoggerOperator<TYqlLog>::Log();
-            logger.ResetBackend(THolder(backend.Release()));
-        } else {
-            TLoggerOperator<TYqlLog>::Set(new TYqlLog(logType, levels));
+        std::vector<THolder<TLogBackend>> backends;
+
+        // Set stderr log destination if none was described in config
+        if (config.LogDestSize() == 0) {
+            backends.emplace_back(CreateLogBackend("stderr", LOG_MAX_PRIORITY, false));
+        }
+
+        for (const auto& logDestionation : config.GetLogDest()) {
+            // Generate the backend we need and temporary store it
+            switch (logDestionation.GetType()) {
+                case NProto::TLoggingConfig::STDERR:
+                case NProto::TLoggingConfig::STDOUT:
+                case NProto::TLoggingConfig::CONSOLE: {
+                    if (!startAsDaemon) {
+                        backends.emplace_back(CreateLogBackend(ConvertDestinationType(logDestionation.GetType()), LOG_MAX_PRIORITY, false));
+                    }
+                    break;
+                }
+                case NProto::TLoggingConfig::FILE: {
+                    backends.emplace_back(CreateLogBackend(logDestionation.GetTarget(), LOG_MAX_PRIORITY, false));
+                    break;
+                }
+                case NProto::TLoggingConfig::SYSLOG: {
+                    backends.emplace_back(MakeHolder<TSysLogBackend>(GetProgramName().data(), TSysLogBackend::TSYSLOG_LOCAL1));
+                    break;
+                }
+            }
+        }
+
+        // Combine created backends and set them for logger
+        auto& logger = TLoggerOperator<TYqlLog>::Log();
+        if (backends.size() == 1) {
+            logger.ResetBackend(std::move(backends[0]));
+        } else if (backends.size() > 1) {
+            THolder<TCompositeLogBackend> compositeBackend = MakeHolder<TCompositeLogBackend>();
+            for (auto& backend : backends) {
+                compositeBackend->AddLogBackend(std::move(backend));
+            }
+            logger.ResetBackend(std::move(compositeBackend));
         }
     }
 }
diff --git a/ydb/library/yql/utils/log/log.h b/ydb/library/yql/utils/log/log.h
index 4fdcebecf99..6ab38b98db1 100644
--- a/ydb/library/yql/utils/log/log.h
+++ b/ydb/library/yql/utils/log/log.h
@@ -85,6 +85,11 @@
 
 
 namespace NYql {
+
+namespace NProto {
+    class TLoggingConfig;
+} // NProto
+
 namespace NLog {
 
 using TComponentLevels =
@@ -150,6 +155,11 @@ inline bool IsYqlLoggerInitialized() {
 void InitLogger(const TString& log, bool startAsDaemon = false);
 
 /**
+ * @brief Initialize logger with backends described in config.
+*/
+void InitLogger(const NProto::TLoggingConfig& loggingConfig, bool startAsDaemon = false);
+
+/**
  * @brief Initialize logger with concrete backend.
  *
  * @param backend - logger backend
diff --git a/ydb/library/yql/utils/log/proto/CMakeLists.darwin.txt b/ydb/library/yql/utils/log/proto/CMakeLists.darwin.txt
new file mode 100644
index 00000000000..07fd0f3c5ef
--- /dev/null
+++ b/ydb/library/yql/utils/log/proto/CMakeLists.darwin.txt
@@ -0,0 +1,31 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(utils-log-proto)
+target_link_libraries(utils-log-proto PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  contrib-libs-protobuf
+)
+target_proto_messages(utils-log-proto PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/utils/log/proto/logger_config.proto
+)
+target_proto_addincls(utils-log-proto
+  ./
+  ${CMAKE_SOURCE_DIR}/
+  ${CMAKE_BINARY_DIR}
+  ${CMAKE_SOURCE_DIR}
+  ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+  ${CMAKE_BINARY_DIR}
+  ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(utils-log-proto
+  --cpp_out=${CMAKE_BINARY_DIR}/
+  --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/ydb/library/yql/utils/log/proto/CMakeLists.linux-aarch64.txt b/ydb/library/yql/utils/log/proto/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..b4e3c43b264
--- /dev/null
+++ b/ydb/library/yql/utils/log/proto/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(utils-log-proto)
+target_link_libraries(utils-log-proto PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  contrib-libs-protobuf
+)
+target_proto_messages(utils-log-proto PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/utils/log/proto/logger_config.proto
+)
+target_proto_addincls(utils-log-proto
+  ./
+  ${CMAKE_SOURCE_DIR}/
+  ${CMAKE_BINARY_DIR}
+  ${CMAKE_SOURCE_DIR}
+  ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+  ${CMAKE_BINARY_DIR}
+  ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(utils-log-proto
+  --cpp_out=${CMAKE_BINARY_DIR}/
+  --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/ydb/library/yql/utils/log/proto/CMakeLists.linux.txt b/ydb/library/yql/utils/log/proto/CMakeLists.linux.txt
new file mode 100644
index 00000000000..b4e3c43b264
--- /dev/null
+++ b/ydb/library/yql/utils/log/proto/CMakeLists.linux.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(utils-log-proto)
+target_link_libraries(utils-log-proto PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  contrib-libs-protobuf
+)
+target_proto_messages(utils-log-proto PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/utils/log/proto/logger_config.proto
+)
+target_proto_addincls(utils-log-proto
+  ./
+  ${CMAKE_SOURCE_DIR}/
+  ${CMAKE_BINARY_DIR}
+  ${CMAKE_SOURCE_DIR}
+  ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+  ${CMAKE_BINARY_DIR}
+  ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(utils-log-proto
+  --cpp_out=${CMAKE_BINARY_DIR}/
+  --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/ydb/library/yql/utils/log/proto/CMakeLists.txt b/ydb/library/yql/utils/log/proto/CMakeLists.txt
new file mode 100644
index 00000000000..5bb4faffb40
--- /dev/null
+++ b/ydb/library/yql/utils/log/proto/CMakeLists.txt
@@ -0,0 +1,15 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+  include(CMakeLists.linux-aarch64.txt)
+elseif (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+  include(CMakeLists.darwin.txt)
+elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+  include(CMakeLists.linux.txt)
+endif()
diff --git a/ydb/library/yql/utils/log/proto/logger_config.proto b/ydb/library/yql/utils/log/proto/logger_config.proto
new file mode 100644
index 00000000000..668a2d23053
--- /dev/null
+++ b/ydb/library/yql/utils/log/proto/logger_config.proto
@@ -0,0 +1,62 @@
+package NYql.NProto;
+option java_package = "ru.yandex.yql.proto";
+
+message TLoggingConfig {
+    enum ELogTo {
+        STDERR = 1;
+        STDOUT = 2;
+        CONSOLE = 3;
+        FILE = 4;
+        SYSLOG = 5;
+    }
+
+    enum ELevel {
+        FATAL = 0;
+        ERROR = 1;
+        WARN = 2;
+        INFO = 3;
+        DEBUG = 4;
+        TRACE = 5;
+    }
+
+    enum EComponent {
+        DEFAULT = 0;
+        CORE = 1;
+        CORE_EXECUTION = 2;
+        SQL = 3;
+        PROVIDER_COMMON = 4;
+        PROVIDER_CONFIG = 5;
+        PROVIDER_RESULT = 6;
+        PROVIDER_YT = 7;
+        PROVIDER_KIKIMR = 8;
+        PROVIDER_KQP = 9;
+        PROVIDER_RTMR = 10;
+        PERFORMANCE = 11;
+        NET = 12;
+        PROVIDER_STAT = 13;
+        PROVIDER_SOLOMON = 14;
+        CORE_EVAL = 15;
+        CORE_PEEPHOLE = 16;
+        PROVIDER_DQ = 17;
+        PROVIDER_CLICKHOUSE = 18;
+        PROVIDER_YDB = 19;
+        PROVIDER_PQ = 20;
+        PROVIDER_S3 = 21;
+        CORE_DQ = 22;
+        HTTP_GATEWAY = 23;
+    }
+
+    message TComponentLevel {
+        optional EComponent C = 1;
+        optional ELevel L = 2;
+    }
+
+    message TLogDestination {
+        optional ELogTo Type = 1;
+        optional string Target = 2;
+    }
+
+    repeated TLogDestination LogDest = 2; // If none is passed InitLogger sets default STDERR backend
+    repeated TComponentLevel Levels = 3;
+    optional ELevel AllComponentsLevel = 4 [default = INFO];
+}