YT-19324: move YT provider to ydb/library/yql

This commit is formed by the following script: https://paste.yandex-team.ru/6f92e4b8-efc5-4d34-948b-15ee2accd7e7/text.
This commit has zero effect on all projects that depend on YQL.

The summary of changes:
- `yql/providers/yt -> ydb/library/yql/providers/yt `- the whole implementation of YT provider is moved into YDB code base for further export as a part of YT YQL plugin shared library;
- `yql/providers/stat/{expr_nodes,uploader} -> ydb/library/yql/providers/stat/{expr_nodes,uploader}` - a small interface without implementation and the description of stat expr nodes;
- `yql/core/extract_predicate/ut -> ydb/library/yql/core/extract_predicate/ut`;
- `yql/core/{ut,ut_common} -> ydb/library/yql/core/{ut,ut_common}`;
- `yql/core` is gone;
- `yql/library/url_preprocessing -> ydb/library/yql/core/url_preprocessing`.

**NB**: all new targets inside `ydb/` are under `IF (NOT CMAKE_EXPORT)` clause which disables them from open-source cmake generation and ya make build. They will be enabled in the subsequent commits.

5 files changed, 390 insertions, 0 deletions

diff --git a/yt/cpp/mapreduce/interface/logging/logger.cpp b/yt/cpp/mapreduce/interface/logging/logger.cpp
new file mode 100644
index 00000000000..bfa56b94f6d
--- /dev/null
+++ b/yt/cpp/mapreduce/interface/logging/logger.cpp
@@ -0,0 +1,188 @@
+#include "logger.h"
+
+#include <util/datetime/base.h>
+
+#include <util/stream/file.h>
+#include <util/stream/format.h>
+#include <util/stream/printf.h>
+#include <util/stream/str.h>
+
+#include <util/system/mutex.h>
+#include <util/system/rwlock.h>
+#include <util/system/thread.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+static TStringBuf StripFileName(TStringBuf path) {
+    TStringBuf l, r;
+    if (path.TryRSplit('/', l, r) || path.TryRSplit('\\', l, r)) {
+        return r;
+    } else {
+        return path;
+    }
+}
+
+static char GetLogLevelCode(ILogger::ELevel level) {
+    switch (level) {
+        case ILogger::FATAL: return 'F';
+        case ILogger::ERROR: return 'E';
+        case ILogger::INFO: return 'I';
+        case ILogger::DEBUG: return 'D';
+    }
+    Y_UNREACHABLE();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TNullLogger
+    : public ILogger
+{
+public:
+    void Log(ELevel level, const TSourceLocation& sourceLocation, const char* format, va_list args) override
+    {
+        Y_UNUSED(level);
+        Y_UNUSED(sourceLocation);
+        Y_UNUSED(format);
+        Y_UNUSED(args);
+    }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TLoggerBase
+    : public ILogger
+{
+public:
+    TLoggerBase(ELevel cutLevel)
+        : CutLevel_(cutLevel)
+    { }
+
+    virtual void OutputLine(const TString& line) = 0;
+
+    void Log(ELevel level, const TSourceLocation& sourceLocation, const char* format, va_list args) override
+    {
+        if (level > CutLevel_) {
+            return;
+        }
+
+        TStringStream stream;
+        stream << TInstant::Now().ToStringLocal()
+            << " " << GetLogLevelCode(level)
+            << " [" << Hex(TThread::CurrentThreadId(), HF_FULL) << "] ";
+        Printf(stream, format, args);
+        stream << " - " << StripFileName(sourceLocation.File) << ':' << sourceLocation.Line << Endl;
+
+        TGuard<TMutex> guard(Mutex_);
+        OutputLine(stream.Str());
+    }
+
+private:
+    ELevel CutLevel_;
+    TMutex Mutex_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TStdErrLogger
+    : public TLoggerBase
+{
+public:
+    TStdErrLogger(ELevel cutLevel)
+        : TLoggerBase(cutLevel)
+    { }
+
+    void OutputLine(const TString& line) override
+    {
+        Cerr << line;
+    }
+};
+
+ILoggerPtr CreateStdErrLogger(ILogger::ELevel cutLevel)
+{
+    return new TStdErrLogger(cutLevel);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TFileLogger
+    : public TLoggerBase
+{
+public:
+    TFileLogger(ELevel cutLevel, const TString& path, bool append)
+        : TLoggerBase(cutLevel)
+        , Stream_(TFile(path, OpenAlways | WrOnly | Seq | (append ? ForAppend : EOpenMode())))
+    { }
+
+    void OutputLine(const TString& line) override
+    {
+        Stream_ << line;
+    }
+
+private:
+    TUnbufferedFileOutput Stream_;
+};
+
+ILoggerPtr CreateFileLogger(ILogger::ELevel cutLevel, const TString& path, bool append)
+{
+    return new TFileLogger(cutLevel, path, append);
+}
+////////////////////////////////////////////////////////////////////////////////
+
+class TBufferedFileLogger
+    : public TLoggerBase
+{
+public:
+    TBufferedFileLogger(ELevel cutLevel, const TString& path, bool append)
+        : TLoggerBase(cutLevel)
+        , Stream_(TFile(path, OpenAlways | WrOnly | Seq | (append ? ForAppend : EOpenMode())))
+    { }
+
+    void OutputLine(const TString& line) override
+    {
+        Stream_ << line;
+    }
+
+private:
+    TFileOutput Stream_;
+};
+
+ILoggerPtr CreateBufferedFileLogger(ILogger::ELevel cutLevel, const TString& path, bool append)
+{
+    return new TBufferedFileLogger(cutLevel, path, append);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+static TRWMutex LoggerMutex;
+static ILoggerPtr Logger;
+
+struct TLoggerInitializer
+{
+    TLoggerInitializer()
+    {
+        Logger = new TNullLogger;
+    }
+} LoggerInitializer;
+
+void SetLogger(ILoggerPtr logger)
+{
+    auto guard = TWriteGuard(LoggerMutex);
+    if (logger) {
+        Logger = logger;
+    } else {
+        Logger = new TNullLogger;
+    }
+}
+
+ILoggerPtr GetLogger()
+{
+    auto guard = TReadGuard(LoggerMutex);
+    return Logger;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+}
+
diff --git a/yt/cpp/mapreduce/interface/logging/logger.h b/yt/cpp/mapreduce/interface/logging/logger.h
new file mode 100644
index 00000000000..2b5aae87d14
--- /dev/null
+++ b/yt/cpp/mapreduce/interface/logging/logger.h
@@ -0,0 +1,43 @@
+#pragma once
+
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+#include <util/system/compat.h>
+#include <util/system/src_location.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class ILogger
+    : public TThrRefBase
+{
+public:
+    enum ELevel
+    {
+        FATAL /* "fatal", "FATAL" */,
+        // We don't have such level as `warning', but we support it for compatibility with other APIs.
+        ERROR /* "error", "warning", "ERROR", "WARNING" */,
+        INFO /* "info", "INFO" */,
+        DEBUG /* "debug", "DEBUG" */
+    };
+
+    virtual void Log(ELevel level, const ::TSourceLocation& sourceLocation, const char* format, va_list args) = 0;
+};
+
+using ILoggerPtr = ::TIntrusivePtr<ILogger>;
+
+void SetLogger(ILoggerPtr logger);
+ILoggerPtr GetLogger();
+
+ILoggerPtr CreateStdErrLogger(ILogger::ELevel cutLevel);
+ILoggerPtr CreateFileLogger(ILogger::ELevel cutLevel, const TString& path, bool append = false);
+
+/**
+ * Create logger that writes to a file in a buffered manner.
+ * It should result in fewer system calls (useful if you expect a lot of log messages),
+ * but in case of a crash, you would lose some log messages that haven't been flushed yet.
+ */
+ILoggerPtr CreateBufferedFileLogger(ILogger::ELevel cutLevel, const TString& path, bool append = false);
+
+} // namespace NYT
diff --git a/yt/cpp/mapreduce/interface/logging/ya.make b/yt/cpp/mapreduce/interface/logging/ya.make
new file mode 100644
index 00000000000..8095bfe4bae
--- /dev/null
+++ b/yt/cpp/mapreduce/interface/logging/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+SRCS(
+    logger.cpp
+    yt_log.cpp
+)
+
+PEERDIR(
+    library/cpp/yt/logging
+)
+
+GENERATE_ENUM_SERIALIZATION(logger.h)
+
+END()
diff --git a/yt/cpp/mapreduce/interface/logging/yt_log.cpp b/yt/cpp/mapreduce/interface/logging/yt_log.cpp
new file mode 100644
index 00000000000..9fa7b915802
--- /dev/null
+++ b/yt/cpp/mapreduce/interface/logging/yt_log.cpp
@@ -0,0 +1,126 @@
+#include "yt_log.h"
+
+#include "logger.h"
+
+#include <util/generic/guid.h>
+
+#include <util/system/mutex.h>
+
+namespace NYT {
+
+using namespace NLogging;
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace {
+
+class TLogManager
+    : public ILogManager
+{
+public:
+    static constexpr TStringBuf CategoryName = "Wrapper";
+
+public:
+    void RegisterStaticAnchor(
+        TLoggingAnchor* anchor,
+        ::TSourceLocation sourceLocation,
+        TStringBuf anchorMessage) override
+    {
+        if (anchor->Registered.exchange(true)) {
+            return;
+        }
+
+        anchor->Enabled.store(true);
+
+        auto guard = Guard(Mutex_);
+        anchor->SourceLocation = sourceLocation;
+        anchor->AnchorMessage = anchorMessage;
+    }
+
+    void UpdateAnchor(TLoggingAnchor* /*position*/) override
+    { }
+
+    void Enqueue(TLogEvent&& event) override
+    {
+        auto message = TString(event.MessageRef.ToStringBuf());
+        LogMessage(
+            ToImplLevel(event.Level),
+            ::TSourceLocation(event.SourceFile, event.SourceLine),
+            "%.*s",
+            event.MessageRef.size(),
+            event.MessageRef.begin());
+    }
+
+    const TLoggingCategory* GetCategory(TStringBuf categoryName) override
+    {
+        Y_VERIFY(categoryName == CategoryName);
+        return &Category_;
+    }
+
+    void UpdateCategory(TLoggingCategory* /*category*/) override
+    {
+        Y_FAIL();
+    }
+
+    bool GetAbortOnAlert() const override
+    {
+        return false;
+    }
+
+private:
+    static ILogger::ELevel ToImplLevel(ELogLevel level)
+    {
+        switch (level) {
+            case ELogLevel::Minimum:
+            case ELogLevel::Trace:
+            case ELogLevel::Debug:
+                return ILogger::ELevel::DEBUG;
+            case ELogLevel::Info:
+                return ILogger::ELevel::INFO;
+            case ELogLevel::Warning:
+            case ELogLevel::Error:
+                return ILogger::ELevel::ERROR;
+            case ELogLevel::Alert:
+            case ELogLevel::Fatal:
+            case ELogLevel::Maximum:
+                return ILogger::ELevel::FATAL;
+        }
+    }
+
+    static void LogMessage(ILogger::ELevel level, const ::TSourceLocation& sourceLocation, const char* format, ...)
+    {
+        va_list args;
+        va_start(args, format);
+        GetLogger()->Log(level, sourceLocation, format, args);
+        va_end(args);
+    }
+
+private:
+    ::TMutex Mutex_;
+    std::atomic<int> ActualVersion_{1};
+    const TLoggingCategory Category_{
+        .Name{CategoryName},
+        .MinPlainTextLevel{ELogLevel::Minimum},
+        .CurrentVersion{1},
+        .ActualVersion = &ActualVersion_,
+    };
+};
+
+TLogManager LogManager;
+
+} // namespace
+
+////////////////////////////////////////////////////////////////////////////////
+
+TLogger Logger(&LogManager, TLogManager::CategoryName);
+
+////////////////////////////////////////////////////////////////////////////////
+
+void FormatValue(TStringBuilderBase* builder, const TGUID& value, TStringBuf /*format*/)
+{
+    builder->AppendString(GetGuidAsString(value));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/cpp/mapreduce/interface/logging/yt_log.h b/yt/cpp/mapreduce/interface/logging/yt_log.h
new file mode 100644
index 00000000000..4cf93a6ba1d
--- /dev/null
+++ b/yt/cpp/mapreduce/interface/logging/yt_log.h
@@ -0,0 +1,17 @@
+#pragma once
+
+#include <library/cpp/yt/logging/logger.h>
+
+struct TGUID;
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+extern NLogging::TLogger Logger;
+
+void FormatValue(TStringBuilderBase* builder, const TGUID& value, TStringBuf format);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT