YQL-19206: Move contrib/ydb/library/yql/tools/ to yql/essentials/tools/

YQL-19206: Move contrib/ydb/library/yql/tools/ to yql/essentials/tools/
commit_hash:db43e12cb7f8d79b470589676fd98198a9c94318

10 files changed, 948 insertions, 6 deletions

diff --git a/build/conf/project_specific/yql_udf.conf b/build/conf/project_specific/yql_udf.conf
index 785b93c99bf..1cdbc45db85 100644
--- a/build/conf/project_specific/yql_udf.conf
+++ b/build/conf/project_specific/yql_udf.conf
@@ -38,15 +38,15 @@ module YQL_UDF_TEST: PY3TEST_BIN {
 
     PEERDIR(contrib/ydb/library/yql/tests/common/udf_test)
 
-    DEPENDS(contrib/ydb/library/yql/tools/astdiff)
+    DEPENDS(yql/essentials/tools/astdiff)
     DEPENDS(contrib/ydb/library/yql/tools/yqlrun)
     DEPENDS(contrib/ydb/library/yql/tools/udf_resolver)
     DATA(arcadia/contrib/ydb/library/yql/mount)
     DATA(arcadia/contrib/ydb/library/yql/cfg/udf_test)
-    ENV(YQL_ASTDIFF_PATH="contrib/ydb/library/yql/tools/astdiff/astdiff")
+    ENV(YQL_ASTDIFF_PATH="yql/essentials/tools/astdiff/astdiff")
     ENV(YQL_CONFIG_DIR="contrib/ydb/library/yql/cfg/udf_test")
     ENV(YQL_YQLRUN_PATH="contrib/ydb/library/yql/tools/yqlrun/yqlrun")
-    ENV(YQL_SQL2YQL_PATH="contrib/ydb/library/yql/tools/sql2yql/sql2yql")
+    ENV(YQL_SQL2YQL_PATH="yql/essentials/tools/sql2yql/sql2yql")
     ENV(YQL_UDFRESOLVER_PATH="contrib/ydb/library/yql/tools/udf_resolver/udf_resolver")
 }
 
@@ -67,15 +67,15 @@ module YQL_UDF_TEST_CONTRIB: PY3TEST_BIN {
 
     PEERDIR(contrib/ydb/library/yql/tests/common/udf_test)
 
-    DEPENDS(contrib/ydb/library/yql/tools/astdiff)
+    DEPENDS(yql/essentials/tools/astdiff)
     DEPENDS(contrib/ydb/library/yql/tools/yqlrun)
     DEPENDS(contrib/ydb/library/yql/tools/udf_resolver)
     DATA(arcadia/contrib/ydb/library/yql/mount)
     DATA(arcadia/contrib/ydb/library/yql/cfg/udf_test)
-    ENV(YQL_ASTDIFF_PATH="contrib/ydb/library/yql/tools/astdiff/astdiff")
+    ENV(YQL_ASTDIFF_PATH="yql/essentials/tools/astdiff/astdiff")
     ENV(YQL_CONFIG_DIR="contrib/ydb/library/yql/cfg/udf_test")
     ENV(YQL_YQLRUN_PATH="contrib/ydb/library/yql/tools/yqlrun/yqlrun")
-    ENV(YQL_SQL2YQL_PATH="contrib/ydb/library/yql/tools/sql2yql/sql2yql")
+    ENV(YQL_SQL2YQL_PATH="yql/essentials/tools/sql2yql/sql2yql")
     ENV(YQL_UDFRESOLVER_PATH="contrib/ydb/library/yql/tools/udf_resolver/udf_resolver")
 }
 
diff --git a/yql/essentials/tools/astdiff/astdiff.cpp b/yql/essentials/tools/astdiff/astdiff.cpp
new file mode 100644
index 00000000000..5cd8342d363
--- /dev/null
+++ b/yql/essentials/tools/astdiff/astdiff.cpp
@@ -0,0 +1,135 @@
+
+#include <yql/essentials/utils/backtrace/backtrace.h>
+
+#include <contrib/ydb/library/yql/ast/yql_expr.h>
+
+#include <library/cpp/svnversion/svnversion.h>
+
+#include <util/stream/file.h>
+#include <util/folder/path.h>
+#include <util/string/split.h>
+#include <util/generic/yexception.h>
+
+#include <sstream>
+
+#include <contrib/libs/dtl/dtl/dtl.hpp>
+
+using namespace NYql;
+
+std::string CalculateDiff(const TString& oldAst, const TString& newAst) {
+    auto oldLines = StringSplitter(oldAst).Split('\n').ToList<std::string>();
+    auto newLines = StringSplitter(newAst).Split('\n').ToList<std::string>();
+
+    dtl::Diff<std::string, TVector<std::string>> d(oldLines, newLines);
+    d.compose();
+    d.composeUnifiedHunks();
+
+    std::ostringstream ss;
+    d.printUnifiedFormat(ss);
+    return ss.str();
+}
+
+
+const int DIFF_LINES_LIMIT = 16;
+
+void DumpSmallNodes(const TExprNode* rootOne, const TExprNode* rootTwo) {
+    const auto isDumpSmall = [] (const TString& dump) {
+        return std::count(dump.begin(), dump.end(), '\n') < DIFF_LINES_LIMIT;
+    };
+    const auto rootOneDump = rootOne->Dump();
+    if (!isDumpSmall(rootOneDump)) {
+        return;
+    }
+    const auto rootTwoDump = rootTwo->Dump();
+    if (!isDumpSmall(rootTwoDump)) {
+        return;
+    }
+
+    Cerr << rootOneDump << '\n' << rootTwoDump;
+}
+
+int Main(int argc, const char *argv[])
+{
+    if (argc != 3) {
+        PrintProgramSvnVersion();
+        Cout << Endl << "Usage: " << argv[0] << " <fileone> <filetwo>" << Endl;
+        return 2;
+    }
+
+    const TString fileOne(argv[1]), fileTwo(argv[2]);
+    const TString progOneAst = TFileInput(fileOne).ReadAll();
+    const TString progTwoAst = TFileInput(fileTwo).ReadAll();
+    const auto progOne(ParseAst(progOneAst)), progTwo(ParseAst(progTwoAst));
+
+    if (!(progOne.IsOk() && progTwo.IsOk())) {
+        if (!progOne.IsOk()) {
+            Cerr << "Errors in " << fileOne << Endl;
+            progOne.Issues.PrintTo(Cerr);
+        }
+        if (!progTwo.IsOk()) {
+            Cerr << "Errors in " << fileTwo << Endl;
+            progTwo.Issues.PrintTo(Cerr);
+        }
+        return 3;
+    }
+
+    TExprContext ctxOne, ctxTwo;
+    TExprNode::TPtr exprOne, exprTwo;
+
+    const bool okOne = CompileExpr(*progOne.Root, exprOne, ctxOne, nullptr, nullptr);
+    const bool okTwo = CompileExpr(*progTwo.Root, exprTwo, ctxTwo, nullptr, nullptr);
+
+    if (!(okOne && okTwo)) {
+        if (!okOne) {
+            Cerr << "Errors on compile " << fileOne << Endl;
+            ctxOne.IssueManager.GetIssues().PrintTo(Cerr);
+        }
+        if (!okTwo) {
+            Cerr << "Errors on compile " << fileTwo << Endl;
+            ctxTwo.IssueManager.GetIssues().PrintTo(Cerr);
+        }
+        return 4;
+    }
+
+    const TExprNode* rootOne = exprOne.Get();
+    const TExprNode* rootTwo = exprTwo.Get();
+
+    auto rootOnePos = ctxOne.GetPosition(rootOne->Pos());
+    auto rootTwoPos = ctxTwo.GetPosition(rootTwo->Pos());
+    if (!CompareExprTrees(rootOne, rootTwo)) {
+        const auto diff = CalculateDiff(progOneAst, progTwoAst);
+
+        Cerr << "Programs are not equal!" << Endl;
+        if (rootOne->Type() != rootTwo->Type()) {
+            Cerr << "Node in " << fileOne << " at [" << rootOnePos.Row << ":" << rootOnePos.Column << "] type is " << rootOne->Type() << Endl;
+            Cerr << "Node in " << fileTwo << " at [" << rootTwoPos.Row << ":" << rootTwoPos.Column << "] type is " << rootTwo->Type() << Endl;
+            Cerr << "\nFile diff:\n" << diff;
+        } else if (rootOne->ChildrenSize() != rootTwo->ChildrenSize()) {
+            Cerr << "Node '" << rootOne->Content() << "' in " << fileOne << " at [" << rootOnePos.Row << ":" << rootOnePos.Column << "] has " << rootOne->ChildrenSize() << " children." << Endl;
+            Cerr << "Node '" << rootTwo->Content() << "' in " << fileTwo << " at [" << rootTwoPos.Row << ":" << rootTwoPos.Column << "] has " << rootTwo->ChildrenSize() << " children." << Endl;
+            DumpSmallNodes(rootOne, rootTwo);
+            Cerr << "\nFile diff:\n" << diff;
+        } else {
+            Cerr << "Node in " << fileOne << " at [" << rootOnePos.Row << ":" << rootOnePos.Column << "]:";
+            Cerr << "Node in " << fileTwo << " at [" << rootTwoPos.Row << ":" << rootTwoPos.Column << "]:";
+            DumpSmallNodes(rootOne, rootTwo);
+            Cerr << "\nFile diff:\n" << diff;
+        }
+        return 5;
+    }
+
+    return 0;
+}
+
+int main(int argc, const char *argv[]) {
+    NYql::NBacktrace::RegisterKikimrFatalActions();
+    NYql::NBacktrace::EnableKikimrSymbolize();
+
+    try {
+        return Main(argc, argv);
+    }
+    catch (...) {
+        Cerr << CurrentExceptionMessage() << Endl;
+        return 1;
+    }
+}
diff --git a/yql/essentials/tools/astdiff/ya.make b/yql/essentials/tools/astdiff/ya.make
new file mode 100644
index 00000000000..0856b27b2c6
--- /dev/null
+++ b/yql/essentials/tools/astdiff/ya.make
@@ -0,0 +1,16 @@
+PROGRAM(astdiff)
+
+SRCS(
+    astdiff.cpp
+)
+
+PEERDIR(
+    library/cpp/getopt
+    library/cpp/svnversion
+    contrib/ydb/library/yql/ast
+    yql/essentials/utils/backtrace
+
+    contrib/libs/dtl
+)
+
+END()
diff --git a/yql/essentials/tools/purebench/purebench.cpp b/yql/essentials/tools/purebench/purebench.cpp
new file mode 100644
index 00000000000..763a323a6b5
--- /dev/null
+++ b/yql/essentials/tools/purebench/purebench.cpp
@@ -0,0 +1,273 @@
+#include <library/cpp/svnversion/svnversion.h>
+#include <library/cpp/getopt/last_getopt.h>
+
+#include <contrib/ydb/library/yql/public/purecalc/purecalc.h>
+#include <contrib/ydb/library/yql/public/purecalc/io_specs/mkql/spec.h>
+#include <contrib/ydb/library/yql/public/purecalc/io_specs/arrow/spec.h>
+#include <contrib/ydb/library/yql/public/purecalc/helpers/stream/stream_from_vector.h>
+
+#include <yql/essentials/utils/log/log.h>
+#include <yql/essentials/utils/backtrace/backtrace.h>
+#include <contrib/ydb/library/yql/public/udf/arrow/util.h>
+#include <contrib/ydb/library/yql/public/udf/udf_registrator.h>
+#include <contrib/ydb/library/yql/public/udf/udf_version.h>
+
+#include <library/cpp/skiff/skiff.h>
+#include <library/cpp/yson/writer.h>
+
+#include <util/datetime/cputimer.h>
+#include <util/stream/file.h>
+#include <util/stream/format.h>
+#include <util/stream/null.h>
+
+#include <algorithm>
+#include <cmath>
+
+using namespace NYql;
+using namespace NYql::NPureCalc;
+
+TStringStream MakeGenInput(ui64 count) {
+    TStringStream stream;
+    NSkiff::TUncheckedSkiffWriter writer{&stream};
+    for (ui64 i = 0; i < count; ++i) {
+        writer.WriteVariant16Tag(0);
+        writer.WriteInt64(i);
+    }
+    writer.Finish();
+    return stream;
+}
+
+template <typename TInputSpec, typename TOutputSpec>
+using TRunCallable = std::function<void (const THolder<TPullListProgram<TInputSpec, TOutputSpec>>&)>;
+
+template <typename TOutputSpec>
+NYT::TNode RunGenSql(
+    const IProgramFactoryPtr factory,
+    const TVector<NYT::TNode>& inputSchema,
+    const TString& sql,
+    ETranslationMode isPg,
+    TRunCallable<TSkiffInputSpec, TOutputSpec> runCallable
+) {
+    auto inputSpec = TSkiffInputSpec(inputSchema);
+    auto outputSpec = TOutputSpec({NYT::TNode::CreateEntity()});
+    auto program = factory->MakePullListProgram(inputSpec, outputSpec, sql, isPg);
+
+    runCallable(program);
+
+    return program->MakeOutputSchema();
+}
+
+template <typename TInputSpec, typename TStream>
+void ShowResults(
+    const IProgramFactoryPtr factory,
+    const TVector<NYT::TNode>& inputSchema,
+    const TString& sql,
+    ETranslationMode isPg,
+    TStream* input
+) {
+    auto inputSpec = TInputSpec(inputSchema);
+    auto outputSpec = TYsonOutputSpec({NYT::TNode::CreateEntity()});
+    auto program = factory->MakePullListProgram(inputSpec, outputSpec, sql, isPg);
+    auto handle = program->Apply(input);
+    TStringStream output;
+    handle->Run(&output);
+    TStringInput in(output.Str());
+    NYson::ReformatYsonStream(&in, &Cerr, NYson::EYsonFormat::Pretty, NYson::EYsonType::ListFragment);
+}
+
+template <typename TInputSpec, typename TOutputSpec>
+double RunBenchmarks(
+    const IProgramFactoryPtr factory,
+    const TVector<NYT::TNode>& inputSchema,
+    const TString& sql,
+    ETranslationMode isPg,
+    ui32 repeats,
+    TRunCallable<TInputSpec, TOutputSpec> runCallable
+) {
+    auto inputSpec = TInputSpec(inputSchema);
+    auto outputSpec = TOutputSpec({NYT::TNode::CreateEntity()});
+    auto program = factory->MakePullListProgram(inputSpec, outputSpec, sql, isPg);
+
+    Cerr << "Dry run of test sql...\n";
+
+    runCallable(program);
+
+    Cerr << "Run benchmark...\n";
+
+    TVector<TDuration> times;
+    TSimpleTimer allTimer;
+    for (ui32 i = 0; i < repeats; ++i) {
+        TSimpleTimer timer;
+        runCallable(program);
+        times.push_back(timer.Get());
+    }
+
+    Cout << "Elapsed: " << allTimer.Get() << "\n";
+
+    Sort(times);
+    times.erase(times.end() - times.size() / 3, times.end());
+
+    double sum = std::transform_reduce(times.cbegin(), times.cend(),
+        .0, std::plus{}, [](auto t) { return std::log(t.MicroSeconds()); });
+
+    return std::exp(sum / times.size());
+}
+
+int Main(int argc, const char *argv[])
+{
+    Y_UNUSED(NUdf::GetStaticSymbols());
+    using namespace NLastGetopt;
+    TOpts opts = TOpts::Default();
+    ui64 count;
+    ui32 repeats;
+    TString genSql, testSql;
+    bool showResults;
+    TString udfsDir;
+    TString LLVMSettings;
+    TString blockEngineSettings;
+    TString exprFile;
+    opts.AddHelpOption();
+    opts.AddLongOption("ndebug", "should be at first argument, do not show debug info in error output").NoArgument();
+    opts.AddLongOption('b', "blocks-engine", "Block engine settings").StoreResult(&blockEngineSettings).DefaultValue("disable");
+    opts.AddLongOption('c', "count", "count of input rows").StoreResult(&count).DefaultValue(1000000);
+    opts.AddLongOption('g', "gen-sql", "SQL query to generate data").StoreResult(&genSql).DefaultValue("select index from Input");
+    opts.AddLongOption('t', "test-sql", "SQL query to test").StoreResult(&testSql).DefaultValue("select count(*) as count from Input");
+    opts.AddLongOption('r', "repeats", "number of iterations").StoreResult(&repeats).DefaultValue(10);
+    opts.AddLongOption('w', "show-results", "show results of test SQL").StoreResult(&showResults).DefaultValue(true);
+    opts.AddLongOption("pg", "use PG syntax for generate query").NoArgument();
+    opts.AddLongOption("pt", "use PG syntax for test query").NoArgument();
+    opts.AddLongOption("udfs-dir", "directory with UDFs").StoreResult(&udfsDir).DefaultValue("");
+    opts.AddLongOption("llvm-settings", "LLVM settings").StoreResult(&LLVMSettings).DefaultValue("");
+    opts.AddLongOption("print-expr", "print rebuild AST before execution").NoArgument();
+    opts.AddLongOption("expr-file", "print AST to that file instead of stdout").StoreResult(&exprFile);
+    opts.SetFreeArgsMax(0);
+    TOptsParseResult res(&opts, argc, argv);
+
+    auto factoryOptions = TProgramFactoryOptions();
+    factoryOptions.SetUDFsDir(udfsDir);
+    factoryOptions.SetLLVMSettings(LLVMSettings);
+    factoryOptions.SetBlockEngineSettings(blockEngineSettings);
+
+    IOutputStream* exprOut = nullptr;
+    THolder<TFixedBufferFileOutput> exprFileHolder;
+    if (res.Has("print-expr")) {
+        exprOut = &Cout;
+    } else if (!exprFile.empty()) {
+        exprFileHolder.Reset(new TFixedBufferFileOutput(exprFile));
+        exprOut = exprFileHolder.Get();
+    }
+    factoryOptions.SetExprOutputStream(exprOut);
+
+    auto factory = MakeProgramFactory(factoryOptions);
+
+    NYT::TNode members{NYT::TNode::CreateList()};
+    auto typeNode = NYT::TNode::CreateList()
+                        .Add("DataType")
+                        .Add("Int64");
+
+    members.Add(NYT::TNode::CreateList()
+                    .Add("index")
+                    .Add(typeNode));
+    NYT::TNode schema = NYT::TNode::CreateList()
+                            .Add("StructType")
+                            .Add(members);
+
+    auto inputGenSchema = TVector<NYT::TNode>{schema};
+    auto inputGenStream = MakeGenInput(count);
+    Cerr << "Input data size: " << inputGenStream.Size() << "\n";
+    ETranslationMode isPgGen = res.Has("pg") ? ETranslationMode::PG : ETranslationMode::SQL;
+    ETranslationMode isPgTest = res.Has("pt") ? ETranslationMode::PG : ETranslationMode::SQL;
+    double normalizedTime;
+    size_t inputBenchSize;
+
+    if (blockEngineSettings == "disable") {
+        TStringStream outputGenStream;
+        auto outputGenSchema = RunGenSql<TSkiffOutputSpec>(
+            factory, inputGenSchema, genSql, isPgGen,
+            [&](const auto& program) {
+                auto handle = program->Apply(&inputGenStream);
+                handle->Run(&outputGenStream);
+                Cerr << "Generated data size: " << outputGenStream.Size() << "\n";
+            });
+
+        if (showResults) {
+            auto inputResStream = TStringStream(outputGenStream);
+            ShowResults<TSkiffInputSpec>(
+                factory, {outputGenSchema}, testSql, isPgTest, &inputResStream);
+        }
+
+        inputBenchSize = outputGenStream.Size();
+        normalizedTime = RunBenchmarks<TSkiffInputSpec, TSkiffOutputSpec>(
+            factory, {outputGenSchema}, testSql, isPgTest, repeats,
+            [&](const auto& program) {
+                auto inputBorrowed = TStringStream(outputGenStream);
+                auto handle = program->Apply(&inputBorrowed);
+                TNullOutput output;
+                handle->Run(&output);
+            });
+    } else {
+        auto inputGenSpec = TSkiffInputSpec(inputGenSchema);
+        auto outputGenSpec = TArrowOutputSpec({NYT::TNode::CreateEntity()});
+        // XXX: <RunGenSql> cannot be used for this case, since all buffers
+        // from the Datums in the obtained batches are owned by the worker's
+        // allocator. Hence, the program (i.e. worker) object should be created
+        // at the very beginning of the block, or at least prior to all the
+        // temporary batch storages (mind outputGenStream below).
+        auto program = factory->MakePullListProgram(
+            inputGenSpec, outputGenSpec, genSql, isPgGen);
+
+        auto handle = program->Apply(&inputGenStream);
+        auto outputGenSchema = program->MakeOutputSchema();
+
+        TVector<arrow::compute::ExecBatch> outputGenStream;
+        while (arrow::compute::ExecBatch* batch = handle->Fetch()) {
+            outputGenStream.push_back(*batch);
+        }
+
+        ui64 outputGenSize = std::transform_reduce(
+            outputGenStream.cbegin(), outputGenStream.cend(),
+            0l, std::plus{}, [](const auto& b) {
+                return NYql::NUdf::GetSizeOfArrowExecBatchInBytes(b);
+            });
+
+        Cerr << "Generated data size: " << outputGenSize << "\n";
+
+        if (showResults) {
+            auto inputResStreamHolder = StreamFromVector(outputGenStream);
+            auto inputResStream = inputResStreamHolder.Get();
+            ShowResults<TArrowInputSpec>(
+                factory, {outputGenSchema}, testSql, isPgTest, inputResStream);
+        }
+
+        inputBenchSize = outputGenSize;
+        normalizedTime = RunBenchmarks<TArrowInputSpec, TArrowOutputSpec>(
+            factory, {outputGenSchema}, testSql, isPgTest, repeats,
+            [&](const auto& program) {
+                auto handle = program->Apply(StreamFromVector(outputGenStream));
+                while (arrow::compute::ExecBatch* batch = handle->Fetch()) {}
+            });
+    }
+
+    Cout << "Bench score: " << Prec(inputBenchSize / normalizedTime, 4) << "\n";
+
+    NLog::CleanupLogger();
+    return 0;
+}
+
+int main(int argc, const char *argv[]) {
+    if (argc > 1 && TString(argv[1]) != TStringBuf("--ndebug")) {
+        Cerr << "purebench ABI version: " << NKikimr::NUdf::CurrentAbiVersionStr() << Endl;
+    }
+
+    NYql::NBacktrace::RegisterKikimrFatalActions();
+    NYql::NBacktrace::EnableKikimrSymbolize();
+
+    try {
+        return Main(argc, argv);
+    } catch (const TCompileError& e) {
+        Cerr << e.what() << "\n" << e.GetIssues();
+    } catch (...) {
+        Cerr << CurrentExceptionMessage() << Endl;
+        return 1;
+    }
+}
diff --git a/yql/essentials/tools/purebench/ya.make b/yql/essentials/tools/purebench/ya.make
new file mode 100644
index 00000000000..410dd6bba7d
--- /dev/null
+++ b/yql/essentials/tools/purebench/ya.make
@@ -0,0 +1,31 @@
+PROGRAM(purebench)
+
+ALLOCATOR(J)
+
+SRCS(
+    purebench.cpp
+)
+
+IF (OS_LINUX)
+    # prevent external python extensions to lookup protobuf symbols (and maybe
+    # other common stuff) in main binary
+    EXPORTS_SCRIPT(${ARCADIA_ROOT}/contrib/ydb/library/yql/tools/exports.symlist)
+ENDIF()
+
+PEERDIR(
+    library/cpp/getopt
+    library/cpp/svnversion
+    yql/essentials/utils/backtrace
+    yql/essentials/utils/log
+    contrib/ydb/library/yql/public/udf
+    contrib/ydb/library/yql/public/udf/service/exception_policy
+    library/cpp/skiff
+    library/cpp/yson
+    contrib/ydb/library/yql/public/purecalc/io_specs/mkql
+    contrib/ydb/library/yql/public/purecalc/io_specs/arrow
+    contrib/ydb/library/yql/public/purecalc
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/yql/essentials/tools/sql2yql/sql2yql.cpp b/yql/essentials/tools/sql2yql/sql2yql.cpp
new file mode 100644
index 00000000000..31925812bb6
--- /dev/null
+++ b/yql/essentials/tools/sql2yql/sql2yql.cpp
@@ -0,0 +1,375 @@
+#include <contrib/ydb/library/yql/ast/yql_ast.h>
+#include <contrib/ydb/library/yql/ast/yql_ast_annotation.h>
+#include <contrib/ydb/library/yql/ast/yql_expr.h>
+
+#include <contrib/ydb/library/yql/parser/lexer_common/hints.h>
+
+#include <contrib/ydb/library/yql/sql/sql.h>
+#include <contrib/ydb/library/yql/providers/common/provider/yql_provider_names.h>
+#include <contrib/ydb/library/yql/parser/pg_wrapper/interface/parser.h>
+
+#include <library/cpp/getopt/last_getopt.h>
+#include <contrib/ydb/library/yql/sql/v1/format/sql_format.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/file.h>
+#include <util/generic/hash.h>
+#include <util/generic/hash_set.h>
+#include <util/generic/string.h>
+#include <util/string/escape.h>
+
+#include <google/protobuf/message.h>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/repeated_field.h>
+
+struct TPosOutput {
+    IOutputStream& Out;
+    ui32 Line;
+    ui32 Column;
+
+    TPosOutput(IOutputStream& out)
+        : Out(out)
+        , Line(1)
+        , Column(0)
+    {
+    }
+
+    void Output(ui32 line, ui32 column, const TString& value) {
+        while (Line < line) {
+            Out << Endl;
+            ++Line;
+            Column = 0;
+        }
+        while (Column < column) {
+            Out << " ";
+            ++Column;
+        }
+        if (value != "<EOF>") {
+            Out << value;
+            Column += value.size();
+        }
+    }
+};
+
+static void ExtractQuery(TPosOutput& out, const google::protobuf::Message& node);
+
+static void VisitField(TPosOutput& out, const google::protobuf::FieldDescriptor& descr, const google::protobuf::Message& field) {
+    using namespace google::protobuf;
+    const Descriptor* d = descr.message_type();
+    if (!d) {
+        ythrow yexception() << "Invalid AST: non-message node encountered";
+    }
+    if (d->name() == "TToken") {
+        const Reflection* r = field.GetReflection();
+        out.Output(r->GetUInt32(field, d->field(0)), r->GetUInt32(field, d->field(1)), r->GetString(field, d->field(2)));
+    } else {
+        ExtractQuery(out, field);
+    }
+}
+
+static void ExtractQuery(TPosOutput& out, const google::protobuf::Message& node) {
+    using namespace google::protobuf;
+    TVector<const FieldDescriptor*> fields;
+    const Reflection* ref = node.GetReflection();
+    ref->ListFields(node, &fields);
+
+    for (auto it = fields.begin(); it != fields.end(); ++it) {
+        if ((*it)->is_repeated()) {
+            const ui32 fieldSize = ref->FieldSize(node, *it);
+            for (ui32 i = 0; i < fieldSize; ++i) {
+                VisitField(out, **it, ref->GetRepeatedMessage(node, *it, i));
+            }
+        } else {
+            VisitField(out, **it, ref->GetMessage(node, *it));
+        }
+    }
+}
+
+bool TestFormat(
+    const TString& query,
+    const NSQLTranslation::TTranslationSettings& settings,
+    const TString& queryFile,
+    const NYql::TAstParseResult& parseRes,
+    const TString& outFileName,
+    const bool checkDoubleFormatting
+) {
+    TStringStream yqlProgram;
+    parseRes.Root->PrettyPrintTo(yqlProgram, NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::ShortQuote);
+
+    TString frmQuery;
+    NYql::TIssues issues;
+    auto formatter = NSQLFormat::MakeSqlFormatter(settings);
+    if (!formatter->Format(query, frmQuery, issues)) {
+        Cerr << "Failed to format query: " << issues.ToString() << Endl;
+        return false;
+    }
+    NYql::TAstParseResult frmParseRes = NSQLTranslation::SqlToYql(query, settings);
+    TStringStream frmYqlProgram;
+    frmParseRes.Root->PrettyPrintTo(frmYqlProgram, NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::ShortQuote);
+    if (!frmParseRes.Issues.Empty()) {
+        frmParseRes.Issues.PrintWithProgramTo(Cerr, queryFile, frmQuery);
+        if (AnyOf(frmParseRes.Issues, [](const auto& issue) { return issue.GetSeverity() == NYql::TSeverityIds::S_ERROR;})) {
+            return false;
+        }
+    }
+    if (!frmParseRes.IsOk()) {
+        Cerr << "No error reported, but no yql compiled result!" << Endl << Endl;
+        return false;
+    }
+    if (yqlProgram.Str() != frmYqlProgram.Str()) {
+        Cerr << "source query's AST and formatted query's AST are not same\n";
+        return false;
+    }
+
+    TString frmQuery2;
+    if (!formatter->Format(frmQuery, frmQuery2, issues)) {
+        Cerr << "Failed to format already formatted query: " << issues.ToString() << Endl;
+        return false;
+    }
+
+    if (checkDoubleFormatting && frmQuery != frmQuery2) {
+        Cerr << "Formatting an already formatted query yielded a different resut" << Endl
+             << "Add /* skip double format */ to suppress" << Endl;
+        return false;
+    }
+
+    if (!outFileName.empty()) {
+        TFixedBufferFileOutput out{outFileName};
+        out << frmQuery;
+    }
+    return true;
+}
+
+class TStoreMappingFunctor: public NLastGetopt::IOptHandler {
+public:
+    TStoreMappingFunctor(THashMap<TString, TString>* target, char delim = '@')
+        : Target(target)
+        , Delim(delim)
+    {
+    }
+
+    void HandleOpt(const NLastGetopt::TOptsParser* parser) final {
+        const TStringBuf val(parser->CurValOrDef());
+        const auto service = TString(val.After(Delim));
+        auto res = Target->emplace(TString(val.Before(Delim)), service);
+        if (!res.second) {
+            /// force replace already exist parametr
+            res.first->second = service;
+        }
+    }
+
+private:
+    THashMap<TString, TString>* Target;
+    char Delim;
+};
+
+int BuildAST(int argc, char* argv[]) {
+    NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
+
+    TString outFileName;
+    TString queryString;
+    ui16 syntaxVersion;
+    TString outFileNameFormat;
+    THashMap<TString, TString> clusterMapping;
+    clusterMapping["plato"] = NYql::YtProviderName;
+    clusterMapping["pg_catalog"] = NYql::PgProviderName;
+    clusterMapping["information_schema"] = NYql::PgProviderName;
+
+    THashMap<TString, TString> tables;
+    THashSet<TString> flags;
+
+    opts.AddLongOption('o', "output", "save output to file").RequiredArgument("file").StoreResult(&outFileName);
+    opts.AddLongOption('q', "query", "query string").RequiredArgument("query").StoreResult(&queryString);
+    opts.AddLongOption('t', "tree", "print AST proto text").NoArgument();
+    opts.AddLongOption('d', "diff", "print inlined diff for original query and query build from AST if they differ").NoArgument();
+    opts.AddLongOption('D', "dump", "dump inlined diff for original query and query build from AST").NoArgument();
+    opts.AddLongOption('p', "print-query", "print given query before parsing").NoArgument();
+    opts.AddLongOption('y', "yql", "translate result to Yql and print it").NoArgument();
+    opts.AddLongOption('l', "lexer", "print query token stream").NoArgument();
+    opts.AddLongOption("ansi-lexer", "use ansi lexer").NoArgument();
+    opts.AddLongOption("pg", "use pg_query parser").NoArgument();
+    opts.AddLongOption('a', "ann", "print Yql annotations").NoArgument();
+    opts.AddLongOption('C', "cluster", "set cluster to service mapping").RequiredArgument("name@service").Handler(new TStoreMappingFunctor(&clusterMapping));
+    opts.AddLongOption('T', "table", "set table to filename mapping").RequiredArgument("table@path").Handler(new TStoreMappingFunctor(&tables));
+    opts.AddLongOption('R', "replace", "replace Output table with each statement result").NoArgument();
+    opts.AddLongOption("sqllogictest", "input files are in sqllogictest format").NoArgument();
+    opts.AddLongOption("syntax-version", "SQL syntax version").StoreResult(&syntaxVersion).DefaultValue(1);
+    opts.AddLongOption('F', "flags", "SQL pragma flags").SplitHandler(&flags, ',');
+    opts.AddLongOption("assume-ydb-on-slash", "Assume YDB provider if cluster name starts with '/'").NoArgument();
+    opts.AddLongOption("test-format", "compare formatted query's AST with the original query's AST (only syntaxVersion=1 is supported).").NoArgument();
+    opts.AddLongOption("test-double-format", "check if formatting already formatted query produces the same result").NoArgument();
+    opts.AddLongOption("test-antlr4", "check antlr4 parser").NoArgument();
+    opts.AddLongOption("format-output", "Saves formatted query to it").RequiredArgument("format-output").StoreResult(&outFileNameFormat);
+    opts.SetFreeArgDefaultTitle("query file");
+    opts.AddHelpOption();
+
+    NLastGetopt::TOptsParseResult res(&opts, argc, argv);
+    TVector<TString> queryFiles(res.GetFreeArgs());
+
+    THolder<TFixedBufferFileOutput> outFile;
+    if (!outFileName.empty()) {
+        outFile.Reset(new TFixedBufferFileOutput(outFileName));
+    }
+    IOutputStream& out = outFile ? *outFile.Get() : Cout;
+
+    if (!res.Has("query") && queryFiles.empty()) {
+        Cerr << "No --query nor query file was specified" << Endl << Endl;
+        opts.PrintUsage(argv[0], Cerr);
+    }
+
+    TVector<TString> queries;
+    int errors = 0;
+    for (ui32 i = 0; i <= queryFiles.size(); ++i) {
+        queries.clear();
+        TString queryFile("query");
+        if (i < queryFiles.size()) {
+            queryFile = queryFiles[i];
+            TAutoPtr<TFileInput> filePtr;
+            if (queryFile != "-") {
+                filePtr.Reset(new TFileInput(queryFile));
+            }
+            IInputStream& in = filePtr.Get() ? *filePtr : Cin;
+            if (res.Has("sqllogictest")) {
+                ui32 lineNum = 1;
+                TString line;
+                bool take = false;
+                while (in.ReadLine(line)) {
+                    if (line.StartsWith("statement") || line.StartsWith("query")) {
+                        take = true;
+                        queries.emplace_back();
+                        queryFile = queryFiles[i] + " line " + ToString(lineNum + 1);
+                    } else if (line.StartsWith("----") || line.empty()) {
+                        take = false;
+                    } else if (take) {
+                        queries.back().append(line).append("\n");
+                    }
+                    ++lineNum;
+
+                }
+            } else {
+                queries.push_back(in.ReadAll());
+            }
+        } else {
+            queries.push_back(queryString);
+        }
+
+        for (const auto& query: queries) {
+            if (query.empty()) {
+                continue;
+            }
+            if (res.Has("print-query")) {
+                out << query << Endl;
+            }
+
+            google::protobuf::Arena arena;
+            NSQLTranslation::TTranslationSettings settings;
+            settings.Arena = &arena;
+            settings.ClusterMapping = clusterMapping;
+            settings.Flags = flags;
+            settings.SyntaxVersion = syntaxVersion;
+            settings.AnsiLexer = res.Has("ansi-lexer");
+            settings.WarnOnV0 = false;
+            settings.V0ForceDisable = false;
+            settings.AssumeYdbOnClusterWithSlash = res.Has("assume-ydb-on-slash");
+            settings.TestAntlr4 = res.Has("test-antlr4");
+
+            if (res.Has("lexer")) {
+                NYql::TIssues issues;
+                auto lexer = NSQLTranslation::SqlLexer(query, issues, settings);
+                NSQLTranslation::TParsedTokenList tokens;
+                if (lexer && NSQLTranslation::Tokenize(*lexer, query, queryFile, tokens, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+                    for (auto& token : tokens) {
+                        out << token.Line << ":" << token.LinePos << "\t\t" << token.Name << "(" << EscapeC(token.Content) << ")\n";
+                    }
+                }
+                if (!issues.Empty()) {
+                    issues.PrintTo(Cerr);
+                }
+
+                bool hasError = AnyOf(issues, [](const auto& issue) { return issue.GetSeverity() == NYql::TSeverityIds::S_ERROR;});
+                if (hasError) {
+                    ++errors;
+                }
+                continue;
+            }
+
+            NYql::TAstParseResult parseRes;
+            if (res.Has("pg")) {
+                parseRes = NSQLTranslationPG::PGToYql(query, settings);
+            } else {
+                if (res.Has("tree") || res.Has("diff") || res.Has("dump")) {
+                    google::protobuf::Message* ast(NSQLTranslation::SqlAST(query, queryFile, parseRes.Issues,
+                        NSQLTranslation::SQL_MAX_PARSER_ERRORS, settings));
+                    if (ast) {
+                        if (res.Has("tree")) {
+                            out << ast->DebugString() << Endl;
+                        }
+                        if (res.Has("diff") || res.Has("dump")) {
+                            TStringStream result;
+                            TPosOutput posOut(result);
+                            ExtractQuery(posOut, *ast);
+                            if (res.Has("dump") || query != result.Str()) {
+                              out << NUnitTest::ColoredDiff(query, result.Str()) << Endl;
+                           }
+                        }
+
+                        NSQLTranslation::TSQLHints hints;
+                        auto lexer = SqlLexer(query, parseRes.Issues, settings);
+                        if (lexer && CollectSqlHints(*lexer, query, queryFile, settings.File, hints, parseRes.Issues,
+                            settings.MaxErrors, settings.Antlr4Parser)) {
+                            parseRes = NSQLTranslation::SqlASTToYql(*ast, hints, settings);
+                        }
+                   }
+                } else {
+                   parseRes = NSQLTranslation::SqlToYql(query, settings);
+                }
+            }
+
+            if (parseRes.Root) {
+                TStringStream yqlProgram;
+                parseRes.Root->PrettyPrintTo(yqlProgram, NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::ShortQuote);
+                if (res.Has("yql")) {
+                    out << yqlProgram.Str();
+                }
+                if (res.Has("ann")) {
+                    TMemoryPool pool(1024);
+                    NYql::AnnotatePositions(*parseRes.Root, pool)->PrettyPrintTo(out, NYql::TAstPrintFlags::PerLine);
+                }
+            }
+
+            bool hasError = false;
+            if (!parseRes.Issues.Empty()) {
+                parseRes.Issues.PrintWithProgramTo(Cerr, queryFile, query);
+                hasError = AnyOf(parseRes.Issues, [](const auto& issue) { return issue.GetSeverity() == NYql::TSeverityIds::S_ERROR;});
+            }
+
+            if (!parseRes.IsOk() && !hasError) {
+                hasError = true;
+                Cerr << "No error reported, but no yql compiled result!" << Endl << Endl;
+            }
+
+            if (res.Has("test-format") && syntaxVersion == 1 && !hasError && parseRes.Root) {
+                hasError = !TestFormat(query, settings, queryFile, parseRes, outFileNameFormat, res.Has("test-double-format"));
+            }
+
+            if (hasError) {
+                ++errors;
+            }
+        }
+    }
+
+    return errors;
+}
+
+int main(int argc, char* argv[]) {
+    try {
+        return BuildAST(argc, argv);
+    } catch (const yexception& e) {
+        Cerr << "Caught exception:" << e.what() << Endl;
+        return 1;
+    } catch (...) {
+        Cerr << "Caught exception" << Endl;
+        return 1;
+    }
+    return 0;
+}
diff --git a/yql/essentials/tools/sql2yql/ya.make b/yql/essentials/tools/sql2yql/ya.make
new file mode 100644
index 00000000000..20d79c00395
--- /dev/null
+++ b/yql/essentials/tools/sql2yql/ya.make
@@ -0,0 +1,23 @@
+PROGRAM()
+
+PEERDIR(
+    contrib/libs/antlr3_cpp_runtime
+    library/cpp/getopt
+    library/cpp/testing/unittest
+    contrib/ydb/library/yql/parser/lexer_common
+    contrib/ydb/library/yql/parser/pg_wrapper
+    contrib/ydb/library/yql/public/udf/service/stub
+    contrib/ydb/library/yql/sql
+    contrib/ydb/library/yql/sql/pg
+    contrib/ydb/library/yql/sql/v1/format
+)
+
+ADDINCL(
+    GLOBAL contrib/libs/antlr3_cpp_runtime/include
+)
+
+SRCS(
+    sql2yql.cpp
+)
+
+END()
diff --git a/yql/essentials/tools/sql_formatter/sql_formatter.cpp b/yql/essentials/tools/sql_formatter/sql_formatter.cpp
new file mode 100644
index 00000000000..820afff807b
--- /dev/null
+++ b/yql/essentials/tools/sql_formatter/sql_formatter.cpp
@@ -0,0 +1,72 @@
+#include <contrib/ydb/library/yql/sql/v1/format/sql_format.h>
+
+#include <library/cpp/getopt/last_getopt.h>
+#include <google/protobuf/arena.h>
+
+#include <util/stream/file.h>
+
+int RunFormat(int argc, char* argv[]) {
+    NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
+
+    TString outFileName;
+    TString inFileName;
+    TString queryString;
+
+    opts.AddLongOption('o', "output", "save output to file").RequiredArgument("file").StoreResult(&outFileName);
+    opts.AddLongOption('i', "input", "input file").RequiredArgument("input").StoreResult(&inFileName);
+    opts.AddLongOption('p', "print-query", "print given query before parsing").NoArgument();
+    opts.AddLongOption('f', "obfuscate", "obfuscate query").NoArgument();
+    opts.AddLongOption("ansi-lexer", "use ansi lexer").NoArgument();
+    opts.AddHelpOption();
+
+    NLastGetopt::TOptsParseResult res(&opts, argc, argv);
+
+    THolder<TFixedBufferFileOutput> outFile;
+    if (!outFileName.empty()) {
+        outFile.Reset(new TFixedBufferFileOutput(outFileName));
+    }
+    IOutputStream& out = outFile ? *outFile.Get() : Cout;
+
+    THolder<TUnbufferedFileInput> inFile;
+    if (!inFileName.empty()) {
+        inFile.Reset(new TUnbufferedFileInput(inFileName));
+    }
+    IInputStream& in = inFile ? *inFile.Get() : Cin;
+
+    queryString = in.ReadAll();
+    int errors = 0;
+    TString queryFile("query");
+    if (res.Has("print-query")) {
+        out << queryString << Endl;
+    }
+    google::protobuf::Arena arena;
+    NSQLTranslation::TTranslationSettings settings;
+    settings.Arena = &arena;
+    settings.AnsiLexer = res.Has("ansi-lexer");
+    auto formatter = NSQLFormat::MakeSqlFormatter(settings);
+    TString frm_query;
+    TString error;
+    NYql::TIssues issues;
+    if (!formatter->Format(queryString, frm_query, issues, res.Has("obfuscate") ?
+        NSQLFormat::EFormatMode::Obfuscate : NSQLFormat::EFormatMode::Pretty)) {
+        ++errors;
+        Cerr << "Error formatting query: " << issues.ToString() << Endl;
+    } else {
+        out << frm_query << Endl;
+    }
+
+    return errors;
+}
+
+int main(int argc, char* argv[]) {
+    try {
+        return RunFormat(argc, argv);
+    } catch (const yexception& e) {
+        Cerr << "Caught exception:" << e.what() << Endl;
+        return 1;
+    } catch (...) {
+        Cerr << CurrentExceptionMessage() << Endl;
+        return 1;
+    }
+    return 0;
+}
diff --git a/yql/essentials/tools/sql_formatter/ya.make b/yql/essentials/tools/sql_formatter/ya.make
new file mode 100644
index 00000000000..4c361e32df0
--- /dev/null
+++ b/yql/essentials/tools/sql_formatter/ya.make
@@ -0,0 +1,13 @@
+PROGRAM()
+
+PEERDIR(
+    library/cpp/getopt
+    contrib/libs/protobuf
+    contrib/ydb/library/yql/sql/v1/format
+)
+
+SRCS(
+    sql_formatter.cpp
+)
+
+END()
diff --git a/yql/essentials/tools/ya.make b/yql/essentials/tools/ya.make
index 2c8a3945a2a..53ace3f512a 100644
--- a/yql/essentials/tools/ya.make
+++ b/yql/essentials/tools/ya.make
@@ -1,5 +1,9 @@
 RECURSE(
+    astdiff    
     pg_catalog_dump
     pg-make-test
     pgrun
+    purebench
+    sql2yql
+    sql_formatter
 )