YQL-19553 CLI for yql linter

init
commit_hash:a21a71769ad2095e40909c69255a3cf38eabc179

16 files changed, 291 insertions, 38 deletions

diff --git a/yql/essentials/public/fastcheck/format.cpp b/yql/essentials/public/fastcheck/format.cpp
index 8370ffdfb12..d7fbce148c5 100644
--- a/yql/essentials/public/fastcheck/format.cpp
+++ b/yql/essentials/public/fastcheck/format.cpp
@@ -54,9 +54,8 @@ private:
         settings.AnsiLexer = request.IsAnsiLexer;
 
         auto formatter = NSQLFormat::MakeSqlFormatter(settings);
-        NYql::TIssues issues;
         TString formattedQuery;
-        res.Success = formatter->Format(request.Program, formattedQuery, issues);
+        res.Success = formatter->Format(request.Program, formattedQuery, res.Issues);
         if (res.Success && formattedQuery != request.Program) {
             res.Success = false;
             TPosition origPos(0, 1, request.File);
diff --git a/yql/essentials/public/fastcheck/linter.cpp b/yql/essentials/public/fastcheck/linter.cpp
index 971e6fd9777..a32814e3ec3 100644
--- a/yql/essentials/public/fastcheck/linter.cpp
+++ b/yql/essentials/public/fastcheck/linter.cpp
@@ -4,11 +4,15 @@
 #include <yql/essentials/ast/yql_ast.h>
 #include <yql/essentials/ast/yql_expr.h>
 
+#include <util/string/split.h>
+
 #include <functional>
 
 namespace NYql {
 namespace NFastCheck {
 
+namespace {
+
 class TCheckRunnerFactory : public ICheckRunnerFactory {
 public:
     using TRunnerFactoryFunction = std::function<std::unique_ptr<ICheckRunner>()>;
@@ -41,22 +45,13 @@ private:
     TSet<TString> CheckNames_;
 };
 
-TCheckRunnerFactory& GetCheckRunnerFactory() {
-    return *Singleton<TCheckRunnerFactory>();
-};
-
-const TSet<TString>& ListChecks() {
-    return GetCheckRunnerFactory().ListChecks();
-}
-
-TChecksResponse RunChecks(const TChecksRequest& request) {
-    auto filters = request.Filters.GetOrElse(TVector<TCheckFilter>(1, TCheckFilter{.Include = true, .CheckNameGlob = "*"}));
-    const auto& fullChecks = ListChecks();
+TSet<TString> GetEnabledChecks(const TSet<TString>& allChecks, const TMaybe<TVector<TCheckFilter>>& filters) {
+    auto usedFilters = filters.GetOrElse(TVector<TCheckFilter>(1, TCheckFilter{.Include = true, .CheckNameGlob = "*"}));
     TSet<TString> enabledChecks;
-    for (const auto& f : filters) {
+    for (const auto& f : usedFilters) {
         if (f.CheckNameGlob == "*") {
             if (f.Include) {
-                enabledChecks = fullChecks;
+                enabledChecks = allChecks;
             } else {
                 enabledChecks.clear();
             }
@@ -65,7 +60,7 @@ TChecksResponse RunChecks(const TChecksRequest& request) {
             Y_ENSURE(f.CheckNameGlob.find('*') == TString::npos);
             Y_ENSURE(f.CheckNameGlob.find('?') == TString::npos);
             if (f.Include) {
-                if (fullChecks.contains(f.CheckNameGlob)) {
+                if (allChecks.contains(f.CheckNameGlob)) {
                     enabledChecks.insert(f.CheckNameGlob);
                 }
             } else {
@@ -74,6 +69,37 @@ TChecksResponse RunChecks(const TChecksRequest& request) {
         }
     }
 
+    return enabledChecks;
+}
+
+TCheckRunnerFactory& GetCheckRunnerFactory() {
+    return *Singleton<TCheckRunnerFactory>();
+};
+
+}
+
+TVector<TCheckFilter> ParseChecks(const TString& checks) {
+    TVector<TCheckFilter> res;
+    for (TStringBuf one: StringSplitter(checks).SplitByString(",")) {
+        TCheckFilter f;
+        TStringBuf afterPrefix = one;
+        if (one.AfterPrefix("-", afterPrefix)) {
+            f.Include = false;
+        }
+
+        f.CheckNameGlob = afterPrefix;
+        res.push_back(f);
+    }
+
+    return res;
+}
+
+TSet<TString> ListChecks(const TMaybe<TVector<TCheckFilter>>& filters) {
+    return GetEnabledChecks(GetCheckRunnerFactory().ListChecks(), filters);
+}
+
+TChecksResponse RunChecks(const TChecksRequest& request) {
+    auto enabledChecks = GetEnabledChecks(GetCheckRunnerFactory().ListChecks(), request.Filters);
     TChecksResponse res;
     for (const auto& c : enabledChecks) {
         auto checkRunner = GetCheckRunnerFactory().MakeRunner(c);
diff --git a/yql/essentials/public/fastcheck/linter.h b/yql/essentials/public/fastcheck/linter.h
index aa4355b597c..7af5e624551 100644
--- a/yql/essentials/public/fastcheck/linter.h
+++ b/yql/essentials/public/fastcheck/linter.h
@@ -8,8 +8,6 @@
 namespace NYql {
 namespace NFastCheck {
 
-const TSet<TString>& ListChecks();
-
 enum class ESyntax {
     SExpr,
     YQL,
@@ -49,6 +47,8 @@ struct TChecksResponse {
     TVector<TCheckResponse> Checks;
 };
 
+TVector<TCheckFilter> ParseChecks(const TString& checks);
+TSet<TString> ListChecks(const TMaybe<TVector<TCheckFilter>>& filters = Nothing());
 TChecksResponse RunChecks(const TChecksRequest& request);
 
 }
diff --git a/yql/essentials/public/fastcheck/parser.cpp b/yql/essentials/public/fastcheck/parser.cpp
index 82fca672b48..3e065be19eb 100644
--- a/yql/essentials/public/fastcheck/parser.cpp
+++ b/yql/essentials/public/fastcheck/parser.cpp
@@ -1,4 +1,5 @@
 #include "check_runner.h"
+#include <yql/essentials/sql/v1/lexer/lexer.h>
 #include <yql/essentials/sql/v1/proto_parser/proto_parser.h>
 #include <yql/essentials/sql/settings/translation_settings.h>
 #include <yql/essentials/parser/pg_wrapper/interface/raw_parser.h>
@@ -72,11 +73,18 @@ private:
             return res;
         }
 
-        google::protobuf::Arena arena;
-        auto msg = NSQLTranslationV1::SqlAST(request.Program, request.File, res.Issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS,
-            settings.AnsiLexer, true, false, &arena);
-        if (msg) {
-            res.Success = true;
+        auto lexer = NSQLTranslationV1::MakeLexer(settings.AnsiLexer, true);
+        auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) {
+            Y_UNUSED(token);
+        };
+
+        if (lexer->Tokenize(request.Program, request.File, onNextToken, res.Issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+            google::protobuf::Arena arena;
+            auto msg = NSQLTranslationV1::SqlAST(request.Program, request.File, res.Issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS,
+                settings.AnsiLexer, true, false, &arena);
+            if (msg) {
+                res.Success = true;
+            }
         }
 
         return res;
diff --git a/yql/essentials/public/fastcheck/ya.make b/yql/essentials/public/fastcheck/ya.make
index 732e5d011ac..d1574d5707b 100644
--- a/yql/essentials/public/fastcheck/ya.make
+++ b/yql/essentials/public/fastcheck/ya.make
@@ -25,6 +25,8 @@ PEERDIR(
     yql/essentials/parser/pg_wrapper/interface
 )
 
+GENERATE_ENUM_SERIALIZATION(linter.h)
+
 END()
 
 RECURSE_FOR_TESTS(
diff --git a/yql/essentials/public/issue/yql_issue.cpp b/yql/essentials/public/issue/yql_issue.cpp
index 50589e99763..bb171a78692 100644
--- a/yql/essentials/public/issue/yql_issue.cpp
+++ b/yql/essentials/public/issue/yql_issue.cpp
@@ -199,7 +199,8 @@ void TIssues::PrintTo(IOutputStream& out, bool oneLine) const
 void TIssues::PrintWithProgramTo(
         IOutputStream& out,
         const TString& programFilename,
-        const TString& programText) const
+        const TString& programText,
+        bool colorize) const
 {
     using namespace NColorizer;
 
@@ -210,11 +211,41 @@ void TIssues::PrintWithProgramTo(
         WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) {
             auto shift = level * 4;
             Indent(out, shift);
-            out << DarkGray() << programFilename << Old() << ':';
-            out << Purple() << issue.Range() << Old();
-            auto color = (issue.GetSeverity() ==  TSeverityIds::S_WARNING) ? Yellow() : LightRed();
+            if (colorize) {
+                out << DarkGray();
+            }
+            out << programFilename;
+            if (colorize) {
+                out << Old();
+            }
+
+            out << ':';
+            if (colorize) {
+                out << Purple();
+            }
+
+            out << issue.Range();
+            if (colorize) {
+                out << Old();
+            }
+
             auto severityName = SeverityToString(issue.GetSeverity());
-            out << color << ": "<< severityName << ": " << issue.GetMessage() << Old() << '\n';
+            if (colorize) {
+                if (issue.GetSeverity() == TSeverityIds::S_INFO) {
+                    out << LightGreen();
+                } else if (issue.GetSeverity() == TSeverityIds::S_WARNING) {
+                    out << Yellow();
+                } else {
+                    out << LightRed();
+                }
+            }
+
+            out << ": "<< severityName << ": " << issue.GetMessage();
+            if (colorize) {
+                out << Old();
+            }
+
+            out << '\n';
             Indent(out, shift);
             if (issue.Position.HasValue()) {
                 out << '\t' << lines[issue.Position.Row] << '\n';
diff --git a/yql/essentials/public/issue/yql_issue.h b/yql/essentials/public/issue/yql_issue.h
index 00c49fb1fca..07fcdfed86e 100644
--- a/yql/essentials/public/issue/yql_issue.h
+++ b/yql/essentials/public/issue/yql_issue.h
@@ -315,7 +315,8 @@ public:
     void PrintWithProgramTo(
             IOutputStream& out,
             const TString& programFilename,
-            const TString& programText) const;
+            const TString& programText,
+            bool colorize = true) const;
 
     inline TString ToString(bool oneLine = false) const {
         TStringStream out;
diff --git a/yql/essentials/sql/v1/format/sql_format.cpp b/yql/essentials/sql/v1/format/sql_format.cpp
index 0ede01dd8fd..c531d249775 100644
--- a/yql/essentials/sql/v1/format/sql_format.cpp
+++ b/yql/essentials/sql/v1/format/sql_format.cpp
@@ -3071,7 +3071,7 @@ public:
         }
 
         if (mode == EFormatMode::Obfuscate) {
-            auto message = NSQLTranslationV1::SqlAST(query, "Query", issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser, parsedSettings.TestAntlr4, parsedSettings.Arena);
+            auto message = NSQLTranslationV1::SqlAST(query, parsedSettings.File, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser, parsedSettings.TestAntlr4, parsedSettings.Arena);
             if (!message) {
                 return false;
             }
@@ -3082,7 +3082,7 @@ public:
 
         auto lexer = NSQLTranslationV1::MakeLexer(parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser);
         TVector<TString> statements;
-        if (!NSQLTranslationV1::SplitQueryToStatements(query, lexer, statements, issues)) {
+        if (!NSQLTranslationV1::SplitQueryToStatements(query, lexer, statements, issues, parsedSettings.File)) {
             return false;
         }
 
@@ -3101,12 +3101,12 @@ public:
                 }
             };
 
-            if (!lexer->Tokenize(currentQuery, "Query", onNextRawToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+            if (!lexer->Tokenize(currentQuery, parsedSettings.File, onNextRawToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
                 return false;
             }
 
             NYql::TIssues parserIssues;
-            auto message = NSQLTranslationV1::SqlAST(currentQuery, "Query", parserIssues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser, parsedSettings.TestAntlr4, parsedSettings.Arena);
+            auto message = NSQLTranslationV1::SqlAST(currentQuery, parsedSettings.File, parserIssues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser, parsedSettings.TestAntlr4, parsedSettings.Arena);
             if (!message) {
                 finalFormattedQuery << currentQuery;
                 if (!currentQuery.EndsWith("\n")) {
@@ -3127,7 +3127,7 @@ public:
                 stmtFormattedTokens.push_back(token);
             };
 
-            if (!lexer->Tokenize(currentFormattedQuery, "Query", onNextFormattedToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+            if (!lexer->Tokenize(currentFormattedQuery, parsedSettings.File, onNextFormattedToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
                 return false;
             }
 
@@ -3178,7 +3178,7 @@ TString MutateQuery(const TString& query, const NSQLTranslation::TTranslationSet
         }
     };
 
-    if (!lexer->Tokenize(query, "Query", onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+    if (!lexer->Tokenize(query, parsedSettings.File, onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
         throw yexception() << issues.ToString();
     }
 
diff --git a/yql/essentials/sql/v1/lexer/lexer.cpp b/yql/essentials/sql/v1/lexer/lexer.cpp
index 91142870fdf..fb9ca54f358 100644
--- a/yql/essentials/sql/v1/lexer/lexer.cpp
+++ b/yql/essentials/sql/v1/lexer/lexer.cpp
@@ -242,7 +242,7 @@ void SplitByStatements(TTokenIterator begin, TTokenIterator end, TVector<TTokenI
 
 }
 
-bool SplitQueryToStatements(const TString& query, NSQLTranslation::ILexer::TPtr& lexer, TVector<TString>& statements, NYql::TIssues& issues) {
+bool SplitQueryToStatements(const TString& query, NSQLTranslation::ILexer::TPtr& lexer, TVector<TString>& statements, NYql::TIssues& issues, const TString& file) {
     TParsedTokenList allTokens;
     auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) {
         if (token.Name != "EOF") {
@@ -250,7 +250,7 @@ bool SplitQueryToStatements(const TString& query, NSQLTranslation::ILexer::TPtr&
         }
     };
 
-    if (!lexer->Tokenize(query, "Query", onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+    if (!lexer->Tokenize(query, file, onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
         return false;
     }
 
diff --git a/yql/essentials/sql/v1/lexer/lexer.h b/yql/essentials/sql/v1/lexer/lexer.h
index d43efc6387f..3ce780686c6 100644
--- a/yql/essentials/sql/v1/lexer/lexer.h
+++ b/yql/essentials/sql/v1/lexer/lexer.h
@@ -13,5 +13,5 @@ NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi, bool antlr4);
 bool IsProbablyKeyword(const NSQLTranslation::TParsedToken& token);
 
 bool SplitQueryToStatements(const TString& query, NSQLTranslation::ILexer::TPtr& lexer,
-    TVector<TString>& statements, NYql::TIssues& issues);
+    TVector<TString>& statements, NYql::TIssues& issues, const TString& file = "");
 }
diff --git a/yql/essentials/tools/ya.make b/yql/essentials/tools/ya.make
index c8b02dfa7af..becd8907f6a 100644
--- a/yql/essentials/tools/ya.make
+++ b/yql/essentials/tools/ya.make
@@ -12,4 +12,5 @@ RECURSE(
     udf_probe
     udf_resolver
     yql_facade_run
+    yql_linter
 )
diff --git a/yql/essentials/tools/yql_linter/ya.make b/yql/essentials/tools/yql_linter/ya.make
new file mode 100644
index 00000000000..a8053c04d1c
--- /dev/null
+++ b/yql/essentials/tools/yql_linter/ya.make
@@ -0,0 +1,13 @@
+PROGRAM()
+
+PEERDIR(
+    library/cpp/getopt
+    yql/essentials/public/fastcheck
+    library/cpp/colorizer
+)
+
+SRCS(
+    yql_linter.cpp
+)
+
+END()
diff --git a/yql/essentials/tools/yql_linter/yql_linter.cpp b/yql/essentials/tools/yql_linter/yql_linter.cpp
new file mode 100644
index 00000000000..7e392b7fc0a
--- /dev/null
+++ b/yql/essentials/tools/yql_linter/yql_linter.cpp
@@ -0,0 +1,122 @@
+#include <yql/essentials/public/fastcheck/linter.h>
+#include <yql/essentials/utils/tty.h>
+
+#include <library/cpp/getopt/last_getopt.h>
+#include <google/protobuf/arena.h>
+
+#include <library/cpp/colorizer/output.h>
+
+#include <util/generic/serialized_enum.h>
+#include <util/stream/file.h>
+
+int Run(int argc, char* argv[]) {
+    NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
+
+    TString inFileName;
+    TString queryString;
+    TString checks;
+    THashMap<TString, TString> clusterMapping;
+    TString modeStr = "Default";
+    TString syntaxStr = "YQL";
+
+    opts.AddLongOption('i', "input", "input file").RequiredArgument("input").StoreResult(&inFileName);
+    opts.AddLongOption('v', "verbose", "show lint issues").NoArgument();
+    opts.AddLongOption("list-checks", "list all enabled checks and exit").NoArgument();
+    opts.AddLongOption("checks", "comma-separated list of globs with optional '-' prefix").StoreResult(&checks);
+    opts.AddLongOption('C', "cluster", "cluster to service mapping").RequiredArgument("name@service")
+        .KVHandler([&](TString cluster, TString provider) {
+            if (cluster.empty() || provider.empty()) {
+                throw yexception() << "Incorrect service mapping, expected form cluster@provider, e.g. plato@yt";
+            }
+            clusterMapping[cluster] = provider;
+        }, '@');
+
+    opts.AddLongOption('m', "mode", "query mode, allowed values: " + GetEnumAllNames<NYql::NFastCheck::EMode>()).StoreResult(&modeStr);
+    opts.AddLongOption('s', "syntax", "query syntax, allowed values: " + GetEnumAllNames<NYql::NFastCheck::ESyntax>()).StoreResult(&syntaxStr);
+    opts.AddLongOption("ansi-lexer", "use ansi lexer").NoArgument();
+    opts.AddLongOption("no-colors", "disable colors for output").NoArgument();
+    opts.SetFreeArgsNum(0);
+    opts.AddHelpOption();
+
+    NLastGetopt::TOptsParseResult res(&opts, argc, argv);
+
+    const bool colorize = !res.Has("no-colors") && IsTty(NYql::EStdStream::Out);
+    NYql::NFastCheck::TChecksRequest checkReq;
+    if (res.Has("checks")) {
+        checkReq.Filters = NYql::NFastCheck::ParseChecks(checks);
+    }
+
+    if (res.Has("list-checks")) {
+        for (const auto& c : NYql::NFastCheck::ListChecks(checkReq.Filters)) {
+            if (colorize) {
+                Cout << NColorizer::LightCyan();
+            }
+
+            Cout << c;
+            if (colorize) {
+                Cout << NColorizer::Old();
+            }
+
+            Cout << '\n';
+        }
+
+        return 0;
+    }
+
+    THolder<TUnbufferedFileInput> inFile;
+    if (!inFileName.empty()) {
+        inFile.Reset(new TUnbufferedFileInput(inFileName));
+    }
+    IInputStream& in = inFile ? *inFile.Get() : Cin;
+
+    queryString = in.ReadAll();
+    int errors = 0;
+    TString queryFile("query");
+
+    checkReq.IsAnsiLexer = res.Has("ansi-lexer");
+    checkReq.Program = queryString;
+    checkReq.Syntax = NYql::NFastCheck::ESyntax::YQL;
+    checkReq.ClusterMapping = clusterMapping;
+    checkReq.Mode =  FromString<NYql::NFastCheck::EMode>(modeStr);
+    checkReq.Syntax =  FromString<NYql::NFastCheck::ESyntax>(syntaxStr);
+    auto checkResp = NYql::NFastCheck::RunChecks(checkReq);
+    for (const auto& c : checkResp.Checks) {
+        if (!c.Success) {
+            errors = 1;
+        }
+
+        if (colorize) {
+            Cout << NColorizer::LightCyan();
+        }
+
+        Cout << c.CheckName << " ";
+        if (colorize) {
+            Cout << (c.Success ? NColorizer::Green() : NColorizer::LightRed());
+        }
+
+        Cout << (c.Success ? "PASSED" : "FAIL");
+        if (colorize) {
+            Cout << NColorizer::Old();
+        }
+
+        Cout << "\n";
+        if (res.Has("verbose")) {
+            c.Issues.PrintWithProgramTo(Cout, inFileName, queryString, colorize);
+        }
+    }
+
+    return errors;
+}
+
+int main(int argc, char* argv[]) {
+    try {
+        return Run(argc, argv);
+    } catch (const yexception& e) {
+        Cerr << "Caught exception:" << e.what() << Endl;
+        return 1;
+    } catch (...) {
+        Cerr << CurrentExceptionMessage() << Endl;
+        return 1;
+    }
+    return 0;
+}
diff --git a/yql/essentials/utils/tty.cpp b/yql/essentials/utils/tty.cpp
new file mode 100644
index 00000000000..57db2d9bf31
--- /dev/null
+++ b/yql/essentials/utils/tty.cpp
@@ -0,0 +1,35 @@
+#include "tty.h"
+#include <util/system/platform.h>
+
+#ifdef _win_
+#include <io.h>
+#include <stdio.h>
+#else
+#include <unistd.h>
+#endif
+
+namespace NYql {
+
+bool IsTty(EStdStream stream) {
+#ifdef _win_
+    switch (stream) {
+    case EStdStream::In:
+        return _isatty(_fileno(stdin));
+    case EStdStream::Out:
+        return _isatty(_fileno(stdout));
+    case EStdStream::Err:
+        return _isatty(_fileno(stderr));
+    }
+#else
+    switch (stream) {
+    case EStdStream::In:
+        return isatty(STDIN_FILENO);
+    case EStdStream::Out:
+        return isatty(STDOUT_FILENO);
+    case EStdStream::Err:
+        return isatty(STDERR_FILENO);
+    }
+#endif
+}
+
+} // namespace NYql
diff --git a/yql/essentials/utils/tty.h b/yql/essentials/utils/tty.h
new file mode 100644
index 00000000000..41341d6311a
--- /dev/null
+++ b/yql/essentials/utils/tty.h
@@ -0,0 +1,13 @@
+#pragma once
+
+namespace NYql {
+
+enum class EStdStream {
+    In,
+    Out,
+    Err
+};
+
+bool IsTty(EStdStream stream);
+
+} // namespace NYql
diff --git a/yql/essentials/utils/ya.make b/yql/essentials/utils/ya.make
index e42fe3e369c..12634d6d647 100644
--- a/yql/essentials/utils/ya.make
+++ b/yql/essentials/utils/ya.make
@@ -33,6 +33,8 @@ SRCS(
     sort.h
     swap_bytes.cpp
     swap_bytes.h
+    tty.cpp
+    tty.h
     url_builder.cpp
     utf8.cpp
     yql_panic.cpp