Intermediate changes

commit_hash:91f77329ea1087b0436a5c5ee5fcd7273aeb7b27

6 files changed, 118 insertions, 30 deletions

diff --git a/yql/tools/yqlrun/http/sql_tokens_servlet.cpp b/yql/tools/yqlrun/http/sql_tokens_servlet.cpp
new file mode 100644
index 00000000000..aa0fc2fbfca
--- /dev/null
+++ b/yql/tools/yqlrun/http/sql_tokens_servlet.cpp
@@ -0,0 +1,85 @@
+#include "sql_tokens_servlet.h"
+
+#include <library/cpp/json/json_reader.h>
+#include <yql/essentials/sql/v1/format/sql_format.h>
+
+namespace {
+
+THashSet<TString> ExtractNamesFromJson(const NJson::TJsonValue& json) {
+    THashSet<TString> names;
+    for (const auto& value : json.GetArraySafe()) {
+        names.insert(value["name"].GetStringSafe());
+    }
+    return names;
+}
+
+NJson::TJsonValue LoadJsonResource(TStringBuf filename) {
+    TString text;
+    Y_ENSURE(NResource::FindExact(filename, &text));
+    return NJson::ReadJsonFastTree(text);
+}
+
+void OutputJsArray(TStringStream& out, TStringBuf varName, const THashSet<TString>& names) {
+    out << varName << " = [";
+    for (const auto& name : names) {
+        out << '"' << name << "\",";
+    }
+    out << "];\n";
+}
+
+void OutputKnownTokens(TStringStream& out)
+{
+    // See type_id in the grammar.
+    THashSet<TString> compositeTypes = {
+        "OPTIONAL",
+        "TUPLE",
+        "STRUCT",
+        "VARIANT",
+        "LIST",
+        "STREAM",
+        "FLOW",
+        "DICT",
+        "SET",
+        "ENUM",
+        "RESOURCE",
+        "TAGGED",
+        "CALLABLE",
+    };
+
+    out << "window.sql = {};\n";
+
+    auto kws = NSQLFormat::GetKeywords();
+    for (const auto& ty : compositeTypes) {
+        kws.erase(ty);
+    }
+    OutputJsArray(out, "window.sql.keywords", kws);
+
+    THashSet<TString> types = ExtractNamesFromJson(LoadJsonResource("types.json"));
+    types.insert(compositeTypes.begin(), compositeTypes.end());
+    OutputJsArray(out, "window.sql.types", types);
+
+    THashSet<TString> builtinFuncs = ExtractNamesFromJson(LoadJsonResource("sql_functions.json"));
+    OutputJsArray(out, "window.sql.builtinFunctions", builtinFuncs);
+}
+
+} // namespace
+
+namespace NYql {
+namespace NHttp {
+
+TSqlTokensServlet::TSqlTokensServlet()
+{
+    TStringStream out;
+    OutputKnownTokens(out);
+    Script_ = out.Str();
+}
+
+void TSqlTokensServlet::DoGet(const TRequest& req, TResponse& resp) const {
+    Y_UNUSED(req);
+
+    resp.Body = TBlob::FromString(Script_);
+    resp.Headers.AddHeader(THttpInputHeader("Content-Type: text/javascript"));
+}
+
+} // namespace NHttp
+} // namespace NYql
diff --git a/yql/tools/yqlrun/http/sql_tokens_servlet.h b/yql/tools/yqlrun/http/sql_tokens_servlet.h
new file mode 100644
index 00000000000..049432410f0
--- /dev/null
+++ b/yql/tools/yqlrun/http/sql_tokens_servlet.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "servlet.h"
+
+namespace NYql {
+namespace NHttp {
+
+///////////////////////////////////////////////////////////////////////////////
+// TSqlTokensServlet
+///////////////////////////////////////////////////////////////////////////////
+class TSqlTokensServlet: public IServlet {
+public:
+    TSqlTokensServlet();
+    void DoGet(const TRequest& req, TResponse& resp) const override final;
+
+private:
+    TString Script_;
+};
+
+} // namespace NHttp
+} // namespace NYql
diff --git a/yql/tools/yqlrun/http/www/file-index.html b/yql/tools/yqlrun/http/www/file-index.html
index 1d3d5172c68..681d7256711 100644
--- a/yql/tools/yqlrun/http/www/file-index.html
+++ b/yql/tools/yqlrun/http/www/file-index.html
@@ -131,6 +131,7 @@
 <script type='text/javascript' src='/js/dagre-d3.core.min.js'></script>
 <script type='text/javascript' src='/js/ace.min.js'></script>
 <script type='text/javascript' src='/js/theme-tomorrow.min.js'></script>
+<script type='text/javascript' src='/js/sql-tokens.js'></script>
 <script type='text/javascript' src='/js/yql-functions.js'></script>
 <script type='text/javascript' src='/js/mode-sql.js'></script>
 <script type='text/javascript' src='/js/mode-yql.js'></script>
diff --git a/yql/tools/yqlrun/http/www/js/mode-sql.js b/yql/tools/yqlrun/http/www/js/mode-sql.js
index 81448df47e0..a071689013a 100644
--- a/yql/tools/yqlrun/http/www/js/mode-sql.js
+++ b/yql/tools/yqlrun/http/www/js/mode-sql.js
@@ -6,37 +6,10 @@ var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
 
 var SqlHighlightRules = function() {
 
-    var keywords = (
-        "abort|action|all|and|as|asc|ast|attach|auth|avg|begin|between|case|cli|update|clusters|columns|compile|cross|cube|default|define|in|" +
-        "delete|desc|describe|distinct|discard|do|drop|else|end|exclusion|explain|fields|file|from|full|join|get|if|for|evaluate|" +
-        "group|by|grouping|having|help|history|inner|insert|into|is|join|left|limitmeta|" +
-        "not|null|offset|on|only|open|operations|optimize|or|order|by|parallel|parse|pragma|preview|process|" +
-        "progress|put|queries|query|quickstart|reduce|remove|replace|into|restart|result|results|right|join|rollup|run|running|" +
-        "schema|scheme|select|semi|set|sets|show|status|stream|subquery|table|tables|then|union|use|using|validate|values|version|when|where|limit|" +
-        "with|yamr|yt|null|revert|ignore|upsert|erase|presort|assume|any|without|window|partition|rows|range|groups|" +
-        "unbounded|following|preceding|current|row|sample|tablesample|flatten|view|bernoulli|system|repeatable|over|return"
-    );
-
-    var builtinConstants = (
-        "true|false"
-    );
-
-    var builtinFunctions = (
-        "avg|cast|coalesce|likely|random|randomnumber|filecontent|filepath|length|max|median|count|count_if|" +
-        "grouping|min|percentile|sum|min_by|max_by|min_of|max_of|stddev|variance|" +
-        "stddev_sample|stddev_population|variance_sample|variance_population|" +
-        "bool_and|bool_or|bit_and|bit_or|bit_xor|some|list|unique"
-    );
-
-    var dataTypes = (
-        "string|byte|double|float|int32|uint32|int64|uint64|bool"
-    );
-
     var keywordMapper = this.createKeywordMapper({
-        "keyword.operator": builtinFunctions,
-        "keyword": keywords,
-        "constant.language": builtinConstants,
-        "storage.type": dataTypes
+        "keyword.operator": window.sql.builtinFunctions.join("|"),
+        "keyword": window.sql.keywords.join("|"),
+        "storage.type": window.sql.types.join("|")
     }, "identifier", true);
 
     this.$rules = {
diff --git a/yql/tools/yqlrun/http/ya.make b/yql/tools/yqlrun/http/ya.make
index 3b4a4f5fb63..635b3b5554a 100644
--- a/yql/tools/yqlrun/http/ya.make
+++ b/yql/tools/yqlrun/http/ya.make
@@ -4,6 +4,7 @@ SRCS(
     assets_servlet.cpp
     server.cpp
     servlet.cpp
+    sql_tokens_servlet.cpp
     yql_functions_servlet.cpp
     yql_servlet.cpp
     yql_server.cpp
@@ -73,6 +74,11 @@ FILES(
     www/js/theme-tomorrow.min.js
 )
 
+RESOURCE(
+    yql/essentials/data/language/types.json types.json
+    yql/essentials/data/language/sql_functions.json sql_functions.json
+)
+
 YQL_LAST_ABI_VERSION()
 
 END()
diff --git a/yql/tools/yqlrun/http/yql_server.cpp b/yql/tools/yqlrun/http/yql_server.cpp
index 6467eee80f1..0b8eb95dae2 100644
--- a/yql/tools/yqlrun/http/yql_server.cpp
+++ b/yql/tools/yqlrun/http/yql_server.cpp
@@ -1,4 +1,5 @@
 #include "yql_server.h"
+#include "sql_tokens_servlet.h"
 
 #include <yql/essentials/core/cbo/simple/cbo_simple.h>
 #include <yql/essentials/providers/common/proto/gateways_config.pb.h>
@@ -655,6 +656,7 @@ TAutoPtr<TYqlServer> CreateYqlServer(
     server->RegisterAction<TYqlActionOptimizeOrValidateFile>("/api/yql/peephole");
 
     server->RegisterServlet("/js/yql-functions.js", new TYqlFunctoinsServlet());
+    server->RegisterServlet("/js/sql-tokens.js", new TSqlTokensServlet());
 
     server->RegisterAction<TYqlActionFileRun>("/api/yql/lineage");
     server->RegisterAction<TYqlActionFileRun>("/api/yql/run");