diff options
author | vvvv <vvvv@yandex-team.com> | 2025-04-21 18:26:53 +0300 |
---|---|---|
committer | vvvv <vvvv@yandex-team.com> | 2025-04-21 18:42:33 +0300 |
commit | 731eb6bc12b7509ccf4471a360f2b99a2daa45d9 (patch) | |
tree | 5ced0866106d119a9a568dd5a24b56e22860386d | |
parent | 868c99fb96a5b19e28ba29ca40db7e4fb1b729a6 (diff) | |
download | ydb-731eb6bc12b7509ccf4471a360f2b99a2daa45d9.tar.gz |
YQL-19845 introduced max released version, used in fastcheck/linter/purecalc & embedded
commit_hash:b5d1a0cb6f6eedc8b2a450091bde452ac628406d
23 files changed, 264 insertions, 55 deletions
diff --git a/yql/essentials/public/fastcheck/check_runner.cpp b/yql/essentials/public/fastcheck/check_runner.cpp new file mode 100644 index 00000000000..97889d18858 --- /dev/null +++ b/yql/essentials/public/fastcheck/check_runner.cpp @@ -0,0 +1,31 @@ +#include "check_runner.h" +#include <yql/essentials/core/langver/yql_core_langver.h> + +namespace NYql { +namespace NFastCheck { + +TCheckResponse TCheckRunnerBase::Run(const TChecksRequest& request) { + TMaybe<TIssue> verIssue; + if (!CheckLangVersion(request.LangVer, GetMaxReleasedLangVersion(), verIssue)) { + TCheckResponse response; + response.Success = false; + response.CheckName = GetCheckName(); + response.Issues.AddIssue(*verIssue); + return response; + } + + auto ret = DoRun(request); + if (!verIssue) { + return ret; + } + + TCheckResponse response; + response.Success = ret.Success; + response.CheckName = GetCheckName(); + response.Issues.AddIssue(*verIssue); + response.Issues.AddIssues(ret.Issues); + return response; +} + +} +} diff --git a/yql/essentials/public/fastcheck/check_runner.h b/yql/essentials/public/fastcheck/check_runner.h index 012f3761021..74edd882ef0 100644 --- a/yql/essentials/public/fastcheck/check_runner.h +++ b/yql/essentials/public/fastcheck/check_runner.h @@ -12,6 +12,14 @@ public: virtual TCheckResponse Run(const TChecksRequest& request) = 0; }; +class TCheckRunnerBase : public ICheckRunner { +public: + virtual TCheckResponse Run(const TChecksRequest& request) final; + +protected: + virtual TCheckResponse DoRun(const TChecksRequest& request) = 0; +}; + class ICheckRunnerFactory { public: virtual ~ICheckRunnerFactory() = default; diff --git a/yql/essentials/public/fastcheck/fastcheck.cpp b/yql/essentials/public/fastcheck/fastcheck.cpp index 6f4274752b8..8b97278887a 100644 --- a/yql/essentials/public/fastcheck/fastcheck.cpp +++ b/yql/essentials/public/fastcheck/fastcheck.cpp @@ -12,11 +12,22 @@ #include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h> #include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h> #include <yql/essentials/parser/pg_wrapper/interface/parser.h> +#include <yql/essentials/core/langver/yql_core_langver.h> namespace NYql { namespace NFastCheck { bool CheckProgram(const TString& program, const TOptions& options, TIssues& errors) { + TMaybe<TIssue> verIssue; + auto verCheck = CheckLangVersion(options.LangVer, GetMaxReleasedLangVersion(), verIssue); + if (verIssue) { + errors.AddIssue(*verIssue); + } + + if (!verCheck) { + return false; + } + NSQLTranslationV1::TLexers lexers; lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); @@ -33,6 +44,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro TAstParseResult astRes; if (options.IsSql) { NSQLTranslation::TTranslationSettings settings; + settings.LangVer = options.LangVer; settings.ClusterMapping = options.ClusterMapping; settings.SyntaxVersion = options.SyntaxVersion; settings.V0Behavior = NSQLTranslation::EV0Behavior::Disable; @@ -47,7 +59,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro } if (!astRes.IsOk()) { - errors = std::move(astRes.Issues); + errors.AddIssues(astRes.Issues); return false; } @@ -59,6 +71,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro // parse SQL libs for (const auto& x : options.SqlLibs) { NSQLTranslation::TTranslationSettings settings; + settings.LangVer = options.LangVer; settings.ClusterMapping = options.ClusterMapping; settings.SyntaxVersion = options.SyntaxVersion; settings.V0Behavior = NSQLTranslation::EV0Behavior::Disable; @@ -67,7 +80,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro astRes = SqlToYql(translators, x.second, settings); if (!astRes.IsOk()) { - errors = std::move(astRes.Issues); + errors.AddIssues(astRes.Issues); return false; } } @@ -90,7 +103,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro IModuleResolver::TPtr moduleResolver; TUserDataTable userDataTable = GetYqlModuleResolver(libCtx, moduleResolver, userData, options.ClusterMapping, {}); if (!userDataTable) { - errors = libCtx.IssueManager.GetIssues(); + errors.AddIssues(libCtx.IssueManager.GetIssues()); libCtx.IssueManager.Reset(); return false; } @@ -103,7 +116,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro TExprContext exprCtx(libCtx.NextUniqueId); TExprNode::TPtr exprRoot; if (!CompileExpr(*astRes.Root, exprRoot, exprCtx, moduleResolver.get(), nullptr, false, Max<ui32>(), options.SyntaxVersion)) { - errors = exprCtx.IssueManager.GetIssues(); + errors.AddIssues(exprCtx.IssueManager.GetIssues()); exprCtx.IssueManager.Reset(); return false; } diff --git a/yql/essentials/public/fastcheck/fastcheck.h b/yql/essentials/public/fastcheck/fastcheck.h index c2043026153..8edb554e922 100644 --- a/yql/essentials/public/fastcheck/fastcheck.h +++ b/yql/essentials/public/fastcheck/fastcheck.h @@ -2,6 +2,7 @@ #include <yql/essentials/ast/yql_errors.h> #include <util/generic/hash.h> #include <yql/essentials/providers/common/provider/yql_provider_names.h> +#include <yql/essentials/public/langver/yql_langver.h> namespace NYql { namespace NFastCheck { @@ -11,6 +12,7 @@ struct TOptions { bool ParseOnly = false; THashMap<TString, TString> ClusterMapping; ui16 SyntaxVersion = 1; + TLangVersion LangVer = MinLangVersion; bool IsLibrary = false; THashMap<TString, TString> SqlLibs = {}; // mapping file name => SQL }; diff --git a/yql/essentials/public/fastcheck/fastcheck_ut.cpp b/yql/essentials/public/fastcheck/fastcheck_ut.cpp index 79c4d30d4f3..f6a7bf6f2b8 100644 --- a/yql/essentials/public/fastcheck/fastcheck_ut.cpp +++ b/yql/essentials/public/fastcheck/fastcheck_ut.cpp @@ -7,6 +7,7 @@ using namespace NYql::NFastCheck; Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(ParsePureYqlGood) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = false; options.ParseOnly = true; TIssues errors; @@ -16,6 +17,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(ParsePureYqlBad) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = false; options.ParseOnly = true; TIssues errors; @@ -25,6 +27,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(ParsePureSqlGood) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = true; options.ParseOnly = true; TIssues errors; @@ -34,6 +37,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(ParsePureSqlBad) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = true; options.ParseOnly = true; TIssues errors; @@ -43,6 +47,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(CompilePureYqlBad) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = false; options.ParseOnly = false; TIssues errors; @@ -52,6 +57,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(CompileTableSqlGood) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = true; options.ParseOnly = false; options.ClusterMapping["plato"] = YtProviderName; @@ -62,6 +68,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(CompileTableSqlBad) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = true; options.ParseOnly = false; TIssues errors; @@ -71,6 +78,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(CompileLibrary) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = true; options.IsLibrary = true; TIssues errors; @@ -80,6 +88,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(CompileSqlWithLibsGood) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = true; options.ParseOnly = false; options.SqlLibs["foo.sql"] = "$x = 1; export $x;"; @@ -90,6 +99,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(ParseSqlWithBadLib) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = true; options.ParseOnly = true; options.SqlLibs["foo.sql"] = "$x = 1; zexport $x;"; @@ -100,6 +110,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(CompileSqlWithUnresolvedLib) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = true; options.ParseOnly = false; options.SqlLibs["foo.sql"] = "$x = 1; export $x;"; @@ -110,6 +121,7 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { Y_UNIT_TEST(ParseSqlWithUnresolvedLib) { TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); options.IsSql = true; options.ParseOnly = true; options.SqlLibs["foo.sql"] = "$x = 1; export $x;"; @@ -117,4 +129,14 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { UNIT_ASSERT(CheckProgram("pragma library('foo.sql');import foo symbols $y; select $y", options, errors)); UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); } + + Y_UNIT_TEST(TooHighLangVer) { + TOptions options; + options.LangVer = GetMaxLangVersion(); + options.IsSql = false; + options.ParseOnly = true; + TIssues errors; + UNIT_ASSERT(!CheckProgram("(return world)", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); + } } diff --git a/yql/essentials/public/fastcheck/format.cpp b/yql/essentials/public/fastcheck/format.cpp index b487c56fc19..d016b959777 100644 --- a/yql/essentials/public/fastcheck/format.cpp +++ b/yql/essentials/public/fastcheck/format.cpp @@ -40,13 +40,13 @@ TString ReplaceHidden(TStringBuf input) { return res; } -class TFormatRunner : public ICheckRunner { +class TFormatRunner : public TCheckRunnerBase { public: TString GetCheckName() const final { return "format"; } - TCheckResponse Run(const TChecksRequest& request) final { + TCheckResponse DoRun(const TChecksRequest& request) final { switch (request.Syntax) { case ESyntax::SExpr: return RunSExpr(request); diff --git a/yql/essentials/public/fastcheck/lexer.cpp b/yql/essentials/public/fastcheck/lexer.cpp index d09a2b79e97..b1f26361704 100644 --- a/yql/essentials/public/fastcheck/lexer.cpp +++ b/yql/essentials/public/fastcheck/lexer.cpp @@ -9,13 +9,13 @@ namespace NFastCheck { namespace { -class TLexerRunner : public ICheckRunner { +class TLexerRunner : public TCheckRunnerBase { public: TString GetCheckName() const final { return "lexer"; } - TCheckResponse Run(const TChecksRequest& request) final { + TCheckResponse DoRun(const TChecksRequest& request) final { switch (request.Syntax) { case ESyntax::SExpr: return RunSExpr(request); diff --git a/yql/essentials/public/fastcheck/linter.h b/yql/essentials/public/fastcheck/linter.h index 455a734333b..1314f7c065f 100644 --- a/yql/essentials/public/fastcheck/linter.h +++ b/yql/essentials/public/fastcheck/linter.h @@ -4,6 +4,7 @@ #include <util/generic/hash.h> #include <util/generic/set.h> #include <yql/essentials/providers/common/provider/yql_provider_names.h> +#include <yql/essentials/public/langver/yql_langver.h> namespace NYql { namespace NFastCheck { @@ -40,6 +41,7 @@ struct TChecksRequest { THashMap<TString, TString> ClusterMapping; ESyntax Syntax = ESyntax::YQL; ui16 SyntaxVersion = 1; + TLangVersion LangVer = MinLangVersion; bool IsAnsiLexer = false; EMode Mode = EMode::Default; TMaybe<TVector<TCheckFilter>> Filters; diff --git a/yql/essentials/public/fastcheck/linter_ut.cpp b/yql/essentials/public/fastcheck/linter_ut.cpp index c7f7cb45d20..ecccc7945cf 100644 --- a/yql/essentials/public/fastcheck/linter_ut.cpp +++ b/yql/essentials/public/fastcheck/linter_ut.cpp @@ -4,6 +4,16 @@ using namespace NYql; using namespace NYql::NFastCheck; +namespace { + +TChecksRequest MakeCheckRequest() { + return TChecksRequest{ + .LangVer = GetMaxReleasedLangVersion() + }; +} + +} + Y_UNIT_TEST_SUITE(TLinterTests) { Y_UNIT_TEST(ListChecksResult) { auto res = ListChecks(); @@ -11,7 +21,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(DummyLexerSExpr) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "((return world))"; request.Syntax = ESyntax::SExpr; request.Filters.ConstructInPlace(); @@ -24,7 +34,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(DummyLexerPg) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select 1::text"; request.Syntax = ESyntax::PG; request.Filters.ConstructInPlace(); @@ -37,7 +47,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodLexerYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "1"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -50,7 +60,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadLexerYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "Я"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -64,7 +74,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(DummyFormatSExpr) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "((return world))"; request.Syntax = ESyntax::SExpr; request.Filters.ConstructInPlace(); @@ -77,7 +87,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(DummyFormatPg) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select 1::text"; request.Syntax = ESyntax::PG; request.Filters.ConstructInPlace(); @@ -90,7 +100,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodFormatYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "SELECT\n 1\n;\n"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -103,7 +113,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodFormatYqlWithWinEOL) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "SELECT\r\n 1\r\n;\r\n"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -116,7 +126,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodFormatYqlWithWinEOLInComment) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "--\r\nSELECT\n 1\n;\n\nSELECT\n 2\n;\n"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -129,7 +139,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(UnparsedFormatYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select1\n"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -142,7 +152,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadFormatYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select 1"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -156,7 +166,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(ContextForBadFormatYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "SELECT\n 'привет',1;"; request.File = "myFile.sql"; request.Syntax = ESyntax::YQL; @@ -174,7 +184,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadFormatYqlHidden) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select\t1 "; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -194,7 +204,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodParserSExpr) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "((return world))"; request.Syntax = ESyntax::SExpr; request.Filters.ConstructInPlace(); @@ -207,7 +217,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadParserSExpr) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = ")"; request.Syntax = ESyntax::SExpr; request.Filters.ConstructInPlace(); @@ -221,7 +231,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodParserPg) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select 1::text"; request.Syntax = ESyntax::PG; request.Filters.ConstructInPlace(); @@ -234,7 +244,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadParserPg) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "sel"; request.Syntax = ESyntax::PG; request.Filters.ConstructInPlace(); @@ -248,7 +258,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(DummyParserPg) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select 1::text"; request.Syntax = ESyntax::PG; request.Filters.ConstructInPlace(); @@ -261,7 +271,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodParserYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "SELECT 1"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -274,7 +284,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadParserYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "1"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -288,7 +298,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(DummyTranslatorSExpr) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "((return world))"; request.Syntax = ESyntax::SExpr; request.Filters.ConstructInPlace(); @@ -301,7 +311,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodTranslatorPg) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.ClusterMapping["plato"] = TString(YtProviderName); request.Program = "select * from plato.\"Input\""; request.Syntax = ESyntax::PG; @@ -315,7 +325,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadTranslatorPg) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select * from \"Input\""; request.Syntax = ESyntax::PG; request.Filters.ConstructInPlace(); @@ -329,7 +339,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodTranslatorYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.ClusterMapping["plato"] = TString(YtProviderName); request.Program = "use plato; select * from Input"; request.Syntax = ESyntax::YQL; @@ -343,7 +353,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadTranslatorYql) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select ListLengggth([1,2,3])"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -357,7 +367,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(AllowYqlExportsForLibrary) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "$a = 1; export $a"; request.Mode = EMode::Library; request.Syntax = ESyntax::YQL; @@ -371,7 +381,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(AllowYqlExportsForDefault) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "$a = 1; export $a"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -384,7 +394,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(DisallowYqlExportsForMain) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "$a = 1; export $a"; request.Syntax = ESyntax::YQL; request.Mode = EMode::Main; @@ -399,7 +409,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(DisallowYqlExportsForView) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "$a = 1; export $a"; request.Syntax = ESyntax::YQL; request.Mode = EMode::View; @@ -414,7 +424,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodYqlView) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select 1"; request.Syntax = ESyntax::YQL; request.Mode = EMode::View; @@ -428,7 +438,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadYqlView) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select 1;select 2"; request.Syntax = ESyntax::YQL; request.Mode = EMode::View; @@ -443,7 +453,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(AllChecks) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "SELECT\n 1\n;\n"; request.Syntax = ESyntax::YQL; auto res = RunChecks(request); @@ -460,7 +470,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(AllChecksByStar) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "SELECT\n 1\n;\n"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -479,7 +489,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(NoChecksByStarWithSecondFilter) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "1"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -492,7 +502,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadTranslatorYqlWithoutUseMany) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select * from Input"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -506,7 +516,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodTranslatorYqlWithoutUseSingle) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select * from Input"; request.ClusterMode = EClusterMode::Single; request.ClusterSystem = YtProviderName; @@ -521,7 +531,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodTranslatorYqlWithoutUseUnknown) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select * from Input"; request.ClusterMode = EClusterMode::Unknown; request.Syntax = ESyntax::YQL; @@ -535,7 +545,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(BadTranslatorYqlAnotherClusterMany) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select * from foo.Input"; request.Syntax = ESyntax::YQL; request.Filters.ConstructInPlace(); @@ -549,7 +559,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodTranslatorYqlAnotherClusterMany) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "select * from foo.Input"; request.ClusterSystem = YtProviderName; request.Syntax = ESyntax::YQL; @@ -563,7 +573,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemUpdate) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "update foo set value = 1"; request.ClusterMode = EClusterMode::Unknown; request.Syntax = ESyntax::YQL; @@ -577,7 +587,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemDelete) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "delete from foo where value = 1"; request.ClusterMode = EClusterMode::Unknown; request.Syntax = ESyntax::YQL; @@ -591,7 +601,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { } Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemReplaceInto) { - TChecksRequest request; + TChecksRequest request = MakeCheckRequest(); request.Program = "replace into foo select 1"; request.ClusterMode = EClusterMode::Unknown; request.Syntax = ESyntax::YQL; @@ -603,4 +613,20 @@ Y_UNIT_TEST_SUITE(TLinterTests) { UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); } + + Y_UNIT_TEST(TooHighLangVersion) { + TChecksRequest request; + request.LangVer = GetMaxLangVersion(); + request.ClusterMapping["plato"] = TString(YtProviderName); + request.Program = "use plato; select * from Input"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); + } } diff --git a/yql/essentials/public/fastcheck/parser.cpp b/yql/essentials/public/fastcheck/parser.cpp index 01f24a63b0b..9ab04728744 100644 --- a/yql/essentials/public/fastcheck/parser.cpp +++ b/yql/essentials/public/fastcheck/parser.cpp @@ -14,13 +14,13 @@ namespace NFastCheck { namespace { -class TParserRunner : public ICheckRunner { +class TParserRunner : public TCheckRunnerBase { public: TString GetCheckName() const final { return "parser"; } - TCheckResponse Run(const TChecksRequest& request) final { + TCheckResponse DoRun(const TChecksRequest& request) final { switch (request.Syntax) { case ESyntax::SExpr: return RunSExpr(request); diff --git a/yql/essentials/public/fastcheck/translator.cpp b/yql/essentials/public/fastcheck/translator.cpp index 3b5f0b9b073..432c58fff0c 100644 --- a/yql/essentials/public/fastcheck/translator.cpp +++ b/yql/essentials/public/fastcheck/translator.cpp @@ -13,13 +13,13 @@ namespace NFastCheck { namespace { -class TTranslatorRunner : public ICheckRunner { +class TTranslatorRunner : public TCheckRunnerBase { public: TString GetCheckName() const final { return "translator"; } - TCheckResponse Run(const TChecksRequest& request) final { + TCheckResponse DoRun(const TChecksRequest& request) final { switch (request.Syntax) { case ESyntax::SExpr: return RunSExpr(request); diff --git a/yql/essentials/public/fastcheck/ya.make b/yql/essentials/public/fastcheck/ya.make index 02488500a8a..57b804a2eb6 100644 --- a/yql/essentials/public/fastcheck/ya.make +++ b/yql/essentials/public/fastcheck/ya.make @@ -1,6 +1,7 @@ LIBRARY() SRCS( + check_runner.cpp fastcheck.cpp linter.cpp lexer.cpp @@ -19,6 +20,8 @@ PEERDIR( yql/essentials/sql/pg yql/essentials/parser/pg_wrapper yql/essentials/providers/common/provider + yql/essentials/public/langver + yql/essentials/core/langver yql/essentials/sql/v1/lexer yql/essentials/sql/v1/lexer/antlr4 yql/essentials/sql/v1/lexer/antlr4_ansi diff --git a/yql/essentials/public/langver/ut/yql_langver_ut.cpp b/yql/essentials/public/langver/ut/yql_langver_ut.cpp index 7467a7c5183..0c289a0942c 100644 --- a/yql/essentials/public/langver/ut/yql_langver_ut.cpp +++ b/yql/essentials/public/langver/ut/yql_langver_ut.cpp @@ -51,6 +51,18 @@ Y_UNIT_TEST_SUITE(TLangVerTests) { UNIT_ASSERT(IsAvalableLangVersion(MakeLangVersion(2025,2),MakeLangVersion(2025,2))); UNIT_ASSERT(!IsAvalableLangVersion(MakeLangVersion(2025,3),MakeLangVersion(2025,2))); } + + Y_UNIT_TEST(MaxReleasedLangVersionIsValid) { + UNIT_ASSERT(IsValidLangVersion(GetMaxReleasedLangVersion())); + } + + Y_UNIT_TEST(MaxLangVersionIsValid) { + UNIT_ASSERT(IsValidLangVersion(GetMaxLangVersion())); + } + + Y_UNIT_TEST(MaxVersionIsAboveThanReleased) { + UNIT_ASSERT(GetMaxLangVersion() > GetMaxReleasedLangVersion()); + } } } diff --git a/yql/essentials/public/langver/yql_langver.cpp b/yql/essentials/public/langver/yql_langver.cpp index 02584e81b6a..52da1ab57b0 100644 --- a/yql/essentials/public/langver/yql_langver.cpp +++ b/yql/essentials/public/langver/yql_langver.cpp @@ -9,6 +9,8 @@ namespace NYql { namespace { +constexpr TLangVersion MaxReleasedLangVersion = MakeLangVersion(2025, 1); + const std::pair<ui32,ui32> Versions[] = { #include "yql_langver_list.inc" }; @@ -67,4 +69,18 @@ bool FormatLangVersion(TLangVersion ver, TLangVersionBuffer& buffer, TStringBuf& return true; } +TLangVersion GetMaxReleasedLangVersion() { + return MaxReleasedLangVersion; +} + +TLangVersion GetMaxLangVersion() { + TLangVersion max = 0; + for (size_t i = 0; i < Y_ARRAY_SIZE(Versions); ++i) { + auto v = MakeLangVersion(Versions[i].first, Versions[i].second); + max = Max(max, v); + } + + return max; +} + } diff --git a/yql/essentials/public/langver/yql_langver.h b/yql/essentials/public/langver/yql_langver.h index e14d5c91b12..48fec3a1ded 100644 --- a/yql/essentials/public/langver/yql_langver.h +++ b/yql/essentials/public/langver/yql_langver.h @@ -48,6 +48,9 @@ constexpr inline bool IsUnsupportedLangVersion(TLangVersion ver, TLangVersion ma constexpr TLangVersion MinLangVersion = MakeLangVersion(2025, 1); +TLangVersion GetMaxReleasedLangVersion(); +TLangVersion GetMaxLangVersion(); + constexpr ui32 LangVersionBufferSize = 4 + 1 + 2 + 1; // year.minor\0 using TLangVersionBuffer = std::array<char, LangVersionBufferSize>; diff --git a/yql/essentials/public/purecalc/common/interface.cpp b/yql/essentials/public/purecalc/common/interface.cpp index c1a55d63a9c..9c95c0454ad 100644 --- a/yql/essentials/public/purecalc/common/interface.cpp +++ b/yql/essentials/public/purecalc/common/interface.cpp @@ -34,9 +34,15 @@ TProgramFactoryOptions::TProgramFactoryOptions() , UseSystemColumns(false) , UseWorkerPool(true) , UseAntlr4(true) + , LangVer(MinLangVersion) { } +TProgramFactoryOptions& TProgramFactoryOptions::SetLanguageVersion(TLangVersion langver) { + LangVer = langver; + return *this; +} + TProgramFactoryOptions& TProgramFactoryOptions::SetUDFsDir(TStringBuf dir) { UdfsDir_ = dir; return *this; diff --git a/yql/essentials/public/purecalc/common/interface.h b/yql/essentials/public/purecalc/common/interface.h index 629f9fb249f..f16f9fca238 100644 --- a/yql/essentials/public/purecalc/common/interface.h +++ b/yql/essentials/public/purecalc/common/interface.h @@ -10,6 +10,7 @@ #include <yql/essentials/public/udf/udf_registrator.h> #include <yql/essentials/public/issue/yql_issue.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <library/cpp/yson/node/node.h> #include <library/cpp/logger/priority.h> @@ -271,11 +272,21 @@ namespace NYql { /// Use Antlr4 parser (for migration) bool UseAntlr4; + /// Language version + TLangVersion LangVer; + public: TProgramFactoryOptions(); public: /** + * Set language version for queries + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetLanguageVersion(TLangVersion langver); + + /** * Set a new path to a directory with UDFs. * * @return reference to self, to allow method chaining. diff --git a/yql/essentials/public/purecalc/common/program_factory.cpp b/yql/essentials/public/purecalc/common/program_factory.cpp index c27c082b83b..dae0b52be95 100644 --- a/yql/essentials/public/purecalc/common/program_factory.cpp +++ b/yql/essentials/public/purecalc/common/program_factory.cpp @@ -90,6 +90,7 @@ IPullStreamWorkerFactoryPtr TProgramFactory::MakePullStreamWorkerFactory( CountersProvider_, mode, syntaxVersion, + Options_.LangVer, Options_.NativeYtTypeFlags, Options_.DeterministicTimeProviderSeed, Options_.UseSystemColumns, @@ -120,6 +121,7 @@ IPullListWorkerFactoryPtr TProgramFactory::MakePullListWorkerFactory( CountersProvider_, mode, syntaxVersion, + Options_.LangVer, Options_.NativeYtTypeFlags, Options_.DeterministicTimeProviderSeed, Options_.UseSystemColumns, @@ -154,6 +156,7 @@ IPushStreamWorkerFactoryPtr TProgramFactory::MakePushStreamWorkerFactory( CountersProvider_, mode, syntaxVersion, + Options_.LangVer, Options_.NativeYtTypeFlags, Options_.DeterministicTimeProviderSeed, Options_.UseSystemColumns, diff --git a/yql/essentials/public/purecalc/common/worker_factory.cpp b/yql/essentials/public/purecalc/common/worker_factory.cpp index 7597c7fdd29..ae9bcb3ea10 100644 --- a/yql/essentials/public/purecalc/common/worker_factory.cpp +++ b/yql/essentials/public/purecalc/common/worker_factory.cpp @@ -16,6 +16,7 @@ #include <yql/essentials/core/yql_expr_optimize.h> #include <yql/essentials/core/yql_type_helpers.h> #include <yql/essentials/core/peephole_opt/yql_opt_peephole_physical.h> +#include <yql/essentials/core/langver/yql_core_langver.h> #include <yql/essentials/providers/common/codec/yql_codec.h> #include <yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.h> #include <yql/essentials/providers/common/arrow_resolve/yql_simple_arrow_resolver.h> @@ -110,7 +111,7 @@ TWorkerFactory<TBase>::TWorkerFactory(TWorkerFactoryOptions options, EProcessorM SerializedProgram_ = TString{options.Query}; } else { ExprRoot_ = Compile(options.Query, options.TranslationMode_, - options.ModuleResolver, options.SyntaxVersion_, options.Modules, + options.ModuleResolver, options.SyntaxVersion_, options.LangVer_, options.Modules, options.InputSpec, options.OutputSpec, options.UseAntlr4, processorMode); RawOutputType_ = GetSequenceItemType(ExprRoot_->Pos(), ExprRoot_->GetTypeAnn(), true, ExprContext_); @@ -139,6 +140,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( ETranslationMode mode, IModuleResolver::TPtr factoryModuleResolver, ui16 syntaxVersion, + NYql::TLangVersion langver, const THashMap<TString, TString>& modules, const TInputSpecBase& inputSpec, const TOutputSpecBase& outputSpec, @@ -150,12 +152,20 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( ythrow TCompileError("", "") << "only PullList mode is compatible to PostgreSQL syntax"; } + TMaybe<TIssue> verIssue; + if (!CheckLangVersion(langver, GetMaxReleasedLangVersion(), verIssue)) { + TIssues issues; + issues.AddIssue(*verIssue); + ythrow TCompileError("", issues.ToString()); + } + // Prepare type annotation context TTypeAnnotationContextPtr typeContext; IModuleResolver::TPtr moduleResolver = factoryModuleResolver ? factoryModuleResolver->CreateMutableChild() : nullptr; typeContext = MakeIntrusive<TTypeAnnotationContext>(); + typeContext->LangVer = langver; typeContext->RandomProvider = CreateDefaultRandomProvider(); typeContext->TimeProvider = DeterministicTimeProviderSeed_ ? CreateDeterministicTimeProvider(*DeterministicTimeProviderSeed_) : @@ -185,6 +195,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( settings.PgParser = true; } + settings.LangVer = langver; settings.SyntaxVersion = syntaxVersion; settings.V0Behavior = NSQLTranslation::EV0Behavior::Disable; settings.EmitReadsForExists = true; @@ -232,6 +243,10 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( astRes = ParseAst(TString(query)); } + if (verIssue) { + ExprContext_.IssueManager.RaiseIssue(*verIssue); + } + ExprContext_.IssueManager.AddIssues(astRes.Issues); if (!astRes.IsOk()) { ythrow TCompileError(TString(query), GetIssues().ToString()) << "failed to parse " << mode; diff --git a/yql/essentials/public/purecalc/common/worker_factory.h b/yql/essentials/public/purecalc/common/worker_factory.h index c48fc32f951..8588e1e0f51 100644 --- a/yql/essentials/public/purecalc/common/worker_factory.h +++ b/yql/essentials/public/purecalc/common/worker_factory.h @@ -28,6 +28,7 @@ namespace NYql { NKikimr::NUdf::ICountersProvider* CountersProvider_; ETranslationMode TranslationMode_; ui16 SyntaxVersion_; + TLangVersion LangVer_; ui64 NativeYtTypeFlags_; TMaybe<ui64> DeterministicTimeProviderSeed_; bool UseSystemColumns; @@ -49,6 +50,7 @@ namespace NYql { NKikimr::NUdf::ICountersProvider* CountersProvider, ETranslationMode translationMode, ui16 syntaxVersion, + TLangVersion langver, ui64 nativeYtTypeFlags, TMaybe<ui64> deterministicTimeProviderSeed, bool useSystemColumns, @@ -69,6 +71,7 @@ namespace NYql { , CountersProvider_(CountersProvider) , TranslationMode_(translationMode) , SyntaxVersion_(syntaxVersion) + , LangVer_(langver) , NativeYtTypeFlags_(nativeYtTypeFlags) , DeterministicTimeProviderSeed_(deterministicTimeProviderSeed) , UseSystemColumns(useSystemColumns) @@ -129,6 +132,7 @@ namespace NYql { ETranslationMode mode, IModuleResolver::TPtr moduleResolver, ui16 syntaxVersion, + NYql::TLangVersion langver, const THashMap<TString, TString>& modules, const TInputSpecBase& inputSpec, const TOutputSpecBase& outputSpec, diff --git a/yql/essentials/public/purecalc/common/ya.make.inc b/yql/essentials/public/purecalc/common/ya.make.inc index 0f3e3d93702..8aaf8e7c3f7 100644 --- a/yql/essentials/public/purecalc/common/ya.make.inc +++ b/yql/essentials/public/purecalc/common/ya.make.inc @@ -35,6 +35,8 @@ PEERDIR( yql/essentials/core/user_data yql/essentials/utils/backtrace yql/essentials/utils/log + yql/essentials/public/langver + yql/essentials/core/langver yql/essentials/core yql/essentials/core/type_ann yql/essentials/sql diff --git a/yql/essentials/public/purecalc/ut/test_langver.cpp b/yql/essentials/public/purecalc/ut/test_langver.cpp new file mode 100644 index 00000000000..1b3cfa2ff46 --- /dev/null +++ b/yql/essentials/public/purecalc/ut/test_langver.cpp @@ -0,0 +1,29 @@ +#include <yql/essentials/public/purecalc/purecalc.h> +#include <yql/essentials/public/purecalc/io_specs/protobuf/spec.h> +#include <yql/essentials/public/purecalc/ut/protos/test_structs.pb.h> +#include <yql/essentials/public/purecalc/ut/empty_stream.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include "fake_spec.h" + +Y_UNIT_TEST_SUITE(TestLangVer) { + Y_UNIT_TEST(TooHighLangVer) { + using namespace NYql::NPureCalc; + + auto options = TProgramFactoryOptions(); + options.SetLanguageVersion(NYql::GetMaxLangVersion()); + auto factory = MakeProgramFactory(options); + + try { + auto sql = TString(R"( + SELECT * FROM Input; + )"); + + factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + UNIT_FAIL("Exception is expected"); + } catch (const TCompileError& e) { + UNIT_ASSERT_C(e.GetIssues().Contains("version"), e.GetIssues()); + } + } +} diff --git a/yql/essentials/public/purecalc/ut/ya.make b/yql/essentials/public/purecalc/ut/ya.make index 701d0cb37e2..2f4882c1291 100644 --- a/yql/essentials/public/purecalc/ut/ya.make +++ b/yql/essentials/public/purecalc/ut/ya.make @@ -4,6 +4,7 @@ SRCS( empty_stream.h fake_spec.cpp fake_spec.h + test_langver.cpp test_schema.cpp test_sexpr.cpp test_sql.cpp |