diff options
author | Alexander Smirnov <alex@ydb.tech> | 2025-03-14 00:51:45 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2025-03-14 00:51:45 +0000 |
commit | 3e3d50dea42f66b1ba457411b8864990f90bbe21 (patch) | |
tree | 7d75df352fc045a84d46764b96b496b5775bbf44 /yql | |
parent | 7778cd274683ce11e318b799ea12c7bc0b3a4bdd (diff) | |
parent | 422642b601155a296cb0a69eb9b1f7ba146ffa49 (diff) | |
download | ydb-3e3d50dea42f66b1ba457411b8864990f90bbe21.tar.gz |
Merge branch 'rightlib' into merge-libs-250314-0050
Diffstat (limited to 'yql')
-rw-r--r-- | yql/essentials/core/dq_expr_nodes/ya.make | 54 | ||||
-rw-r--r-- | yql/essentials/core/dqs_expr_nodes/ya.make | 54 | ||||
-rw-r--r-- | yql/essentials/core/expr_nodes/ya.make | 51 | ||||
-rw-r--r-- | yql/essentials/core/yql_execution.cpp | 15 | ||||
-rw-r--r-- | yql/essentials/core/yql_execution.h | 7 | ||||
-rw-r--r-- | yql/essentials/core/yql_opt_utils.cpp | 48 | ||||
-rw-r--r-- | yql/essentials/core/yql_opt_utils.h | 3 | ||||
-rw-r--r-- | yql/essentials/providers/common/provider/yql_provider_names.h | 1 | ||||
-rw-r--r-- | yql/essentials/providers/pg/expr_nodes/ya.make | 51 | ||||
-rw-r--r-- | yql/essentials/providers/result/expr_nodes/ya.make | 51 | ||||
-rw-r--r-- | yql/essentials/public/fastcheck/linter.h | 8 | ||||
-rw-r--r-- | yql/essentials/public/fastcheck/linter_ut.cpp | 161 | ||||
-rw-r--r-- | yql/essentials/public/fastcheck/translator.cpp | 36 | ||||
-rw-r--r-- | yql/essentials/sql/v1/sql_into_tables.cpp | 6 | ||||
-rw-r--r-- | yql/essentials/sql/v1/sql_query.cpp | 4 | ||||
-rw-r--r-- | yql/essentials/tools/yql_linter/yql_linter.cpp | 6 | ||||
-rw-r--r-- | yql/essentials/udfs/common/datetime2/datetime_udf.cpp | 89 | ||||
-rw-r--r-- | yql/providers/stat/expr_nodes/ya.make | 51 |
18 files changed, 392 insertions, 304 deletions
diff --git a/yql/essentials/core/dq_expr_nodes/ya.make b/yql/essentials/core/dq_expr_nodes/ya.make index e1acd2003e..190ce26c77 100644 --- a/yql/essentials/core/dq_expr_nodes/ya.make +++ b/yql/essentials/core/dq_expr_nodes/ya.make @@ -12,42 +12,22 @@ SRCDIR( yql/essentials/core/expr_nodes_gen ) -IF(EXPORT_CMAKE) - RUN_PYTHON3( - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/gen/__main__.py - yql_expr_nodes_gen.jnj - dq_expr_nodes.json - dq_expr_nodes.gen.h - dq_expr_nodes.decl.inl.h - dq_expr_nodes.defs.inl.h - NDq - IN yql_expr_nodes_gen.jnj - IN dq_expr_nodes.json - OUT dq_expr_nodes.gen.h - OUT dq_expr_nodes.decl.inl.h - OUT dq_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ELSE() - RUN_PROGRAM( - yql/essentials/core/expr_nodes_gen/gen - yql_expr_nodes_gen.jnj - dq_expr_nodes.json - dq_expr_nodes.gen.h - dq_expr_nodes.decl.inl.h - dq_expr_nodes.defs.inl.h - NDq - IN yql_expr_nodes_gen.jnj - IN dq_expr_nodes.json - OUT dq_expr_nodes.gen.h - OUT dq_expr_nodes.decl.inl.h - OUT dq_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ENDIF() +RUN_PY3_PROGRAM( + yql/essentials/core/expr_nodes_gen/gen + yql_expr_nodes_gen.jnj + dq_expr_nodes.json + dq_expr_nodes.gen.h + dq_expr_nodes.decl.inl.h + dq_expr_nodes.defs.inl.h + NDq + IN yql_expr_nodes_gen.jnj + IN dq_expr_nodes.json + OUT dq_expr_nodes.gen.h + OUT dq_expr_nodes.decl.inl.h + OUT dq_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h +) END() diff --git a/yql/essentials/core/dqs_expr_nodes/ya.make b/yql/essentials/core/dqs_expr_nodes/ya.make index 35519a2d71..b4648fa68e 100644 --- a/yql/essentials/core/dqs_expr_nodes/ya.make +++ b/yql/essentials/core/dqs_expr_nodes/ya.make @@ -12,42 +12,22 @@ SRCDIR( yql/essentials/core/expr_nodes_gen ) -IF(EXPORT_CMAKE) - RUN_PYTHON3( - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/gen/__main__.py - yql_expr_nodes_gen.jnj - dqs_expr_nodes.json - dqs_expr_nodes.gen.h - dqs_expr_nodes.decl.inl.h - dqs_expr_nodes.defs.inl.h - NDq - IN yql_expr_nodes_gen.jnj - IN dqs_expr_nodes.json - OUT dqs_expr_nodes.gen.h - OUT dqs_expr_nodes.decl.inl.h - OUT dqs_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ELSE() - RUN_PROGRAM( - yql/essentials/core/expr_nodes_gen/gen - yql_expr_nodes_gen.jnj - dqs_expr_nodes.json - dqs_expr_nodes.gen.h - dqs_expr_nodes.decl.inl.h - dqs_expr_nodes.defs.inl.h - NDq - IN yql_expr_nodes_gen.jnj - IN dqs_expr_nodes.json - OUT dqs_expr_nodes.gen.h - OUT dqs_expr_nodes.decl.inl.h - OUT dqs_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ENDIF() +RUN_PY3_PROGRAM( + yql/essentials/core/expr_nodes_gen/gen + yql_expr_nodes_gen.jnj + dqs_expr_nodes.json + dqs_expr_nodes.gen.h + dqs_expr_nodes.decl.inl.h + dqs_expr_nodes.defs.inl.h + NDq + IN yql_expr_nodes_gen.jnj + IN dqs_expr_nodes.json + OUT dqs_expr_nodes.gen.h + OUT dqs_expr_nodes.decl.inl.h + OUT dqs_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h +) END() diff --git a/yql/essentials/core/expr_nodes/ya.make b/yql/essentials/core/expr_nodes/ya.make index 0ee410d90a..2b9ad6a3e1 100644 --- a/yql/essentials/core/expr_nodes/ya.make +++ b/yql/essentials/core/expr_nodes/ya.make @@ -13,40 +13,21 @@ PEERDIR( SRCDIR(yql/essentials/core/expr_nodes_gen) -IF(EXPORT_CMAKE) - RUN_PYTHON3( - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/gen/__main__.py - yql_expr_nodes_gen.jnj - yql_expr_nodes.json - yql_expr_nodes.gen.h - yql_expr_nodes.decl.inl.h - yql_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_expr_nodes.json - OUT yql_expr_nodes.gen.h - OUT yql_expr_nodes.decl.inl.h - OUT yql_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ELSE() - RUN_PROGRAM( - yql/essentials/core/expr_nodes_gen/gen - yql_expr_nodes_gen.jnj - yql_expr_nodes.json - yql_expr_nodes.gen.h - yql_expr_nodes.decl.inl.h - yql_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_expr_nodes.json - OUT yql_expr_nodes.gen.h - OUT yql_expr_nodes.decl.inl.h - OUT yql_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ENDIF() +RUN_PY3_PROGRAM( + yql/essentials/core/expr_nodes_gen/gen + yql_expr_nodes_gen.jnj + yql_expr_nodes.json + yql_expr_nodes.gen.h + yql_expr_nodes.decl.inl.h + yql_expr_nodes.defs.inl.h + IN yql_expr_nodes_gen.jnj + IN yql_expr_nodes.json + OUT yql_expr_nodes.gen.h + OUT yql_expr_nodes.decl.inl.h + OUT yql_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h +) END() diff --git a/yql/essentials/core/yql_execution.cpp b/yql/essentials/core/yql_execution.cpp index 5b34866487..57892229b3 100644 --- a/yql/essentials/core/yql_execution.cpp +++ b/yql/essentials/core/yql_execution.cpp @@ -961,3 +961,18 @@ void Out<NYql::TOperationProgress::EState>(class IOutputStream &o, NYql::TOperat return; } } + +template<> +void Out<NYql::TOperationProgress::EOpBlockStatus>(class IOutputStream &o, NYql::TOperationProgress::EOpBlockStatus x) { +#define YQL_OPERATION_BLOCK_STATUS_MAP_TO_STRING_IMPL(name, ...) \ + case NYql::TOperationProgress::EOpBlockStatus::name: \ + o << #name; \ + return; + + switch (x) { + YQL_OPERATION_BLOCK_STATUS_MAP(YQL_OPERATION_BLOCK_STATUS_MAP_TO_STRING_IMPL) + default: + o << static_cast<int>(x); + return; + } +} diff --git a/yql/essentials/core/yql_execution.h b/yql/essentials/core/yql_execution.h index e1714d5eda..cf1476402d 100644 --- a/yql/essentials/core/yql_execution.h +++ b/yql/essentials/core/yql_execution.h @@ -77,6 +77,13 @@ namespace NYql { , Stage(stage, TInstant::Now()) { } + + static EOpBlockStatus CombineBlockStatuses(EOpBlockStatus lhs, EOpBlockStatus rhs) { + if (lhs == rhs) { + return lhs; + } + return EOpBlockStatus::Partial; + } }; struct TOperationStatistics { diff --git a/yql/essentials/core/yql_opt_utils.cpp b/yql/essentials/core/yql_opt_utils.cpp index 73f0bf064c..9a51b06801 100644 --- a/yql/essentials/core/yql_opt_utils.cpp +++ b/yql/essentials/core/yql_opt_utils.cpp @@ -2423,4 +2423,52 @@ TExprNode::TPtr KeepWorld(TExprNode::TPtr node, const TExprNode& src, TExprConte } } +TOperationProgress::EOpBlockStatus DetermineProgramBlockStatus(const TExprNode& root) { + auto pRoot = &root; + + // TODO: remove after block IO transition to Stream + if (pRoot->IsCallable("ToFlow")) { + pRoot = &pRoot->Head(); + } + + if (pRoot->IsCallable("WideFromBlocks")) { + // Assume Full block status even if block output is not applied + pRoot = &pRoot->Head(); + } + + auto rootType = pRoot->GetTypeAnn(); + YQL_ENSURE(rootType); + + auto status = IsWideSequenceBlockType(*rootType) ? TOperationProgress::EOpBlockStatus::Full : TOperationProgress::EOpBlockStatus::None; + bool stop = false; + VisitExpr(*pRoot, [&](const TExprNode& node) { + if (stop || node.IsLambda()) { + return false; + } + + const TTypeAnnotationNode* nodeType = node.GetTypeAnn(); + YQL_ENSURE(nodeType); + + if (nodeType->GetKind() != ETypeAnnotationKind::Stream && nodeType->GetKind() != ETypeAnnotationKind::Flow) { + return false; + } + + const bool isBlock = IsWideSequenceBlockType(*nodeType); + if (status == TOperationProgress::EOpBlockStatus::Full && !isBlock || + status == TOperationProgress::EOpBlockStatus::None && isBlock) + { + status = TOperationProgress::EOpBlockStatus::Partial; + } + + if (status == TOperationProgress::EOpBlockStatus::Partial) { + stop = true; + return false; + } + + return true; + }); + + return status; +} + } diff --git a/yql/essentials/core/yql_opt_utils.h b/yql/essentials/core/yql_opt_utils.h index aa83773087..fd818dbd35 100644 --- a/yql/essentials/core/yql_opt_utils.h +++ b/yql/essentials/core/yql_opt_utils.h @@ -1,6 +1,7 @@ #pragma once #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h> +#include <yql/essentials/core/yql_execution.h> #include <yql/essentials/core/yql_graph_transformer.h> #include <yql/essentials/core/yql_opt_window.h> #include <yql/essentials/core/yql_type_annotation.h> @@ -192,4 +193,6 @@ bool IsOptimizerDisabled(const TTypeAnnotationContext& types) { extern const char KeepWorldOptName[]; +TOperationProgress::EOpBlockStatus DetermineProgramBlockStatus(const TExprNode& root); + } diff --git a/yql/essentials/providers/common/provider/yql_provider_names.h b/yql/essentials/providers/common/provider/yql_provider_names.h index de54138b28..37c49a4fff 100644 --- a/yql/essentials/providers/common/provider/yql_provider_names.h +++ b/yql/essentials/providers/common/provider/yql_provider_names.h @@ -5,6 +5,7 @@ namespace NYql { +constexpr TStringBuf UnknownProviderName = "unknown"; constexpr TStringBuf ConfigProviderName = "config"; constexpr TStringBuf KikimrProviderName = "kikimr"; constexpr TStringBuf ResultProviderName = "result"; diff --git a/yql/essentials/providers/pg/expr_nodes/ya.make b/yql/essentials/providers/pg/expr_nodes/ya.make index 6195b92abb..8dc6632159 100644 --- a/yql/essentials/providers/pg/expr_nodes/ya.make +++ b/yql/essentials/providers/pg/expr_nodes/ya.make @@ -13,40 +13,21 @@ SRCDIR( yql/essentials/core/expr_nodes_gen ) -IF(EXPORT_CMAKE) - RUN_PYTHON3( - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/gen/__main__.py - yql_expr_nodes_gen.jnj - yql_pg_expr_nodes.json - yql_pg_expr_nodes.gen.h - yql_pg_expr_nodes.decl.inl.h - yql_pg_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_pg_expr_nodes.json - OUT yql_pg_expr_nodes.gen.h - OUT yql_pg_expr_nodes.decl.inl.h - OUT yql_pg_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ELSE() - RUN_PROGRAM( - yql/essentials/core/expr_nodes_gen/gen - yql_expr_nodes_gen.jnj - yql_pg_expr_nodes.json - yql_pg_expr_nodes.gen.h - yql_pg_expr_nodes.decl.inl.h - yql_pg_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_pg_expr_nodes.json - OUT yql_pg_expr_nodes.gen.h - OUT yql_pg_expr_nodes.decl.inl.h - OUT yql_pg_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ENDIF() +RUN_PY3_PROGRAM( + yql/essentials/core/expr_nodes_gen/gen + yql_expr_nodes_gen.jnj + yql_pg_expr_nodes.json + yql_pg_expr_nodes.gen.h + yql_pg_expr_nodes.decl.inl.h + yql_pg_expr_nodes.defs.inl.h + IN yql_expr_nodes_gen.jnj + IN yql_pg_expr_nodes.json + OUT yql_pg_expr_nodes.gen.h + OUT yql_pg_expr_nodes.decl.inl.h + OUT yql_pg_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h +) END() diff --git a/yql/essentials/providers/result/expr_nodes/ya.make b/yql/essentials/providers/result/expr_nodes/ya.make index db67925532..ca3a6bf63f 100644 --- a/yql/essentials/providers/result/expr_nodes/ya.make +++ b/yql/essentials/providers/result/expr_nodes/ya.make @@ -13,40 +13,21 @@ SRCDIR( yql/essentials/core/expr_nodes_gen ) -IF(EXPORT_CMAKE) - RUN_PYTHON3( - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/gen/__main__.py - yql_expr_nodes_gen.jnj - yql_res_expr_nodes.json - yql_res_expr_nodes.gen.h - yql_res_expr_nodes.decl.inl.h - yql_res_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_res_expr_nodes.json - OUT yql_res_expr_nodes.gen.h - OUT yql_res_expr_nodes.decl.inl.h - OUT yql_res_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ELSE() - RUN_PROGRAM( - yql/essentials/core/expr_nodes_gen/gen - yql_expr_nodes_gen.jnj - yql_res_expr_nodes.json - yql_res_expr_nodes.gen.h - yql_res_expr_nodes.decl.inl.h - yql_res_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_res_expr_nodes.json - OUT yql_res_expr_nodes.gen.h - OUT yql_res_expr_nodes.decl.inl.h - OUT yql_res_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ENDIF() +RUN_PY3_PROGRAM( + yql/essentials/core/expr_nodes_gen/gen + yql_expr_nodes_gen.jnj + yql_res_expr_nodes.json + yql_res_expr_nodes.gen.h + yql_res_expr_nodes.decl.inl.h + yql_res_expr_nodes.defs.inl.h + IN yql_expr_nodes_gen.jnj + IN yql_res_expr_nodes.json + OUT yql_res_expr_nodes.gen.h + OUT yql_res_expr_nodes.decl.inl.h + OUT yql_res_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h +) END() diff --git a/yql/essentials/public/fastcheck/linter.h b/yql/essentials/public/fastcheck/linter.h index 7af5e62455..455a734333 100644 --- a/yql/essentials/public/fastcheck/linter.h +++ b/yql/essentials/public/fastcheck/linter.h @@ -21,6 +21,12 @@ enum class EMode { View }; +enum EClusterMode { + Many, + Single, + Unknown +}; + struct TCheckFilter { bool Include = true; TString CheckNameGlob; @@ -29,6 +35,8 @@ struct TCheckFilter { struct TChecksRequest { TString Program; TString File; + EClusterMode ClusterMode = Many; + TString ClusterSystem; THashMap<TString, TString> ClusterMapping; ESyntax Syntax = ESyntax::YQL; ui16 SyntaxVersion = 1; diff --git a/yql/essentials/public/fastcheck/linter_ut.cpp b/yql/essentials/public/fastcheck/linter_ut.cpp index be44f88c50..2c5c0bbd0a 100644 --- a/yql/essentials/public/fastcheck/linter_ut.cpp +++ b/yql/essentials/public/fastcheck/linter_ut.cpp @@ -19,7 +19,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); - UNIT_ASSERT(res.Checks[0].Success); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -32,7 +32,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); - UNIT_ASSERT(res.Checks[0].Success); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -45,7 +45,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); - UNIT_ASSERT(res.Checks[0].Success); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -72,7 +72,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT(res.Checks[0].Success); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -85,7 +85,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT(res.Checks[0].Success); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -98,8 +98,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -112,8 +111,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -126,8 +124,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -172,8 +169,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); } @@ -200,8 +196,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); } @@ -228,7 +223,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT(res.Checks[0].Success); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -241,7 +236,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT(res.Checks[0].Success); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -268,7 +263,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(res.Checks[0].Success); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } @@ -282,8 +277,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); } @@ -311,8 +305,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); } @@ -340,8 +333,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); } @@ -354,8 +346,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); } @@ -399,8 +390,7 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); } @@ -465,6 +455,119 @@ Y_UNIT_TEST_SUITE(TLinterTests) { auto res = RunChecks(request); UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); - UNIT_ASSERT(res.Checks[0].Success); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + } + + Y_UNIT_TEST(BadTranslatorYqlWithoutUseMany) { + TChecksRequest request; + request.Program = "select * from Input"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); + } + + Y_UNIT_TEST(GoodTranslatorYqlWithoutUseSingle) { + TChecksRequest request; + request.Program = "select * from Input"; + request.ClusterMode = EClusterMode::Single; + request.ClusterSystem = YtProviderName; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); + } + + Y_UNIT_TEST(GoodTranslatorYqlWithoutUseUnknown) { + TChecksRequest request; + request.Program = "select * from Input"; + request.ClusterMode = EClusterMode::Unknown; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); + } + + Y_UNIT_TEST(BadTranslatorYqlAnotherClusterMany) { + TChecksRequest request; + request.Program = "select * from foo.Input"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); + } + + Y_UNIT_TEST(GoodTranslatorYqlAnotherClusterMany) { + TChecksRequest request; + request.Program = "select * from foo.Input"; + request.ClusterSystem = YtProviderName; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); + } + + Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemUpdate) { + TChecksRequest request; + request.Program = "update foo set value = 1"; + request.ClusterMode = EClusterMode::Unknown; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); + } + + Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemDelete) { + TChecksRequest request; + request.Program = "delete from foo where value = 1"; + request.ClusterMode = EClusterMode::Unknown; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); + } + + Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemReplaceInto) { + TChecksRequest request; + request.Program = "replace into foo select 1"; + request.ClusterMode = EClusterMode::Unknown; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); } } diff --git a/yql/essentials/public/fastcheck/translator.cpp b/yql/essentials/public/fastcheck/translator.cpp index 8350dd9b4f..3b5f0b9b07 100644 --- a/yql/essentials/public/fastcheck/translator.cpp +++ b/yql/essentials/public/fastcheck/translator.cpp @@ -6,6 +6,7 @@ #include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h> #include <yql/essentials/sql/settings/translation_settings.h> #include <yql/essentials/parser/pg_wrapper/interface/parser.h> +#include <yql/essentials/providers/common/provider/yql_provider_names.h> namespace NYql { namespace NFastCheck { @@ -41,7 +42,8 @@ private: NSQLTranslation::TTranslationSettings settings; settings.Arena = &arena; settings.PgParser = true; - settings.ClusterMapping = request.ClusterMapping; + FillClusters(request, settings); + auto astRes = NSQLTranslationPG::PGToYql(request.Program, settings); return TCheckResponse{ .CheckName = GetCheckName(), @@ -56,7 +58,7 @@ private: NSQLTranslation::TTranslationSettings settings; settings.Arena = &arena; settings.File = request.File; - settings.ClusterMapping = request.ClusterMapping; + FillClusters(request, settings); settings.EmitReadsForExists = true; settings.Antlr4Parser = true; settings.AnsiLexer = request.IsAnsiLexer; @@ -102,6 +104,36 @@ private: res.Issues = astRes.Issues; return res; } + + void FillClusters(const TChecksRequest& request, NSQLTranslation::TTranslationSettings& settings) { + if (!request.ClusterSystem.empty()) { + Y_ENSURE(AnyOf(Providers, [&](const auto& p) { return p == request.ClusterSystem; }), + "Invalid ClusterSystem value: " + request.ClusterSystem); + } + + switch (request.ClusterMode) { + case EClusterMode::Many: + for (const auto& x : request.ClusterMapping) { + Y_ENSURE(AnyOf(Providers, [&](const auto& p) { return p == x.second; }), + "Invalid system: " + x.second); + } + + settings.ClusterMapping = request.ClusterMapping; + settings.DynamicClusterProvider = request.ClusterSystem; + break; + case EClusterMode::Single: + Y_ENSURE(!request.ClusterSystem.empty(), "Missing ClusterSystem parameter"); + settings.DefaultCluster = "single"; + settings.ClusterMapping["single"] = request.ClusterSystem; + settings.DynamicClusterProvider = request.ClusterSystem; + break; + case EClusterMode::Unknown: + settings.DefaultCluster = "single"; + settings.ClusterMapping["single"] = UnknownProviderName; + settings.DynamicClusterProvider = UnknownProviderName; + break; + } + } }; } diff --git a/yql/essentials/sql/v1/sql_into_tables.cpp b/yql/essentials/sql/v1/sql_into_tables.cpp index 77498af54a..ff7dd271cd 100644 --- a/yql/essentials/sql/v1/sql_into_tables.cpp +++ b/yql/essentials/sql/v1/sql_into_tables.cpp @@ -156,7 +156,7 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { TVector<TString> eraseColumns; if (intoTableRef.HasBlock2()) { - if (service != StatProviderName) { + if (service != StatProviderName && service != UnknownProviderName) { Ctx.Error() << "ERASE BY is unsupported for " << service; return nullptr; } @@ -217,6 +217,10 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const ESQLWriteColumnMode mode, const TPosition& pos) { Y_UNUSED(node); auto serviceName = table.Service; + if (serviceName == UnknownProviderName) { + return true; + } + const bool isMapReduce = serviceName == YtProviderName; const bool isKikimr = serviceName == KikimrProviderName || serviceName == YdbProviderName; const bool isRtmr = serviceName == RtmrProviderName; diff --git a/yql/essentials/sql/v1/sql_query.cpp b/yql/essentials/sql/v1/sql_query.cpp index 6dc510bf57..e919630a03 100644 --- a/yql/essentials/sql/v1/sql_query.cpp +++ b/yql/essentials/sql/v1/sql_query.cpp @@ -3486,7 +3486,7 @@ TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) { } const bool isKikimr = table.Service == KikimrProviderName; - if (!isKikimr) { + if (!isKikimr && table.Service != UnknownProviderName) { Ctx.Error(GetPos(stmt.GetToken2())) << "DELETE is unsupported for " << table.Service; return nullptr; } @@ -3552,7 +3552,7 @@ TNodePtr TSqlQuery::Build(const TRule_update_stmt& stmt) { const bool isKikimr = table.Service == KikimrProviderName; - if (!isKikimr) { + if (!isKikimr && table.Service != UnknownProviderName) { Ctx.Error(GetPos(stmt.GetToken2())) << "UPDATE is unsupported for " << table.Service; return nullptr; } diff --git a/yql/essentials/tools/yql_linter/yql_linter.cpp b/yql/essentials/tools/yql_linter/yql_linter.cpp index 7e392b7fc0..458496c21e 100644 --- a/yql/essentials/tools/yql_linter/yql_linter.cpp +++ b/yql/essentials/tools/yql_linter/yql_linter.cpp @@ -18,6 +18,8 @@ int Run(int argc, char* argv[]) { THashMap<TString, TString> clusterMapping; TString modeStr = "Default"; TString syntaxStr = "YQL"; + TString clusterModeStr = "Many"; + TString clusterSystem; opts.AddLongOption('i', "input", "input file").RequiredArgument("input").StoreResult(&inFileName); opts.AddLongOption('v', "verbose", "show lint issues").NoArgument(); @@ -33,6 +35,8 @@ int Run(int argc, char* argv[]) { opts.AddLongOption('m', "mode", "query mode, allowed values: " + GetEnumAllNames<NYql::NFastCheck::EMode>()).StoreResult(&modeStr); opts.AddLongOption('s', "syntax", "query syntax, allowed values: " + GetEnumAllNames<NYql::NFastCheck::ESyntax>()).StoreResult(&syntaxStr); + opts.AddLongOption("cluster-mode", "cluster mode, allowed values: " + GetEnumAllNames<NYql::NFastCheck::EClusterMode>()).StoreResult(&clusterModeStr); + opts.AddLongOption("cluster-system", "cluster system").StoreResult(&clusterSystem); opts.AddLongOption("ansi-lexer", "use ansi lexer").NoArgument(); opts.AddLongOption("no-colors", "disable colors for output").NoArgument(); opts.SetFreeArgsNum(0); @@ -79,6 +83,8 @@ int Run(int argc, char* argv[]) { checkReq.ClusterMapping = clusterMapping; checkReq.Mode = FromString<NYql::NFastCheck::EMode>(modeStr); checkReq.Syntax = FromString<NYql::NFastCheck::ESyntax>(syntaxStr); + checkReq.ClusterMode = FromString<NYql::NFastCheck::EClusterMode>(clusterModeStr); + checkReq.ClusterSystem = clusterSystem; auto checkResp = NYql::NFastCheck::RunChecks(checkReq); for (const auto& c : checkResp.Checks) { if (!c.Success) { diff --git a/yql/essentials/udfs/common/datetime2/datetime_udf.cpp b/yql/essentials/udfs/common/datetime2/datetime_udf.cpp index 8dc7513288..35e4ade0a8 100644 --- a/yql/essentials/udfs/common/datetime2/datetime_udf.cpp +++ b/yql/essentials/udfs/common/datetime2/datetime_udf.cpp @@ -563,20 +563,14 @@ private: namespace { -// FIXME: The default value for TResourceName allows to omit -// explicit specialization in functions that still doesn't support -// big datetime types. Should be removed in future. -template<const char* TResourceName = TMResourceName, typename TValue, +template<const char* TResourceName, typename TValue, typename TStorage = std::conditional_t<TResourceName == TMResourceName, TTMStorage, TTM64Storage>> const TStorage& Reference(const TValue& value) { return *reinterpret_cast<const TStorage*>(value.GetRawPtr()); } -// FIXME: The default value for TResourceName allows to omit -// explicit specialization in functions that still doesn't support -// big datetime types. Should be removed in future. -template<const char* TResourceName = TMResourceName, typename TValue, +template<const char* TResourceName, typename TValue, typename TStorage = std::conditional_t<TResourceName == TMResourceName, TTMStorage, TTM64Storage>> TStorage& Reference(TValue& value) { @@ -608,20 +602,15 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: return result; } -// FIXME: The default value for TResourceName allows to omit -// explicit specialization in functions that still doesn't support -// big datetime types. Should be removed in future. #define ACCESSORS_POLY(field, type, wtype) \ - template<const char* TResourceName = TMResourceName, typename TValue, \ - typename rtype = std::conditional_t<TResourceName == TMResourceName, \ - type, wtype>> \ - inline rtype Get##field(const TValue& tm) { \ - return (rtype)Reference<TResourceName>(tm).field; \ + template<const char* TResourceName, typename TValue, typename TRetType \ + = std::conditional_t<TResourceName == TMResourceName, type, wtype>> \ + inline TRetType Get##field(const TValue& tm) { \ + return (TRetType)Reference<TResourceName>(tm).field; \ } \ - template<const char* TResourceName = TMResourceName, typename TValue, \ - typename itype = std::conditional_t<TResourceName == TMResourceName, \ - type, wtype>> \ - inline void Set##field(TValue& tm, itype value) { \ + template<const char* TResourceName, typename TValue, typename TArgType \ + = std::conditional_t<TResourceName == TMResourceName, type, wtype>> \ + inline void Set##field(TValue& tm, TArgType value) { \ Reference<TResourceName>(tm).field = value; \ } \ @@ -644,10 +633,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: #undef ACCESSORS #undef ACCESSORS_POLY - // FIXME: The default value for TResourceName allows to omit - // explicit specialization in functions that still doesn't support - // big datetime types. Should be removed in future. - template<const char* TResourceName = TMResourceName> + template<const char* TResourceName> inline bool ValidateYear(std::conditional_t<TResourceName == TMResourceName, ui16, i32> year) { if constexpr (TResourceName == TMResourceName) { return year >= NUdf::MIN_YEAR || year < NUdf::MAX_YEAR; @@ -781,7 +767,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: static void Process(const IValueBuilder* valueBuilder, TBlockItem arg, const TSink& sink) { try { TBlockItem res {0}; - Split(arg, Reference(res), *valueBuilder); + Split(arg, Reference<TMResourceName>(res), *valueBuilder); sink(res); } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << e.what()).data()); @@ -925,7 +911,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); - auto& storage = Reference(result); + auto& storage = Reference<TMResourceName>(result); storage.FromDate(builder, args[0].Get<ui16>()); return result; } catch (const std::exception& e) { @@ -960,7 +946,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); - auto& storage = Reference(result); + auto& storage = Reference<TMResourceName>(result); storage.FromDatetime(builder, args[0].Get<ui32>()); return result; } catch (const std::exception& e) { @@ -995,7 +981,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); - auto& storage = Reference(result); + auto& storage = Reference<TMResourceName>(result); storage.FromTimestamp(builder, args[0].Get<ui64>()); return result; } catch (const std::exception& e) { @@ -1030,7 +1016,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); - auto& storage = Reference(result); + auto& storage = Reference<TMResourceName>(result); storage.FromDate(builder, args[0].Get<ui16>(), args[0].GetTimezoneId()); return result; } catch (const std::exception& e) { @@ -1067,7 +1053,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); - auto& storage = Reference(result); + auto& storage = Reference<TMResourceName>(result); storage.FromDatetime(builder, args[0].Get<ui32>(), args[0].GetTimezoneId()); return result; } catch (const std::exception& e) { @@ -1104,7 +1090,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: auto& builder = valueBuilder->GetDateBuilder(); TUnboxedValuePod result(0); - auto& storage = Reference(result); + auto& storage = Reference<TMResourceName>(result); storage.FromTimestamp(builder, args[0].Get<ui64>(), args[0].GetTimezoneId()); return result; } catch (const std::exception& e) { @@ -1143,7 +1129,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: template<typename TSink> static void Process(const IValueBuilder* valueBuilder, TBlockItem item, const TSink& sink) { - auto& storage = Reference(item); + auto& storage = Reference<TMResourceName>(item); sink(TBlockItem(Make(storage, *valueBuilder))); } }; @@ -1183,28 +1169,28 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: BEGIN_SIMPLE_STRICT_ARROW_UDF(TMakeDate, TDate(TAutoMap<TResource<TMResourceName>>)) { auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = Reference(args[0]); + auto& storage = Reference<TMResourceName>(args[0]); return TUnboxedValuePod(storage.ToDate(builder, false)); } END_SIMPLE_ARROW_UDF(TMakeDate, TMakeDateKernelExec<TDate>::Do); BEGIN_SIMPLE_STRICT_ARROW_UDF(TMakeDatetime, TDatetime(TAutoMap<TResource<TMResourceName>>)) { auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = Reference(args[0]); + auto& storage = Reference<TMResourceName>(args[0]); return TUnboxedValuePod(storage.ToDatetime(builder)); } END_SIMPLE_ARROW_UDF(TMakeDatetime, TMakeDateKernelExec<TDatetime>::Do); BEGIN_SIMPLE_STRICT_ARROW_UDF(TMakeTimestamp, TTimestamp(TAutoMap<TResource<TMResourceName>>)) { auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = Reference(args[0]); + auto& storage = Reference<TMResourceName>(args[0]); return TUnboxedValuePod(storage.ToTimestamp(builder)); } END_SIMPLE_ARROW_UDF(TMakeTimestamp, TMakeDateKernelExec<TTimestamp>::Do); BEGIN_SIMPLE_STRICT_ARROW_UDF(TMakeTzDate, TTzDate(TAutoMap<TResource<TMResourceName>>)) { auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = Reference(args[0]); + auto& storage = Reference<TMResourceName>(args[0]); try { TUnboxedValuePod result(storage.ToDate(builder, true)); result.SetTimezoneId(storage.TimezoneId); @@ -1220,7 +1206,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: BEGIN_SIMPLE_STRICT_ARROW_UDF(TMakeTzDatetime, TTzDatetime(TAutoMap<TResource<TMResourceName>>)) { auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = Reference(args[0]); + auto& storage = Reference<TMResourceName>(args[0]); TUnboxedValuePod result(storage.ToDatetime(builder)); result.SetTimezoneId(storage.TimezoneId); return result; @@ -1229,7 +1215,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: BEGIN_SIMPLE_STRICT_ARROW_UDF(TMakeTzTimestamp, TTzTimestamp(TAutoMap<TResource<TMResourceName>>)) { auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = Reference(args[0]); + auto& storage = Reference<TMResourceName>(args[0]); TUnboxedValuePod result(storage.ToTimestamp(builder)); result.SetTimezoneId(storage.TimezoneId); return result; @@ -1240,7 +1226,7 @@ TUnboxedValuePod DoAddYears(const TUnboxedValuePod& date, i64 years, const NUdf: SIMPLE_STRICT_UDF(TConvert, TResource<TM64ResourceName>(TAutoMap<TResource<TMResourceName>>)) { Y_UNUSED(valueBuilder); TUnboxedValuePod result(0); - auto& arg = Reference(args[0]); + auto& arg = Reference<TMResourceName>(args[0]); auto& storage = Reference<TM64ResourceName>(result); storage.From(arg); return result; @@ -1611,7 +1597,7 @@ TUnboxedValue GetDayOfWeekName(const IValueBuilder* valueBuilder, const TUnboxed template<typename TSink> static void Process(const IValueBuilder* valueBuilder, TBlockItem item, const TSink& sink) { Y_UNUSED(valueBuilder); - auto timezoneId = GetTimezoneId(item); + auto timezoneId = GetTimezoneId<TMResourceName>(item); if (timezoneId >= NUdf::GetTimezones().size()) { sink(TBlockItem{}); } else { @@ -1620,15 +1606,6 @@ TUnboxedValue GetDayOfWeekName(const IValueBuilder* valueBuilder, const TUnboxed } }; - BEGIN_SIMPLE_STRICT_ARROW_UDF(TGetTimezoneName, char*(TAutoMap<TResource<TMResourceName>>)) { - auto timezoneId = GetTimezoneId(args[0]); - if (timezoneId >= NUdf::GetTimezones().size()) { - return TUnboxedValuePod(); - } - return valueBuilder->NewString(NUdf::GetTimezones()[timezoneId]); - } - END_SIMPLE_ARROW_UDF(TGetTimezoneName, TTGetTimezoneNameKernelExec::Do); - template<const char* TResourceName> TUnboxedValue GetTimezoneName(const IValueBuilder* valueBuilder, const TUnboxedValuePod& arg) { const ui16 tzId = GetTimezoneId<TResourceName>(arg); @@ -1969,8 +1946,8 @@ private: struct TStartOfKernelExec : TUnaryKernelExec<TStartOfKernelExec<Core>, TResourceBlockReader<false>, TResourceArrayBuilder<true>> { template<typename TSink> static void Process(const IValueBuilder* valueBuilder, TBlockItem item, const TSink& sink) { - if (auto res = Core(Reference(item), *valueBuilder)) { - Reference(item) = res.GetRef(); + if (auto res = Core(Reference<TMResourceName>(item), *valueBuilder)) { + Reference<TMResourceName>(item) = res.GetRef(); sink(item); } else { sink(TBlockItem{}); @@ -2281,7 +2258,7 @@ private: struct TStartEndOfBinaryKernelExec : TBinaryKernelExec<TStartEndOfBinaryKernelExec<UseEnd>> { template<typename TSink> static void Process(const IValueBuilder* valueBuilder, TBlockItem arg1, TBlockItem arg2, const TSink& sink) { - auto& storage = Reference(arg1); + auto& storage = Reference<TMResourceName>(arg1); ui64 interval = std::abs(arg2.Get<i64>()); if (interval == 0) { sink(arg1); @@ -2424,7 +2401,7 @@ private: template<typename TSink> static void Process(const IValueBuilder* valueBuilder, TBlockItem item, const TSink& sink) { Y_UNUSED(valueBuilder); - auto& storage = Reference(item); + auto& storage = Reference<TMResourceName>(item); sink(TBlockItem{(TDataType<TInterval>::TLayout)storage.ToTimeOfDay()}); } }; @@ -3147,7 +3124,7 @@ private: if constexpr (TResourceName == TMResourceName) { static constexpr size_t size = 4; ui32 year = 0U; - if (limit < size || !ParseNDigits<size>::Do(it, year) || !ValidateYear(year)) { + if (limit < size || !ParseNDigits<size>::Do(it, year) || !ValidateYear<TMResourceName>(year)) { return false; } SetYear<TMResourceName>(result, year); @@ -3159,7 +3136,7 @@ private: negative = -1LL; it++; } - if (!ParseNDigits<size, true>::Do(it, year) || !ValidateYear(negative * year)) { + if (!ParseNDigits<size, true>::Do(it, year) || !ValidateYear<TM64ResourceName>(negative * year)) { return false; } SetYear<TM64ResourceName>(result, negative * year); @@ -3333,7 +3310,7 @@ private: } \ auto& builder = valueBuilder->GetDateBuilder(); \ TUnboxedValuePod result(0); \ - auto& storage = Reference(result); \ + auto& storage = Reference<TMResourceName>(result); \ storage.FromTimestamp(builder, instant.MicroSeconds()); \ return result; \ } diff --git a/yql/providers/stat/expr_nodes/ya.make b/yql/providers/stat/expr_nodes/ya.make index 478e92f4d3..0d4b19adad 100644 --- a/yql/providers/stat/expr_nodes/ya.make +++ b/yql/providers/stat/expr_nodes/ya.make @@ -13,40 +13,21 @@ SRCDIR( yql/essentials/core/expr_nodes_gen ) -IF(EXPORT_CMAKE) - RUN_PYTHON3( - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/gen/__main__.py - yql_expr_nodes_gen.jnj - yql_stat_expr_nodes.json - yql_stat_expr_nodes.gen.h - yql_stat_expr_nodes.decl.inl.h - yql_stat_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_stat_expr_nodes.json - OUT yql_stat_expr_nodes.gen.h - OUT yql_stat_expr_nodes.decl.inl.h - OUT yql_stat_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ELSE() - RUN_PROGRAM( - yql/essentials/core/expr_nodes_gen/gen - yql_expr_nodes_gen.jnj - yql_stat_expr_nodes.json - yql_stat_expr_nodes.gen.h - yql_stat_expr_nodes.decl.inl.h - yql_stat_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_stat_expr_nodes.json - OUT yql_stat_expr_nodes.gen.h - OUT yql_stat_expr_nodes.decl.inl.h - OUT yql_stat_expr_nodes.defs.inl.h - OUTPUT_INCLUDES - ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h - ) -ENDIF() +RUN_PY3_PROGRAM( + yql/essentials/core/expr_nodes_gen/gen + yql_expr_nodes_gen.jnj + yql_stat_expr_nodes.json + yql_stat_expr_nodes.gen.h + yql_stat_expr_nodes.decl.inl.h + yql_stat_expr_nodes.defs.inl.h + IN yql_expr_nodes_gen.jnj + IN yql_stat_expr_nodes.json + OUT yql_stat_expr_nodes.gen.h + OUT yql_stat_expr_nodes.decl.inl.h + OUT yql_stat_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h +) END() |