aboutsummaryrefslogtreecommitdiffstats
path: root/yql/essentials
diff options
context:
space:
mode:
authorAlexander Smirnov <alex@ydb.tech>2024-12-24 22:01:20 +0000
committerAlexander Smirnov <alex@ydb.tech>2024-12-24 22:01:20 +0000
commitbd0e2de0b1035962a4d5b9e847eaa6508fad7fcf (patch)
tree79878ca309f9f7fada064f9b78b4223af4635f28 /yql/essentials
parentbe43a4691ebdd4dbe260a8d77df4cd8423b14c05 (diff)
parente6bd80ded127cd064560f7ea471974b602770cb1 (diff)
downloadydb-bd0e2de0b1035962a4d5b9e847eaa6508fad7fcf.tar.gz
Merge branch 'PR'
Diffstat (limited to 'yql/essentials')
-rw-r--r--yql/essentials/core/cbo/cbo_hints.cpp39
-rw-r--r--yql/essentials/core/services/mounts/ya.make1
-rw-r--r--yql/essentials/core/services/mounts/yql_mounts.cpp2
-rw-r--r--yql/essentials/core/type_ann/type_ann_core.cpp5
-rw-r--r--yql/essentials/core/ut_common/ya.make24
-rw-r--r--yql/essentials/core/ut_common/yql_ut_common.cpp55
-rw-r--r--yql/essentials/core/ut_common/yql_ut_common.h23
-rw-r--r--yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp17
-rw-r--r--yql/essentials/minikql/dom/make.cpp1
-rw-r--r--yql/essentials/minikql/dom/node.cpp4
-rw-r--r--yql/essentials/minikql/dom/node.h2
-rw-r--r--yql/essentials/minikql/mkql_program_builder.cpp2
-rw-r--r--yql/essentials/minikql/mkql_runtime_version.h2
-rw-r--r--yql/essentials/parser/pg_catalog/catalog.cpp3
-rw-r--r--yql/essentials/parser/pg_catalog/ya.make1
-rw-r--r--yql/essentials/parser/pg_wrapper/ut/codegen_ut.cpp22
-rw-r--r--yql/essentials/parser/pg_wrapper/ut/ya.make6
-rw-r--r--yql/essentials/parser/pg_wrapper/ya.make2
-rw-r--r--yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp5
-rw-r--r--yql/essentials/providers/pure/ut/ya.make23
-rw-r--r--yql/essentials/providers/pure/ut/yql_pure_provider_ut.cpp270
-rw-r--r--yql/essentials/providers/pure/ya.make4
-rw-r--r--yql/essentials/providers/pure/yql_pure_provider.cpp50
-rw-r--r--yql/essentials/sql/v1/SQLv1.g.in14
-rw-r--r--yql/essentials/sql/v1/SQLv1Antlr4.g.in14
-rw-r--r--yql/essentials/sql/v1/format/sql_format.cpp5
-rw-r--r--yql/essentials/sql/v1/format/sql_format_ut.h12
-rw-r--r--yql/essentials/sql/v1/node.h8
-rw-r--r--yql/essentials/sql/v1/select.cpp4
-rw-r--r--yql/essentials/sql/v1/sql_query.cpp30
-rw-r--r--yql/essentials/sql/v1/sql_translation.cpp81
-rw-r--r--yql/essentials/sql/v1/sql_translation.h2
-rw-r--r--yql/essentials/sql/v1/sql_ut.cpp76
-rw-r--r--yql/essentials/sql/v1/sql_ut_antlr4.cpp76
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/result.json18
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-group_by_session_nopush_/formatted.sql3
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-add_bitcast_/formatted.sql9
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-div_bitcast_/formatted.sql9
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mod_bitcast_/formatted.sql9
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mul_bitcast_/formatted.sql9
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-sub_bitcast_/formatted.sql9
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_column_group-hint_/formatted.sql12
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_compare_diff_zones_/formatted.sql3
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-cast_string_implicit_/formatted.sql37
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-dict_builtins_null_lookup_/formatted.sql6
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_ansi_join_/formatted.sql37
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_noansi_join_/formatted.sql37
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_/formatted.sql3
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_ansi_/formatted.sql3
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_/formatted.sql4
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_only_sorted_merge_/formatted.sql4
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_cbo_3_tables_/formatted.sql4
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_left_cbo_/formatted.sql4
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_right_cbo_/formatted.sql4
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-decimal_/formatted.sql9
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-tzdate_/formatted.sql9
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-uuid_/formatted.sql9
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_match_recognize-all_rows_per_match_/formatted.sql6
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_order_by-order_by_missing_project_column_nosimple_/formatted.sql46
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_params-primitives_/formatted.sql16
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_/formatted.sql3
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_udf_/formatted.sql3
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-subquery_expr_/formatted.sql3
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_schema-user_schema_with_sort_/formatted.sql3
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-named_args_for_script_with_posargs_reuse_args_fail_/formatted.sql3
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-wrong_args_fail_/formatted.sql12
-rw-r--r--yql/essentials/tests/sql/sql2yql/ya.make4
-rw-r--r--yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.cfg1
-rw-r--r--yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.sql32
-rwxr-xr-xyql/essentials/tools/minirun/apply_patches.sh14
-rw-r--r--yql/essentials/tools/minirun/minirun.cpp1
-rw-r--r--yql/essentials/tools/minirun/patches/01_no_icu.patch93
-rw-r--r--yql/essentials/tools/udf_resolver/udf_resolver.cpp1
-rw-r--r--yql/essentials/tools/yql_facade_run/yql_facade_run.cpp2
-rw-r--r--yql/essentials/tools/yql_facade_run/yql_facade_run.h1
-rw-r--r--yql/essentials/udfs/common/digest/digest_udf.cpp165
-rw-r--r--yql/essentials/udfs/common/digest/test/canondata/result.json5
-rw-r--r--yql/essentials/udfs/common/digest/test/canondata/test.test_Init_/results.txt202
-rw-r--r--yql/essentials/udfs/common/digest/test/cases/Init.cfg1
-rw-r--r--yql/essentials/udfs/common/digest/test/cases/Init.sql11
-rw-r--r--yql/essentials/udfs/common/hyperscan/test/canondata/test.test_Error_/extracted2
-rw-r--r--yql/essentials/udfs/common/yson2/test/canondata/test.test_From_/results.txt10
-rw-r--r--yql/essentials/udfs/common/yson2/test/cases/From.sql1
-rw-r--r--yql/essentials/udfs/common/yson2/yson2_udf.cpp1
-rw-r--r--yql/essentials/utils/log/profile.cpp47
85 files changed, 1388 insertions, 437 deletions
diff --git a/yql/essentials/core/cbo/cbo_hints.cpp b/yql/essentials/core/cbo/cbo_hints.cpp
index 630bb3e6b0..3a022b36d0 100644
--- a/yql/essentials/core/cbo/cbo_hints.cpp
+++ b/yql/essentials/core/cbo/cbo_hints.cpp
@@ -6,9 +6,16 @@
using namespace NYql;
+TString ToLower(TString s) {
+ for (char& c: s) {
+ c = std::tolower(c);
+ }
+ return s;
+}
+
class TOptimizerHintsParser {
public:
- TOptimizerHintsParser(const TString& text)
+ TOptimizerHintsParser(const TString& text)
: Pos(-1)
, Size(static_cast<i32>(text.size()) - 1)
, Text(text)
@@ -45,9 +52,9 @@ private:
return labels;
}
- void JoinType() {
+ void JoinType() {
i32 beginPos = Pos + 1;
-
+
Keyword({"("});
i32 labelsBeginPos = Pos + 1;
@@ -59,17 +66,17 @@ private:
labels.pop_back();
Keyword({")"});
-
+
TVector<EJoinAlgoType> joinAlgos = {EJoinAlgoType::GraceJoin, EJoinAlgoType::LookupJoin, EJoinAlgoType::MapJoin};
- TVector<TString> joinAlgosStr = {"Shuffle", "Lookup", "Broadcast"};
+ TVector<TString> joinAlgosStr = {"shuffle", "lookup", "broadcast"};
for (const auto& [JoinType, joinAlgoStr]: Zip(joinAlgos, joinAlgosStr)) {
- if (reqJoinAlgoStr == joinAlgoStr) {
+ if (ToLower(reqJoinAlgoStr) == joinAlgoStr) {
Hints.JoinAlgoHints->PushBack(std::move(labels), JoinType, "JoinType" + Text.substr(beginPos, Pos - beginPos + 1));
return;
}
}
-
+
ParseError(Sprintf("Unknown JoinType: '%s', supported algos: [%s]", reqJoinAlgoStr.c_str(), JoinSeq(", ", joinAlgosStr).c_str()), Pos - reqJoinAlgoStr.size());
Y_UNREACHABLE();
}
@@ -82,7 +89,7 @@ private:
Keyword({")"});
Hints.JoinOrderHints->PushBack(
- std::move(joinOrderHintTree),
+ std::move(joinOrderHintTree),
leading? "Leading" : "JoinOrder" + Text.substr(beginPos, Pos - beginPos + 1)
);
}
@@ -100,7 +107,7 @@ private:
auto join = JoinOrderLabels();
Keyword({")"});
return join;
- }
+ }
ParseError(Sprintf("JoinOrder args must be either a relation, either a join, example of the format: JoinOrder(t1 (t2 t3))"), Pos);
Y_UNREACHABLE();
@@ -116,7 +123,7 @@ private:
char sign = signStr[0];
auto value = Number();
Keyword({")"});
-
+
TCardinalityHints::ECardOperation op;
switch (sign) {
case '+': { op = TCardinalityHints::ECardOperation::Add; break; }
@@ -138,7 +145,7 @@ private:
}
TString Label() {
- return Term(Letters() | Digits());
+ return Term(LabelAllowedSymbols());
}
std::optional<TString> MaybeLabel() {
@@ -170,7 +177,7 @@ private:
char Char(unsigned char c) {
std::bitset<256> allowed;
- allowed[c] = 1;
+ allowed[c] = 1;
return Char(allowed);
}
@@ -181,7 +188,7 @@ private:
}
return Char(allowed);
}
-
+
char Char(const std::bitset<256>& allowedSymbols = {}) {
Y_ENSURE(Pos < Size, Sprintf("Expected [%s], but got end of the string.", ""));
@@ -281,6 +288,12 @@ private:
return res;
}
+ constexpr std::bitset<256> LabelAllowedSymbols() {
+ auto labelSymbols = Digits() | Letters();
+ labelSymbols['_'] = 1;
+ return labelSymbols;
+ }
+
void SkipWhiteSpaces() {
for (; Pos < Size && isspace(Text[Pos + 1]); ++Pos) {
}
diff --git a/yql/essentials/core/services/mounts/ya.make b/yql/essentials/core/services/mounts/ya.make
index 660204f365..f6180eef6e 100644
--- a/yql/essentials/core/services/mounts/ya.make
+++ b/yql/essentials/core/services/mounts/ya.make
@@ -9,6 +9,7 @@ PEERDIR(
library/cpp/resource
yql/essentials/core/user_data
yql/essentials/core
+ yql/essentials/utils/log
)
YQL_LAST_ABI_VERSION()
diff --git a/yql/essentials/core/services/mounts/yql_mounts.cpp b/yql/essentials/core/services/mounts/yql_mounts.cpp
index 7d4052e8c8..683a7fe84b 100644
--- a/yql/essentials/core/services/mounts/yql_mounts.cpp
+++ b/yql/essentials/core/services/mounts/yql_mounts.cpp
@@ -1,6 +1,7 @@
#include "yql_mounts.h"
#include <yql/essentials/core/yql_library_compiler.h>
+#include <yql/essentials/utils/log/profile.h>
#include <library/cpp/resource/resource.h>
@@ -117,6 +118,7 @@ namespace NYql {
bool optimizeLibraries,
THolder<TExprContext> ownedCtx)
{
+ YQL_PROFILE_FUNC(DEBUG);
auto ctx = rawCtx ? rawCtx : ownedCtx.Get();
Y_ENSURE(ctx);
TUserDataTable mounts;
diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp
index 3b6a626e07..d448c7ed54 100644
--- a/yql/essentials/core/type_ann/type_ann_core.cpp
+++ b/yql/essentials/core/type_ann/type_ann_core.cpp
@@ -5930,6 +5930,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (const auto status = ConvertChildrenToType(input, commonType, ctx.Expr); status != IGraphTransformer::TStatus::Ok)
return status;
+ if (warn) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
const auto dictType = IsSet ?
ctx.Expr.MakeType<TDictExprType>(commonType, ctx.Expr.MakeType<TVoidExprType>()):
ctx.Expr.MakeType<TDictExprType>(commonType->Cast<TTupleExprType>()->GetItems().front(), commonType->Cast<TTupleExprType>()->GetItems().back());
@@ -5943,7 +5946,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- return warn ? IGraphTransformer::TStatus::Repeat : IGraphTransformer::TStatus::Ok;
+ return IGraphTransformer::TStatus::Ok;
}
IGraphTransformer::TStatus DictFromKeysWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
diff --git a/yql/essentials/core/ut_common/ya.make b/yql/essentials/core/ut_common/ya.make
deleted file mode 100644
index 04c59aaf3d..0000000000
--- a/yql/essentials/core/ut_common/ya.make
+++ /dev/null
@@ -1,24 +0,0 @@
-IF (NOT OPENSOURCE)
-
-LIBRARY()
-
-SRCS(
- yql_ut_common.cpp
- yql_ut_common.h
-)
-
-PEERDIR(
- yql/essentials/core
- yql/essentials/core/expr_nodes
- contrib/ydb/library/yql/dq/proto
- contrib/ydb/library/yql/dq/expr_nodes
-)
-
-
-YQL_LAST_ABI_VERSION()
-
-
-END()
-
-ENDIF()
-
diff --git a/yql/essentials/core/ut_common/yql_ut_common.cpp b/yql/essentials/core/ut_common/yql_ut_common.cpp
deleted file mode 100644
index cef3f2723c..0000000000
--- a/yql/essentials/core/ut_common/yql_ut_common.cpp
+++ /dev/null
@@ -1,55 +0,0 @@
-#include "yql_ut_common.h"
-
-#include <library/cpp/random_provider/random_provider.h>
-#include <library/cpp/time_provider/time_provider.h>
-
-#include <util/generic/guid.h>
-#include <util/system/user.h>
-#include <util/stream/file.h>
-
-namespace NYql {
-
-TTestTablesMapping::TTestTablesMapping()
- : TmpInput()
- , TmpInputAttr(TmpInput.Name() + ".attr")
- , TmpOutput()
- , TmpOutputAttr(TmpOutput.Name() + ".attr")
-{
- {
- TUnbufferedFileOutput tmpInput(TmpInput);
- tmpInput << "{\"key\"=\"\";\"subkey\"=\"\";\"value\"=\"\"}" << Endl;
- TUnbufferedFileOutput tmpInputAttr(TmpInputAttr);
- tmpInputAttr << "{\"_yql_row_spec\" = {\"Type\" = [\"StructType\";["
- << "[\"key\";[\"DataType\";\"String\"]];"
- << "[\"subkey\";[\"DataType\";\"String\"]];"
- << "[\"value\";[\"DataType\";\"String\"]]"
- << "]]}}" << Endl;
- }
- insert(std::make_pair("yt.plato.Input", TmpInput.Name()));
-
- {
- TUnbufferedFileOutput tmpOutput(TmpOutput);
- tmpOutput << "{\"key\"=\"\";\"subkey\"=\"\";\"value\"=\"\"}" << Endl;
- TUnbufferedFileOutput tmpOutputAttr(TmpOutputAttr);
- tmpOutputAttr << "{\"_yql_row_spec\" = {\"Type\" = [\"StructType\";["
- << "[\"key\";[\"DataType\";\"String\"]];"
- << "[\"subkey\";[\"DataType\";\"String\"]];"
- << "[\"value\";[\"DataType\";\"String\"]]"
- << "]]}}" << Endl;
- }
- insert(std::make_pair("yt.plato.Output", TmpOutput.Name()));
-}
-
-void InitializeYtGateway(IYtGateway::TPtr gateway, TYtState::TPtr ytState) {
- ytState->SessionId = CreateGuidAsString();
- gateway->OpenSession(
- IYtGateway::TOpenSessionOptions(ytState->SessionId)
- .UserName(GetUsername())
- .ProgressWriter(&NullProgressWriter)
- .OperationOptions(TYqlOperationOptions())
- .RandomProvider(CreateDeterministicRandomProvider(1))
- .TimeProvider(CreateDeterministicTimeProvider(10000000))
- );
-}
-
-}
diff --git a/yql/essentials/core/ut_common/yql_ut_common.h b/yql/essentials/core/ut_common/yql_ut_common.h
deleted file mode 100644
index ddee02690a..0000000000
--- a/yql/essentials/core/ut_common/yql_ut_common.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#pragma once
-
-#include <yql/essentials/core/yql_expr_type_annotation.h>
-
-#include <yt/yql/providers/yt/gateway/file/yql_yt_file.h>
-#include <yt/yql/providers/yt/provider/yql_yt_provider.h>
-
-#include <util/system/tempfile.h>
-
-namespace NYql {
-
-struct TTestTablesMapping: public THashMap<TString, TString> {
- TTempFileHandle TmpInput;
- TTempFileHandle TmpInputAttr;
- TTempFileHandle TmpOutput;
- TTempFileHandle TmpOutputAttr;
-
- TTestTablesMapping();
-};
-
-void InitializeYtGateway(IYtGateway::TPtr gateway, TYtState::TPtr ytState);
-
-}
diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp
index f1cb1ea1aa..f4d60c3ef5 100644
--- a/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp
+++ b/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp
@@ -796,14 +796,17 @@ Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinTestOptional) {
expectedSubkey.push_back(leftSubkeyInit[i]);
expectedValue.push_back(leftValueInit[i]);
- const auto& found = rightMap.find(*leftKeyInit[i]);
- if (found != rightMap.cend()) {
- expectedRightKey.push_back(found->first);
- expectedRightValue.push_back(found->second);
- } else {
- expectedRightKey.push_back(std::nullopt);
- expectedRightValue.push_back(std::nullopt);
+ if (leftKeyInit[i].has_value()) {
+ const auto& found = rightMap.find(*leftKeyInit[i]);
+ if (found != rightMap.cend()) {
+ expectedRightKey.push_back(found->first);
+ expectedRightValue.push_back(found->second);
+ continue;
+ }
}
+
+ expectedRightKey.push_back(std::nullopt);
+ expectedRightValue.push_back(std::nullopt);
}
auto [leftType, leftList] = ConvertVectorsToTuples(setup,
diff --git a/yql/essentials/minikql/dom/make.cpp b/yql/essentials/minikql/dom/make.cpp
index ca6864f759..8c1bfecfd0 100644
--- a/yql/essentials/minikql/dom/make.cpp
+++ b/yql/essentials/minikql/dom/make.cpp
@@ -133,6 +133,7 @@ TUnboxedValuePod MakeVariant(const ITypeInfoHelper* typeHelper, const TType* sha
TUnboxedValuePod MakeDom(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) {
switch (const auto kind = typeHelper->GetTypeKind(shape)) {
case ETypeKind::Null:
+ case ETypeKind::Void:
return MakeEntity();
case ETypeKind::EmptyList:
return SetNodeType<ENodeType::List>(TUnboxedValuePod::Void());
diff --git a/yql/essentials/minikql/dom/node.cpp b/yql/essentials/minikql/dom/node.cpp
index 6eabde3e4e..64458dea8d 100644
--- a/yql/essentials/minikql/dom/node.cpp
+++ b/yql/essentials/minikql/dom/node.cpp
@@ -133,6 +133,10 @@ bool TMapNode::IsSortedDict() const {
return true;
}
+TStringRef TMapNode::GetResourceTag() const {
+ return NodeResourceName;
+}
+
void* TMapNode::GetResource() {
return Items_;
}
diff --git a/yql/essentials/minikql/dom/node.h b/yql/essentials/minikql/dom/node.h
index 04a211fd09..5719709655 100644
--- a/yql/essentials/minikql/dom/node.h
+++ b/yql/essentials/minikql/dom/node.h
@@ -94,6 +94,8 @@ private:
bool IsSortedDict() const final;
+ TStringRef GetResourceTag() const final;
+
void* GetResource() final;
ui32 Count_;
diff --git a/yql/essentials/minikql/mkql_program_builder.cpp b/yql/essentials/minikql/mkql_program_builder.cpp
index 0b9197024d..895f2d590a 100644
--- a/yql/essentials/minikql/mkql_program_builder.cpp
+++ b/yql/essentials/minikql/mkql_program_builder.cpp
@@ -6243,7 +6243,7 @@ TRuntimeNode TProgramBuilder::MatchRecognizeCore(
callableBuilder.Add(NewDataLiteral(static_cast<i32>(skipTo.To)));
callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(skipTo.Var));
}
- if constexpr (RuntimeVersion >= 53U) {
+ if constexpr (RuntimeVersion >= 54U) {
callableBuilder.Add(NewDataLiteral(static_cast<i32>(rowsPerMatch)));
callableBuilder.Add(NewList(outputColumnEntryType, outputColumnOrder));
}
diff --git a/yql/essentials/minikql/mkql_runtime_version.h b/yql/essentials/minikql/mkql_runtime_version.h
index b4fbc1951e..26cd1a8fcc 100644
--- a/yql/essentials/minikql/mkql_runtime_version.h
+++ b/yql/essentials/minikql/mkql_runtime_version.h
@@ -24,7 +24,7 @@ namespace NMiniKQL {
// 1. Bump this version every time incompatible runtime nodes are introduced.
// 2. Make sure you provide runtime node generation for previous runtime versions.
#ifndef MKQL_RUNTIME_VERSION
-#define MKQL_RUNTIME_VERSION 53U
+#define MKQL_RUNTIME_VERSION 54U
#endif
// History:
diff --git a/yql/essentials/parser/pg_catalog/catalog.cpp b/yql/essentials/parser/pg_catalog/catalog.cpp
index c631fca46b..40a2d3a52b 100644
--- a/yql/essentials/parser/pg_catalog/catalog.cpp
+++ b/yql/essentials/parser/pg_catalog/catalog.cpp
@@ -1,5 +1,6 @@
#include "catalog.h"
#include <yql/essentials/parser/pg_catalog/proto/pg_catalog.pb.h>
+#include <yql/essentials/utils/log/profile.h>
#include <util/generic/array_size.h>
#include <util/generic/utility.h>
#include <util/generic/hash.h>
@@ -3649,6 +3650,7 @@ ISqlLanguageParser* GetSqlLanguageParser() {
}
void LoadSystemFunctions(ISystemFunctionsParser& parser) {
+ YQL_PROFILE_FUNC(DEBUG);
auto& catalog = TCatalog::MutableInstance();
with_lock (catalog.ExtensionsGuard) {
if (catalog.State->SystemFunctionInit) {
@@ -3684,6 +3686,7 @@ void LoadSystemFunctions(ISystemFunctionsParser& parser) {
void RegisterExtensions(const TVector<TExtensionDesc>& extensions, bool typesOnly,
IExtensionSqlParser& parser, IExtensionLoader* loader) {
+ YQL_PROFILE_FUNC(DEBUG);
if (extensions.size() > MaximumExtensionsCount) {
throw yexception() << "Too many extensions: " << extensions.size();
}
diff --git a/yql/essentials/parser/pg_catalog/ya.make b/yql/essentials/parser/pg_catalog/ya.make
index f9aefb43aa..815bcaa340 100644
--- a/yql/essentials/parser/pg_catalog/ya.make
+++ b/yql/essentials/parser/pg_catalog/ya.make
@@ -23,6 +23,7 @@ PEERDIR(
yql/essentials/public/issue
library/cpp/digest/md5
yql/essentials/parser/pg_catalog/proto
+ yql/essentials/utils/log
)
END()
diff --git a/yql/essentials/parser/pg_wrapper/ut/codegen_ut.cpp b/yql/essentials/parser/pg_wrapper/ut/codegen_ut.cpp
index 50320e27dd..91536d2872 100644
--- a/yql/essentials/parser/pg_wrapper/ut/codegen_ut.cpp
+++ b/yql/essentials/parser/pg_wrapper/ut/codegen_ut.cpp
@@ -45,7 +45,7 @@ Y_UNIT_TEST_SUITE(TPgCodegen) {
void PgFuncImpl(EKernelFlavor flavor, bool constArg, bool fixed) {
const TString& name = fixed ? "date_eq" : "textout";
ICodegen::TPtr codegen;
- TExecFunc execFunc;
+ TExecFunc execFunc;
switch (flavor) {
case EKernelFlavor::Indirect: {
if (fixed) {
@@ -94,10 +94,10 @@ Y_UNIT_TEST_SUITE(TPgCodegen) {
nullBuilder.Reserve(length);
auto out = dataBuilder.MutableData();
auto outNulls = nullBuilder.MutableData();
- NUdf::TFixedSizeBlockReader<ui64, false> reader1;
- NUdf::TFixedSizeBlockReader<ui64, false> reader2;
+ NUdf::TFixedSizeBlockReader<ui64, false> reader1;
+ NUdf::TFixedSizeBlockReader<ui64, false> reader2;
const auto& array1 = *batch.values[0].array();
- const auto ptr1 = array1.GetValues<ui64>(1);
+ const auto ptr1 = array1.GetValues<ui64>(1);
if (batch.values[1].is_array()) {
const auto& array2 = *batch.values[1].array();
const auto ptr2 = array2.GetValues<ui64>(1);
@@ -222,7 +222,7 @@ Y_UNIT_TEST_SUITE(TPgCodegen) {
arrow::compute::ExecBatch batch(std::move(batchArgs), N);
{
- Cout << "begin...\n";
+ Cout << "begin...\n";
TSimpleTimer timer;
for (size_t count = 0; count < (fixed ? 10000 : 1000); ++count) {
arrow::Datum res;
@@ -247,7 +247,7 @@ Y_UNIT_TEST_SUITE(TPgCodegen) {
Y_UNIT_TEST(PgFixedFuncIndirect) {
PgFuncImpl(EKernelFlavor::Indirect, false, true);
PgFuncImpl(EKernelFlavor::Indirect, true, true);
- }
+ }
#if !defined(USE_SLOW_PG_KERNELS)
Y_UNIT_TEST(PgFixedFuncCpp) {
@@ -255,19 +255,21 @@ Y_UNIT_TEST_SUITE(TPgCodegen) {
PgFuncImpl(EKernelFlavor::Cpp, true, true);
}
+#if defined(YQL_USE_PG_BC)
Y_UNIT_TEST(PgFixedFuncBC) {
PgFuncImpl(EKernelFlavor::BitCode, false, true);
PgFuncImpl(EKernelFlavor::BitCode, true, true);
}
-#endif
+#endif
+#endif
Y_UNIT_TEST(PgStrFuncIdeal) {
PgFuncImpl(EKernelFlavor::Ideal, false, false);
- }
+ }
Y_UNIT_TEST(PgStrFuncDefArg) {
PgFuncImpl(EKernelFlavor::DefArg, false, false);
- }
+ }
Y_UNIT_TEST(PgStrFuncIndirect) {
PgFuncImpl(EKernelFlavor::Indirect, false, false);
@@ -278,9 +280,11 @@ Y_UNIT_TEST_SUITE(TPgCodegen) {
PgFuncImpl(EKernelFlavor::Cpp, false, false);
}
+#if defined(YQL_USE_PG_BC)
Y_UNIT_TEST(PgStrFuncBC) {
PgFuncImpl(EKernelFlavor::BitCode, false, false);
}
#endif
+#endif
}
diff --git a/yql/essentials/parser/pg_wrapper/ut/ya.make b/yql/essentials/parser/pg_wrapper/ut/ya.make
index 4dc451e52f..37d3514757 100644
--- a/yql/essentials/parser/pg_wrapper/ut/ya.make
+++ b/yql/essentials/parser/pg_wrapper/ut/ya.make
@@ -35,6 +35,12 @@ PEERDIR(
YQL_LAST_ABI_VERSION()
+IF (YQL_USE_PG_BC)
+ CFLAGS(
+ -DYQL_USE_PG_BC
+ )
+ENDIF()
+
IF (MKQL_RUNTIME_VERSION)
CFLAGS(
-DMKQL_RUNTIME_VERSION=$MKQL_RUNTIME_VERSION
diff --git a/yql/essentials/parser/pg_wrapper/ya.make b/yql/essentials/parser/pg_wrapper/ya.make
index cf31405feb..7ea0ecfa7a 100644
--- a/yql/essentials/parser/pg_wrapper/ya.make
+++ b/yql/essentials/parser/pg_wrapper/ya.make
@@ -64,9 +64,11 @@ ENDIF()
IF (NOT OS_WINDOWS AND NOT SANITIZER_TYPE AND NOT BUILD_TYPE == "DEBUG")
IF (NOT BUILD_POSTGRES_ONLY)
+IF (YQL_USE_PG_BC)
USE_LLVM_BC14()
INCLUDE(pg_bc.all.inc)
ENDIF()
+ENDIF()
ELSE()
CFLAGS(-DUSE_SLOW_PG_KERNELS)
ENDIF()
diff --git a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp
index f02a666e5c..9e13ad766b 100644
--- a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp
+++ b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp
@@ -174,8 +174,9 @@ public:
bool needLibrary = false;
for (auto& m : *import->Modules) {
if (requiredLoadedModules.contains(m)) {
- YQL_ENSURE(import->Block->Type == EUserDataType::PATH);
- path2LoadedImport[import->Block->Data] = import;
+ if (import->Block->Type == EUserDataType::PATH) {
+ path2LoadedImport[import->Block->Data] = import;
+ }
}
if (requiredExternalModules.contains(m)) {
diff --git a/yql/essentials/providers/pure/ut/ya.make b/yql/essentials/providers/pure/ut/ya.make
new file mode 100644
index 0000000000..770076756e
--- /dev/null
+++ b/yql/essentials/providers/pure/ut/ya.make
@@ -0,0 +1,23 @@
+UNITTEST_FOR(yql/essentials/providers/pure)
+
+SIZE(MEDIUM)
+
+SRCS(
+ yql_pure_provider_ut.cpp
+)
+
+PEERDIR(
+ yql/essentials/core/facade
+ yql/essentials/public/result_format
+ yql/essentials/minikql/invoke_builtins
+ yql/essentials/minikql
+ yql/essentials/minikql/comp_nodes/llvm14
+ yql/essentials/minikql/invoke_builtins/llvm14
+ yql/essentials/sql/pg_dummy
+ yql/essentials/public/udf/service/exception_policy
+ library/cpp/yson/node
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/yql/essentials/providers/pure/ut/yql_pure_provider_ut.cpp b/yql/essentials/providers/pure/ut/yql_pure_provider_ut.cpp
new file mode 100644
index 0000000000..c2d66c002a
--- /dev/null
+++ b/yql/essentials/providers/pure/ut/yql_pure_provider_ut.cpp
@@ -0,0 +1,270 @@
+#include "yql_pure_provider.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <yql/essentials/core/facade/yql_facade.h>
+#include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h>
+#include <yql/essentials/minikql/mkql_function_registry.h>
+#include <yql/essentials/public/result_format/yql_result_format_response.h>
+
+#include <library/cpp/yson/node/node_io.h>
+
+#include <util/system/user.h>
+#include <util/string/strip.h>
+
+namespace NYql {
+
+namespace {
+
+struct TSettings {
+ bool SExpr = false;
+ bool Pretty = false;
+};
+
+TString Run(const TString& query, TSettings settings = {}) {
+ auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry());
+ TVector<TDataProviderInitializer> dataProvidersInit;
+ dataProvidersInit.push_back(GetPureDataProviderInitializer());
+ TProgramFactory factory(true, functionRegistry.Get(), 0ULL, dataProvidersInit, "ut");
+ TProgramPtr program = factory.Create("-stdin-", query);
+ program->ConfigureYsonResultFormat(settings.Pretty ? NYson::EYsonFormat::Pretty : NYson::EYsonFormat::Text);
+ bool parseRes;
+ if (settings.SExpr) {
+ parseRes = program->ParseYql();
+ } else {
+ parseRes = program->ParseSql();
+ }
+
+ if (!parseRes) {
+ TStringStream err;
+ program->PrintErrorsTo(err);
+ UNIT_FAIL(err.Str());
+ }
+
+ if (!program->Compile(GetUsername())) {
+ TStringStream err;
+ program->PrintErrorsTo(err);
+ UNIT_FAIL(err.Str());
+ }
+
+ TProgram::TStatus status = program->Run(GetUsername());
+ if (status == TProgram::TStatus::Error) {
+ TStringStream err;
+ program->PrintErrorsTo(err);
+ UNIT_FAIL(err.Str());
+ }
+
+ return program->ResultsAsString();
+}
+
+}
+
+Y_UNIT_TEST_SUITE(TPureProviderTests) {
+ Y_UNIT_TEST(SExpr) {
+ const auto s = R"(
+ (
+ (let result_sink (DataSink 'result))
+ (let output (Int32 '1))
+ (let world (Write! world result_sink (Key) output '('('type))))
+ (return (Commit! world result_sink))
+ )
+
+ )";
+ const auto expectedRes = R"(
+[
+ {
+ "Write" = [
+ {
+ "Type" = [
+ "DataType";
+ "Int32"
+ ];
+ "Data" = "1"
+ }
+ ]
+ }
+]
+ )";
+ auto res = Run(s, TSettings{.SExpr = true, .Pretty = true});
+ UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes));
+ }
+
+ Y_UNIT_TEST(Sql0Rows) {
+ const auto s = "select * from (select 1 as x) limit 0";
+ const auto expectedRes = R"(
+[
+ {
+ "Write" = [
+ {
+ "Type" = [
+ "ListType";
+ [
+ "StructType";
+ [
+ [
+ "x";
+ [
+ "DataType";
+ "Int32"
+ ]
+ ]
+ ]
+ ]
+ ];
+ "Data" = []
+ }
+ ]
+ }
+]
+ )";
+ auto res = Run(s, TSettings{.Pretty = true});
+ UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes));
+ }
+
+ void Sql1RowImpl(const TString& query) {
+ const auto expectedRes = R"(
+[
+ {
+ "Write" = [
+ {
+ "Type" = [
+ "ListType";
+ [
+ "StructType";
+ [
+ [
+ "x";
+ [
+ "DataType";
+ "Int32"
+ ]
+ ]
+ ]
+ ]
+ ];
+ "Data" = [
+ [
+ "1"
+ ]
+ ]
+ }
+ ]
+ }
+]
+ )";
+ auto res = Run(query, TSettings{.Pretty = true});
+ UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes));
+ }
+
+ Y_UNIT_TEST(Sql1Row_LLVM_On) {
+ const auto s = "pragma config.flags(\"LLVM\",\"--dump-stats\");select 1 as x";
+ Sql1RowImpl(s);
+ }
+
+ Y_UNIT_TEST(Sql1Row_LLVM_Off) {
+ const auto s = "pragma config.flags(\"LLVM\",\"OFF\");select 1 as x";
+ Sql1RowImpl(s);
+ }
+
+ Y_UNIT_TEST(Sql2Rows) {
+ const auto s = "select 1 as x union all select 2 as x order by x";
+ const auto expectedRes = R"(
+[
+ {
+ "Write" = [
+ {
+ "Type" = [
+ "ListType";
+ [
+ "StructType";
+ [
+ [
+ "x";
+ [
+ "DataType";
+ "Int32"
+ ]
+ ]
+ ]
+ ]
+ ];
+ "Data" = [
+ [
+ "1"
+ ];
+ [
+ "2"
+ ]
+ ]
+ }
+ ]
+ }
+]
+ )";
+ auto res = Run(s, TSettings{.Pretty = true});
+ UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes));
+ }
+
+ Y_UNIT_TEST(TruncateRows) {
+ const auto s = "select x from (select ListFromRange(1,2000) as x) flatten by x";
+ auto res = Run(s);
+ auto respList = NResult::ParseResponse(NYT::NodeFromYsonString(res));
+ UNIT_ASSERT_VALUES_EQUAL(respList.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(respList[0].Writes.size(), 1);
+ UNIT_ASSERT(respList[0].Writes[0].IsTruncated);
+ }
+
+ Y_UNIT_TEST(TruncateBytes) {
+ const auto s = "select '" + TString(1000000, 'a') + "' as x, 1 as y union all select '' as x, 2 as y order by y";
+ auto res = Run(s);
+ auto respList = NResult::ParseResponse(NYT::NodeFromYsonString(res));
+ UNIT_ASSERT_VALUES_EQUAL(respList.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(respList[0].Writes.size(), 1);
+ UNIT_ASSERT(respList[0].Writes[0].IsTruncated);
+ }
+
+ Y_UNIT_TEST(ColumnOrder) {
+ const auto s = "pragma OrderedColumns;select 1 as y, 2 as x";
+ const auto expectedRes = R"(
+[
+ {
+ "Write" = [
+ {
+ "Type" = [
+ "ListType";
+ [
+ "StructType";
+ [
+ [
+ "y";
+ [
+ "DataType";
+ "Int32"
+ ]
+ ];
+ [
+ "x";
+ [
+ "DataType";
+ "Int32"
+ ]
+ ]
+ ]
+ ]
+ ];
+ "Data" = [
+ [
+ "1";
+ "2"
+ ]
+ ]
+ }
+ ]
+ }
+]
+ )";
+ auto res = Run(s, TSettings{.Pretty = true});
+ UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes));
+ }
+}
+
+} // namespace NYql
diff --git a/yql/essentials/providers/pure/ya.make b/yql/essentials/providers/pure/ya.make
index 2ce55fd588..d7bbf0e5ab 100644
--- a/yql/essentials/providers/pure/ya.make
+++ b/yql/essentials/providers/pure/ya.make
@@ -26,3 +26,7 @@ PEERDIR(
YQL_LAST_ABI_VERSION()
END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/yql/essentials/providers/pure/yql_pure_provider.cpp b/yql/essentials/providers/pure/yql_pure_provider.cpp
index b1c65f71e1..8257951be0 100644
--- a/yql/essentials/providers/pure/yql_pure_provider.cpp
+++ b/yql/essentials/providers/pure/yql_pure_provider.cpp
@@ -62,7 +62,32 @@ public:
return SyncError();
}
+ const bool isList = lambda.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List;
auto optimized = lambda.Ptr();
+ auto source1 = ctx.Builder(lambda.Pos())
+ .Callable("Take")
+ .Callable(0, "SourceOf")
+ .Callable(0, "StreamType")
+ .Callable(0, "NullType")
+ .Seal()
+ .Seal()
+ .Seal()
+ .Callable(1, "Uint64")
+ .Atom(0, "1")
+ .Seal()
+ .Seal()
+ .Build();
+
+ optimized = ctx.Builder(lambda.Pos())
+ .Callable(isList ? "FlatMap" : "Map")
+ .Add(0, source1)
+ .Lambda(1)
+ .Param("x")
+ .Set(optimized)
+ .Seal()
+ .Seal()
+ .Build();
+
bool hasNonDeterministicFunctions;
auto status = PeepHoleOptimizeNode(optimized, optimized, ctx, *State_->Types, nullptr, hasNonDeterministicFunctions);
if (status.Level == IGraphTransformer::TStatus::Error) {
@@ -124,26 +149,37 @@ public:
TStringOutput dataOut(data);
TCountingOutput dataCountingOut(&dataOut);
NYson::TYsonWriter dataWriter(&dataCountingOut, NCommon::GetYsonFormat(fillSettings), ::NYson::EYsonType::Node, false);
- if (type->IsList()) {
- auto inputType = AS_TYPE(TListType, type)->GetItemType();
+ YQL_ENSURE(type->IsStream());
+ auto itemType = AS_TYPE(TStreamType, type)->GetItemType();
+ if (isList) {
TMaybe<ui64> rowsLimit = fillSettings.RowsLimitPerWrite;
TMaybe<ui64> bytesLimit = fillSettings.AllResultsBytesLimit;
- TMaybe<TVector<ui32>> structPositions = NCommon::CreateStructPositions(inputType, &columns);
+ TMaybe<TVector<ui32>> structPositions = NCommon::CreateStructPositions(itemType, &columns);
dataWriter.OnBeginList();
- const auto it = value.GetListIterator();
ui64 rows = 0;
- for (NUdf::TUnboxedValue item; it.Next(item); ++rows) {
+ for (;;) {
+ NUdf::TUnboxedValue item;
+ auto status = value.Fetch(item);
+ if (status == NUdf::EFetchStatus::Finish) {
+ break;
+ }
+
+ YQL_ENSURE(status == NUdf::EFetchStatus::Ok);
if ((rowsLimit && rows >= *rowsLimit) || (bytesLimit && dataCountingOut.Counter() >= *bytesLimit)) {
truncated = true;
break;
}
dataWriter.OnListItem();
- NCommon::WriteYsonValue(dataWriter, item, inputType, structPositions.Get());
+ NCommon::WriteYsonValue(dataWriter, item, itemType, structPositions.Get());
+ ++rows;
}
dataWriter.OnEndList();
} else {
- NCommon::WriteYsonValue(dataWriter, value, type, nullptr);
+ NUdf::TUnboxedValue item;
+ YQL_ENSURE(value.Fetch(item) == NUdf::EFetchStatus::Ok);
+ NCommon::WriteYsonValue(dataWriter, item, itemType, nullptr);
+ YQL_ENSURE(value.Fetch(item) == NUdf::EFetchStatus::Finish);
}
writer.OnKeyedItem("Data");
diff --git a/yql/essentials/sql/v1/SQLv1.g.in b/yql/essentials/sql/v1/SQLv1.g.in
index 670ad27e3e..61e7dc6f25 100644
--- a/yql/essentials/sql/v1/SQLv1.g.in
+++ b/yql/essentials/sql/v1/SQLv1.g.in
@@ -825,8 +825,8 @@ alter_table_alter_index_action:
drop_table_stmt: DROP (TABLE | TABLESTORE | EXTERNAL TABLE) (IF EXISTS)? simple_table_ref;
-create_user_stmt: CREATE USER role_name create_user_option?;
-alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name);
+create_user_stmt: CREATE USER role_name create_user_option*;
+alter_user_stmt: ALTER USER role_name (WITH? create_user_option+ | RENAME TO role_name);
create_group_stmt: CREATE GROUP role_name (WITH USER role_name (COMMA role_name)* COMMA?)?;
alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_name)* COMMA? | RENAME TO role_name);
@@ -834,7 +834,9 @@ alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_n
drop_role_stmt: DROP (USER|GROUP) (IF EXISTS)? role_name (COMMA role_name)* COMMA?;
role_name: an_id_or_type | bind_parameter;
-create_user_option: ENCRYPTED? PASSWORD expr;
+create_user_option: password_option | login_option;
+password_option: ENCRYPTED? PASSWORD expr;
+login_option: LOGIN | NOLOGIN;
grant_permissions_stmt: GRANT permission_name_target ON an_id_schema (COMMA an_id_schema)* TO role_name (COMMA role_name)* COMMA? (WITH GRANT OPTION)?;
revoke_permissions_stmt: REVOKE (GRANT OPTION FOR)? permission_name_target ON an_id_schema (COMMA an_id_schema)* FROM role_name (COMMA role_name)*;
@@ -1368,6 +1370,7 @@ keyword_as_compat:
| LEGACY
| LIKE
| LOCAL
+ | LOGIN
| MANAGE
| MATCH
| MATCHES
@@ -1380,6 +1383,7 @@ keyword_as_compat:
// | NATURAL
| NEXT
| NO
+ | NOLOGIN
// | NOTNULL
| NULLS
| OBJECT
@@ -1592,6 +1596,7 @@ keyword_compat: (
| LEGACY
| LIKE
| LOCAL
+ | LOGIN
| MANAGE
| MATCH
| MATCHES
@@ -1604,6 +1609,7 @@ keyword_compat: (
| NATURAL
| NEXT
| NO
+ | NOLOGIN
| NOTNULL
| NULLS
| OBJECT
@@ -1949,6 +1955,7 @@ LIKE: L I K E;
LIMIT: L I M I T;
LIST: L I S T;
LOCAL: L O C A L;
+LOGIN: L O G I N;
MANAGE: M A N A G E;
MATCH: M A T C H;
MATCHES: M A T C H E S;
@@ -1961,6 +1968,7 @@ NANOSECONDS: N A N O S E C O N D S;
NATURAL: N A T U R A L;
NEXT: N E X T;
NO: N O;
+NOLOGIN: N O L O G I N;
NOT: N O T;
NOTNULL: N O T N U L L;
NULL: N U L L;
diff --git a/yql/essentials/sql/v1/SQLv1Antlr4.g.in b/yql/essentials/sql/v1/SQLv1Antlr4.g.in
index 89131437e9..96e450501c 100644
--- a/yql/essentials/sql/v1/SQLv1Antlr4.g.in
+++ b/yql/essentials/sql/v1/SQLv1Antlr4.g.in
@@ -824,8 +824,8 @@ alter_table_alter_index_action:
drop_table_stmt: DROP (TABLE | TABLESTORE | EXTERNAL TABLE) (IF EXISTS)? simple_table_ref;
-create_user_stmt: CREATE USER role_name create_user_option?;
-alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name);
+create_user_stmt: CREATE USER role_name create_user_option*;
+alter_user_stmt: ALTER USER role_name (WITH? create_user_option+ | RENAME TO role_name);
create_group_stmt: CREATE GROUP role_name (WITH USER role_name (COMMA role_name)* COMMA?)?;
alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_name)* COMMA? | RENAME TO role_name);
@@ -833,7 +833,9 @@ alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_n
drop_role_stmt: DROP (USER|GROUP) (IF EXISTS)? role_name (COMMA role_name)* COMMA?;
role_name: an_id_or_type | bind_parameter;
-create_user_option: ENCRYPTED? PASSWORD expr;
+create_user_option: password_option | login_option;
+password_option: ENCRYPTED? PASSWORD expr;
+login_option: LOGIN | NOLOGIN;
grant_permissions_stmt: GRANT permission_name_target ON an_id_schema (COMMA an_id_schema)* TO role_name (COMMA role_name)* COMMA? (WITH GRANT OPTION)?;
revoke_permissions_stmt: REVOKE (GRANT OPTION FOR)? permission_name_target ON an_id_schema (COMMA an_id_schema)* FROM role_name (COMMA role_name)*;
@@ -1367,6 +1369,7 @@ keyword_as_compat:
| LEGACY
| LIKE
| LOCAL
+ | LOGIN
| MANAGE
| MATCH
| MATCHES
@@ -1379,6 +1382,7 @@ keyword_as_compat:
// | NATURAL
| NEXT
| NO
+ | NOLOGIN
// | NOTNULL
| NULLS
| OBJECT
@@ -1591,6 +1595,7 @@ keyword_compat: (
| LEGACY
| LIKE
| LOCAL
+ | LOGIN
| MANAGE
| MATCH
| MATCHES
@@ -1603,6 +1608,7 @@ keyword_compat: (
| NATURAL
| NEXT
| NO
+ | NOLOGIN
| NOTNULL
| NULLS
| OBJECT
@@ -1948,6 +1954,7 @@ LIKE: L I K E;
LIMIT: L I M I T;
LIST: L I S T;
LOCAL: L O C A L;
+LOGIN: L O G I N;
MANAGE: M A N A G E;
MATCH: M A T C H;
MATCHES: M A T C H E S;
@@ -1960,6 +1967,7 @@ NANOSECONDS: N A N O S E C O N D S;
NATURAL: N A T U R A L;
NEXT: N E X T;
NO: N O;
+NOLOGIN: N O L O G I N;
NOT: N O T;
NOTNULL: N O T N U L L;
NULL: N U L L;
diff --git a/yql/essentials/sql/v1/format/sql_format.cpp b/yql/essentials/sql/v1/format/sql_format.cpp
index b330800734..b7a8d5c48d 100644
--- a/yql/essentials/sql/v1/format/sql_format.cpp
+++ b/yql/essentials/sql/v1/format/sql_format.cpp
@@ -276,6 +276,11 @@ TTokenIterator GetNextStatementBegin(TTokenIterator begin, TTokenIterator end) {
continue;
}
if (curr->Name == "SEMICOLON") {
+ auto next = SkipWS(curr + 1, end);
+ while (next != end && next->Name == "COMMENT" && curr->Line == next->Line) {
+ curr = next;
+ next = SkipWS(next + 1, end);
+ }
++curr;
break;
}
diff --git a/yql/essentials/sql/v1/format/sql_format_ut.h b/yql/essentials/sql/v1/format/sql_format_ut.h
index f44ab557c3..bd62ddf368 100644
--- a/yql/essentials/sql/v1/format/sql_format_ut.h
+++ b/yql/essentials/sql/v1/format/sql_format_ut.h
@@ -73,8 +73,13 @@ Y_UNIT_TEST(DropRole) {
Y_UNIT_TEST(CreateUser) {
TCases cases = {
- {"use plato;create user user;","USE plato;\n\nCREATE USER user;\n"},
- {"use plato;create user user encrypted password 'foo';","USE plato;\n\nCREATE USER user ENCRYPTED PASSWORD 'foo';\n"},
+ {"use plato;create user user;", "USE plato;\n\nCREATE USER user;\n"},
+ {"use plato;create user user encrypted password 'foo';", "USE plato;\n\nCREATE USER user ENCRYPTED PASSWORD 'foo';\n"},
+ {"use plato;CREATE USER user1;", "USE plato;\n\nCREATE USER user1;\n"},
+ {"use plato;create user user1 encrypted password '123' login;", "USE plato;\n\nCREATE USER user1 ENCRYPTED PASSWORD '123' LOGIN;\n"},
+ {"use plato;cREATE USER user1 PASSWORD '123' NOLOGIN;", "USE plato;\n\nCREATE USER user1 PASSWORD '123' NOLOGIN;\n"},
+ {"use plato;CREATE USER user1 LOGIN;", "USE plato;\n\nCREATE USER user1 LOGIN;\n"},
+ {"use plato;CREATE USER user1 NOLOGIN;", "USE plato;\n\nCREATE USER user1 NOLOGIN;\n"},
};
TSetup setup;
@@ -97,6 +102,7 @@ Y_UNIT_TEST(AlterUser) {
{"use plato;alter user user rename to user;","USE plato;\n\nALTER USER user RENAME TO user;\n"},
{"use plato;alter user user encrypted password 'foo';","USE plato;\n\nALTER USER user ENCRYPTED PASSWORD 'foo';\n"},
{"use plato;alter user user with encrypted password 'foo';","USE plato;\n\nALTER USER user WITH ENCRYPTED PASSWORD 'foo';\n"},
+ {"use plato;ALTER USER user1 NOLOGIN;", "USE plato;\n\nALTER USER user1 NOLOGIN;\n"},
};
TSetup setup;
@@ -211,6 +217,8 @@ Y_UNIT_TEST(NamedNode) {
"$a = (\n\tSELECT\n\t\t*\n\tFROM\n\t\t$t -- comment\n);\n"},
{"-- comment\r\r\r$a=1;",
"-- comment\r\n$a = 1;\n"},
+ {"$a=1;-- comment\n$b=2;/* comment */ /* comment */\n$c = 3;/* comment */ -- comment",
+ "$a = 1; -- comment\n$b = 2; /* comment */ /* comment */\n$c = 3; /* comment */ -- comment\n"},
};
TSetup setup;
diff --git a/yql/essentials/sql/v1/node.h b/yql/essentials/sql/v1/node.h
index fe6ddb08ac..609cd82dd4 100644
--- a/yql/essentials/sql/v1/node.h
+++ b/yql/essentials/sql/v1/node.h
@@ -1305,6 +1305,14 @@ namespace NSQLTranslationV1 {
TMaybe<TDeferredAtom> Password;
bool IsPasswordEncrypted = false;
TVector<TDeferredAtom> Roles;
+
+ enum class ETypeOfLogin {
+ Undefined,
+ Login,
+ NoLogin
+ };
+
+ ETypeOfLogin CanLogin = ETypeOfLogin::Undefined;
};
struct TSequenceParameters {
diff --git a/yql/essentials/sql/v1/select.cpp b/yql/essentials/sql/v1/select.cpp
index 1442215e15..4f5db44069 100644
--- a/yql/essentials/sql/v1/select.cpp
+++ b/yql/essentials/sql/v1/select.cpp
@@ -1869,6 +1869,10 @@ public:
return false;
}
+ if (!ctx.SimpleColumns && Columns.QualifiedAll && !columnName.Contains('.')) {
+ return false;
+ }
+
if (!Columns.IsColumnPossible(ctx, columnName)) {
return true;
}
diff --git a/yql/essentials/sql/v1/sql_query.cpp b/yql/essentials/sql/v1/sql_query.cpp
index 5252417526..a2dd6cc68a 100644
--- a/yql/essentials/sql/v1/sql_query.cpp
+++ b/yql/essentials/sql/v1/sql_query.cpp
@@ -583,7 +583,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore22: {
- // create_user_stmt: CREATE USER role_name create_user_option?;
+ // create_user_stmt: CREATE USER role_name (create_user_option)*;
Ctx.BodyPart();
auto& node = core.GetAlt_sql_stmt_core22().GetRule_create_user_stmt1();
@@ -604,9 +604,19 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
}
TMaybe<TRoleParameters> roleParams;
- if (node.HasBlock4()) {
+ const auto& options = node.GetBlock4();
+
+ if (options.size() > 0) {
roleParams.ConstructInPlace();
- if (!RoleParameters(node.GetBlock4().GetRule_create_user_option1(), *roleParams)) {
+ std::vector<TRule_create_user_option> opts;
+ opts.reserve(options.size());
+ for (const auto& opt : options) {
+ opts.push_back(opt.GetRule_create_user_option1());
+ }
+
+ bool isCreateUser = true;
+
+ if (!RoleParameters(opts, *roleParams, isCreateUser)) {
return false;
}
}
@@ -615,7 +625,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore23: {
- // alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name);
+ // alter_user_stmt: ALTER USER role_name (WITH? create_user_option+ | RENAME TO role_name);
Ctx.BodyPart();
auto& node = core.GetAlt_sql_stmt_core23().GetRule_alter_user_stmt1();
@@ -641,7 +651,17 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
switch (node.GetBlock4().Alt_case()) {
case TRule_alter_user_stmt_TBlock4::kAlt1: {
TRoleParameters roleParams;
- if (!RoleParameters(node.GetBlock4().GetAlt1().GetRule_create_user_option2(), roleParams)) {
+
+ auto options = node.GetBlock4().GetAlt1().GetBlock2();
+ std::vector<TRule_create_user_option> opts;
+ opts.reserve(options.size());
+ for (const auto& opt : options) {
+ opts.push_back(opt.GetRule_create_user_option1());
+ }
+
+ bool isCreateUser = false;
+
+ if (!RoleParameters(opts, roleParams, isCreateUser)) {
return false;
}
stmt = BuildAlterUser(pos, service, cluster, roleName, roleParams, Ctx.Scoped);
diff --git a/yql/essentials/sql/v1/sql_translation.cpp b/yql/essentials/sql/v1/sql_translation.cpp
index 61a273b260..16de88ffe0 100644
--- a/yql/essentials/sql/v1/sql_translation.cpp
+++ b/yql/essentials/sql/v1/sql_translation.cpp
@@ -3818,19 +3818,82 @@ bool TSqlTranslation::RoleNameClause(const TRule_role_name& node, TDeferredAtom&
return true;
}
-bool TSqlTranslation::RoleParameters(const TRule_create_user_option& node, TRoleParameters& result) {
- // create_user_option: ENCRYPTED? PASSWORD expr;
+bool TSqlTranslation::RoleParameters(const std::vector<TRule_create_user_option>& optionsList, TRoleParameters& result, bool isCreateUser) {
+ enum class ECreateUserOption {
+ Login,
+ Password
+ };
+
+ std::set<ECreateUserOption> used = {};
+
+ auto ParseCreateUserOption = [&used, this](const TRule_create_user_option& option, TRoleParameters& result) -> bool {
+ // create_user_option: password_option | login_option;
+ // password_option: ENCRYPTED? PASSWORD expr;
+ // login_option: LOGIN | NOLOGIN;
+
+ switch (option.Alt_case()) {
+ case TRule_create_user_option::kAltCreateUserOption1:
+ {
+ TSqlExpression expr(Ctx, Mode);
+ TNodePtr password = expr.Build(option.GetAlt_create_user_option1().GetRule_password_option1().GetRule_expr3());
+ if (!password) {
+ Error() << "Couldn't parse the password";
+ return false;
+ }
+
+ result.IsPasswordEncrypted = option.GetAlt_create_user_option1().GetRule_password_option1().HasBlock1();
+ if (!password->IsNull()) {
+ result.Password = MakeAtomFromExpression(Ctx.Pos(), Ctx, password);
+ }
+
+ if (used.contains(ECreateUserOption::Password)) {
+ Error() << "Conflicting or redundant options";
+ return false;
+ }
+
+ used.insert(ECreateUserOption::Password);
+
+ break;
+ }
+ case TRule_create_user_option::kAltCreateUserOption2:
+ {
+ if (used.contains(ECreateUserOption::Login)) {
+ Error() << "Conflicting or redundant options";
+ return false;
+ }
+
+ used.insert(ECreateUserOption::Login);
+
+ const auto token = option.GetAlt_create_user_option2().GetRule_login_option1().GetToken1().GetId();
+ if (IS_TOKEN(token, LOGIN)) {
+ result.CanLogin = TRoleParameters::ETypeOfLogin::Login;
+ } else if (IS_TOKEN(token, NOLOGIN)) {
+ result.CanLogin = TRoleParameters::ETypeOfLogin::NoLogin;
+ } else {
+ Y_ABORT("You should change implementation according to grammar changes");
+ }
+
+ break;
+ }
+ case TRule_create_user_option::ALT_NOT_SET:
+ {
+ Y_ABORT("You should change implementation according to grammar changes");
+ }
+ }
+
+ return true;
+ };
+
result = TRoleParameters{};
- TSqlExpression expr(Ctx, Mode);
- TNodePtr password = expr.Build(node.GetRule_expr3());
- if (!password) {
- return false;
+ if (isCreateUser) {
+ result.CanLogin = TRoleParameters::ETypeOfLogin::NoLogin;
}
- result.IsPasswordEncrypted = node.HasBlock1();
- if (!password->IsNull()) {
- result.Password = MakeAtomFromExpression(Ctx.Pos(), Ctx, password);
+ for (const auto& option : optionsList) {
+ if (!ParseCreateUserOption(option, result)) {
+ return false;
+ }
}
return true;
diff --git a/yql/essentials/sql/v1/sql_translation.h b/yql/essentials/sql/v1/sql_translation.h
index 325640d74a..3587b57a7a 100644
--- a/yql/essentials/sql/v1/sql_translation.h
+++ b/yql/essentials/sql/v1/sql_translation.h
@@ -242,7 +242,7 @@ protected:
bool ParseResourcePoolClassifierSettings(std::map<TString, TDeferredAtom>& result, const TRule_with_table_settings& settings);
bool ParseResourcePoolClassifierSettings(std::map<TString, TDeferredAtom>& result, std::set<TString>& toReset, const TRule_alter_resource_pool_classifier_action& alterAction);
bool RoleNameClause(const TRule_role_name& node, TDeferredAtom& result, bool allowSystemRoles);
- bool RoleParameters(const TRule_create_user_option& node, TRoleParameters& result);
+ bool RoleParameters(const std::vector<TRule_create_user_option>& optionsList, TRoleParameters& result, bool isCreateUser);
bool PermissionNameClause(const TRule_permission_name_target& node, TVector<TDeferredAtom>& result, bool withGrantOption);
bool PermissionNameClause(const TRule_permission_name& node, TDeferredAtom& result);
bool PermissionNameClause(const TRule_permission_id& node, TDeferredAtom& result);
diff --git a/yql/essentials/sql/v1/sql_ut.cpp b/yql/essentials/sql/v1/sql_ut.cpp
index 1c87beee05..efbed0e665 100644
--- a/yql/essentials/sql/v1/sql_ut.cpp
+++ b/yql/essentials/sql/v1/sql_ut.cpp
@@ -4957,6 +4957,82 @@ select FormatType($f());
"<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n");
}
+ Y_UNIT_TEST(CreateAlterUserWithLoginNoLogin) {
+ auto reqCreateUser = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1;
+ )");
+
+ UNIT_ASSERT(reqCreateUser.IsOk());
+ UNIT_ASSERT(reqCreateUser.Root);
+
+ auto reqAlterUser = SqlToYql(R"(
+ USE plato;
+ ALTER USER user1;
+ )");
+
+ UNIT_ASSERT(!reqAlterUser.IsOk());
+ UNIT_ASSERT_STRING_CONTAINS(reqAlterUser.Issues.ToString(), "Error: Unexpected token ';' : cannot match to any predicted input...");
+
+ auto reqPasswordAndLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 PASSWORD '123' LOGIN;
+ )");
+
+ UNIT_ASSERT(reqPasswordAndLogin.IsOk());
+ UNIT_ASSERT(reqPasswordAndLogin.Root);
+
+ auto reqPasswordAndNoLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 PASSWORD '123' NOLOGIN;
+ )");
+
+ UNIT_ASSERT(reqPasswordAndNoLogin.IsOk());
+ UNIT_ASSERT(reqPasswordAndNoLogin.Root);
+
+ auto reqLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 LOGIN;
+ )");
+
+ UNIT_ASSERT(reqLogin.IsOk());
+ UNIT_ASSERT(reqLogin.Root);
+
+ auto reqNoLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 NOLOGIN;
+ )");
+
+ UNIT_ASSERT(reqNoLogin.IsOk());
+ UNIT_ASSERT(reqNoLogin.Root);
+
+ auto reqLoginNoLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 LOGIN NOLOGIN;
+ )");
+
+ UNIT_ASSERT(!reqLoginNoLogin.IsOk());
+ UNIT_ASSERT_STRING_CONTAINS(reqLoginNoLogin.Issues.ToString(), "Error: Conflicting or redundant options");
+
+ auto reqAlterLoginNoLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 LOGIN;
+ ALTER USER user1 NOLOGIN;
+ )");
+
+ UNIT_ASSERT(reqAlterLoginNoLogin.IsOk());
+ UNIT_ASSERT(reqAlterLoginNoLogin.Root);
+
+ auto reqAlterLoginNoLoginWithPassword = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 LOGIN;
+ ALTER USER user1 PASSWORD '321' NOLOGIN;
+ )");
+
+ UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.IsOk());
+ UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.Root);
+ }
+
Y_UNIT_TEST(CreateAlterUserWithoutCluster) {
ExpectFailWithError("\n CREATE USER user ENCRYPTED PASSWORD 'foobar';", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n");
ExpectFailWithError("ALTER USER CURRENT_USER RENAME TO $foo;", "<main>:1:1: Error: USE statement is missing - no default cluster is selected\n");
diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.cpp b/yql/essentials/sql/v1/sql_ut_antlr4.cpp
index 705ca3d19c..c5df28bbaa 100644
--- a/yql/essentials/sql/v1/sql_ut_antlr4.cpp
+++ b/yql/essentials/sql/v1/sql_ut_antlr4.cpp
@@ -471,6 +471,82 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["EquiJoin"]);
}
+ Y_UNIT_TEST(CreateAlterUserWithLoginNoLogin) {
+ auto reqCreateUser = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1;
+ )");
+
+ UNIT_ASSERT(reqCreateUser.IsOk());
+ UNIT_ASSERT(reqCreateUser.Root);
+
+ auto reqAlterUser = SqlToYql(R"(
+ USE plato;
+ ALTER USER user1;
+ )");
+
+ UNIT_ASSERT(!reqAlterUser.IsOk());
+ UNIT_ASSERT_STRING_CONTAINS(reqAlterUser.Issues.ToString(), "Error: mismatched input ';' expecting {ENCRYPTED, LOGIN, NOLOGIN, PASSWORD, RENAME, WITH}");
+
+ auto reqPasswordAndLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 LOgin;
+ )");
+
+ UNIT_ASSERT(reqPasswordAndLogin.IsOk());
+ UNIT_ASSERT(reqPasswordAndLogin.Root);
+
+ auto reqPasswordAndNoLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 PASSWORD '123' NOLOGIN;
+ )");
+
+ UNIT_ASSERT(reqPasswordAndNoLogin.IsOk());
+ UNIT_ASSERT(reqPasswordAndNoLogin.Root);
+
+ auto reqLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 LOGIN;
+ )");
+
+ UNIT_ASSERT(reqLogin.IsOk());
+ UNIT_ASSERT(reqLogin.Root);
+
+ auto reqNoLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 NOLOGIN;
+ )");
+
+ UNIT_ASSERT(reqNoLogin.IsOk());
+ UNIT_ASSERT(reqNoLogin.Root);
+
+ auto reqLoginNoLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 LOGIN NOLOGIN;
+ )");
+
+ UNIT_ASSERT(!reqLoginNoLogin.IsOk());
+ UNIT_ASSERT_STRING_CONTAINS(reqLoginNoLogin.Issues.ToString(), "Error: Conflicting or redundant options");
+
+ auto reqAlterLoginNoLogin = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 LOGIN;
+ ALTER USER user1 NOLOGIN;
+ )");
+
+ UNIT_ASSERT(reqAlterLoginNoLogin.IsOk());
+ UNIT_ASSERT(reqAlterLoginNoLogin.Root);
+
+ auto reqAlterLoginNoLoginWithPassword = SqlToYql(R"(
+ USE plato;
+ CREATE USER user1 LOGIN;
+ ALTER USER user1 PASSWORD '321' NOLOGIN;
+ )");
+
+ UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.IsOk());
+ UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.Root);
+ }
+
Y_UNIT_TEST(JoinWithoutConcreteColumns) {
NYql::TAstParseResult res = SqlToYql(
" use plato;"
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json
index f687bca643..da8e4a69ce 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/result.json
+++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json
@@ -11222,9 +11222,9 @@
],
"test_sql2yql.test[match_recognize-all_rows_per_match]": [
{
- "checksum": "31a940b1bbcea146ae9383e278326c7a",
- "size": 6729,
- "uri": "https://{canondata_backend}/1889210/954e2f1656d98697ece5794c59acf75dd1d40612/resource.tar.gz#test_sql2yql.test_match_recognize-all_rows_per_match_/sql.yql"
+ "checksum": "ee193116db0e897c8a3fbb98a75caaac",
+ "size": 6732,
+ "uri": "https://{canondata_backend}/1889210/a0cc9e3113699f51443284ed56291923dfc3735d/resource.tar.gz#test_sql2yql.test_match_recognize-all_rows_per_match_/sql.yql"
}
],
"test_sql2yql.test[match_recognize-greedy_quantifiers]": [
@@ -12067,6 +12067,13 @@
"uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_order_by-order_by_missing_project_column_join_types_/sql.yql"
}
],
+ "test_sql2yql.test[order_by-order_by_missing_project_column_nosimple]": [
+ {
+ "checksum": "ea2472dbe1cea58561e9a506a692738f",
+ "size": 3172,
+ "uri": "https://{canondata_backend}/1599023/02c3082e8ed2b6689e833c3b6594da85e12a01be/resource.tar.gz#test_sql2yql.test_order_by-order_by_missing_project_column_nosimple_/sql.yql"
+ }
+ ],
"test_sql2yql.test[order_by-order_by_mul_columns]": [
{
"checksum": "88f079be2cb6e1e762d50772e8fbc12d",
@@ -28625,6 +28632,11 @@
"uri": "file://test_sql_format.test_order_by-order_by_missing_project_column_join_types_/formatted.sql"
}
],
+ "test_sql_format.test[order_by-order_by_missing_project_column_nosimple]": [
+ {
+ "uri": "file://test_sql_format.test_order_by-order_by_missing_project_column_nosimple_/formatted.sql"
+ }
+ ],
"test_sql_format.test[order_by-order_by_mul_columns]": [
{
"uri": "file://test_sql_format.test_order_by-order_by_mul_columns_/formatted.sql"
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-group_by_session_nopush_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-group_by_session_nopush_/formatted.sql
index 8c15f9f952..55b25c01db 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-group_by_session_nopush_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-group_by_session_nopush_/formatted.sql
@@ -17,5 +17,4 @@ FROM (
)
WHERE
ss != 100500
-;
--- should not push down
+; -- should not push down
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-add_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-add_bitcast_/formatted.sql
index 066eecc165..ef99f355e0 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-add_bitcast_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-add_bitcast_/formatted.sql
@@ -2,15 +2,12 @@ USE plato;
SELECT
1ul + 1
-;
+; -- warn
--- warn
SELECT
1u + 1
-;
+; -- warn
--- warn
SELECT
1l + 1u
-;
--- ok
+; -- ok
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-div_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-div_bitcast_/formatted.sql
index 9597182864..946e613c55 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-div_bitcast_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-div_bitcast_/formatted.sql
@@ -2,15 +2,12 @@ USE plato;
SELECT
1ul / 1
-;
+; -- warn
--- warn
SELECT
1u / 1
-;
+; -- warn
--- warn
SELECT
1l / 1u
-;
--- ok
+; -- ok
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mod_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mod_bitcast_/formatted.sql
index efa491555d..c3d6cc2b1d 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mod_bitcast_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mod_bitcast_/formatted.sql
@@ -2,15 +2,12 @@ USE plato;
SELECT
1ul % 1
-;
+; -- warn
--- warn
SELECT
1u % 1
-;
+; -- warn
--- warn
SELECT
1l % 1u
-;
--- ok
+; -- ok
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mul_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mul_bitcast_/formatted.sql
index 8fbc996355..51daa0e7ff 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mul_bitcast_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mul_bitcast_/formatted.sql
@@ -2,15 +2,12 @@ USE plato;
SELECT
1ul * 1
-;
+; -- warn
--- warn
SELECT
1u * 1
-;
+; -- warn
--- warn
SELECT
1l * 1u
-;
--- ok
+; -- ok
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-sub_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-sub_bitcast_/formatted.sql
index 8d28bd557c..b414769fd7 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-sub_bitcast_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-sub_bitcast_/formatted.sql
@@ -2,15 +2,12 @@ USE plato;
SELECT
1ul - 1
-;
+; -- warn
--- warn
SELECT
1u - 1
-;
+; -- warn
--- warn
SELECT
1l - 1u
-;
--- ok
+; -- ok
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_column_group-hint_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_column_group-hint_/formatted.sql
index ae5cb7e3dc..1c102e4f17 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_column_group-hint_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_column_group-hint_/formatted.sql
@@ -7,9 +7,8 @@ $i1 = (
Input
WHERE
a > 'a'
-);
+); -- several publish consumers with same groups
--- several publish consumers with same groups
$i2 = (
SELECT
*
@@ -17,9 +16,8 @@ $i2 = (
Input
WHERE
a > 'a1'
-);
+); -- several publish consumers with different groups
--- several publish consumers with different groups
$i3 = (
SELECT
*
@@ -27,9 +25,8 @@ $i3 = (
Input
WHERE
a < 'a2'
-);
+); -- several consumers including publish
--- several consumers including publish
$i4 = (
SELECT
*
@@ -37,9 +34,8 @@ $i4 = (
Input
WHERE
a != 'a'
-);
+); -- several publish consumers with and without groups
--- several publish consumers with and without groups
-- test column group spec normalization
INSERT INTO Output1 WITH column_groups = '{g1=[a;b;c];def=#}'
SELECT
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_compare_diff_zones_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_compare_diff_zones_/formatted.sql
index 5fe7c180ba..01eff73bc1 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_compare_diff_zones_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_compare_diff_zones_/formatted.sql
@@ -13,9 +13,8 @@ SELECT
SELECT
CAST('2000-01-01,GMT' AS tzdate) == tzdate('2000-01-01,America/Los_Angeles')
-;
+; -- same time value
--- same time value
SELECT
RemoveTimezone(CAST('2000-01-01,GMT' AS tzdate)) == RemoveTimezone(tzdate('2000-01-01,America/Los_Angeles'))
;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-cast_string_implicit_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-cast_string_implicit_/formatted.sql
index d34761cff9..3b7ae94bf9 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-cast_string_implicit_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-cast_string_implicit_/formatted.sql
@@ -1,60 +1,47 @@
SELECT
AsList('aaa', 'aaa'u)
-;
+; -- List<String>
--- List<String>
SELECT
AsList('aaa', '[1, 2, 3]'j)
-;
+; -- List<String>
--- List<String>
SELECT
AsList('aaa', '[1; 2; 3]'y)
-;
+; -- List<String>
--- List<String>
SELECT
AsList('aaa'u, 'aaa')
-;
+; -- List<String>
--- List<String>
SELECT
AsList('aaa'u, '[1, 2, 3]'j)
-;
+; -- List<Utf8>
--- List<Utf8>
SELECT
AsList('aaa'u, '[1; 2; 3]'y)
-;
+; -- List<String>
--- List<String>
SELECT
AsList('[1, 2, 3]'j, 'aaa')
-;
+; -- List<String>
--- List<String>
SELECT
AsList('[1, 2, 3]'j, 'aaa'u)
-;
+; -- List<Utf8>
--- List<Utf8>
SELECT
AsList('[1, 2, 3]'j, '[1; 2; 3]'y)
-;
+; -- List<String>
--- List<String>
SELECT
AsList('[1; 2; 3]'y, 'aaa')
-;
+; -- List<String>
--- List<String>
SELECT
AsList('[1; 2; 3]'y, 'aaa'u)
-;
+; -- List<String>
--- List<String>
SELECT
AsList('[1; 2; 3]'y, '[1, 2, 3]'j)
-;
--- List<String>
-
+; -- List<String>
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-dict_builtins_null_lookup_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-dict_builtins_null_lookup_/formatted.sql
index 053a8e0f2c..9fa9484936 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-dict_builtins_null_lookup_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-dict_builtins_null_lookup_/formatted.sql
@@ -10,9 +10,8 @@ SELECT
DictContains($d3, NULL), -- true, null is convertible to Nothing<T> for any T
DictLookup($d2, NULL), -- Nothing(String?), no such key
DictLookup($d3, NULL)
-;
+; -- Just("baz"), null is convertible to Nothing<T> for any T
--- Just("baz"), null is convertible to Nothing<T> for any T
SELECT
DictContains($t1, AsTuple(1, 'keyy')), -- false, missing key
DictContains($t1, AsTuple(1, 'key')), -- true, match
@@ -21,5 +20,4 @@ SELECT
DictContains($t3, AsTuple(NULL, 'key')), -- true, null is convertible to Nothing<T> for any T
DictLookup($t2, AsTuple(NULL, 'key')), -- Nothing(Tuple<Int32?, String>?), no such key
DictLookup($t3, AsTuple(NULL, 'key'))
-;
--- Just(AsTuple(123, "value")), null is convertible to Nothing<T> for any T
+; -- Just(AsTuple(123, "value")), null is convertible to Nothing<T> for any T
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_ansi_join_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_ansi_join_/formatted.sql
index 3dec26b8c0..cf090dac37 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_ansi_join_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_ansi_join_/formatted.sql
@@ -66,9 +66,8 @@ WHERE
optkey NOT IN $rp
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -77,9 +76,8 @@ WHERE
optkey NOT IN $rp
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -88,9 +86,8 @@ WHERE
optkey NOT IN $rp
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
-- Right is O
SELECT
optkey
@@ -100,9 +97,8 @@ WHERE
optkey NOT IN $ro
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -111,9 +107,8 @@ WHERE
optkey NOT IN $ro
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -122,9 +117,8 @@ WHERE
optkey NOT IN $ro
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
-- Right is N
SELECT
optkey
@@ -134,9 +128,8 @@ WHERE
optkey NOT IN $rn
ORDER BY
optkey
-;
+; -- []
--- []
SELECT
optkey
FROM
@@ -145,9 +138,8 @@ WHERE
optkey NOT IN $rn
ORDER BY
optkey
-;
+; -- []
--- []
SELECT
optkey
FROM
@@ -156,9 +148,8 @@ WHERE
optkey NOT IN $rn
ORDER BY
optkey
-;
+; -- []
--- []
-- 2, 4, 6, null
$extraDict = (
SELECT
@@ -182,9 +173,8 @@ WHERE
optkey != 10 AND optkey NOT IN $ro AND optkey IN $extraDict AND optkey != 4
ORDER BY
optkey
-;
+; -- [2,6]
--- [2,6]
SELECT
optkey
FROM
@@ -193,9 +183,8 @@ WHERE
optkey != 10 AND optkey NOT IN $rn AND optkey IN $extraDict AND optkey != 4
ORDER BY
optkey
-;
+; -- []
--- []
-- Empty dict
SELECT
optkey
@@ -211,6 +200,4 @@ WHERE
)
ORDER BY
optkey
-;
--- [1-10,null]
-
+; -- [1-10,null]
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_noansi_join_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_noansi_join_/formatted.sql
index 9fbe514124..4f02d53d68 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_noansi_join_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_noansi_join_/formatted.sql
@@ -66,9 +66,8 @@ WHERE
optkey NOT IN $rp
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -77,9 +76,8 @@ WHERE
optkey NOT IN $rp
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -88,9 +86,8 @@ WHERE
optkey NOT IN $rp
ORDER BY
optkey
-;
+; -- [null,2,3,4,6,8,10]
--- [null,2,3,4,6,8,10]
-- Right is O
SELECT
optkey
@@ -100,9 +97,8 @@ WHERE
optkey NOT IN $ro
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -111,9 +107,8 @@ WHERE
optkey NOT IN $ro
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -122,9 +117,8 @@ WHERE
optkey NOT IN $ro
ORDER BY
optkey
-;
+; -- [null,2,3,4,6,8,10]
--- [null,2,3,4,6,8,10]
-- Right is N
SELECT
optkey
@@ -134,9 +128,8 @@ WHERE
optkey NOT IN $rn
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -145,9 +138,8 @@ WHERE
optkey NOT IN $rn
ORDER BY
optkey
-;
+; -- [2,3,4,6,8,10]
--- [2,3,4,6,8,10]
SELECT
optkey
FROM
@@ -156,9 +148,8 @@ WHERE
optkey NOT IN $rn
ORDER BY
optkey
-;
+; -- [null,2,3,4,6,8,10]
--- [null,2,3,4,6,8,10]
-- 2, 4, 6, null
$extraDict = (
SELECT
@@ -182,9 +173,8 @@ WHERE
optkey != 10 AND optkey NOT IN $ro AND optkey IN $extraDict AND optkey != 4
ORDER BY
optkey
-;
+; -- [2,6]
--- [2,6]
SELECT
optkey
FROM
@@ -193,9 +183,8 @@ WHERE
optkey != 10 AND optkey NOT IN $rn AND optkey IN $extraDict AND optkey != 4
ORDER BY
optkey
-;
+; -- [2,6]
--- [2,6]
-- Empty dict
SELECT
optkey
@@ -211,6 +200,4 @@ WHERE
)
ORDER BY
optkey
-;
--- [1-10,null]
-
+; -- [1-10,null]
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_/formatted.sql
index 7cc64316c7..02d8df23eb 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_/formatted.sql
@@ -10,5 +10,4 @@ SELECT
(1, NULL) IN ((1, 2), (1, 3)), -- Nothing<Bool?>
(2, NULL) IN ((1, 2), (1, 3)), -- Nothing<Bool?>
(1, NULL) IN ()
-;
--- Nothing<Bool?>
+; -- Nothing<Bool?>
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_ansi_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_ansi_/formatted.sql
index 326a709927..cd79c2a96e 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_ansi_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_ansi_/formatted.sql
@@ -10,5 +10,4 @@ SELECT
(1, NULL) IN ((1, 2), (1, 3)), -- Nothing<Bool?>
(2, NULL) IN ((1, 2), (1, 3)), -- Just(false)
(1, NULL) IN ()
-;
--- Just(false)
+; -- Just(false)
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_/formatted.sql
index 37e8cecb5a..f9f38135f4 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_/formatted.sql
@@ -1,9 +1,7 @@
/* ytfile can not */
USE plato;
-PRAGMA warning('disable', '8001');
-
--- CBO_MISSING_TABLE_STATS
+PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS
PRAGMA CostBasedOptimizer = 'native';
PRAGMA yt.MapJoinLimit = '1000';
PRAGMA yt.LookupJoinLimit = '1000';
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_only_sorted_merge_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_only_sorted_merge_/formatted.sql
index 59a75e45c9..3773cd9c62 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_only_sorted_merge_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_only_sorted_merge_/formatted.sql
@@ -1,9 +1,7 @@
/* ytfile can not */
USE plato;
-PRAGMA warning('disable', '8001');
-
--- CBO_MISSING_TABLE_STATS
+PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS
PRAGMA CostBasedOptimizer = 'native';
PRAGMA yt.MapJoinLimit = '1000';
PRAGMA yt.LookupJoinLimit = '1000';
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_cbo_3_tables_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_cbo_3_tables_/formatted.sql
index ecbc9980f9..db7bf3d339 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_cbo_3_tables_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_cbo_3_tables_/formatted.sql
@@ -1,8 +1,6 @@
USE plato;
-PRAGMA warning('disable', '8001');
-
--- CBO_MISSING_TABLE_STATS
+PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS
PRAGMA CostBasedOptimizer = 'pg';
SELECT
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_left_cbo_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_left_cbo_/formatted.sql
index e3287b0ccb..f84bb6ef38 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_left_cbo_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_left_cbo_/formatted.sql
@@ -1,8 +1,6 @@
USE plato;
-PRAGMA warning('disable', '8001');
-
--- CBO_MISSING_TABLE_STATS
+PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS
PRAGMA CostBasedOptimizer = 'PG';
SELECT
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_right_cbo_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_right_cbo_/formatted.sql
index 89c9d30704..b1ab35595c 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_right_cbo_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_right_cbo_/formatted.sql
@@ -1,8 +1,6 @@
USE plato;
-PRAGMA warning('disable', '8001');
-
--- CBO_MISSING_TABLE_STATS
+PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS
PRAGMA CostBasedOptimizer = 'PG';
SELECT
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-decimal_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-decimal_/formatted.sql
index 1c8ffe23b4..a1866f61c1 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-decimal_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-decimal_/formatted.sql
@@ -84,23 +84,20 @@ FROM
@decimal
WHERE
value == CAST('6.6' AS Decimal (15, 10))
-;
+; -- Safe key filter calc
--- Safe key filter calc
SELECT
*
FROM
@decimal
WHERE
value == CAST($asIs('3.3') AS Decimal (15, 10))
-;
+; -- Unsafe key filter calc
--- Unsafe key filter calc
SELECT
*
FROM
@decimal
WHERE
value == CAST($asIs('bad') AS Decimal (15, 10))
-;
--- Unsafe key filter calc
+; -- Unsafe key filter calc
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-tzdate_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-tzdate_/formatted.sql
index db97e0a135..262e38eeec 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-tzdate_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-tzdate_/formatted.sql
@@ -77,23 +77,20 @@ FROM
@tzdate
WHERE
value == CAST('1999-01-01,Europe/Moscow' AS TzDate)
-;
+; -- Safe key filter calc
--- Safe key filter calc
SELECT
*
FROM
@tzdate
WHERE
value == CAST($asIs('2105-12-30,America/Los_Angeles') AS TzDate)
-;
+; -- Unsafe key filter calc
--- Unsafe key filter calc
SELECT
*
FROM
@tzdate
WHERE
value == CAST($asIs('bad') AS TzDate)
-;
--- Unsafe key filter calc
+; -- Unsafe key filter calc
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-uuid_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-uuid_/formatted.sql
index b4fc19dd85..5ca4a5e7ee 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-uuid_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-uuid_/formatted.sql
@@ -98,23 +98,20 @@ FROM
@uuid
WHERE
value == CAST('00000000-0000-0000-0000-100000000000' AS Uuid)
-;
+; -- Safe key filter calc
--- Safe key filter calc
SELECT
*
FROM
@uuid
WHERE
value == CAST($asIs('00000000-0000-0000-0000-200000000000') AS Uuid)
-;
+; -- Unsafe key filter calc
--- Unsafe key filter calc
SELECT
*
FROM
@uuid
WHERE
value == CAST($asIs('bad') AS Uuid)
-;
--- Unsafe key filter calc
+; -- Unsafe key filter calc
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_match_recognize-all_rows_per_match_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_match_recognize-all_rows_per_match_/formatted.sql
index edcc854cfa..ae64953618 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_match_recognize-all_rows_per_match_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_match_recognize-all_rows_per_match_/formatted.sql
@@ -1,6 +1,6 @@
-PRAGMA FeatureR010 = "prototype";
+PRAGMA FeatureR010 = 'prototype';
-$input =
+$input = (
SELECT
*
FROM
@@ -19,7 +19,7 @@ $input =
<|time: 1100, value: 5|>,
<|time: 1200, value: 0|>,
])
-;
+);
SELECT
*
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_order_by-order_by_missing_project_column_nosimple_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_order_by-order_by_missing_project_column_nosimple_/formatted.sql
new file mode 100644
index 0000000000..5bb4d66337
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_order_by-order_by_missing_project_column_nosimple_/formatted.sql
@@ -0,0 +1,46 @@
+/* syntax version 1 */
+/* postgres can not */
+/* dq can not */
+/* dqfile can not */
+/* yt can not */
+PRAGMA DisableSimpleColumns;
+
+$src = [
+ <|a: 5, b: 50, zz: 500|>,
+ <|a: 4, b: 40, zz: 400|>,
+ <|a: 3, b: 30, zz: 300|>,
+ <|a: 2, b: 20, zz: 200|>,
+ <|a: 1, b: 10, zz: 100|>,
+];
+
+$src1 = [
+ <|a: 4, f: 40|>,
+ <|a: 3, f: 30|>,
+ <|a: 2, f: 20|>,
+ <|a: 1, f: 10|>,
+];
+
+$src = (
+ SELECT
+ *
+ FROM
+ as_table($src)
+);
+
+$src1 = (
+ SELECT
+ *
+ FROM
+ as_table($src1)
+);
+
+SELECT
+ x.*
+FROM
+ $src AS x
+LEFT ONLY JOIN
+ $src1 AS y
+USING (a)
+ORDER BY
+ random(a)
+;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_params-primitives_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_params-primitives_/formatted.sql
index 19fc917dc6..0c5da65033 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_params-primitives_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_params-primitives_/formatted.sql
@@ -3,22 +3,14 @@ DECLARE $x1 AS int;
DECLARE $x2 AS Double;
DECLARE $x3 AS Bool;
DECLARE $x4 AS Float;
-DECLARE $x5 AS String;
-
--- unicode
+DECLARE $x5 AS String; -- unicode
DECLARE $x6 AS Int64;
DECLARE $x7 AS Uint64;
-DECLARE $x8 AS String;
-
--- binary
+DECLARE $x8 AS String; -- binary
DECLARE $x9 AS Utf8;
DECLARE $x10 AS Uuid;
-DECLARE $x11 AS String?;
-
--- null
-DECLARE $x12 AS String?;
-
--- not null
+DECLARE $x11 AS String?; -- null
+DECLARE $x12 AS String?; -- not null
DECLARE $x13 AS Yson;
DECLARE $x14 AS Json;
DECLARE $x15 AS datetime;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_/formatted.sql
index 08d9a3b581..0a9259aed4 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_/formatted.sql
@@ -8,9 +8,8 @@ $count = (
COUNT(*)
FROM
Input
-);
+); -- $count = 10
--- $count = 10
$var = (
SELECT
*
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_udf_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_udf_/formatted.sql
index 7a2f11fb41..7da3c0e4d7 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_udf_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_udf_/formatted.sql
@@ -11,5 +11,4 @@ FROM
TABLESAMPLE BERNOULLI (Math::Ceil(100 * $percent))
ORDER BY
key
-;
--- 100% sample
+; -- 100% sample
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-subquery_expr_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-subquery_expr_/formatted.sql
index 337f365d5e..6b7041a3f1 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-subquery_expr_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-subquery_expr_/formatted.sql
@@ -6,9 +6,8 @@ $count = (
COUNT(*)
FROM
plato.Input
-);
+); -- $count = 10
--- $count = 10
SELECT
*
FROM (
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_schema-user_schema_with_sort_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_schema-user_schema_with_sort_/formatted.sql
index 6b9523050c..38c1e0f904 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_schema-user_schema_with_sort_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_schema-user_schema_with_sort_/formatted.sql
@@ -15,9 +15,8 @@ FROM
Input1 WITH SCHEMA Struct<key: String?, subkey: String>
ORDER BY
key
-;
+; -- should reset sort
--- should reset sort
SELECT
*
FROM
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-named_args_for_script_with_posargs_reuse_args_fail_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-named_args_for_script_with_posargs_reuse_args_fail_/formatted.sql
index b1cc429950..3a7b94d707 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-named_args_for_script_with_posargs_reuse_args_fail_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-named_args_for_script_with_posargs_reuse_args_fail_/formatted.sql
@@ -26,5 +26,4 @@ SELECT
$udf(name, age, age AS age) AS val
FROM
$data
-;
--- age is reused as named after positional
+; -- age is reused as named after positional
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-wrong_args_fail_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-wrong_args_fail_/formatted.sql
index 67a6d45b69..2aeb8fc7ed 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-wrong_args_fail_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-wrong_args_fail_/formatted.sql
@@ -3,20 +3,16 @@
-- Find has optional args
SELECT
String::ReplaceAll()
-;
+; -- too few
--- too few
SELECT
String::ReplaceAll('abc')
-;
+; -- too few
--- too few
SELECT
String::ReplaceAll('abc', 'b', 2, 4)
-;
+; -- too many
--- too many
SELECT
String::ReplaceAll('abc', 'b', 2, 4, 44)
-;
--- too many
+; -- too many
diff --git a/yql/essentials/tests/sql/sql2yql/ya.make b/yql/essentials/tests/sql/sql2yql/ya.make
index f6f5df53a0..cff97cecd9 100644
--- a/yql/essentials/tests/sql/sql2yql/ya.make
+++ b/yql/essentials/tests/sql/sql2yql/ya.make
@@ -1,3 +1,5 @@
+IF (NOT OPENSOURCE)
+
PY3TEST()
TEST_SRCS(
test_sql2yql.py
@@ -42,3 +44,5 @@ NO_CHECK_IMPORTS()
END()
+ENDIF()
+
diff --git a/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.cfg b/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.cfg
new file mode 100644
index 0000000000..bb349dd8ab
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.cfg
@@ -0,0 +1 @@
+providers yt
diff --git a/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.sql b/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.sql
new file mode 100644
index 0000000000..f1434c8c08
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.sql
@@ -0,0 +1,32 @@
+/* syntax version 1 */
+/* postgres can not */
+/* dq can not */
+/* dqfile can not */
+/* yt can not */
+pragma DisableSimpleColumns;
+
+$src = [
+<|a:5, b:50, zz:500|>,
+<|a:4, b:40, zz:400|>,
+<|a:3, b:30, zz:300|>,
+<|a:2, b:20, zz:200|>,
+<|a:1, b:10, zz:100|>,
+];
+
+$src1 = [
+<|a:4, f:40|>,
+<|a:3, f:30|>,
+<|a:2, f:20|>,
+<|a:1, f:10|>,
+];
+
+
+$src = select * from as_table($src);
+$src1 = select * from as_table($src1);
+
+select
+ x.*
+from $src as x
+left only join $src1 as y
+using(a)
+order by random(a);
diff --git a/yql/essentials/tools/minirun/apply_patches.sh b/yql/essentials/tools/minirun/apply_patches.sh
new file mode 100755
index 0000000000..ca2e4f445f
--- /dev/null
+++ b/yql/essentials/tools/minirun/apply_patches.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -eu
+
+errexit() {
+ echo $1
+ exit 1
+ }
+
+echo patching
+cd ../../../../
+patch -p0 < yql/essentials/tools/minirun/patches/01_no_icu.patch || errexit "Source patching failed"
+echo done
+
+
diff --git a/yql/essentials/tools/minirun/minirun.cpp b/yql/essentials/tools/minirun/minirun.cpp
index 0d4fedfd9a..1282212bb2 100644
--- a/yql/essentials/tools/minirun/minirun.cpp
+++ b/yql/essentials/tools/minirun/minirun.cpp
@@ -11,6 +11,7 @@ public:
{
GetRunOptions().UseRepeatableRandomAndTimeProviders = true;
GetRunOptions().ResultsFormat = NYson::EYsonFormat::Pretty;
+ GetRunOptions().OptimizeLibs = false;
GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
opts.AddLongOption("ndebug", "Do not show debug info in error output").NoArgument().SetFlag(&GetRunOptions().NoDebug);
diff --git a/yql/essentials/tools/minirun/patches/01_no_icu.patch b/yql/essentials/tools/minirun/patches/01_no_icu.patch
new file mode 100644
index 0000000000..95a2723287
--- /dev/null
+++ b/yql/essentials/tools/minirun/patches/01_no_icu.patch
@@ -0,0 +1,93 @@
+--- contrib/restricted/boost/regex/ya.make (index)
++++ contrib/restricted/boost/regex/ya.make (working tree)
+@@ -16,7 +16,6 @@ VERSION(1.87.0)
+ ORIGINAL_SOURCE(https://github.com/boostorg/regex/archive/boost-1.87.0.tar.gz)
+
+ PEERDIR(
+- contrib/libs/icu
+ contrib/restricted/boost/assert
+ contrib/restricted/boost/concept_check
+ contrib/restricted/boost/config
+@@ -40,7 +39,6 @@ NO_COMPILER_WARNINGS()
+ NO_UTIL()
+
+ CFLAGS(
+- -DBOOST_HAS_ICU
+ -DBOOST_NO_CXX98_BINDERS
+ )
+
+--- contrib/restricted/thrift/ya.make (index)
++++ contrib/restricted/thrift/ya.make (working tree)
+@@ -24,7 +24,6 @@ PEERDIR(
+ contrib/libs/openssl
+ contrib/libs/zlib
+ contrib/restricted/boost/interprocess
+- contrib/restricted/boost/locale
+ contrib/restricted/boost/math
+ contrib/restricted/boost/system
+ contrib/restricted/boost/thread
+@@ -49,7 +48,6 @@ SRCS(
+ thrift/protocol/TBase64Utils.cpp
+ thrift/protocol/TDebugProtocol.cpp
+ thrift/protocol/THeaderProtocol.cpp
+- thrift/protocol/TJSONProtocol.cpp
+ thrift/protocol/TMultiplexedProtocol.cpp
+ thrift/protocol/TProtocol.cpp
+ thrift/server/TConnectedClient.cpp
+--- yql/essentials/parser/pg_wrapper/postgresql/src/backend/utils/adt/formatting.c (index)
++++ yql/essentials/parser/pg_wrapper/postgresql/src/backend/utils/adt/formatting.c (working tree)
+@@ -72,7 +72,7 @@
+ #include <wctype.h>
+
+ #ifdef USE_ICU
+-#include <unicode/ustring.h>
++#error #include <unicode/ustring.h>
+ #endif
+
+ #include "catalog/pg_collation.h"
+--- yql/essentials/parser/pg_wrapper/postgresql/src/backend/utils/adt/pg_locale.c (index)
++++ yql/essentials/parser/pg_wrapper/postgresql/src/backend/utils/adt/pg_locale.c (working tree)
+@@ -69,8 +69,8 @@
+ #include "utils/syscache.h"
+
+ #ifdef USE_ICU
+-#include <unicode/ucnv.h>
+-#include <unicode/ustring.h>
++#error #include <unicode/ucnv.h>
++#error #include <unicode/ustring.h>
+ #endif
+
+ #ifdef __GLIBC__
+--- yql/essentials/parser/pg_wrapper/postgresql/src/include/pg_config-linux.h (index)
++++ yql/essentials/parser/pg_wrapper/postgresql/src/include/pg_config-linux.h (working tree)
+@@ -707,7 +707,8 @@
+ /* #undef USE_BSD_AUTH */
+
+ /* Define to build with ICU support. (--with-icu) */
+-#define USE_ICU 1
++//#define USE_ICU 1
++#undef USE_ICU
+
+ /* Define to 1 to build with LDAP support. (--with-ldap) */
+ /* #undef USE_LDAP */
+--- yql/essentials/parser/pg_wrapper/postgresql/src/include/utils/pg_locale.h (index)
++++ yql/essentials/parser/pg_wrapper/postgresql/src/include/utils/pg_locale.h (working tree)
+@@ -16,7 +16,7 @@
+ #include <xlocale.h>
+ #endif
+ #ifdef USE_ICU
+-#include <unicode/ucol.h>
++#error include <unicode/ucol.h>
+ #endif
+
+ #ifdef USE_ICU
+--- yql/essentials/parser/pg_wrapper/ya.make (index)
++++ yql/essentials/parser/pg_wrapper/ya.make (working tree)
+@@ -95,7 +95,6 @@ PEERDIR(
+ yql/essentials/types/dynumber
+ yql/essentials/types/uuid
+
+- contrib/libs/icu
+ contrib/libs/libc_compat
+ contrib/libs/libxml
+ contrib/libs/lz4
diff --git a/yql/essentials/tools/udf_resolver/udf_resolver.cpp b/yql/essentials/tools/udf_resolver/udf_resolver.cpp
index 8d2b8ec6cd..5b8717bfe3 100644
--- a/yql/essentials/tools/udf_resolver/udf_resolver.cpp
+++ b/yql/essentials/tools/udf_resolver/udf_resolver.cpp
@@ -456,6 +456,7 @@ int main(int argc, char **argv) {
Allow(stat),
#endif
Allow(sysinfo),
+ Allow(sigaltstack),
Allow(uname),
#ifndef _arm64_
Allow(unlink),
diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
index 6e6167582f..2db2f62607 100644
--- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
+++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
@@ -573,7 +573,7 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
moduleResolver = std::make_shared<TModuleResolver>(std::move(modules), ctx.NextUniqueId, ClusterMapping_, RunOptions_.SqlFlags, RunOptions_.Mode >= ERunMode::Validate);
} else {
- if (!GetYqlDefaultModuleResolver(ctx, moduleResolver, ClusterMapping_, RunOptions_.Mode >= ERunMode::Validate)) {
+ if (!GetYqlDefaultModuleResolver(ctx, moduleResolver, ClusterMapping_, RunOptions_.OptimizeLibs && RunOptions_.Mode >= ERunMode::Validate)) {
*RunOptions_.ErrStream << "Errors loading default YQL libraries:" << Endl;
ctx.IssueManager.GetIssues().PrintTo(*RunOptions_.ErrStream);
return -1;
diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.h b/yql/essentials/tools/yql_facade_run/yql_facade_run.h
index f5e1fd5bd2..b586370c2d 100644
--- a/yql/essentials/tools/yql_facade_run/yql_facade_run.h
+++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.h
@@ -127,6 +127,7 @@ public:
bool EnableResultPosition = false;
bool EnableCredentials = false;
bool EnableQPlayer = false;
+ bool OptimizeLibs = true;
void Parse(int argc, const char *argv[]);
diff --git a/yql/essentials/udfs/common/digest/digest_udf.cpp b/yql/essentials/udfs/common/digest/digest_udf.cpp
index 491fe7a66c..1925052200 100644
--- a/yql/essentials/udfs/common/digest/digest_udf.cpp
+++ b/yql/essentials/udfs/common/digest/digest_udf.cpp
@@ -26,6 +26,68 @@ using namespace NKikimr;
using namespace NUdf;
namespace {
+ enum EDigestType {
+ CRC32C, CRC64, FNV32, FNV64, MURMUR, MURMUR32, MURMUR2A, MURMUR2A32, CITY
+ };
+ const char* DigestNames[] = {
+ "Crc32c", "Crc64", "Fnv32", "Fnv64", "MurMurHash", "MurMurHash32", "MurMurHash2A", "MurMurHash2A32", "CityHash"
+ };
+
+ template<typename TResult>
+ using TDigestGenerator = TResult(const TStringRef&, TMaybe<TResult> init);
+
+ template<EDigestType DigestType, typename TResult, TDigestGenerator<TResult>* Generator>
+ class TDigestFunctionUdf: public TBoxedValue {
+ public:
+ TDigestFunctionUdf(TSourcePosition pos) : Pos_(pos) {}
+
+ static TStringRef Name() {
+ static TString name = DigestNames[DigestType];
+ return TStringRef(name);
+ }
+
+ static bool DeclareSignature(
+ const TStringRef& name,
+ TType*,
+ IFunctionTypeInfoBuilder& builder,
+ bool typesOnly)
+ {
+ if (Name() != name) {
+ return false;
+ }
+
+ auto args = builder.Args();
+ args->Add(builder.SimpleType<char *>()).Flags(ICallablePayload::TArgumentFlags::AutoMap);
+ args->Add(builder.Optional()->Item(builder.SimpleType<TResult>()).Build()).Name("Init");
+ args->Done();
+ builder.OptionalArgs(1);
+ builder.Returns(builder.SimpleType<TResult>());
+
+ if (!typesOnly) {
+ builder.Implementation(new TDigestFunctionUdf<DigestType, TResult, Generator>(GetSourcePosition(builder)));
+ }
+
+ return true;
+ }
+
+ private:
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const final try {
+ TMaybe<TResult> init = Nothing();
+ if (auto val = args[1]) {
+ init = val.Get<TResult>();
+ }
+ return TUnboxedValuePod(Generator(args[0].AsStringRef(), init));
+ } catch (const std ::exception&) {
+ TStringBuilder sb;
+ sb << Pos_ << " ";
+ sb << CurrentExceptionMessage();
+ sb << Endl << "[" << TStringBuf(Name()) << "]";
+ UdfTerminate(sb.c_str());
+ }
+
+ TSourcePosition Pos_;
+ };
+
SIMPLE_STRICT_UDF(TCrc32c, ui32(TAutoMap<char*>)) {
Y_UNUSED(valueBuilder);
const auto& inputRef = args[0].AsStringRef();
@@ -33,68 +95,65 @@ namespace {
return TUnboxedValuePod(hash);
}
- SIMPLE_STRICT_UDF(TCrc64, ui64(TAutoMap<char*>)) {
- Y_UNUSED(valueBuilder);
- const auto& inputRef = args[0].AsStringRef();
- ui64 hash = crc64(inputRef.Data(), inputRef.Size());
- return TUnboxedValuePod(hash);
- }
-
- SIMPLE_STRICT_UDF(TFnv32, ui32(TAutoMap<char*>)) {
- Y_UNUSED(valueBuilder);
- const auto& inputRef = args[0].AsStringRef();
- ui32 hash = FnvHash<ui32>(inputRef.Data(), inputRef.Size());
- return TUnboxedValuePod(hash);
- }
+ using TCrc64 = TDigestFunctionUdf<CRC64, ui64, [](auto& inputRef, auto init) {
+ return crc64(inputRef.Data(), inputRef.Size(), init.GetOrElse(CRC64INIT));
+ }>;
- SIMPLE_STRICT_UDF(TFnv64, ui64(TAutoMap<char*>)) {
- Y_UNUSED(valueBuilder);
- const auto& inputRef = args[0].AsStringRef();
- ui64 hash = FnvHash<ui64>(inputRef.Data(), inputRef.Size());
- return TUnboxedValuePod(hash);
- }
+ using TFnv32 = TDigestFunctionUdf<FNV32, ui32, [](auto& inputRef, auto init) {
+ if (init) {
+ return FnvHash<ui32>(inputRef.Data(), inputRef.Size(), *init);
+ } else {
+ return FnvHash<ui32>(inputRef.Data(), inputRef.Size());
+ }
+ }>;
- SIMPLE_STRICT_UDF(TMurMurHash, ui64(TAutoMap<char*>)) {
- Y_UNUSED(valueBuilder);
- const auto& inputRef = args[0].AsStringRef();
- ui64 hash = MurmurHash<ui64>(inputRef.Data(), inputRef.Size());
- return TUnboxedValuePod(hash);
- }
+ using TFnv64 = TDigestFunctionUdf<FNV64, ui64, [](auto& inputRef, auto init) {
+ if (init) {
+ return FnvHash<ui64>(inputRef.Data(), inputRef.Size(), *init);
+ } else {
+ return FnvHash<ui64>(inputRef.Data(), inputRef.Size());
+ }
+ }>;
- SIMPLE_STRICT_UDF(TMurMurHash32, ui32(TAutoMap<char*>)) {
- Y_UNUSED(valueBuilder);
- const auto& inputRef = args[0].AsStringRef();
- ui32 hash = MurmurHash<ui32>(inputRef.Data(), inputRef.Size());
- return TUnboxedValuePod(hash);
- }
+ using TMurMurHash = TDigestFunctionUdf<MURMUR, ui64, [](auto& inputRef, auto init) {
+ if (init) {
+ return MurmurHash<ui64>(inputRef.Data(), inputRef.Size(), *init);
+ } else {
+ return MurmurHash<ui64>(inputRef.Data(), inputRef.Size());
+ }
+ }>;
- SIMPLE_STRICT_UDF(TMurMurHash2A, ui64(TAutoMap<char*>)) {
- Y_UNUSED(valueBuilder);
- const auto& inputRef = args[0].AsStringRef();
- ui64 hash = TMurmurHash2A<ui64>{}.Update(inputRef.Data(), inputRef.Size()).Value();
- return TUnboxedValuePod(hash);
- }
+ using TMurMurHash32 = TDigestFunctionUdf<MURMUR32, ui32, [] (auto& inputRef, auto init) {
+ if (init) {
+ return MurmurHash<ui32>(inputRef.Data(), inputRef.Size(), *init);
+ } else {
+ return MurmurHash<ui32>(inputRef.Data(), inputRef.Size());
+ }
+ }>;
- SIMPLE_STRICT_UDF(TMurMurHash2A32, ui32(TAutoMap<char*>)) {
- Y_UNUSED(valueBuilder);
- const auto& inputRef = args[0].AsStringRef();
- ui32 hash = TMurmurHash2A<ui32>{}.Update(inputRef.Data(), inputRef.Size()).Value();
- return TUnboxedValuePod(hash);
- }
+ using TMurMurHash2A = TDigestFunctionUdf<MURMUR2A, ui64, [] (auto& inputRef, auto init) {
+ if (init) {
+ return TMurmurHash2A<ui64>{*init}.Update(inputRef.Data(), inputRef.Size()).Value();
+ } else {
+ return TMurmurHash2A<ui64>{}.Update(inputRef.Data(), inputRef.Size()).Value();
+ }
+ }>;
- SIMPLE_STRICT_UDF_WITH_OPTIONAL_ARGS(TCityHash, ui64(TAutoMap<char*>, TOptional<ui64>), 1) {
- Y_UNUSED(valueBuilder);
- const auto& inputRef = args[0].AsStringRef();
- ui64 hash;
- if (args[1]) {
- hash = CityHash64WithSeed(inputRef.Data(), inputRef.Size(), args[1].Get<ui64>());
+ using TMurMurHash2A32 = TDigestFunctionUdf<MURMUR2A32, ui32, [] (auto& inputRef, auto init) {
+ if (init) {
+ return TMurmurHash2A<ui32>{*init}.Update(inputRef.Data(), inputRef.Size()).Value();
} else {
- hash = CityHash64(inputRef.Data(), inputRef.Size());
+ return TMurmurHash2A<ui32>{}.Update(inputRef.Data(), inputRef.Size()).Value();
}
- return TUnboxedValuePod(hash);
- }
+ }>;
- using TUi64Pair = NUdf::TTuple<ui64, ui64>;
+ using TCityHash = TDigestFunctionUdf<CITY, ui64, [] (auto& inputRef, auto init) {
+ if (init) {
+ return CityHash64WithSeed(inputRef.Data(), inputRef.Size(), *init);
+ } else {
+ return CityHash64(inputRef.Data(), inputRef.Size());
+ }
+ }>;
class TCityHash128: public TBoxedValue {
public:
diff --git a/yql/essentials/udfs/common/digest/test/canondata/result.json b/yql/essentials/udfs/common/digest/test/canondata/result.json
index fb6112fc5b..5056bb47cb 100644
--- a/yql/essentials/udfs/common/digest/test/canondata/result.json
+++ b/yql/essentials/udfs/common/digest/test/canondata/result.json
@@ -3,5 +3,10 @@
{
"uri": "file://test.test_Basic_/results.txt"
}
+ ],
+ "test.test[Init]": [
+ {
+ "uri": "file://test.test_Init_/results.txt"
+ }
]
}
diff --git a/yql/essentials/udfs/common/digest/test/canondata/test.test_Init_/results.txt b/yql/essentials/udfs/common/digest/test/canondata/test.test_Init_/results.txt
new file mode 100644
index 0000000000..53faaf453b
--- /dev/null
+++ b/yql/essentials/udfs/common/digest/test/canondata/test.test_Init_/results.txt
@@ -0,0 +1,202 @@
+[
+ {
+ "Write" = [
+ {
+ "Type" = [
+ "ListType";
+ [
+ "StructType";
+ [
+ [
+ "column0";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ];
+ [
+ "column1";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ];
+ [
+ "column2";
+ [
+ "DataType";
+ "Uint32"
+ ]
+ ];
+ [
+ "column3";
+ [
+ "DataType";
+ "Uint32"
+ ]
+ ];
+ [
+ "column4";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ];
+ [
+ "column5";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ];
+ [
+ "column6";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ];
+ [
+ "column7";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ];
+ [
+ "column8";
+ [
+ "DataType";
+ "Uint32"
+ ]
+ ];
+ [
+ "column9";
+ [
+ "DataType";
+ "Uint32"
+ ]
+ ];
+ [
+ "column10";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ];
+ [
+ "column11";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ];
+ [
+ "column12";
+ [
+ "DataType";
+ "Uint32"
+ ]
+ ];
+ [
+ "column13";
+ [
+ "DataType";
+ "Uint32"
+ ]
+ ];
+ [
+ "column14";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ];
+ [
+ "column15";
+ [
+ "DataType";
+ "Uint64"
+ ]
+ ]
+ ]
+ ]
+ ];
+ "Data" = [
+ [
+ "12517938706220930219";
+ "12517938706220930219";
+ "151308058";
+ "151308058";
+ "854320535119994";
+ "854320535119994";
+ "9815316673246213022";
+ "9815316673246213022";
+ "3549562424";
+ "3549562424";
+ "7091385679591462881";
+ "7091385679591462881";
+ "2867599578";
+ "2867599578";
+ "9781551444137168419";
+ "9781551444137168419"
+ ];
+ [
+ "7439640649521800617";
+ "7439640649521800617";
+ "151308057";
+ "151308057";
+ "854320535119993";
+ "854320535119993";
+ "3140806664558319421";
+ "3140806664558319421";
+ "4067583852";
+ "4067583852";
+ "8498979662292624923";
+ "8498979662292624923";
+ "1911426935";
+ "1911426935";
+ "16387792208989722131";
+ "16387792208989722131"
+ ];
+ [
+ "9402898472742411560";
+ "9402898472742411560";
+ "151308056";
+ "151308056";
+ "854320535119992";
+ "854320535119992";
+ "5511627967124305793";
+ "5511627967124305793";
+ "4218195213";
+ "4218195213";
+ "4001281192876778049";
+ "4001281192876778049";
+ "1399730784";
+ "1399730784";
+ "3425497360513913945";
+ "3425497360513913945"
+ ];
+ [
+ "777";
+ "777";
+ "777";
+ "777";
+ "777";
+ "777";
+ "16843636611330352828";
+ "16843636611330352828";
+ "2954663174";
+ "2954663174";
+ "13671058521579127884";
+ "13671058521579127884";
+ "2722328547";
+ "2722328547";
+ "15226978636637575808";
+ "15226978636637575808"
+ ]
+ ]
+ }
+ ]
+ }
+] \ No newline at end of file
diff --git a/yql/essentials/udfs/common/digest/test/cases/Init.cfg b/yql/essentials/udfs/common/digest/test/cases/Init.cfg
new file mode 100644
index 0000000000..62e2c58706
--- /dev/null
+++ b/yql/essentials/udfs/common/digest/test/cases/Init.cfg
@@ -0,0 +1 @@
+in plato.Input Basic.in \ No newline at end of file
diff --git a/yql/essentials/udfs/common/digest/test/cases/Init.sql b/yql/essentials/udfs/common/digest/test/cases/Init.sql
new file mode 100644
index 0000000000..7805314d87
--- /dev/null
+++ b/yql/essentials/udfs/common/digest/test/cases/Init.sql
@@ -0,0 +1,11 @@
+SELECT
+ Digest::Crc64(key, 777), Digest::Crc64(key, 777 AS Init),
+ Digest::Fnv32(key, 777), Digest::Fnv32(key, 777 AS Init),
+ Digest::Fnv64(key, 777), Digest::Fnv64(key, 777 AS Init),
+ Digest::MurMurHash(key, 777), Digest::MurMurHash(key, 777 AS Init),
+ Digest::MurMurHash32(key, 777), Digest::MurMurHash32(key, 777 AS Init),
+ Digest::MurMurHash2A(key, 777), Digest::MurMurHash2A(key, 777 AS Init),
+ Digest::MurMurHash2A32(key, 777), Digest::MurMurHash2A32(key, 777 AS Init),
+ Digest::CityHash(key, 777), Digest::CityHash(key, 777 AS Init),
+
+FROM Input; \ No newline at end of file
diff --git a/yql/essentials/udfs/common/hyperscan/test/canondata/test.test_Error_/extracted b/yql/essentials/udfs/common/hyperscan/test/canondata/test.test_Error_/extracted
index d3707ae88c..21ca8cec79 100644
--- a/yql/essentials/udfs/common/hyperscan/test/canondata/test.test_Error_/extracted
+++ b/yql/essentials/udfs/common/hyperscan/test/canondata/test.test_Error_/extracted
@@ -3,6 +3,6 @@
<tmp_path>/program.sql:<main>:8:1: Error: Execution of node: YtMap!
SELECT $match(value) AS match FROM Input;
^
- <tmp_path>/program.sql:<main>:6:21: Error: library/cpp/regex/hyperscan/hyperscan.cpp:102: Failed to compile regex: ^*$. Error message (hyperscan): Invalid repeat at index 1.
+ <tmp_path>/program.sql:<main>:6:21: Error: library/cpp/regex/hyperscan/hyperscan.cpp:92: Failed to compile regex: ^*$. Error message (hyperscan): Invalid repeat at index 1.
$match = Hyperscan::Match("*");
^ \ No newline at end of file
diff --git a/yql/essentials/udfs/common/yson2/test/canondata/test.test_From_/results.txt b/yql/essentials/udfs/common/yson2/test/canondata/test.test_From_/results.txt
index d1e62592cc..ddddb74752 100644
--- a/yql/essentials/udfs/common/yson2/test/canondata/test.test_From_/results.txt
+++ b/yql/essentials/udfs/common/yson2/test/canondata/test.test_From_/results.txt
@@ -32,7 +32,7 @@
"column3";
[
"DataType";
- "Yson"
+ "Bool"
]
];
[
@@ -118,6 +118,13 @@
"DataType";
"Yson"
]
+ ];
+ [
+ "column16";
+ [
+ "DataType";
+ "Yson"
+ ]
]
]
]
@@ -126,6 +133,7 @@
[
%true;
%true;
+ %true;
%false;
{
"$type" = "boolean";
diff --git a/yql/essentials/udfs/common/yson2/test/cases/From.sql b/yql/essentials/udfs/common/yson2/test/cases/From.sql
index 2a1f6ed15a..97b0fff53c 100644
--- a/yql/essentials/udfs/common/yson2/test/cases/From.sql
+++ b/yql/essentials/udfs/common/yson2/test/cases/From.sql
@@ -1,6 +1,7 @@
/* syntax version 1 */
select
Yson::IsEntity(Yson::From(NULL)),
+Yson::IsEntity(Yson::From(Void())),
Yson::IsEntity(Yson::Parse(Yson("#"))),
Yson::IsEntity(Yson::Parse(Yson("1"))),
diff --git a/yql/essentials/udfs/common/yson2/yson2_udf.cpp b/yql/essentials/udfs/common/yson2/yson2_udf.cpp
index 76dbe07c55..997b0cecf1 100644
--- a/yql/essentials/udfs/common/yson2/yson2_udf.cpp
+++ b/yql/essentials/udfs/common/yson2/yson2_udf.cpp
@@ -795,6 +795,7 @@ protected:
static const TType* CheckType(const ITypeInfoHelper::TPtr typeHelper, const TType* shape) {
switch (const auto kind = typeHelper->GetTypeKind(shape)) {
case ETypeKind::Null:
+ case ETypeKind::Void:
case ETypeKind::EmptyList:
case ETypeKind::EmptyDict:
return MoreTypesAllowed ? nullptr : shape;
diff --git a/yql/essentials/utils/log/profile.cpp b/yql/essentials/utils/log/profile.cpp
index 4dd412b8f6..130bb05a4a 100644
--- a/yql/essentials/utils/log/profile.cpp
+++ b/yql/essentials/utils/log/profile.cpp
@@ -17,27 +17,32 @@ TProfilingScope::~TProfilingScope() {
return;
}
- double elapsed = static_cast<double>(::MicroSeconds() - StartedAt_);
- TStringBuf unit("us");
- if (elapsed > 1000000) {
- elapsed /= 1000000;
- unit = TStringBuf("s");
- } else if (elapsed > 1000) {
- elapsed /= 1000;
- unit = TStringBuf("ms");
- }
-
- auto doLog = [&]() {
- YQL_PERF_LOG(Level_, File_, Line_)
- << TStringBuf("Execution of [") << Name_
- << TStringBuf("] took ") << Prec(elapsed, 3) << unit;
- };
-
- if (!LogCtxPath_.first.empty() || !LogCtxPath_.second.empty()) {
- YQL_LOG_CTX_ROOT_SESSION_SCOPE(LogCtxPath_);
- doLog();
- } else {
- doLog();
+ try {
+ double elapsed = static_cast<double>(::MicroSeconds() - StartedAt_);
+ TStringBuf unit("us");
+ if (elapsed > 1000000) {
+ elapsed /= 1000000;
+ unit = TStringBuf("s");
+ } else if (elapsed > 1000) {
+ elapsed /= 1000;
+ unit = TStringBuf("ms");
+ }
+
+ auto doLog = [&]() {
+ YQL_PERF_LOG(Level_, File_, Line_)
+ << TStringBuf("Execution of [") << Name_
+ << TStringBuf("] took ") << Prec(elapsed, 3) << unit;
+ };
+
+ if (!LogCtxPath_.first.empty() || !LogCtxPath_.second.empty()) {
+ YQL_LOG_CTX_ROOT_SESSION_SCOPE(LogCtxPath_);
+ doLog();
+ } else {
+ doLog();
+ }
+ } catch (const std::bad_alloc&) {
+ // no memory, we will not write anything
+ Y_ABORT("No memory");
}
}