diff options
author | Alexander Smirnov <alex@ydb.tech> | 2024-12-24 22:01:20 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2024-12-24 22:01:20 +0000 |
commit | bd0e2de0b1035962a4d5b9e847eaa6508fad7fcf (patch) | |
tree | 79878ca309f9f7fada064f9b78b4223af4635f28 /yql/essentials | |
parent | be43a4691ebdd4dbe260a8d77df4cd8423b14c05 (diff) | |
parent | e6bd80ded127cd064560f7ea471974b602770cb1 (diff) | |
download | ydb-bd0e2de0b1035962a4d5b9e847eaa6508fad7fcf.tar.gz |
Merge branch 'PR'
Diffstat (limited to 'yql/essentials')
85 files changed, 1388 insertions, 437 deletions
diff --git a/yql/essentials/core/cbo/cbo_hints.cpp b/yql/essentials/core/cbo/cbo_hints.cpp index 630bb3e6b0..3a022b36d0 100644 --- a/yql/essentials/core/cbo/cbo_hints.cpp +++ b/yql/essentials/core/cbo/cbo_hints.cpp @@ -6,9 +6,16 @@ using namespace NYql; +TString ToLower(TString s) { + for (char& c: s) { + c = std::tolower(c); + } + return s; +} + class TOptimizerHintsParser { public: - TOptimizerHintsParser(const TString& text) + TOptimizerHintsParser(const TString& text) : Pos(-1) , Size(static_cast<i32>(text.size()) - 1) , Text(text) @@ -45,9 +52,9 @@ private: return labels; } - void JoinType() { + void JoinType() { i32 beginPos = Pos + 1; - + Keyword({"("}); i32 labelsBeginPos = Pos + 1; @@ -59,17 +66,17 @@ private: labels.pop_back(); Keyword({")"}); - + TVector<EJoinAlgoType> joinAlgos = {EJoinAlgoType::GraceJoin, EJoinAlgoType::LookupJoin, EJoinAlgoType::MapJoin}; - TVector<TString> joinAlgosStr = {"Shuffle", "Lookup", "Broadcast"}; + TVector<TString> joinAlgosStr = {"shuffle", "lookup", "broadcast"}; for (const auto& [JoinType, joinAlgoStr]: Zip(joinAlgos, joinAlgosStr)) { - if (reqJoinAlgoStr == joinAlgoStr) { + if (ToLower(reqJoinAlgoStr) == joinAlgoStr) { Hints.JoinAlgoHints->PushBack(std::move(labels), JoinType, "JoinType" + Text.substr(beginPos, Pos - beginPos + 1)); return; } } - + ParseError(Sprintf("Unknown JoinType: '%s', supported algos: [%s]", reqJoinAlgoStr.c_str(), JoinSeq(", ", joinAlgosStr).c_str()), Pos - reqJoinAlgoStr.size()); Y_UNREACHABLE(); } @@ -82,7 +89,7 @@ private: Keyword({")"}); Hints.JoinOrderHints->PushBack( - std::move(joinOrderHintTree), + std::move(joinOrderHintTree), leading? "Leading" : "JoinOrder" + Text.substr(beginPos, Pos - beginPos + 1) ); } @@ -100,7 +107,7 @@ private: auto join = JoinOrderLabels(); Keyword({")"}); return join; - } + } ParseError(Sprintf("JoinOrder args must be either a relation, either a join, example of the format: JoinOrder(t1 (t2 t3))"), Pos); Y_UNREACHABLE(); @@ -116,7 +123,7 @@ private: char sign = signStr[0]; auto value = Number(); Keyword({")"}); - + TCardinalityHints::ECardOperation op; switch (sign) { case '+': { op = TCardinalityHints::ECardOperation::Add; break; } @@ -138,7 +145,7 @@ private: } TString Label() { - return Term(Letters() | Digits()); + return Term(LabelAllowedSymbols()); } std::optional<TString> MaybeLabel() { @@ -170,7 +177,7 @@ private: char Char(unsigned char c) { std::bitset<256> allowed; - allowed[c] = 1; + allowed[c] = 1; return Char(allowed); } @@ -181,7 +188,7 @@ private: } return Char(allowed); } - + char Char(const std::bitset<256>& allowedSymbols = {}) { Y_ENSURE(Pos < Size, Sprintf("Expected [%s], but got end of the string.", "")); @@ -281,6 +288,12 @@ private: return res; } + constexpr std::bitset<256> LabelAllowedSymbols() { + auto labelSymbols = Digits() | Letters(); + labelSymbols['_'] = 1; + return labelSymbols; + } + void SkipWhiteSpaces() { for (; Pos < Size && isspace(Text[Pos + 1]); ++Pos) { } diff --git a/yql/essentials/core/services/mounts/ya.make b/yql/essentials/core/services/mounts/ya.make index 660204f365..f6180eef6e 100644 --- a/yql/essentials/core/services/mounts/ya.make +++ b/yql/essentials/core/services/mounts/ya.make @@ -9,6 +9,7 @@ PEERDIR( library/cpp/resource yql/essentials/core/user_data yql/essentials/core + yql/essentials/utils/log ) YQL_LAST_ABI_VERSION() diff --git a/yql/essentials/core/services/mounts/yql_mounts.cpp b/yql/essentials/core/services/mounts/yql_mounts.cpp index 7d4052e8c8..683a7fe84b 100644 --- a/yql/essentials/core/services/mounts/yql_mounts.cpp +++ b/yql/essentials/core/services/mounts/yql_mounts.cpp @@ -1,6 +1,7 @@ #include "yql_mounts.h" #include <yql/essentials/core/yql_library_compiler.h> +#include <yql/essentials/utils/log/profile.h> #include <library/cpp/resource/resource.h> @@ -117,6 +118,7 @@ namespace NYql { bool optimizeLibraries, THolder<TExprContext> ownedCtx) { + YQL_PROFILE_FUNC(DEBUG); auto ctx = rawCtx ? rawCtx : ownedCtx.Get(); Y_ENSURE(ctx); TUserDataTable mounts; diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp index 3b6a626e07..d448c7ed54 100644 --- a/yql/essentials/core/type_ann/type_ann_core.cpp +++ b/yql/essentials/core/type_ann/type_ann_core.cpp @@ -5930,6 +5930,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (const auto status = ConvertChildrenToType(input, commonType, ctx.Expr); status != IGraphTransformer::TStatus::Ok) return status; + if (warn) { + return IGraphTransformer::TStatus::Repeat; + } const auto dictType = IsSet ? ctx.Expr.MakeType<TDictExprType>(commonType, ctx.Expr.MakeType<TVoidExprType>()): ctx.Expr.MakeType<TDictExprType>(commonType->Cast<TTupleExprType>()->GetItems().front(), commonType->Cast<TTupleExprType>()->GetItems().back()); @@ -5943,7 +5946,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - return warn ? IGraphTransformer::TStatus::Repeat : IGraphTransformer::TStatus::Ok; + return IGraphTransformer::TStatus::Ok; } IGraphTransformer::TStatus DictFromKeysWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { diff --git a/yql/essentials/core/ut_common/ya.make b/yql/essentials/core/ut_common/ya.make deleted file mode 100644 index 04c59aaf3d..0000000000 --- a/yql/essentials/core/ut_common/ya.make +++ /dev/null @@ -1,24 +0,0 @@ -IF (NOT OPENSOURCE) - -LIBRARY() - -SRCS( - yql_ut_common.cpp - yql_ut_common.h -) - -PEERDIR( - yql/essentials/core - yql/essentials/core/expr_nodes - contrib/ydb/library/yql/dq/proto - contrib/ydb/library/yql/dq/expr_nodes -) - - -YQL_LAST_ABI_VERSION() - - -END() - -ENDIF() - diff --git a/yql/essentials/core/ut_common/yql_ut_common.cpp b/yql/essentials/core/ut_common/yql_ut_common.cpp deleted file mode 100644 index cef3f2723c..0000000000 --- a/yql/essentials/core/ut_common/yql_ut_common.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "yql_ut_common.h" - -#include <library/cpp/random_provider/random_provider.h> -#include <library/cpp/time_provider/time_provider.h> - -#include <util/generic/guid.h> -#include <util/system/user.h> -#include <util/stream/file.h> - -namespace NYql { - -TTestTablesMapping::TTestTablesMapping() - : TmpInput() - , TmpInputAttr(TmpInput.Name() + ".attr") - , TmpOutput() - , TmpOutputAttr(TmpOutput.Name() + ".attr") -{ - { - TUnbufferedFileOutput tmpInput(TmpInput); - tmpInput << "{\"key\"=\"\";\"subkey\"=\"\";\"value\"=\"\"}" << Endl; - TUnbufferedFileOutput tmpInputAttr(TmpInputAttr); - tmpInputAttr << "{\"_yql_row_spec\" = {\"Type\" = [\"StructType\";[" - << "[\"key\";[\"DataType\";\"String\"]];" - << "[\"subkey\";[\"DataType\";\"String\"]];" - << "[\"value\";[\"DataType\";\"String\"]]" - << "]]}}" << Endl; - } - insert(std::make_pair("yt.plato.Input", TmpInput.Name())); - - { - TUnbufferedFileOutput tmpOutput(TmpOutput); - tmpOutput << "{\"key\"=\"\";\"subkey\"=\"\";\"value\"=\"\"}" << Endl; - TUnbufferedFileOutput tmpOutputAttr(TmpOutputAttr); - tmpOutputAttr << "{\"_yql_row_spec\" = {\"Type\" = [\"StructType\";[" - << "[\"key\";[\"DataType\";\"String\"]];" - << "[\"subkey\";[\"DataType\";\"String\"]];" - << "[\"value\";[\"DataType\";\"String\"]]" - << "]]}}" << Endl; - } - insert(std::make_pair("yt.plato.Output", TmpOutput.Name())); -} - -void InitializeYtGateway(IYtGateway::TPtr gateway, TYtState::TPtr ytState) { - ytState->SessionId = CreateGuidAsString(); - gateway->OpenSession( - IYtGateway::TOpenSessionOptions(ytState->SessionId) - .UserName(GetUsername()) - .ProgressWriter(&NullProgressWriter) - .OperationOptions(TYqlOperationOptions()) - .RandomProvider(CreateDeterministicRandomProvider(1)) - .TimeProvider(CreateDeterministicTimeProvider(10000000)) - ); -} - -} diff --git a/yql/essentials/core/ut_common/yql_ut_common.h b/yql/essentials/core/ut_common/yql_ut_common.h deleted file mode 100644 index ddee02690a..0000000000 --- a/yql/essentials/core/ut_common/yql_ut_common.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include <yql/essentials/core/yql_expr_type_annotation.h> - -#include <yt/yql/providers/yt/gateway/file/yql_yt_file.h> -#include <yt/yql/providers/yt/provider/yql_yt_provider.h> - -#include <util/system/tempfile.h> - -namespace NYql { - -struct TTestTablesMapping: public THashMap<TString, TString> { - TTempFileHandle TmpInput; - TTempFileHandle TmpInputAttr; - TTempFileHandle TmpOutput; - TTempFileHandle TmpOutputAttr; - - TTestTablesMapping(); -}; - -void InitializeYtGateway(IYtGateway::TPtr gateway, TYtState::TPtr ytState); - -} diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp index f1cb1ea1aa..f4d60c3ef5 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp @@ -796,14 +796,17 @@ Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinTestOptional) { expectedSubkey.push_back(leftSubkeyInit[i]); expectedValue.push_back(leftValueInit[i]); - const auto& found = rightMap.find(*leftKeyInit[i]); - if (found != rightMap.cend()) { - expectedRightKey.push_back(found->first); - expectedRightValue.push_back(found->second); - } else { - expectedRightKey.push_back(std::nullopt); - expectedRightValue.push_back(std::nullopt); + if (leftKeyInit[i].has_value()) { + const auto& found = rightMap.find(*leftKeyInit[i]); + if (found != rightMap.cend()) { + expectedRightKey.push_back(found->first); + expectedRightValue.push_back(found->second); + continue; + } } + + expectedRightKey.push_back(std::nullopt); + expectedRightValue.push_back(std::nullopt); } auto [leftType, leftList] = ConvertVectorsToTuples(setup, diff --git a/yql/essentials/minikql/dom/make.cpp b/yql/essentials/minikql/dom/make.cpp index ca6864f759..8c1bfecfd0 100644 --- a/yql/essentials/minikql/dom/make.cpp +++ b/yql/essentials/minikql/dom/make.cpp @@ -133,6 +133,7 @@ TUnboxedValuePod MakeVariant(const ITypeInfoHelper* typeHelper, const TType* sha TUnboxedValuePod MakeDom(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { switch (const auto kind = typeHelper->GetTypeKind(shape)) { case ETypeKind::Null: + case ETypeKind::Void: return MakeEntity(); case ETypeKind::EmptyList: return SetNodeType<ENodeType::List>(TUnboxedValuePod::Void()); diff --git a/yql/essentials/minikql/dom/node.cpp b/yql/essentials/minikql/dom/node.cpp index 6eabde3e4e..64458dea8d 100644 --- a/yql/essentials/minikql/dom/node.cpp +++ b/yql/essentials/minikql/dom/node.cpp @@ -133,6 +133,10 @@ bool TMapNode::IsSortedDict() const { return true; } +TStringRef TMapNode::GetResourceTag() const { + return NodeResourceName; +} + void* TMapNode::GetResource() { return Items_; } diff --git a/yql/essentials/minikql/dom/node.h b/yql/essentials/minikql/dom/node.h index 04a211fd09..5719709655 100644 --- a/yql/essentials/minikql/dom/node.h +++ b/yql/essentials/minikql/dom/node.h @@ -94,6 +94,8 @@ private: bool IsSortedDict() const final; + TStringRef GetResourceTag() const final; + void* GetResource() final; ui32 Count_; diff --git a/yql/essentials/minikql/mkql_program_builder.cpp b/yql/essentials/minikql/mkql_program_builder.cpp index 0b9197024d..895f2d590a 100644 --- a/yql/essentials/minikql/mkql_program_builder.cpp +++ b/yql/essentials/minikql/mkql_program_builder.cpp @@ -6243,7 +6243,7 @@ TRuntimeNode TProgramBuilder::MatchRecognizeCore( callableBuilder.Add(NewDataLiteral(static_cast<i32>(skipTo.To))); callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(skipTo.Var)); } - if constexpr (RuntimeVersion >= 53U) { + if constexpr (RuntimeVersion >= 54U) { callableBuilder.Add(NewDataLiteral(static_cast<i32>(rowsPerMatch))); callableBuilder.Add(NewList(outputColumnEntryType, outputColumnOrder)); } diff --git a/yql/essentials/minikql/mkql_runtime_version.h b/yql/essentials/minikql/mkql_runtime_version.h index b4fbc1951e..26cd1a8fcc 100644 --- a/yql/essentials/minikql/mkql_runtime_version.h +++ b/yql/essentials/minikql/mkql_runtime_version.h @@ -24,7 +24,7 @@ namespace NMiniKQL { // 1. Bump this version every time incompatible runtime nodes are introduced. // 2. Make sure you provide runtime node generation for previous runtime versions. #ifndef MKQL_RUNTIME_VERSION -#define MKQL_RUNTIME_VERSION 53U +#define MKQL_RUNTIME_VERSION 54U #endif // History: diff --git a/yql/essentials/parser/pg_catalog/catalog.cpp b/yql/essentials/parser/pg_catalog/catalog.cpp index c631fca46b..40a2d3a52b 100644 --- a/yql/essentials/parser/pg_catalog/catalog.cpp +++ b/yql/essentials/parser/pg_catalog/catalog.cpp @@ -1,5 +1,6 @@ #include "catalog.h" #include <yql/essentials/parser/pg_catalog/proto/pg_catalog.pb.h> +#include <yql/essentials/utils/log/profile.h> #include <util/generic/array_size.h> #include <util/generic/utility.h> #include <util/generic/hash.h> @@ -3649,6 +3650,7 @@ ISqlLanguageParser* GetSqlLanguageParser() { } void LoadSystemFunctions(ISystemFunctionsParser& parser) { + YQL_PROFILE_FUNC(DEBUG); auto& catalog = TCatalog::MutableInstance(); with_lock (catalog.ExtensionsGuard) { if (catalog.State->SystemFunctionInit) { @@ -3684,6 +3686,7 @@ void LoadSystemFunctions(ISystemFunctionsParser& parser) { void RegisterExtensions(const TVector<TExtensionDesc>& extensions, bool typesOnly, IExtensionSqlParser& parser, IExtensionLoader* loader) { + YQL_PROFILE_FUNC(DEBUG); if (extensions.size() > MaximumExtensionsCount) { throw yexception() << "Too many extensions: " << extensions.size(); } diff --git a/yql/essentials/parser/pg_catalog/ya.make b/yql/essentials/parser/pg_catalog/ya.make index f9aefb43aa..815bcaa340 100644 --- a/yql/essentials/parser/pg_catalog/ya.make +++ b/yql/essentials/parser/pg_catalog/ya.make @@ -23,6 +23,7 @@ PEERDIR( yql/essentials/public/issue library/cpp/digest/md5 yql/essentials/parser/pg_catalog/proto + yql/essentials/utils/log ) END() diff --git a/yql/essentials/parser/pg_wrapper/ut/codegen_ut.cpp b/yql/essentials/parser/pg_wrapper/ut/codegen_ut.cpp index 50320e27dd..91536d2872 100644 --- a/yql/essentials/parser/pg_wrapper/ut/codegen_ut.cpp +++ b/yql/essentials/parser/pg_wrapper/ut/codegen_ut.cpp @@ -45,7 +45,7 @@ Y_UNIT_TEST_SUITE(TPgCodegen) { void PgFuncImpl(EKernelFlavor flavor, bool constArg, bool fixed) { const TString& name = fixed ? "date_eq" : "textout"; ICodegen::TPtr codegen; - TExecFunc execFunc; + TExecFunc execFunc; switch (flavor) { case EKernelFlavor::Indirect: { if (fixed) { @@ -94,10 +94,10 @@ Y_UNIT_TEST_SUITE(TPgCodegen) { nullBuilder.Reserve(length); auto out = dataBuilder.MutableData(); auto outNulls = nullBuilder.MutableData(); - NUdf::TFixedSizeBlockReader<ui64, false> reader1; - NUdf::TFixedSizeBlockReader<ui64, false> reader2; + NUdf::TFixedSizeBlockReader<ui64, false> reader1; + NUdf::TFixedSizeBlockReader<ui64, false> reader2; const auto& array1 = *batch.values[0].array(); - const auto ptr1 = array1.GetValues<ui64>(1); + const auto ptr1 = array1.GetValues<ui64>(1); if (batch.values[1].is_array()) { const auto& array2 = *batch.values[1].array(); const auto ptr2 = array2.GetValues<ui64>(1); @@ -222,7 +222,7 @@ Y_UNIT_TEST_SUITE(TPgCodegen) { arrow::compute::ExecBatch batch(std::move(batchArgs), N); { - Cout << "begin...\n"; + Cout << "begin...\n"; TSimpleTimer timer; for (size_t count = 0; count < (fixed ? 10000 : 1000); ++count) { arrow::Datum res; @@ -247,7 +247,7 @@ Y_UNIT_TEST_SUITE(TPgCodegen) { Y_UNIT_TEST(PgFixedFuncIndirect) { PgFuncImpl(EKernelFlavor::Indirect, false, true); PgFuncImpl(EKernelFlavor::Indirect, true, true); - } + } #if !defined(USE_SLOW_PG_KERNELS) Y_UNIT_TEST(PgFixedFuncCpp) { @@ -255,19 +255,21 @@ Y_UNIT_TEST_SUITE(TPgCodegen) { PgFuncImpl(EKernelFlavor::Cpp, true, true); } +#if defined(YQL_USE_PG_BC) Y_UNIT_TEST(PgFixedFuncBC) { PgFuncImpl(EKernelFlavor::BitCode, false, true); PgFuncImpl(EKernelFlavor::BitCode, true, true); } -#endif +#endif +#endif Y_UNIT_TEST(PgStrFuncIdeal) { PgFuncImpl(EKernelFlavor::Ideal, false, false); - } + } Y_UNIT_TEST(PgStrFuncDefArg) { PgFuncImpl(EKernelFlavor::DefArg, false, false); - } + } Y_UNIT_TEST(PgStrFuncIndirect) { PgFuncImpl(EKernelFlavor::Indirect, false, false); @@ -278,9 +280,11 @@ Y_UNIT_TEST_SUITE(TPgCodegen) { PgFuncImpl(EKernelFlavor::Cpp, false, false); } +#if defined(YQL_USE_PG_BC) Y_UNIT_TEST(PgStrFuncBC) { PgFuncImpl(EKernelFlavor::BitCode, false, false); } #endif +#endif } diff --git a/yql/essentials/parser/pg_wrapper/ut/ya.make b/yql/essentials/parser/pg_wrapper/ut/ya.make index 4dc451e52f..37d3514757 100644 --- a/yql/essentials/parser/pg_wrapper/ut/ya.make +++ b/yql/essentials/parser/pg_wrapper/ut/ya.make @@ -35,6 +35,12 @@ PEERDIR( YQL_LAST_ABI_VERSION() +IF (YQL_USE_PG_BC) + CFLAGS( + -DYQL_USE_PG_BC + ) +ENDIF() + IF (MKQL_RUNTIME_VERSION) CFLAGS( -DMKQL_RUNTIME_VERSION=$MKQL_RUNTIME_VERSION diff --git a/yql/essentials/parser/pg_wrapper/ya.make b/yql/essentials/parser/pg_wrapper/ya.make index cf31405feb..7ea0ecfa7a 100644 --- a/yql/essentials/parser/pg_wrapper/ya.make +++ b/yql/essentials/parser/pg_wrapper/ya.make @@ -64,9 +64,11 @@ ENDIF() IF (NOT OS_WINDOWS AND NOT SANITIZER_TYPE AND NOT BUILD_TYPE == "DEBUG") IF (NOT BUILD_POSTGRES_ONLY) +IF (YQL_USE_PG_BC) USE_LLVM_BC14() INCLUDE(pg_bc.all.inc) ENDIF() +ENDIF() ELSE() CFLAGS(-DUSE_SLOW_PG_KERNELS) ENDIF() diff --git a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp index f02a666e5c..9e13ad766b 100644 --- a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp +++ b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp @@ -174,8 +174,9 @@ public: bool needLibrary = false; for (auto& m : *import->Modules) { if (requiredLoadedModules.contains(m)) { - YQL_ENSURE(import->Block->Type == EUserDataType::PATH); - path2LoadedImport[import->Block->Data] = import; + if (import->Block->Type == EUserDataType::PATH) { + path2LoadedImport[import->Block->Data] = import; + } } if (requiredExternalModules.contains(m)) { diff --git a/yql/essentials/providers/pure/ut/ya.make b/yql/essentials/providers/pure/ut/ya.make new file mode 100644 index 0000000000..770076756e --- /dev/null +++ b/yql/essentials/providers/pure/ut/ya.make @@ -0,0 +1,23 @@ +UNITTEST_FOR(yql/essentials/providers/pure) + +SIZE(MEDIUM) + +SRCS( + yql_pure_provider_ut.cpp +) + +PEERDIR( + yql/essentials/core/facade + yql/essentials/public/result_format + yql/essentials/minikql/invoke_builtins + yql/essentials/minikql + yql/essentials/minikql/comp_nodes/llvm14 + yql/essentials/minikql/invoke_builtins/llvm14 + yql/essentials/sql/pg_dummy + yql/essentials/public/udf/service/exception_policy + library/cpp/yson/node +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/yql/essentials/providers/pure/ut/yql_pure_provider_ut.cpp b/yql/essentials/providers/pure/ut/yql_pure_provider_ut.cpp new file mode 100644 index 0000000000..c2d66c002a --- /dev/null +++ b/yql/essentials/providers/pure/ut/yql_pure_provider_ut.cpp @@ -0,0 +1,270 @@ +#include "yql_pure_provider.h" + +#include <library/cpp/testing/unittest/registar.h> +#include <yql/essentials/core/facade/yql_facade.h> +#include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h> +#include <yql/essentials/minikql/mkql_function_registry.h> +#include <yql/essentials/public/result_format/yql_result_format_response.h> + +#include <library/cpp/yson/node/node_io.h> + +#include <util/system/user.h> +#include <util/string/strip.h> + +namespace NYql { + +namespace { + +struct TSettings { + bool SExpr = false; + bool Pretty = false; +}; + +TString Run(const TString& query, TSettings settings = {}) { + auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); + TVector<TDataProviderInitializer> dataProvidersInit; + dataProvidersInit.push_back(GetPureDataProviderInitializer()); + TProgramFactory factory(true, functionRegistry.Get(), 0ULL, dataProvidersInit, "ut"); + TProgramPtr program = factory.Create("-stdin-", query); + program->ConfigureYsonResultFormat(settings.Pretty ? NYson::EYsonFormat::Pretty : NYson::EYsonFormat::Text); + bool parseRes; + if (settings.SExpr) { + parseRes = program->ParseYql(); + } else { + parseRes = program->ParseSql(); + } + + if (!parseRes) { + TStringStream err; + program->PrintErrorsTo(err); + UNIT_FAIL(err.Str()); + } + + if (!program->Compile(GetUsername())) { + TStringStream err; + program->PrintErrorsTo(err); + UNIT_FAIL(err.Str()); + } + + TProgram::TStatus status = program->Run(GetUsername()); + if (status == TProgram::TStatus::Error) { + TStringStream err; + program->PrintErrorsTo(err); + UNIT_FAIL(err.Str()); + } + + return program->ResultsAsString(); +} + +} + +Y_UNIT_TEST_SUITE(TPureProviderTests) { + Y_UNIT_TEST(SExpr) { + const auto s = R"( + ( + (let result_sink (DataSink 'result)) + (let output (Int32 '1)) + (let world (Write! world result_sink (Key) output '('('type)))) + (return (Commit! world result_sink)) + ) + + )"; + const auto expectedRes = R"( +[ + { + "Write" = [ + { + "Type" = [ + "DataType"; + "Int32" + ]; + "Data" = "1" + } + ] + } +] + )"; + auto res = Run(s, TSettings{.SExpr = true, .Pretty = true}); + UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes)); + } + + Y_UNIT_TEST(Sql0Rows) { + const auto s = "select * from (select 1 as x) limit 0"; + const auto expectedRes = R"( +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "x"; + [ + "DataType"; + "Int32" + ] + ] + ] + ] + ]; + "Data" = [] + } + ] + } +] + )"; + auto res = Run(s, TSettings{.Pretty = true}); + UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes)); + } + + void Sql1RowImpl(const TString& query) { + const auto expectedRes = R"( +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "x"; + [ + "DataType"; + "Int32" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "1" + ] + ] + } + ] + } +] + )"; + auto res = Run(query, TSettings{.Pretty = true}); + UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes)); + } + + Y_UNIT_TEST(Sql1Row_LLVM_On) { + const auto s = "pragma config.flags(\"LLVM\",\"--dump-stats\");select 1 as x"; + Sql1RowImpl(s); + } + + Y_UNIT_TEST(Sql1Row_LLVM_Off) { + const auto s = "pragma config.flags(\"LLVM\",\"OFF\");select 1 as x"; + Sql1RowImpl(s); + } + + Y_UNIT_TEST(Sql2Rows) { + const auto s = "select 1 as x union all select 2 as x order by x"; + const auto expectedRes = R"( +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "x"; + [ + "DataType"; + "Int32" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "1" + ]; + [ + "2" + ] + ] + } + ] + } +] + )"; + auto res = Run(s, TSettings{.Pretty = true}); + UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes)); + } + + Y_UNIT_TEST(TruncateRows) { + const auto s = "select x from (select ListFromRange(1,2000) as x) flatten by x"; + auto res = Run(s); + auto respList = NResult::ParseResponse(NYT::NodeFromYsonString(res)); + UNIT_ASSERT_VALUES_EQUAL(respList.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(respList[0].Writes.size(), 1); + UNIT_ASSERT(respList[0].Writes[0].IsTruncated); + } + + Y_UNIT_TEST(TruncateBytes) { + const auto s = "select '" + TString(1000000, 'a') + "' as x, 1 as y union all select '' as x, 2 as y order by y"; + auto res = Run(s); + auto respList = NResult::ParseResponse(NYT::NodeFromYsonString(res)); + UNIT_ASSERT_VALUES_EQUAL(respList.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(respList[0].Writes.size(), 1); + UNIT_ASSERT(respList[0].Writes[0].IsTruncated); + } + + Y_UNIT_TEST(ColumnOrder) { + const auto s = "pragma OrderedColumns;select 1 as y, 2 as x"; + const auto expectedRes = R"( +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "y"; + [ + "DataType"; + "Int32" + ] + ]; + [ + "x"; + [ + "DataType"; + "Int32" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "1"; + "2" + ] + ] + } + ] + } +] + )"; + auto res = Run(s, TSettings{.Pretty = true}); + UNIT_ASSERT_NO_DIFF(res, Strip(expectedRes)); + } +} + +} // namespace NYql diff --git a/yql/essentials/providers/pure/ya.make b/yql/essentials/providers/pure/ya.make index 2ce55fd588..d7bbf0e5ab 100644 --- a/yql/essentials/providers/pure/ya.make +++ b/yql/essentials/providers/pure/ya.make @@ -26,3 +26,7 @@ PEERDIR( YQL_LAST_ABI_VERSION() END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/yql/essentials/providers/pure/yql_pure_provider.cpp b/yql/essentials/providers/pure/yql_pure_provider.cpp index b1c65f71e1..8257951be0 100644 --- a/yql/essentials/providers/pure/yql_pure_provider.cpp +++ b/yql/essentials/providers/pure/yql_pure_provider.cpp @@ -62,7 +62,32 @@ public: return SyncError(); } + const bool isList = lambda.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List; auto optimized = lambda.Ptr(); + auto source1 = ctx.Builder(lambda.Pos()) + .Callable("Take") + .Callable(0, "SourceOf") + .Callable(0, "StreamType") + .Callable(0, "NullType") + .Seal() + .Seal() + .Seal() + .Callable(1, "Uint64") + .Atom(0, "1") + .Seal() + .Seal() + .Build(); + + optimized = ctx.Builder(lambda.Pos()) + .Callable(isList ? "FlatMap" : "Map") + .Add(0, source1) + .Lambda(1) + .Param("x") + .Set(optimized) + .Seal() + .Seal() + .Build(); + bool hasNonDeterministicFunctions; auto status = PeepHoleOptimizeNode(optimized, optimized, ctx, *State_->Types, nullptr, hasNonDeterministicFunctions); if (status.Level == IGraphTransformer::TStatus::Error) { @@ -124,26 +149,37 @@ public: TStringOutput dataOut(data); TCountingOutput dataCountingOut(&dataOut); NYson::TYsonWriter dataWriter(&dataCountingOut, NCommon::GetYsonFormat(fillSettings), ::NYson::EYsonType::Node, false); - if (type->IsList()) { - auto inputType = AS_TYPE(TListType, type)->GetItemType(); + YQL_ENSURE(type->IsStream()); + auto itemType = AS_TYPE(TStreamType, type)->GetItemType(); + if (isList) { TMaybe<ui64> rowsLimit = fillSettings.RowsLimitPerWrite; TMaybe<ui64> bytesLimit = fillSettings.AllResultsBytesLimit; - TMaybe<TVector<ui32>> structPositions = NCommon::CreateStructPositions(inputType, &columns); + TMaybe<TVector<ui32>> structPositions = NCommon::CreateStructPositions(itemType, &columns); dataWriter.OnBeginList(); - const auto it = value.GetListIterator(); ui64 rows = 0; - for (NUdf::TUnboxedValue item; it.Next(item); ++rows) { + for (;;) { + NUdf::TUnboxedValue item; + auto status = value.Fetch(item); + if (status == NUdf::EFetchStatus::Finish) { + break; + } + + YQL_ENSURE(status == NUdf::EFetchStatus::Ok); if ((rowsLimit && rows >= *rowsLimit) || (bytesLimit && dataCountingOut.Counter() >= *bytesLimit)) { truncated = true; break; } dataWriter.OnListItem(); - NCommon::WriteYsonValue(dataWriter, item, inputType, structPositions.Get()); + NCommon::WriteYsonValue(dataWriter, item, itemType, structPositions.Get()); + ++rows; } dataWriter.OnEndList(); } else { - NCommon::WriteYsonValue(dataWriter, value, type, nullptr); + NUdf::TUnboxedValue item; + YQL_ENSURE(value.Fetch(item) == NUdf::EFetchStatus::Ok); + NCommon::WriteYsonValue(dataWriter, item, itemType, nullptr); + YQL_ENSURE(value.Fetch(item) == NUdf::EFetchStatus::Finish); } writer.OnKeyedItem("Data"); diff --git a/yql/essentials/sql/v1/SQLv1.g.in b/yql/essentials/sql/v1/SQLv1.g.in index 670ad27e3e..61e7dc6f25 100644 --- a/yql/essentials/sql/v1/SQLv1.g.in +++ b/yql/essentials/sql/v1/SQLv1.g.in @@ -825,8 +825,8 @@ alter_table_alter_index_action: drop_table_stmt: DROP (TABLE | TABLESTORE | EXTERNAL TABLE) (IF EXISTS)? simple_table_ref; -create_user_stmt: CREATE USER role_name create_user_option?; -alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name); +create_user_stmt: CREATE USER role_name create_user_option*; +alter_user_stmt: ALTER USER role_name (WITH? create_user_option+ | RENAME TO role_name); create_group_stmt: CREATE GROUP role_name (WITH USER role_name (COMMA role_name)* COMMA?)?; alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_name)* COMMA? | RENAME TO role_name); @@ -834,7 +834,9 @@ alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_n drop_role_stmt: DROP (USER|GROUP) (IF EXISTS)? role_name (COMMA role_name)* COMMA?; role_name: an_id_or_type | bind_parameter; -create_user_option: ENCRYPTED? PASSWORD expr; +create_user_option: password_option | login_option; +password_option: ENCRYPTED? PASSWORD expr; +login_option: LOGIN | NOLOGIN; grant_permissions_stmt: GRANT permission_name_target ON an_id_schema (COMMA an_id_schema)* TO role_name (COMMA role_name)* COMMA? (WITH GRANT OPTION)?; revoke_permissions_stmt: REVOKE (GRANT OPTION FOR)? permission_name_target ON an_id_schema (COMMA an_id_schema)* FROM role_name (COMMA role_name)*; @@ -1368,6 +1370,7 @@ keyword_as_compat: | LEGACY | LIKE | LOCAL + | LOGIN | MANAGE | MATCH | MATCHES @@ -1380,6 +1383,7 @@ keyword_as_compat: // | NATURAL | NEXT | NO + | NOLOGIN // | NOTNULL | NULLS | OBJECT @@ -1592,6 +1596,7 @@ keyword_compat: ( | LEGACY | LIKE | LOCAL + | LOGIN | MANAGE | MATCH | MATCHES @@ -1604,6 +1609,7 @@ keyword_compat: ( | NATURAL | NEXT | NO + | NOLOGIN | NOTNULL | NULLS | OBJECT @@ -1949,6 +1955,7 @@ LIKE: L I K E; LIMIT: L I M I T; LIST: L I S T; LOCAL: L O C A L; +LOGIN: L O G I N; MANAGE: M A N A G E; MATCH: M A T C H; MATCHES: M A T C H E S; @@ -1961,6 +1968,7 @@ NANOSECONDS: N A N O S E C O N D S; NATURAL: N A T U R A L; NEXT: N E X T; NO: N O; +NOLOGIN: N O L O G I N; NOT: N O T; NOTNULL: N O T N U L L; NULL: N U L L; diff --git a/yql/essentials/sql/v1/SQLv1Antlr4.g.in b/yql/essentials/sql/v1/SQLv1Antlr4.g.in index 89131437e9..96e450501c 100644 --- a/yql/essentials/sql/v1/SQLv1Antlr4.g.in +++ b/yql/essentials/sql/v1/SQLv1Antlr4.g.in @@ -824,8 +824,8 @@ alter_table_alter_index_action: drop_table_stmt: DROP (TABLE | TABLESTORE | EXTERNAL TABLE) (IF EXISTS)? simple_table_ref; -create_user_stmt: CREATE USER role_name create_user_option?; -alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name); +create_user_stmt: CREATE USER role_name create_user_option*; +alter_user_stmt: ALTER USER role_name (WITH? create_user_option+ | RENAME TO role_name); create_group_stmt: CREATE GROUP role_name (WITH USER role_name (COMMA role_name)* COMMA?)?; alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_name)* COMMA? | RENAME TO role_name); @@ -833,7 +833,9 @@ alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_n drop_role_stmt: DROP (USER|GROUP) (IF EXISTS)? role_name (COMMA role_name)* COMMA?; role_name: an_id_or_type | bind_parameter; -create_user_option: ENCRYPTED? PASSWORD expr; +create_user_option: password_option | login_option; +password_option: ENCRYPTED? PASSWORD expr; +login_option: LOGIN | NOLOGIN; grant_permissions_stmt: GRANT permission_name_target ON an_id_schema (COMMA an_id_schema)* TO role_name (COMMA role_name)* COMMA? (WITH GRANT OPTION)?; revoke_permissions_stmt: REVOKE (GRANT OPTION FOR)? permission_name_target ON an_id_schema (COMMA an_id_schema)* FROM role_name (COMMA role_name)*; @@ -1367,6 +1369,7 @@ keyword_as_compat: | LEGACY | LIKE | LOCAL + | LOGIN | MANAGE | MATCH | MATCHES @@ -1379,6 +1382,7 @@ keyword_as_compat: // | NATURAL | NEXT | NO + | NOLOGIN // | NOTNULL | NULLS | OBJECT @@ -1591,6 +1595,7 @@ keyword_compat: ( | LEGACY | LIKE | LOCAL + | LOGIN | MANAGE | MATCH | MATCHES @@ -1603,6 +1608,7 @@ keyword_compat: ( | NATURAL | NEXT | NO + | NOLOGIN | NOTNULL | NULLS | OBJECT @@ -1948,6 +1954,7 @@ LIKE: L I K E; LIMIT: L I M I T; LIST: L I S T; LOCAL: L O C A L; +LOGIN: L O G I N; MANAGE: M A N A G E; MATCH: M A T C H; MATCHES: M A T C H E S; @@ -1960,6 +1967,7 @@ NANOSECONDS: N A N O S E C O N D S; NATURAL: N A T U R A L; NEXT: N E X T; NO: N O; +NOLOGIN: N O L O G I N; NOT: N O T; NOTNULL: N O T N U L L; NULL: N U L L; diff --git a/yql/essentials/sql/v1/format/sql_format.cpp b/yql/essentials/sql/v1/format/sql_format.cpp index b330800734..b7a8d5c48d 100644 --- a/yql/essentials/sql/v1/format/sql_format.cpp +++ b/yql/essentials/sql/v1/format/sql_format.cpp @@ -276,6 +276,11 @@ TTokenIterator GetNextStatementBegin(TTokenIterator begin, TTokenIterator end) { continue; } if (curr->Name == "SEMICOLON") { + auto next = SkipWS(curr + 1, end); + while (next != end && next->Name == "COMMENT" && curr->Line == next->Line) { + curr = next; + next = SkipWS(next + 1, end); + } ++curr; break; } diff --git a/yql/essentials/sql/v1/format/sql_format_ut.h b/yql/essentials/sql/v1/format/sql_format_ut.h index f44ab557c3..bd62ddf368 100644 --- a/yql/essentials/sql/v1/format/sql_format_ut.h +++ b/yql/essentials/sql/v1/format/sql_format_ut.h @@ -73,8 +73,13 @@ Y_UNIT_TEST(DropRole) { Y_UNIT_TEST(CreateUser) { TCases cases = { - {"use plato;create user user;","USE plato;\n\nCREATE USER user;\n"}, - {"use plato;create user user encrypted password 'foo';","USE plato;\n\nCREATE USER user ENCRYPTED PASSWORD 'foo';\n"}, + {"use plato;create user user;", "USE plato;\n\nCREATE USER user;\n"}, + {"use plato;create user user encrypted password 'foo';", "USE plato;\n\nCREATE USER user ENCRYPTED PASSWORD 'foo';\n"}, + {"use plato;CREATE USER user1;", "USE plato;\n\nCREATE USER user1;\n"}, + {"use plato;create user user1 encrypted password '123' login;", "USE plato;\n\nCREATE USER user1 ENCRYPTED PASSWORD '123' LOGIN;\n"}, + {"use plato;cREATE USER user1 PASSWORD '123' NOLOGIN;", "USE plato;\n\nCREATE USER user1 PASSWORD '123' NOLOGIN;\n"}, + {"use plato;CREATE USER user1 LOGIN;", "USE plato;\n\nCREATE USER user1 LOGIN;\n"}, + {"use plato;CREATE USER user1 NOLOGIN;", "USE plato;\n\nCREATE USER user1 NOLOGIN;\n"}, }; TSetup setup; @@ -97,6 +102,7 @@ Y_UNIT_TEST(AlterUser) { {"use plato;alter user user rename to user;","USE plato;\n\nALTER USER user RENAME TO user;\n"}, {"use plato;alter user user encrypted password 'foo';","USE plato;\n\nALTER USER user ENCRYPTED PASSWORD 'foo';\n"}, {"use plato;alter user user with encrypted password 'foo';","USE plato;\n\nALTER USER user WITH ENCRYPTED PASSWORD 'foo';\n"}, + {"use plato;ALTER USER user1 NOLOGIN;", "USE plato;\n\nALTER USER user1 NOLOGIN;\n"}, }; TSetup setup; @@ -211,6 +217,8 @@ Y_UNIT_TEST(NamedNode) { "$a = (\n\tSELECT\n\t\t*\n\tFROM\n\t\t$t -- comment\n);\n"}, {"-- comment\r\r\r$a=1;", "-- comment\r\n$a = 1;\n"}, + {"$a=1;-- comment\n$b=2;/* comment */ /* comment */\n$c = 3;/* comment */ -- comment", + "$a = 1; -- comment\n$b = 2; /* comment */ /* comment */\n$c = 3; /* comment */ -- comment\n"}, }; TSetup setup; diff --git a/yql/essentials/sql/v1/node.h b/yql/essentials/sql/v1/node.h index fe6ddb08ac..609cd82dd4 100644 --- a/yql/essentials/sql/v1/node.h +++ b/yql/essentials/sql/v1/node.h @@ -1305,6 +1305,14 @@ namespace NSQLTranslationV1 { TMaybe<TDeferredAtom> Password; bool IsPasswordEncrypted = false; TVector<TDeferredAtom> Roles; + + enum class ETypeOfLogin { + Undefined, + Login, + NoLogin + }; + + ETypeOfLogin CanLogin = ETypeOfLogin::Undefined; }; struct TSequenceParameters { diff --git a/yql/essentials/sql/v1/select.cpp b/yql/essentials/sql/v1/select.cpp index 1442215e15..4f5db44069 100644 --- a/yql/essentials/sql/v1/select.cpp +++ b/yql/essentials/sql/v1/select.cpp @@ -1869,6 +1869,10 @@ public: return false; } + if (!ctx.SimpleColumns && Columns.QualifiedAll && !columnName.Contains('.')) { + return false; + } + if (!Columns.IsColumnPossible(ctx, columnName)) { return true; } diff --git a/yql/essentials/sql/v1/sql_query.cpp b/yql/essentials/sql/v1/sql_query.cpp index 5252417526..a2dd6cc68a 100644 --- a/yql/essentials/sql/v1/sql_query.cpp +++ b/yql/essentials/sql/v1/sql_query.cpp @@ -583,7 +583,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& break; } case TRule_sql_stmt_core::kAltSqlStmtCore22: { - // create_user_stmt: CREATE USER role_name create_user_option?; + // create_user_stmt: CREATE USER role_name (create_user_option)*; Ctx.BodyPart(); auto& node = core.GetAlt_sql_stmt_core22().GetRule_create_user_stmt1(); @@ -604,9 +604,19 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } TMaybe<TRoleParameters> roleParams; - if (node.HasBlock4()) { + const auto& options = node.GetBlock4(); + + if (options.size() > 0) { roleParams.ConstructInPlace(); - if (!RoleParameters(node.GetBlock4().GetRule_create_user_option1(), *roleParams)) { + std::vector<TRule_create_user_option> opts; + opts.reserve(options.size()); + for (const auto& opt : options) { + opts.push_back(opt.GetRule_create_user_option1()); + } + + bool isCreateUser = true; + + if (!RoleParameters(opts, *roleParams, isCreateUser)) { return false; } } @@ -615,7 +625,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& break; } case TRule_sql_stmt_core::kAltSqlStmtCore23: { - // alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name); + // alter_user_stmt: ALTER USER role_name (WITH? create_user_option+ | RENAME TO role_name); Ctx.BodyPart(); auto& node = core.GetAlt_sql_stmt_core23().GetRule_alter_user_stmt1(); @@ -641,7 +651,17 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& switch (node.GetBlock4().Alt_case()) { case TRule_alter_user_stmt_TBlock4::kAlt1: { TRoleParameters roleParams; - if (!RoleParameters(node.GetBlock4().GetAlt1().GetRule_create_user_option2(), roleParams)) { + + auto options = node.GetBlock4().GetAlt1().GetBlock2(); + std::vector<TRule_create_user_option> opts; + opts.reserve(options.size()); + for (const auto& opt : options) { + opts.push_back(opt.GetRule_create_user_option1()); + } + + bool isCreateUser = false; + + if (!RoleParameters(opts, roleParams, isCreateUser)) { return false; } stmt = BuildAlterUser(pos, service, cluster, roleName, roleParams, Ctx.Scoped); diff --git a/yql/essentials/sql/v1/sql_translation.cpp b/yql/essentials/sql/v1/sql_translation.cpp index 61a273b260..16de88ffe0 100644 --- a/yql/essentials/sql/v1/sql_translation.cpp +++ b/yql/essentials/sql/v1/sql_translation.cpp @@ -3818,19 +3818,82 @@ bool TSqlTranslation::RoleNameClause(const TRule_role_name& node, TDeferredAtom& return true; } -bool TSqlTranslation::RoleParameters(const TRule_create_user_option& node, TRoleParameters& result) { - // create_user_option: ENCRYPTED? PASSWORD expr; +bool TSqlTranslation::RoleParameters(const std::vector<TRule_create_user_option>& optionsList, TRoleParameters& result, bool isCreateUser) { + enum class ECreateUserOption { + Login, + Password + }; + + std::set<ECreateUserOption> used = {}; + + auto ParseCreateUserOption = [&used, this](const TRule_create_user_option& option, TRoleParameters& result) -> bool { + // create_user_option: password_option | login_option; + // password_option: ENCRYPTED? PASSWORD expr; + // login_option: LOGIN | NOLOGIN; + + switch (option.Alt_case()) { + case TRule_create_user_option::kAltCreateUserOption1: + { + TSqlExpression expr(Ctx, Mode); + TNodePtr password = expr.Build(option.GetAlt_create_user_option1().GetRule_password_option1().GetRule_expr3()); + if (!password) { + Error() << "Couldn't parse the password"; + return false; + } + + result.IsPasswordEncrypted = option.GetAlt_create_user_option1().GetRule_password_option1().HasBlock1(); + if (!password->IsNull()) { + result.Password = MakeAtomFromExpression(Ctx.Pos(), Ctx, password); + } + + if (used.contains(ECreateUserOption::Password)) { + Error() << "Conflicting or redundant options"; + return false; + } + + used.insert(ECreateUserOption::Password); + + break; + } + case TRule_create_user_option::kAltCreateUserOption2: + { + if (used.contains(ECreateUserOption::Login)) { + Error() << "Conflicting or redundant options"; + return false; + } + + used.insert(ECreateUserOption::Login); + + const auto token = option.GetAlt_create_user_option2().GetRule_login_option1().GetToken1().GetId(); + if (IS_TOKEN(token, LOGIN)) { + result.CanLogin = TRoleParameters::ETypeOfLogin::Login; + } else if (IS_TOKEN(token, NOLOGIN)) { + result.CanLogin = TRoleParameters::ETypeOfLogin::NoLogin; + } else { + Y_ABORT("You should change implementation according to grammar changes"); + } + + break; + } + case TRule_create_user_option::ALT_NOT_SET: + { + Y_ABORT("You should change implementation according to grammar changes"); + } + } + + return true; + }; + result = TRoleParameters{}; - TSqlExpression expr(Ctx, Mode); - TNodePtr password = expr.Build(node.GetRule_expr3()); - if (!password) { - return false; + if (isCreateUser) { + result.CanLogin = TRoleParameters::ETypeOfLogin::NoLogin; } - result.IsPasswordEncrypted = node.HasBlock1(); - if (!password->IsNull()) { - result.Password = MakeAtomFromExpression(Ctx.Pos(), Ctx, password); + for (const auto& option : optionsList) { + if (!ParseCreateUserOption(option, result)) { + return false; + } } return true; diff --git a/yql/essentials/sql/v1/sql_translation.h b/yql/essentials/sql/v1/sql_translation.h index 325640d74a..3587b57a7a 100644 --- a/yql/essentials/sql/v1/sql_translation.h +++ b/yql/essentials/sql/v1/sql_translation.h @@ -242,7 +242,7 @@ protected: bool ParseResourcePoolClassifierSettings(std::map<TString, TDeferredAtom>& result, const TRule_with_table_settings& settings); bool ParseResourcePoolClassifierSettings(std::map<TString, TDeferredAtom>& result, std::set<TString>& toReset, const TRule_alter_resource_pool_classifier_action& alterAction); bool RoleNameClause(const TRule_role_name& node, TDeferredAtom& result, bool allowSystemRoles); - bool RoleParameters(const TRule_create_user_option& node, TRoleParameters& result); + bool RoleParameters(const std::vector<TRule_create_user_option>& optionsList, TRoleParameters& result, bool isCreateUser); bool PermissionNameClause(const TRule_permission_name_target& node, TVector<TDeferredAtom>& result, bool withGrantOption); bool PermissionNameClause(const TRule_permission_name& node, TDeferredAtom& result); bool PermissionNameClause(const TRule_permission_id& node, TDeferredAtom& result); diff --git a/yql/essentials/sql/v1/sql_ut.cpp b/yql/essentials/sql/v1/sql_ut.cpp index 1c87beee05..efbed0e665 100644 --- a/yql/essentials/sql/v1/sql_ut.cpp +++ b/yql/essentials/sql/v1/sql_ut.cpp @@ -4957,6 +4957,82 @@ select FormatType($f()); "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); } + Y_UNIT_TEST(CreateAlterUserWithLoginNoLogin) { + auto reqCreateUser = SqlToYql(R"( + USE plato; + CREATE USER user1; + )"); + + UNIT_ASSERT(reqCreateUser.IsOk()); + UNIT_ASSERT(reqCreateUser.Root); + + auto reqAlterUser = SqlToYql(R"( + USE plato; + ALTER USER user1; + )"); + + UNIT_ASSERT(!reqAlterUser.IsOk()); + UNIT_ASSERT_STRING_CONTAINS(reqAlterUser.Issues.ToString(), "Error: Unexpected token ';' : cannot match to any predicted input..."); + + auto reqPasswordAndLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 PASSWORD '123' LOGIN; + )"); + + UNIT_ASSERT(reqPasswordAndLogin.IsOk()); + UNIT_ASSERT(reqPasswordAndLogin.Root); + + auto reqPasswordAndNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 PASSWORD '123' NOLOGIN; + )"); + + UNIT_ASSERT(reqPasswordAndNoLogin.IsOk()); + UNIT_ASSERT(reqPasswordAndNoLogin.Root); + + auto reqLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN; + )"); + + UNIT_ASSERT(reqLogin.IsOk()); + UNIT_ASSERT(reqLogin.Root); + + auto reqNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 NOLOGIN; + )"); + + UNIT_ASSERT(reqNoLogin.IsOk()); + UNIT_ASSERT(reqNoLogin.Root); + + auto reqLoginNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN NOLOGIN; + )"); + + UNIT_ASSERT(!reqLoginNoLogin.IsOk()); + UNIT_ASSERT_STRING_CONTAINS(reqLoginNoLogin.Issues.ToString(), "Error: Conflicting or redundant options"); + + auto reqAlterLoginNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN; + ALTER USER user1 NOLOGIN; + )"); + + UNIT_ASSERT(reqAlterLoginNoLogin.IsOk()); + UNIT_ASSERT(reqAlterLoginNoLogin.Root); + + auto reqAlterLoginNoLoginWithPassword = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN; + ALTER USER user1 PASSWORD '321' NOLOGIN; + )"); + + UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.IsOk()); + UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.Root); + } + Y_UNIT_TEST(CreateAlterUserWithoutCluster) { ExpectFailWithError("\n CREATE USER user ENCRYPTED PASSWORD 'foobar';", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); ExpectFailWithError("ALTER USER CURRENT_USER RENAME TO $foo;", "<main>:1:1: Error: USE statement is missing - no default cluster is selected\n"); diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.cpp b/yql/essentials/sql/v1/sql_ut_antlr4.cpp index 705ca3d19c..c5df28bbaa 100644 --- a/yql/essentials/sql/v1/sql_ut_antlr4.cpp +++ b/yql/essentials/sql/v1/sql_ut_antlr4.cpp @@ -471,6 +471,82 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["EquiJoin"]); } + Y_UNIT_TEST(CreateAlterUserWithLoginNoLogin) { + auto reqCreateUser = SqlToYql(R"( + USE plato; + CREATE USER user1; + )"); + + UNIT_ASSERT(reqCreateUser.IsOk()); + UNIT_ASSERT(reqCreateUser.Root); + + auto reqAlterUser = SqlToYql(R"( + USE plato; + ALTER USER user1; + )"); + + UNIT_ASSERT(!reqAlterUser.IsOk()); + UNIT_ASSERT_STRING_CONTAINS(reqAlterUser.Issues.ToString(), "Error: mismatched input ';' expecting {ENCRYPTED, LOGIN, NOLOGIN, PASSWORD, RENAME, WITH}"); + + auto reqPasswordAndLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOgin; + )"); + + UNIT_ASSERT(reqPasswordAndLogin.IsOk()); + UNIT_ASSERT(reqPasswordAndLogin.Root); + + auto reqPasswordAndNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 PASSWORD '123' NOLOGIN; + )"); + + UNIT_ASSERT(reqPasswordAndNoLogin.IsOk()); + UNIT_ASSERT(reqPasswordAndNoLogin.Root); + + auto reqLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN; + )"); + + UNIT_ASSERT(reqLogin.IsOk()); + UNIT_ASSERT(reqLogin.Root); + + auto reqNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 NOLOGIN; + )"); + + UNIT_ASSERT(reqNoLogin.IsOk()); + UNIT_ASSERT(reqNoLogin.Root); + + auto reqLoginNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN NOLOGIN; + )"); + + UNIT_ASSERT(!reqLoginNoLogin.IsOk()); + UNIT_ASSERT_STRING_CONTAINS(reqLoginNoLogin.Issues.ToString(), "Error: Conflicting or redundant options"); + + auto reqAlterLoginNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN; + ALTER USER user1 NOLOGIN; + )"); + + UNIT_ASSERT(reqAlterLoginNoLogin.IsOk()); + UNIT_ASSERT(reqAlterLoginNoLogin.Root); + + auto reqAlterLoginNoLoginWithPassword = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN; + ALTER USER user1 PASSWORD '321' NOLOGIN; + )"); + + UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.IsOk()); + UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.Root); + } + Y_UNIT_TEST(JoinWithoutConcreteColumns) { NYql::TAstParseResult res = SqlToYql( " use plato;" diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index f687bca643..da8e4a69ce 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -11222,9 +11222,9 @@ ], "test_sql2yql.test[match_recognize-all_rows_per_match]": [ { - "checksum": "31a940b1bbcea146ae9383e278326c7a", - "size": 6729, - "uri": "https://{canondata_backend}/1889210/954e2f1656d98697ece5794c59acf75dd1d40612/resource.tar.gz#test_sql2yql.test_match_recognize-all_rows_per_match_/sql.yql" + "checksum": "ee193116db0e897c8a3fbb98a75caaac", + "size": 6732, + "uri": "https://{canondata_backend}/1889210/a0cc9e3113699f51443284ed56291923dfc3735d/resource.tar.gz#test_sql2yql.test_match_recognize-all_rows_per_match_/sql.yql" } ], "test_sql2yql.test[match_recognize-greedy_quantifiers]": [ @@ -12067,6 +12067,13 @@ "uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_order_by-order_by_missing_project_column_join_types_/sql.yql" } ], + "test_sql2yql.test[order_by-order_by_missing_project_column_nosimple]": [ + { + "checksum": "ea2472dbe1cea58561e9a506a692738f", + "size": 3172, + "uri": "https://{canondata_backend}/1599023/02c3082e8ed2b6689e833c3b6594da85e12a01be/resource.tar.gz#test_sql2yql.test_order_by-order_by_missing_project_column_nosimple_/sql.yql" + } + ], "test_sql2yql.test[order_by-order_by_mul_columns]": [ { "checksum": "88f079be2cb6e1e762d50772e8fbc12d", @@ -28625,6 +28632,11 @@ "uri": "file://test_sql_format.test_order_by-order_by_missing_project_column_join_types_/formatted.sql" } ], + "test_sql_format.test[order_by-order_by_missing_project_column_nosimple]": [ + { + "uri": "file://test_sql_format.test_order_by-order_by_missing_project_column_nosimple_/formatted.sql" + } + ], "test_sql_format.test[order_by-order_by_mul_columns]": [ { "uri": "file://test_sql_format.test_order_by-order_by_mul_columns_/formatted.sql" diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-group_by_session_nopush_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-group_by_session_nopush_/formatted.sql index 8c15f9f952..55b25c01db 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-group_by_session_nopush_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-group_by_session_nopush_/formatted.sql @@ -17,5 +17,4 @@ FROM ( ) WHERE ss != 100500 -; --- should not push down +; -- should not push down diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-add_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-add_bitcast_/formatted.sql index 066eecc165..ef99f355e0 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-add_bitcast_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-add_bitcast_/formatted.sql @@ -2,15 +2,12 @@ USE plato; SELECT 1ul + 1 -; +; -- warn --- warn SELECT 1u + 1 -; +; -- warn --- warn SELECT 1l + 1u -; --- ok +; -- ok diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-div_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-div_bitcast_/formatted.sql index 9597182864..946e613c55 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-div_bitcast_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-div_bitcast_/formatted.sql @@ -2,15 +2,12 @@ USE plato; SELECT 1ul / 1 -; +; -- warn --- warn SELECT 1u / 1 -; +; -- warn --- warn SELECT 1l / 1u -; --- ok +; -- ok diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mod_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mod_bitcast_/formatted.sql index efa491555d..c3d6cc2b1d 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mod_bitcast_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mod_bitcast_/formatted.sql @@ -2,15 +2,12 @@ USE plato; SELECT 1ul % 1 -; +; -- warn --- warn SELECT 1u % 1 -; +; -- warn --- warn SELECT 1l % 1u -; --- ok +; -- ok diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mul_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mul_bitcast_/formatted.sql index 8fbc996355..51daa0e7ff 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mul_bitcast_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-mul_bitcast_/formatted.sql @@ -2,15 +2,12 @@ USE plato; SELECT 1ul * 1 -; +; -- warn --- warn SELECT 1u * 1 -; +; -- warn --- warn SELECT 1l * 1u -; --- ok +; -- ok diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-sub_bitcast_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-sub_bitcast_/formatted.sql index 8d28bd557c..b414769fd7 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-sub_bitcast_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bitcast_implicit-sub_bitcast_/formatted.sql @@ -2,15 +2,12 @@ USE plato; SELECT 1ul - 1 -; +; -- warn --- warn SELECT 1u - 1 -; +; -- warn --- warn SELECT 1l - 1u -; --- ok +; -- ok diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_column_group-hint_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_column_group-hint_/formatted.sql index ae5cb7e3dc..1c102e4f17 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_column_group-hint_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_column_group-hint_/formatted.sql @@ -7,9 +7,8 @@ $i1 = ( Input WHERE a > 'a' -); +); -- several publish consumers with same groups --- several publish consumers with same groups $i2 = ( SELECT * @@ -17,9 +16,8 @@ $i2 = ( Input WHERE a > 'a1' -); +); -- several publish consumers with different groups --- several publish consumers with different groups $i3 = ( SELECT * @@ -27,9 +25,8 @@ $i3 = ( Input WHERE a < 'a2' -); +); -- several consumers including publish --- several consumers including publish $i4 = ( SELECT * @@ -37,9 +34,8 @@ $i4 = ( Input WHERE a != 'a' -); +); -- several publish consumers with and without groups --- several publish consumers with and without groups -- test column group spec normalization INSERT INTO Output1 WITH column_groups = '{g1=[a;b;c];def=#}' SELECT diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_compare_diff_zones_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_compare_diff_zones_/formatted.sql index 5fe7c180ba..01eff73bc1 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_compare_diff_zones_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_compare_diff_zones_/formatted.sql @@ -13,9 +13,8 @@ SELECT SELECT CAST('2000-01-01,GMT' AS tzdate) == tzdate('2000-01-01,America/Los_Angeles') -; +; -- same time value --- same time value SELECT RemoveTimezone(CAST('2000-01-01,GMT' AS tzdate)) == RemoveTimezone(tzdate('2000-01-01,America/Los_Angeles')) ; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-cast_string_implicit_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-cast_string_implicit_/formatted.sql index d34761cff9..3b7ae94bf9 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-cast_string_implicit_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-cast_string_implicit_/formatted.sql @@ -1,60 +1,47 @@ SELECT AsList('aaa', 'aaa'u) -; +; -- List<String> --- List<String> SELECT AsList('aaa', '[1, 2, 3]'j) -; +; -- List<String> --- List<String> SELECT AsList('aaa', '[1; 2; 3]'y) -; +; -- List<String> --- List<String> SELECT AsList('aaa'u, 'aaa') -; +; -- List<String> --- List<String> SELECT AsList('aaa'u, '[1, 2, 3]'j) -; +; -- List<Utf8> --- List<Utf8> SELECT AsList('aaa'u, '[1; 2; 3]'y) -; +; -- List<String> --- List<String> SELECT AsList('[1, 2, 3]'j, 'aaa') -; +; -- List<String> --- List<String> SELECT AsList('[1, 2, 3]'j, 'aaa'u) -; +; -- List<Utf8> --- List<Utf8> SELECT AsList('[1, 2, 3]'j, '[1; 2; 3]'y) -; +; -- List<String> --- List<String> SELECT AsList('[1; 2; 3]'y, 'aaa') -; +; -- List<String> --- List<String> SELECT AsList('[1; 2; 3]'y, 'aaa'u) -; +; -- List<String> --- List<String> SELECT AsList('[1; 2; 3]'y, '[1, 2, 3]'j) -; --- List<String> - +; -- List<String> diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-dict_builtins_null_lookup_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-dict_builtins_null_lookup_/formatted.sql index 053a8e0f2c..9fa9484936 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-dict_builtins_null_lookup_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-dict_builtins_null_lookup_/formatted.sql @@ -10,9 +10,8 @@ SELECT DictContains($d3, NULL), -- true, null is convertible to Nothing<T> for any T DictLookup($d2, NULL), -- Nothing(String?), no such key DictLookup($d3, NULL) -; +; -- Just("baz"), null is convertible to Nothing<T> for any T --- Just("baz"), null is convertible to Nothing<T> for any T SELECT DictContains($t1, AsTuple(1, 'keyy')), -- false, missing key DictContains($t1, AsTuple(1, 'key')), -- true, match @@ -21,5 +20,4 @@ SELECT DictContains($t3, AsTuple(NULL, 'key')), -- true, null is convertible to Nothing<T> for any T DictLookup($t2, AsTuple(NULL, 'key')), -- Nothing(Tuple<Int32?, String>?), no such key DictLookup($t3, AsTuple(NULL, 'key')) -; --- Just(AsTuple(123, "value")), null is convertible to Nothing<T> for any T +; -- Just(AsTuple(123, "value")), null is convertible to Nothing<T> for any T diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_ansi_join_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_ansi_join_/formatted.sql index 3dec26b8c0..cf090dac37 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_ansi_join_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_ansi_join_/formatted.sql @@ -66,9 +66,8 @@ WHERE optkey NOT IN $rp ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -77,9 +76,8 @@ WHERE optkey NOT IN $rp ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -88,9 +86,8 @@ WHERE optkey NOT IN $rp ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] -- Right is O SELECT optkey @@ -100,9 +97,8 @@ WHERE optkey NOT IN $ro ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -111,9 +107,8 @@ WHERE optkey NOT IN $ro ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -122,9 +117,8 @@ WHERE optkey NOT IN $ro ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] -- Right is N SELECT optkey @@ -134,9 +128,8 @@ WHERE optkey NOT IN $rn ORDER BY optkey -; +; -- [] --- [] SELECT optkey FROM @@ -145,9 +138,8 @@ WHERE optkey NOT IN $rn ORDER BY optkey -; +; -- [] --- [] SELECT optkey FROM @@ -156,9 +148,8 @@ WHERE optkey NOT IN $rn ORDER BY optkey -; +; -- [] --- [] -- 2, 4, 6, null $extraDict = ( SELECT @@ -182,9 +173,8 @@ WHERE optkey != 10 AND optkey NOT IN $ro AND optkey IN $extraDict AND optkey != 4 ORDER BY optkey -; +; -- [2,6] --- [2,6] SELECT optkey FROM @@ -193,9 +183,8 @@ WHERE optkey != 10 AND optkey NOT IN $rn AND optkey IN $extraDict AND optkey != 4 ORDER BY optkey -; +; -- [] --- [] -- Empty dict SELECT optkey @@ -211,6 +200,4 @@ WHERE ) ORDER BY optkey -; --- [1-10,null] - +; -- [1-10,null] diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_noansi_join_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_noansi_join_/formatted.sql index 9fbe514124..4f02d53d68 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_noansi_join_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_noansi_join_/formatted.sql @@ -66,9 +66,8 @@ WHERE optkey NOT IN $rp ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -77,9 +76,8 @@ WHERE optkey NOT IN $rp ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -88,9 +86,8 @@ WHERE optkey NOT IN $rp ORDER BY optkey -; +; -- [null,2,3,4,6,8,10] --- [null,2,3,4,6,8,10] -- Right is O SELECT optkey @@ -100,9 +97,8 @@ WHERE optkey NOT IN $ro ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -111,9 +107,8 @@ WHERE optkey NOT IN $ro ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -122,9 +117,8 @@ WHERE optkey NOT IN $ro ORDER BY optkey -; +; -- [null,2,3,4,6,8,10] --- [null,2,3,4,6,8,10] -- Right is N SELECT optkey @@ -134,9 +128,8 @@ WHERE optkey NOT IN $rn ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -145,9 +138,8 @@ WHERE optkey NOT IN $rn ORDER BY optkey -; +; -- [2,3,4,6,8,10] --- [2,3,4,6,8,10] SELECT optkey FROM @@ -156,9 +148,8 @@ WHERE optkey NOT IN $rn ORDER BY optkey -; +; -- [null,2,3,4,6,8,10] --- [null,2,3,4,6,8,10] -- 2, 4, 6, null $extraDict = ( SELECT @@ -182,9 +173,8 @@ WHERE optkey != 10 AND optkey NOT IN $ro AND optkey IN $extraDict AND optkey != 4 ORDER BY optkey -; +; -- [2,6] --- [2,6] SELECT optkey FROM @@ -193,9 +183,8 @@ WHERE optkey != 10 AND optkey NOT IN $rn AND optkey IN $extraDict AND optkey != 4 ORDER BY optkey -; +; -- [2,6] --- [2,6] -- Empty dict SELECT optkey @@ -211,6 +200,4 @@ WHERE ) ORDER BY optkey -; --- [1-10,null] - +; -- [1-10,null] diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_/formatted.sql index 7cc64316c7..02d8df23eb 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_/formatted.sql @@ -10,5 +10,4 @@ SELECT (1, NULL) IN ((1, 2), (1, 3)), -- Nothing<Bool?> (2, NULL) IN ((1, 2), (1, 3)), -- Nothing<Bool?> (1, NULL) IN () -; --- Nothing<Bool?> +; -- Nothing<Bool?> diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_ansi_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_ansi_/formatted.sql index 326a709927..cd79c2a96e 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_ansi_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_in-in_with_nulls_and_optionals_extra_ansi_/formatted.sql @@ -10,5 +10,4 @@ SELECT (1, NULL) IN ((1, 2), (1, 3)), -- Nothing<Bool?> (2, NULL) IN ((1, 2), (1, 3)), -- Just(false) (1, NULL) IN () -; --- Just(false) +; -- Just(false) diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_/formatted.sql index 37e8cecb5a..f9f38135f4 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_/formatted.sql @@ -1,9 +1,7 @@ /* ytfile can not */ USE plato; -PRAGMA warning('disable', '8001'); - --- CBO_MISSING_TABLE_STATS +PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS PRAGMA CostBasedOptimizer = 'native'; PRAGMA yt.MapJoinLimit = '1000'; PRAGMA yt.LookupJoinLimit = '1000'; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_only_sorted_merge_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_only_sorted_merge_/formatted.sql index 59a75e45c9..3773cd9c62 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_only_sorted_merge_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-cbo_4tables_only_sorted_merge_/formatted.sql @@ -1,9 +1,7 @@ /* ytfile can not */ USE plato; -PRAGMA warning('disable', '8001'); - --- CBO_MISSING_TABLE_STATS +PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS PRAGMA CostBasedOptimizer = 'native'; PRAGMA yt.MapJoinLimit = '1000'; PRAGMA yt.LookupJoinLimit = '1000'; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_cbo_3_tables_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_cbo_3_tables_/formatted.sql index ecbc9980f9..db7bf3d339 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_cbo_3_tables_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_cbo_3_tables_/formatted.sql @@ -1,8 +1,6 @@ USE plato; -PRAGMA warning('disable', '8001'); - --- CBO_MISSING_TABLE_STATS +PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS PRAGMA CostBasedOptimizer = 'pg'; SELECT diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_left_cbo_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_left_cbo_/formatted.sql index e3287b0ccb..f84bb6ef38 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_left_cbo_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_left_cbo_/formatted.sql @@ -1,8 +1,6 @@ USE plato; -PRAGMA warning('disable', '8001'); - --- CBO_MISSING_TABLE_STATS +PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS PRAGMA CostBasedOptimizer = 'PG'; SELECT diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_right_cbo_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_right_cbo_/formatted.sql index 89c9d30704..b1ab35595c 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_right_cbo_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-join_right_cbo_/formatted.sql @@ -1,8 +1,6 @@ USE plato; -PRAGMA warning('disable', '8001'); - --- CBO_MISSING_TABLE_STATS +PRAGMA warning('disable', '8001'); -- CBO_MISSING_TABLE_STATS PRAGMA CostBasedOptimizer = 'PG'; SELECT diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-decimal_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-decimal_/formatted.sql index 1c8ffe23b4..a1866f61c1 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-decimal_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-decimal_/formatted.sql @@ -84,23 +84,20 @@ FROM @decimal WHERE value == CAST('6.6' AS Decimal (15, 10)) -; +; -- Safe key filter calc --- Safe key filter calc SELECT * FROM @decimal WHERE value == CAST($asIs('3.3') AS Decimal (15, 10)) -; +; -- Unsafe key filter calc --- Unsafe key filter calc SELECT * FROM @decimal WHERE value == CAST($asIs('bad') AS Decimal (15, 10)) -; --- Unsafe key filter calc +; -- Unsafe key filter calc diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-tzdate_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-tzdate_/formatted.sql index db97e0a135..262e38eeec 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-tzdate_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-tzdate_/formatted.sql @@ -77,23 +77,20 @@ FROM @tzdate WHERE value == CAST('1999-01-01,Europe/Moscow' AS TzDate) -; +; -- Safe key filter calc --- Safe key filter calc SELECT * FROM @tzdate WHERE value == CAST($asIs('2105-12-30,America/Los_Angeles') AS TzDate) -; +; -- Unsafe key filter calc --- Unsafe key filter calc SELECT * FROM @tzdate WHERE value == CAST($asIs('bad') AS TzDate) -; --- Unsafe key filter calc +; -- Unsafe key filter calc diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-uuid_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-uuid_/formatted.sql index b4fc19dd85..5ca4a5e7ee 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-uuid_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_key_filter-uuid_/formatted.sql @@ -98,23 +98,20 @@ FROM @uuid WHERE value == CAST('00000000-0000-0000-0000-100000000000' AS Uuid) -; +; -- Safe key filter calc --- Safe key filter calc SELECT * FROM @uuid WHERE value == CAST($asIs('00000000-0000-0000-0000-200000000000') AS Uuid) -; +; -- Unsafe key filter calc --- Unsafe key filter calc SELECT * FROM @uuid WHERE value == CAST($asIs('bad') AS Uuid) -; --- Unsafe key filter calc +; -- Unsafe key filter calc diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_match_recognize-all_rows_per_match_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_match_recognize-all_rows_per_match_/formatted.sql index edcc854cfa..ae64953618 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_match_recognize-all_rows_per_match_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_match_recognize-all_rows_per_match_/formatted.sql @@ -1,6 +1,6 @@ -PRAGMA FeatureR010 = "prototype"; +PRAGMA FeatureR010 = 'prototype'; -$input = +$input = ( SELECT * FROM @@ -19,7 +19,7 @@ $input = <|time: 1100, value: 5|>, <|time: 1200, value: 0|>, ]) -; +); SELECT * diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_order_by-order_by_missing_project_column_nosimple_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_order_by-order_by_missing_project_column_nosimple_/formatted.sql new file mode 100644 index 0000000000..5bb4d66337 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_order_by-order_by_missing_project_column_nosimple_/formatted.sql @@ -0,0 +1,46 @@ +/* syntax version 1 */ +/* postgres can not */ +/* dq can not */ +/* dqfile can not */ +/* yt can not */ +PRAGMA DisableSimpleColumns; + +$src = [ + <|a: 5, b: 50, zz: 500|>, + <|a: 4, b: 40, zz: 400|>, + <|a: 3, b: 30, zz: 300|>, + <|a: 2, b: 20, zz: 200|>, + <|a: 1, b: 10, zz: 100|>, +]; + +$src1 = [ + <|a: 4, f: 40|>, + <|a: 3, f: 30|>, + <|a: 2, f: 20|>, + <|a: 1, f: 10|>, +]; + +$src = ( + SELECT + * + FROM + as_table($src) +); + +$src1 = ( + SELECT + * + FROM + as_table($src1) +); + +SELECT + x.* +FROM + $src AS x +LEFT ONLY JOIN + $src1 AS y +USING (a) +ORDER BY + random(a) +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_params-primitives_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_params-primitives_/formatted.sql index 19fc917dc6..0c5da65033 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_params-primitives_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_params-primitives_/formatted.sql @@ -3,22 +3,14 @@ DECLARE $x1 AS int; DECLARE $x2 AS Double; DECLARE $x3 AS Bool; DECLARE $x4 AS Float; -DECLARE $x5 AS String; - --- unicode +DECLARE $x5 AS String; -- unicode DECLARE $x6 AS Int64; DECLARE $x7 AS Uint64; -DECLARE $x8 AS String; - --- binary +DECLARE $x8 AS String; -- binary DECLARE $x9 AS Utf8; DECLARE $x10 AS Uuid; -DECLARE $x11 AS String?; - --- null -DECLARE $x12 AS String?; - --- not null +DECLARE $x11 AS String?; -- null +DECLARE $x12 AS String?; -- not null DECLARE $x13 AS Yson; DECLARE $x14 AS Json; DECLARE $x15 AS datetime; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_/formatted.sql index 08d9a3b581..0a9259aed4 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_/formatted.sql @@ -8,9 +8,8 @@ $count = ( COUNT(*) FROM Input -); +); -- $count = 10 --- $count = 10 $var = ( SELECT * diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_udf_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_udf_/formatted.sql index 7a2f11fb41..7da3c0e4d7 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_udf_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-bind_expr_udf_/formatted.sql @@ -11,5 +11,4 @@ FROM TABLESAMPLE BERNOULLI (Math::Ceil(100 * $percent)) ORDER BY key -; --- 100% sample +; -- 100% sample diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-subquery_expr_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-subquery_expr_/formatted.sql index 337f365d5e..6b7041a3f1 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-subquery_expr_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_sampling-subquery_expr_/formatted.sql @@ -6,9 +6,8 @@ $count = ( COUNT(*) FROM plato.Input -); +); -- $count = 10 --- $count = 10 SELECT * FROM ( diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_schema-user_schema_with_sort_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_schema-user_schema_with_sort_/formatted.sql index 6b9523050c..38c1e0f904 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_schema-user_schema_with_sort_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_schema-user_schema_with_sort_/formatted.sql @@ -15,9 +15,8 @@ FROM Input1 WITH SCHEMA Struct<key: String?, subkey: String> ORDER BY key -; +; -- should reset sort --- should reset sort SELECT * FROM diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-named_args_for_script_with_posargs_reuse_args_fail_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-named_args_for_script_with_posargs_reuse_args_fail_/formatted.sql index b1cc429950..3a7b94d707 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-named_args_for_script_with_posargs_reuse_args_fail_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-named_args_for_script_with_posargs_reuse_args_fail_/formatted.sql @@ -26,5 +26,4 @@ SELECT $udf(name, age, age AS age) AS val FROM $data -; --- age is reused as named after positional +; -- age is reused as named after positional diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-wrong_args_fail_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-wrong_args_fail_/formatted.sql index 67a6d45b69..2aeb8fc7ed 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-wrong_args_fail_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-wrong_args_fail_/formatted.sql @@ -3,20 +3,16 @@ -- Find has optional args SELECT String::ReplaceAll() -; +; -- too few --- too few SELECT String::ReplaceAll('abc') -; +; -- too few --- too few SELECT String::ReplaceAll('abc', 'b', 2, 4) -; +; -- too many --- too many SELECT String::ReplaceAll('abc', 'b', 2, 4, 44) -; --- too many +; -- too many diff --git a/yql/essentials/tests/sql/sql2yql/ya.make b/yql/essentials/tests/sql/sql2yql/ya.make index f6f5df53a0..cff97cecd9 100644 --- a/yql/essentials/tests/sql/sql2yql/ya.make +++ b/yql/essentials/tests/sql/sql2yql/ya.make @@ -1,3 +1,5 @@ +IF (NOT OPENSOURCE) + PY3TEST() TEST_SRCS( test_sql2yql.py @@ -42,3 +44,5 @@ NO_CHECK_IMPORTS() END() +ENDIF() + diff --git a/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.cfg b/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.cfg new file mode 100644 index 0000000000..bb349dd8ab --- /dev/null +++ b/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.cfg @@ -0,0 +1 @@ +providers yt diff --git a/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.sql b/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.sql new file mode 100644 index 0000000000..f1434c8c08 --- /dev/null +++ b/yql/essentials/tests/sql/suites/order_by/order_by_missing_project_column_nosimple.sql @@ -0,0 +1,32 @@ +/* syntax version 1 */ +/* postgres can not */ +/* dq can not */ +/* dqfile can not */ +/* yt can not */ +pragma DisableSimpleColumns; + +$src = [ +<|a:5, b:50, zz:500|>, +<|a:4, b:40, zz:400|>, +<|a:3, b:30, zz:300|>, +<|a:2, b:20, zz:200|>, +<|a:1, b:10, zz:100|>, +]; + +$src1 = [ +<|a:4, f:40|>, +<|a:3, f:30|>, +<|a:2, f:20|>, +<|a:1, f:10|>, +]; + + +$src = select * from as_table($src); +$src1 = select * from as_table($src1); + +select + x.* +from $src as x +left only join $src1 as y +using(a) +order by random(a); diff --git a/yql/essentials/tools/minirun/apply_patches.sh b/yql/essentials/tools/minirun/apply_patches.sh new file mode 100755 index 0000000000..ca2e4f445f --- /dev/null +++ b/yql/essentials/tools/minirun/apply_patches.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -eu + +errexit() { + echo $1 + exit 1 + } + +echo patching +cd ../../../../ +patch -p0 < yql/essentials/tools/minirun/patches/01_no_icu.patch || errexit "Source patching failed" +echo done + + diff --git a/yql/essentials/tools/minirun/minirun.cpp b/yql/essentials/tools/minirun/minirun.cpp index 0d4fedfd9a..1282212bb2 100644 --- a/yql/essentials/tools/minirun/minirun.cpp +++ b/yql/essentials/tools/minirun/minirun.cpp @@ -11,6 +11,7 @@ public: { GetRunOptions().UseRepeatableRandomAndTimeProviders = true; GetRunOptions().ResultsFormat = NYson::EYsonFormat::Pretty; + GetRunOptions().OptimizeLibs = false; GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { opts.AddLongOption("ndebug", "Do not show debug info in error output").NoArgument().SetFlag(&GetRunOptions().NoDebug); diff --git a/yql/essentials/tools/minirun/patches/01_no_icu.patch b/yql/essentials/tools/minirun/patches/01_no_icu.patch new file mode 100644 index 0000000000..95a2723287 --- /dev/null +++ b/yql/essentials/tools/minirun/patches/01_no_icu.patch @@ -0,0 +1,93 @@ +--- contrib/restricted/boost/regex/ya.make (index) ++++ contrib/restricted/boost/regex/ya.make (working tree) +@@ -16,7 +16,6 @@ VERSION(1.87.0) + ORIGINAL_SOURCE(https://github.com/boostorg/regex/archive/boost-1.87.0.tar.gz) + + PEERDIR( +- contrib/libs/icu + contrib/restricted/boost/assert + contrib/restricted/boost/concept_check + contrib/restricted/boost/config +@@ -40,7 +39,6 @@ NO_COMPILER_WARNINGS() + NO_UTIL() + + CFLAGS( +- -DBOOST_HAS_ICU + -DBOOST_NO_CXX98_BINDERS + ) + +--- contrib/restricted/thrift/ya.make (index) ++++ contrib/restricted/thrift/ya.make (working tree) +@@ -24,7 +24,6 @@ PEERDIR( + contrib/libs/openssl + contrib/libs/zlib + contrib/restricted/boost/interprocess +- contrib/restricted/boost/locale + contrib/restricted/boost/math + contrib/restricted/boost/system + contrib/restricted/boost/thread +@@ -49,7 +48,6 @@ SRCS( + thrift/protocol/TBase64Utils.cpp + thrift/protocol/TDebugProtocol.cpp + thrift/protocol/THeaderProtocol.cpp +- thrift/protocol/TJSONProtocol.cpp + thrift/protocol/TMultiplexedProtocol.cpp + thrift/protocol/TProtocol.cpp + thrift/server/TConnectedClient.cpp +--- yql/essentials/parser/pg_wrapper/postgresql/src/backend/utils/adt/formatting.c (index) ++++ yql/essentials/parser/pg_wrapper/postgresql/src/backend/utils/adt/formatting.c (working tree) +@@ -72,7 +72,7 @@ + #include <wctype.h> + + #ifdef USE_ICU +-#include <unicode/ustring.h> ++#error #include <unicode/ustring.h> + #endif + + #include "catalog/pg_collation.h" +--- yql/essentials/parser/pg_wrapper/postgresql/src/backend/utils/adt/pg_locale.c (index) ++++ yql/essentials/parser/pg_wrapper/postgresql/src/backend/utils/adt/pg_locale.c (working tree) +@@ -69,8 +69,8 @@ + #include "utils/syscache.h" + + #ifdef USE_ICU +-#include <unicode/ucnv.h> +-#include <unicode/ustring.h> ++#error #include <unicode/ucnv.h> ++#error #include <unicode/ustring.h> + #endif + + #ifdef __GLIBC__ +--- yql/essentials/parser/pg_wrapper/postgresql/src/include/pg_config-linux.h (index) ++++ yql/essentials/parser/pg_wrapper/postgresql/src/include/pg_config-linux.h (working tree) +@@ -707,7 +707,8 @@ + /* #undef USE_BSD_AUTH */ + + /* Define to build with ICU support. (--with-icu) */ +-#define USE_ICU 1 ++//#define USE_ICU 1 ++#undef USE_ICU + + /* Define to 1 to build with LDAP support. (--with-ldap) */ + /* #undef USE_LDAP */ +--- yql/essentials/parser/pg_wrapper/postgresql/src/include/utils/pg_locale.h (index) ++++ yql/essentials/parser/pg_wrapper/postgresql/src/include/utils/pg_locale.h (working tree) +@@ -16,7 +16,7 @@ + #include <xlocale.h> + #endif + #ifdef USE_ICU +-#include <unicode/ucol.h> ++#error include <unicode/ucol.h> + #endif + + #ifdef USE_ICU +--- yql/essentials/parser/pg_wrapper/ya.make (index) ++++ yql/essentials/parser/pg_wrapper/ya.make (working tree) +@@ -95,7 +95,6 @@ PEERDIR( + yql/essentials/types/dynumber + yql/essentials/types/uuid + +- contrib/libs/icu + contrib/libs/libc_compat + contrib/libs/libxml + contrib/libs/lz4 diff --git a/yql/essentials/tools/udf_resolver/udf_resolver.cpp b/yql/essentials/tools/udf_resolver/udf_resolver.cpp index 8d2b8ec6cd..5b8717bfe3 100644 --- a/yql/essentials/tools/udf_resolver/udf_resolver.cpp +++ b/yql/essentials/tools/udf_resolver/udf_resolver.cpp @@ -456,6 +456,7 @@ int main(int argc, char **argv) { Allow(stat), #endif Allow(sysinfo), + Allow(sigaltstack), Allow(uname), #ifndef _arm64_ Allow(unlink), diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp index 6e6167582f..2db2f62607 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp @@ -573,7 +573,7 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) { moduleResolver = std::make_shared<TModuleResolver>(std::move(modules), ctx.NextUniqueId, ClusterMapping_, RunOptions_.SqlFlags, RunOptions_.Mode >= ERunMode::Validate); } else { - if (!GetYqlDefaultModuleResolver(ctx, moduleResolver, ClusterMapping_, RunOptions_.Mode >= ERunMode::Validate)) { + if (!GetYqlDefaultModuleResolver(ctx, moduleResolver, ClusterMapping_, RunOptions_.OptimizeLibs && RunOptions_.Mode >= ERunMode::Validate)) { *RunOptions_.ErrStream << "Errors loading default YQL libraries:" << Endl; ctx.IssueManager.GetIssues().PrintTo(*RunOptions_.ErrStream); return -1; diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.h b/yql/essentials/tools/yql_facade_run/yql_facade_run.h index f5e1fd5bd2..b586370c2d 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.h +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.h @@ -127,6 +127,7 @@ public: bool EnableResultPosition = false; bool EnableCredentials = false; bool EnableQPlayer = false; + bool OptimizeLibs = true; void Parse(int argc, const char *argv[]); diff --git a/yql/essentials/udfs/common/digest/digest_udf.cpp b/yql/essentials/udfs/common/digest/digest_udf.cpp index 491fe7a66c..1925052200 100644 --- a/yql/essentials/udfs/common/digest/digest_udf.cpp +++ b/yql/essentials/udfs/common/digest/digest_udf.cpp @@ -26,6 +26,68 @@ using namespace NKikimr; using namespace NUdf; namespace { + enum EDigestType { + CRC32C, CRC64, FNV32, FNV64, MURMUR, MURMUR32, MURMUR2A, MURMUR2A32, CITY + }; + const char* DigestNames[] = { + "Crc32c", "Crc64", "Fnv32", "Fnv64", "MurMurHash", "MurMurHash32", "MurMurHash2A", "MurMurHash2A32", "CityHash" + }; + + template<typename TResult> + using TDigestGenerator = TResult(const TStringRef&, TMaybe<TResult> init); + + template<EDigestType DigestType, typename TResult, TDigestGenerator<TResult>* Generator> + class TDigestFunctionUdf: public TBoxedValue { + public: + TDigestFunctionUdf(TSourcePosition pos) : Pos_(pos) {} + + static TStringRef Name() { + static TString name = DigestNames[DigestType]; + return TStringRef(name); + } + + static bool DeclareSignature( + const TStringRef& name, + TType*, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) + { + if (Name() != name) { + return false; + } + + auto args = builder.Args(); + args->Add(builder.SimpleType<char *>()).Flags(ICallablePayload::TArgumentFlags::AutoMap); + args->Add(builder.Optional()->Item(builder.SimpleType<TResult>()).Build()).Name("Init"); + args->Done(); + builder.OptionalArgs(1); + builder.Returns(builder.SimpleType<TResult>()); + + if (!typesOnly) { + builder.Implementation(new TDigestFunctionUdf<DigestType, TResult, Generator>(GetSourcePosition(builder))); + } + + return true; + } + + private: + TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const final try { + TMaybe<TResult> init = Nothing(); + if (auto val = args[1]) { + init = val.Get<TResult>(); + } + return TUnboxedValuePod(Generator(args[0].AsStringRef(), init)); + } catch (const std ::exception&) { + TStringBuilder sb; + sb << Pos_ << " "; + sb << CurrentExceptionMessage(); + sb << Endl << "[" << TStringBuf(Name()) << "]"; + UdfTerminate(sb.c_str()); + } + + TSourcePosition Pos_; + }; + SIMPLE_STRICT_UDF(TCrc32c, ui32(TAutoMap<char*>)) { Y_UNUSED(valueBuilder); const auto& inputRef = args[0].AsStringRef(); @@ -33,68 +95,65 @@ namespace { return TUnboxedValuePod(hash); } - SIMPLE_STRICT_UDF(TCrc64, ui64(TAutoMap<char*>)) { - Y_UNUSED(valueBuilder); - const auto& inputRef = args[0].AsStringRef(); - ui64 hash = crc64(inputRef.Data(), inputRef.Size()); - return TUnboxedValuePod(hash); - } - - SIMPLE_STRICT_UDF(TFnv32, ui32(TAutoMap<char*>)) { - Y_UNUSED(valueBuilder); - const auto& inputRef = args[0].AsStringRef(); - ui32 hash = FnvHash<ui32>(inputRef.Data(), inputRef.Size()); - return TUnboxedValuePod(hash); - } + using TCrc64 = TDigestFunctionUdf<CRC64, ui64, [](auto& inputRef, auto init) { + return crc64(inputRef.Data(), inputRef.Size(), init.GetOrElse(CRC64INIT)); + }>; - SIMPLE_STRICT_UDF(TFnv64, ui64(TAutoMap<char*>)) { - Y_UNUSED(valueBuilder); - const auto& inputRef = args[0].AsStringRef(); - ui64 hash = FnvHash<ui64>(inputRef.Data(), inputRef.Size()); - return TUnboxedValuePod(hash); - } + using TFnv32 = TDigestFunctionUdf<FNV32, ui32, [](auto& inputRef, auto init) { + if (init) { + return FnvHash<ui32>(inputRef.Data(), inputRef.Size(), *init); + } else { + return FnvHash<ui32>(inputRef.Data(), inputRef.Size()); + } + }>; - SIMPLE_STRICT_UDF(TMurMurHash, ui64(TAutoMap<char*>)) { - Y_UNUSED(valueBuilder); - const auto& inputRef = args[0].AsStringRef(); - ui64 hash = MurmurHash<ui64>(inputRef.Data(), inputRef.Size()); - return TUnboxedValuePod(hash); - } + using TFnv64 = TDigestFunctionUdf<FNV64, ui64, [](auto& inputRef, auto init) { + if (init) { + return FnvHash<ui64>(inputRef.Data(), inputRef.Size(), *init); + } else { + return FnvHash<ui64>(inputRef.Data(), inputRef.Size()); + } + }>; - SIMPLE_STRICT_UDF(TMurMurHash32, ui32(TAutoMap<char*>)) { - Y_UNUSED(valueBuilder); - const auto& inputRef = args[0].AsStringRef(); - ui32 hash = MurmurHash<ui32>(inputRef.Data(), inputRef.Size()); - return TUnboxedValuePod(hash); - } + using TMurMurHash = TDigestFunctionUdf<MURMUR, ui64, [](auto& inputRef, auto init) { + if (init) { + return MurmurHash<ui64>(inputRef.Data(), inputRef.Size(), *init); + } else { + return MurmurHash<ui64>(inputRef.Data(), inputRef.Size()); + } + }>; - SIMPLE_STRICT_UDF(TMurMurHash2A, ui64(TAutoMap<char*>)) { - Y_UNUSED(valueBuilder); - const auto& inputRef = args[0].AsStringRef(); - ui64 hash = TMurmurHash2A<ui64>{}.Update(inputRef.Data(), inputRef.Size()).Value(); - return TUnboxedValuePod(hash); - } + using TMurMurHash32 = TDigestFunctionUdf<MURMUR32, ui32, [] (auto& inputRef, auto init) { + if (init) { + return MurmurHash<ui32>(inputRef.Data(), inputRef.Size(), *init); + } else { + return MurmurHash<ui32>(inputRef.Data(), inputRef.Size()); + } + }>; - SIMPLE_STRICT_UDF(TMurMurHash2A32, ui32(TAutoMap<char*>)) { - Y_UNUSED(valueBuilder); - const auto& inputRef = args[0].AsStringRef(); - ui32 hash = TMurmurHash2A<ui32>{}.Update(inputRef.Data(), inputRef.Size()).Value(); - return TUnboxedValuePod(hash); - } + using TMurMurHash2A = TDigestFunctionUdf<MURMUR2A, ui64, [] (auto& inputRef, auto init) { + if (init) { + return TMurmurHash2A<ui64>{*init}.Update(inputRef.Data(), inputRef.Size()).Value(); + } else { + return TMurmurHash2A<ui64>{}.Update(inputRef.Data(), inputRef.Size()).Value(); + } + }>; - SIMPLE_STRICT_UDF_WITH_OPTIONAL_ARGS(TCityHash, ui64(TAutoMap<char*>, TOptional<ui64>), 1) { - Y_UNUSED(valueBuilder); - const auto& inputRef = args[0].AsStringRef(); - ui64 hash; - if (args[1]) { - hash = CityHash64WithSeed(inputRef.Data(), inputRef.Size(), args[1].Get<ui64>()); + using TMurMurHash2A32 = TDigestFunctionUdf<MURMUR2A32, ui32, [] (auto& inputRef, auto init) { + if (init) { + return TMurmurHash2A<ui32>{*init}.Update(inputRef.Data(), inputRef.Size()).Value(); } else { - hash = CityHash64(inputRef.Data(), inputRef.Size()); + return TMurmurHash2A<ui32>{}.Update(inputRef.Data(), inputRef.Size()).Value(); } - return TUnboxedValuePod(hash); - } + }>; - using TUi64Pair = NUdf::TTuple<ui64, ui64>; + using TCityHash = TDigestFunctionUdf<CITY, ui64, [] (auto& inputRef, auto init) { + if (init) { + return CityHash64WithSeed(inputRef.Data(), inputRef.Size(), *init); + } else { + return CityHash64(inputRef.Data(), inputRef.Size()); + } + }>; class TCityHash128: public TBoxedValue { public: diff --git a/yql/essentials/udfs/common/digest/test/canondata/result.json b/yql/essentials/udfs/common/digest/test/canondata/result.json index fb6112fc5b..5056bb47cb 100644 --- a/yql/essentials/udfs/common/digest/test/canondata/result.json +++ b/yql/essentials/udfs/common/digest/test/canondata/result.json @@ -3,5 +3,10 @@ { "uri": "file://test.test_Basic_/results.txt" } + ], + "test.test[Init]": [ + { + "uri": "file://test.test_Init_/results.txt" + } ] } diff --git a/yql/essentials/udfs/common/digest/test/canondata/test.test_Init_/results.txt b/yql/essentials/udfs/common/digest/test/canondata/test.test_Init_/results.txt new file mode 100644 index 0000000000..53faaf453b --- /dev/null +++ b/yql/essentials/udfs/common/digest/test/canondata/test.test_Init_/results.txt @@ -0,0 +1,202 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "column1"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "column2"; + [ + "DataType"; + "Uint32" + ] + ]; + [ + "column3"; + [ + "DataType"; + "Uint32" + ] + ]; + [ + "column4"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "column5"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "column6"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "column7"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "column8"; + [ + "DataType"; + "Uint32" + ] + ]; + [ + "column9"; + [ + "DataType"; + "Uint32" + ] + ]; + [ + "column10"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "column11"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "column12"; + [ + "DataType"; + "Uint32" + ] + ]; + [ + "column13"; + [ + "DataType"; + "Uint32" + ] + ]; + [ + "column14"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "column15"; + [ + "DataType"; + "Uint64" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "12517938706220930219"; + "12517938706220930219"; + "151308058"; + "151308058"; + "854320535119994"; + "854320535119994"; + "9815316673246213022"; + "9815316673246213022"; + "3549562424"; + "3549562424"; + "7091385679591462881"; + "7091385679591462881"; + "2867599578"; + "2867599578"; + "9781551444137168419"; + "9781551444137168419" + ]; + [ + "7439640649521800617"; + "7439640649521800617"; + "151308057"; + "151308057"; + "854320535119993"; + "854320535119993"; + "3140806664558319421"; + "3140806664558319421"; + "4067583852"; + "4067583852"; + "8498979662292624923"; + "8498979662292624923"; + "1911426935"; + "1911426935"; + "16387792208989722131"; + "16387792208989722131" + ]; + [ + "9402898472742411560"; + "9402898472742411560"; + "151308056"; + "151308056"; + "854320535119992"; + "854320535119992"; + "5511627967124305793"; + "5511627967124305793"; + "4218195213"; + "4218195213"; + "4001281192876778049"; + "4001281192876778049"; + "1399730784"; + "1399730784"; + "3425497360513913945"; + "3425497360513913945" + ]; + [ + "777"; + "777"; + "777"; + "777"; + "777"; + "777"; + "16843636611330352828"; + "16843636611330352828"; + "2954663174"; + "2954663174"; + "13671058521579127884"; + "13671058521579127884"; + "2722328547"; + "2722328547"; + "15226978636637575808"; + "15226978636637575808" + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/digest/test/cases/Init.cfg b/yql/essentials/udfs/common/digest/test/cases/Init.cfg new file mode 100644 index 0000000000..62e2c58706 --- /dev/null +++ b/yql/essentials/udfs/common/digest/test/cases/Init.cfg @@ -0,0 +1 @@ +in plato.Input Basic.in
\ No newline at end of file diff --git a/yql/essentials/udfs/common/digest/test/cases/Init.sql b/yql/essentials/udfs/common/digest/test/cases/Init.sql new file mode 100644 index 0000000000..7805314d87 --- /dev/null +++ b/yql/essentials/udfs/common/digest/test/cases/Init.sql @@ -0,0 +1,11 @@ +SELECT + Digest::Crc64(key, 777), Digest::Crc64(key, 777 AS Init), + Digest::Fnv32(key, 777), Digest::Fnv32(key, 777 AS Init), + Digest::Fnv64(key, 777), Digest::Fnv64(key, 777 AS Init), + Digest::MurMurHash(key, 777), Digest::MurMurHash(key, 777 AS Init), + Digest::MurMurHash32(key, 777), Digest::MurMurHash32(key, 777 AS Init), + Digest::MurMurHash2A(key, 777), Digest::MurMurHash2A(key, 777 AS Init), + Digest::MurMurHash2A32(key, 777), Digest::MurMurHash2A32(key, 777 AS Init), + Digest::CityHash(key, 777), Digest::CityHash(key, 777 AS Init), + +FROM Input;
\ No newline at end of file diff --git a/yql/essentials/udfs/common/hyperscan/test/canondata/test.test_Error_/extracted b/yql/essentials/udfs/common/hyperscan/test/canondata/test.test_Error_/extracted index d3707ae88c..21ca8cec79 100644 --- a/yql/essentials/udfs/common/hyperscan/test/canondata/test.test_Error_/extracted +++ b/yql/essentials/udfs/common/hyperscan/test/canondata/test.test_Error_/extracted @@ -3,6 +3,6 @@ <tmp_path>/program.sql:<main>:8:1: Error: Execution of node: YtMap! SELECT $match(value) AS match FROM Input; ^ - <tmp_path>/program.sql:<main>:6:21: Error: library/cpp/regex/hyperscan/hyperscan.cpp:102: Failed to compile regex: ^*$. Error message (hyperscan): Invalid repeat at index 1. + <tmp_path>/program.sql:<main>:6:21: Error: library/cpp/regex/hyperscan/hyperscan.cpp:92: Failed to compile regex: ^*$. Error message (hyperscan): Invalid repeat at index 1. $match = Hyperscan::Match("*"); ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/yson2/test/canondata/test.test_From_/results.txt b/yql/essentials/udfs/common/yson2/test/canondata/test.test_From_/results.txt index d1e62592cc..ddddb74752 100644 --- a/yql/essentials/udfs/common/yson2/test/canondata/test.test_From_/results.txt +++ b/yql/essentials/udfs/common/yson2/test/canondata/test.test_From_/results.txt @@ -32,7 +32,7 @@ "column3"; [ "DataType"; - "Yson" + "Bool" ] ]; [ @@ -118,6 +118,13 @@ "DataType"; "Yson" ] + ]; + [ + "column16"; + [ + "DataType"; + "Yson" + ] ] ] ] @@ -126,6 +133,7 @@ [ %true; %true; + %true; %false; { "$type" = "boolean"; diff --git a/yql/essentials/udfs/common/yson2/test/cases/From.sql b/yql/essentials/udfs/common/yson2/test/cases/From.sql index 2a1f6ed15a..97b0fff53c 100644 --- a/yql/essentials/udfs/common/yson2/test/cases/From.sql +++ b/yql/essentials/udfs/common/yson2/test/cases/From.sql @@ -1,6 +1,7 @@ /* syntax version 1 */ select Yson::IsEntity(Yson::From(NULL)), +Yson::IsEntity(Yson::From(Void())), Yson::IsEntity(Yson::Parse(Yson("#"))), Yson::IsEntity(Yson::Parse(Yson("1"))), diff --git a/yql/essentials/udfs/common/yson2/yson2_udf.cpp b/yql/essentials/udfs/common/yson2/yson2_udf.cpp index 76dbe07c55..997b0cecf1 100644 --- a/yql/essentials/udfs/common/yson2/yson2_udf.cpp +++ b/yql/essentials/udfs/common/yson2/yson2_udf.cpp @@ -795,6 +795,7 @@ protected: static const TType* CheckType(const ITypeInfoHelper::TPtr typeHelper, const TType* shape) { switch (const auto kind = typeHelper->GetTypeKind(shape)) { case ETypeKind::Null: + case ETypeKind::Void: case ETypeKind::EmptyList: case ETypeKind::EmptyDict: return MoreTypesAllowed ? nullptr : shape; diff --git a/yql/essentials/utils/log/profile.cpp b/yql/essentials/utils/log/profile.cpp index 4dd412b8f6..130bb05a4a 100644 --- a/yql/essentials/utils/log/profile.cpp +++ b/yql/essentials/utils/log/profile.cpp @@ -17,27 +17,32 @@ TProfilingScope::~TProfilingScope() { return; } - double elapsed = static_cast<double>(::MicroSeconds() - StartedAt_); - TStringBuf unit("us"); - if (elapsed > 1000000) { - elapsed /= 1000000; - unit = TStringBuf("s"); - } else if (elapsed > 1000) { - elapsed /= 1000; - unit = TStringBuf("ms"); - } - - auto doLog = [&]() { - YQL_PERF_LOG(Level_, File_, Line_) - << TStringBuf("Execution of [") << Name_ - << TStringBuf("] took ") << Prec(elapsed, 3) << unit; - }; - - if (!LogCtxPath_.first.empty() || !LogCtxPath_.second.empty()) { - YQL_LOG_CTX_ROOT_SESSION_SCOPE(LogCtxPath_); - doLog(); - } else { - doLog(); + try { + double elapsed = static_cast<double>(::MicroSeconds() - StartedAt_); + TStringBuf unit("us"); + if (elapsed > 1000000) { + elapsed /= 1000000; + unit = TStringBuf("s"); + } else if (elapsed > 1000) { + elapsed /= 1000; + unit = TStringBuf("ms"); + } + + auto doLog = [&]() { + YQL_PERF_LOG(Level_, File_, Line_) + << TStringBuf("Execution of [") << Name_ + << TStringBuf("] took ") << Prec(elapsed, 3) << unit; + }; + + if (!LogCtxPath_.first.empty() || !LogCtxPath_.second.empty()) { + YQL_LOG_CTX_ROOT_SESSION_SCOPE(LogCtxPath_); + doLog(); + } else { + doLog(); + } + } catch (const std::bad_alloc&) { + // no memory, we will not write anything + Y_ABORT("No memory"); } } |