diff options
author | ngc224 <ngc224@yandex-team.ru> | 2022-02-10 16:46:20 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:20 +0300 |
commit | 50152f7fd45140de323b88e009d3daa4209b3f07 (patch) | |
tree | 2683706dd2f4f75d50ad7fa0df0bdfa59841c42e | |
parent | c73494e681a4e497ae191ada07a55a6bf55885ff (diff) | |
download | ydb-50152f7fd45140de323b88e009d3daa4209b3f07.tar.gz |
Restoring authorship annotation for <ngc224@yandex-team.ru>. Commit 1 of 2.
50 files changed, 547 insertions, 547 deletions
diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index 8970837f0f..61ac566d27 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -728,20 +728,20 @@ def onadd_pytest_bin(unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return - flat, kws = _common.sort_by_keywords({'RUNNER_BIN': 1}, args) - if flat: - ymake.report_configure_error( - 'Unknown arguments found while processing add_pytest_bin macro: {!r}' - .format(flat) - ) - - runner_bin = kws.get('RUNNER_BIN', [None])[0] + flat, kws = _common.sort_by_keywords({'RUNNER_BIN': 1}, args) + if flat: + ymake.report_configure_error( + 'Unknown arguments found while processing add_pytest_bin macro: {!r}' + .format(flat) + ) + + runner_bin = kws.get('RUNNER_BIN', [None])[0] test_type = 'py3test.bin' if (unit.get("PYTHON3") == 'yes') else "pytest.bin" add_test_to_dart(unit, test_type, runner_bin=runner_bin) + - -def add_test_to_dart(unit, test_type, binary_path=None, runner_bin=None): +def add_test_to_dart(unit, test_type, binary_path=None, runner_bin=None): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return @@ -1016,7 +1016,7 @@ def onsetup_pytest_bin(unit, *args): use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') == "yes" if use_arcadia_python: unit.onresource(['-', 'PY_MAIN={}'.format("library.python.pytest.main:main")]) # XXX - unit.onadd_pytest_bin(list(args)) + unit.onadd_pytest_bin(list(args)) else: unit.onno_platform() unit.onadd_pytest_script(["PY_TEST"]) diff --git a/build/ya.conf.json b/build/ya.conf.json index 5f7cc875d6..90eb7b50cb 100644 --- a/build/ya.conf.json +++ b/build/ya.conf.json @@ -4505,15 +4505,15 @@ "default": true } ] - }, - "run_python_udf": { - "tools": { + }, + "run_python_udf": { + "tools": { "run_python_udf": { "bottle": "run_python_udf", "executable": "run_python_udf" } - }, - "platforms": [ + }, + "platforms": [ { "host": { "os": "LINUX" @@ -4532,7 +4532,7 @@ }, "default": true } - ] + ] }, "graphs_difference": { "tools": { @@ -7248,17 +7248,17 @@ "python3" ] } - }, - "run_python_udf": { - "formula": { - "sandbox_id": 273337583, - "match": "RUN_PYTHON_UDF" - }, - "executable": { + }, + "run_python_udf": { + "formula": { + "sandbox_id": 273337583, + "match": "RUN_PYTHON_UDF" + }, + "executable": { "run_python_udf": [ "run_python_udf" ] - } + } }, "graphs_difference": { "formula": { diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 081833998b..304374a3e4 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -6096,11 +6096,11 @@ YDL_FLAGS= --force-color -I ${ARCADIA_ROOT} YDL_DESC_FLAGS= # tag:src-processing tag:ydl-specific -macro _SRC("ydl", SRC, SRCFLAGS...) { +macro _SRC("ydl", SRC, SRCFLAGS...) { .CMD=${tool:"statbox/ydl/compiler/tooling/ydl/bin"} c $YDL_FLAGS --cpp-output-header ${output;suf=.h:SRC} --cpp-output-source ${output;suf=.cpp:SRC} ${input:SRC} ${output_include;hide:"statbox/ydl/runtime/cpp/gen_support/standard_includes.h"} ${kv;hide:"p YDL"} ${kv;hide:"pc yellow"} - .PEERDIR+=statbox/ydl/runtime/cpp -} - + .PEERDIR+=statbox/ydl/runtime/cpp +} + # tag:ydl-specific ### @usage BUILD_YDL_DESC(Input Symbol Output) ### diff --git a/contrib/restricted/boost/libs/python/arcadia_test/mod/ya.make b/contrib/restricted/boost/libs/python/arcadia_test/mod/ya.make index ff4c0e70e4..1b0e6f37fa 100644 --- a/contrib/restricted/boost/libs/python/arcadia_test/mod/ya.make +++ b/contrib/restricted/boost/libs/python/arcadia_test/mod/ya.make @@ -1,4 +1,4 @@ -PY23_NATIVE_LIBRARY() +PY23_NATIVE_LIBRARY() WITHOUT_LICENSE_TEXTS() diff --git a/contrib/restricted/boost/libs/python/arcadia_test/py2/test_hello.py b/contrib/restricted/boost/libs/python/arcadia_test/py2/test_hello.py index 67d1d8f06a..7034f28ba3 100644 --- a/contrib/restricted/boost/libs/python/arcadia_test/py2/test_hello.py +++ b/contrib/restricted/boost/libs/python/arcadia_test/py2/test_hello.py @@ -1,5 +1,5 @@ -import arcadia_boost_python_test as M - - -def test_hello(): - assert M.hello() == "hello world!" +import arcadia_boost_python_test as M + + +def test_hello(): + assert M.hello() == "hello world!" diff --git a/contrib/restricted/boost/libs/python/arcadia_test/py2/ya.make b/contrib/restricted/boost/libs/python/arcadia_test/py2/ya.make index 2eb3461699..79af53a4d6 100644 --- a/contrib/restricted/boost/libs/python/arcadia_test/py2/ya.make +++ b/contrib/restricted/boost/libs/python/arcadia_test/py2/ya.make @@ -1,17 +1,17 @@ PY2TEST() - + WITHOUT_LICENSE_TEXTS() - + OWNER( antoshkka g:cpp-committee g:cpp-contrib ) - + PEERDIR( contrib/restricted/boost/libs/python/arcadia_test/mod ) -TEST_SRCS(test_hello.py) - -END() +TEST_SRCS(test_hello.py) + +END() diff --git a/contrib/restricted/boost/libs/python/arcadia_test/py3/test_hello.py b/contrib/restricted/boost/libs/python/arcadia_test/py3/test_hello.py index 67d1d8f06a..7034f28ba3 100644 --- a/contrib/restricted/boost/libs/python/arcadia_test/py3/test_hello.py +++ b/contrib/restricted/boost/libs/python/arcadia_test/py3/test_hello.py @@ -1,5 +1,5 @@ -import arcadia_boost_python_test as M - - -def test_hello(): - assert M.hello() == "hello world!" +import arcadia_boost_python_test as M + + +def test_hello(): + assert M.hello() == "hello world!" diff --git a/contrib/restricted/boost/libs/python/arcadia_test/py3/ya.make b/contrib/restricted/boost/libs/python/arcadia_test/py3/ya.make index 5334de5fb0..97f8768548 100644 --- a/contrib/restricted/boost/libs/python/arcadia_test/py3/ya.make +++ b/contrib/restricted/boost/libs/python/arcadia_test/py3/ya.make @@ -1,17 +1,17 @@ -PY3TEST() - +PY3TEST() + WITHOUT_LICENSE_TEXTS() - + OWNER( antoshkka g:cpp-committee g:cpp-contrib ) - + PEERDIR( contrib/restricted/boost/libs/python/arcadia_test/mod ) -TEST_SRCS(test_hello.py) - -END() +TEST_SRCS(test_hello.py) + +END() diff --git a/contrib/restricted/boost/libs/python/arcadia_test/ya.make b/contrib/restricted/boost/libs/python/arcadia_test/ya.make index 4cedc9fb3e..e4c19a7bbe 100644 --- a/contrib/restricted/boost/libs/python/arcadia_test/ya.make +++ b/contrib/restricted/boost/libs/python/arcadia_test/ya.make @@ -1,4 +1,4 @@ -RECURSE( - py2 - py3 -) +RECURSE( + py2 + py3 +) diff --git a/contrib/restricted/boost/libs/python/ya.make b/contrib/restricted/boost/libs/python/ya.make index 8a9607986f..d05d74edb0 100644 --- a/contrib/restricted/boost/libs/python/ya.make +++ b/contrib/restricted/boost/libs/python/ya.make @@ -1,4 +1,4 @@ -PY23_NATIVE_LIBRARY() +PY23_NATIVE_LIBRARY() LICENSE(BSL-1.0) diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json index edfcaa1530..f031b8d5cf 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json @@ -193,11 +193,11 @@ "Match": {"Type": "Callable", "Name": "LMap"} }, { - "Name": "TCoOrderedLMap", - "Base": "TCoMapBase", - "Match": {"Type": "Callable", "Name": "OrderedLMap"} - }, - { + "Name": "TCoOrderedLMap", + "Base": "TCoMapBase", + "Match": {"Type": "Callable", "Name": "OrderedLMap"} + }, + { "Name": "TCoNarrowMap", "Base": "TCoMapBase", "Match": {"Type": "Callable", "Name": "NarrowMap"} @@ -1880,14 +1880,14 @@ "Children": [ {"Index": 0, "Name": "Variant", "Type": "TExprBase"} ] - }, - { - "Name": "TCoEnsurePersistable", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "EnsurePersistable"}, - "Children": [ - {"Index": 0, "Name": "Input", "Type": "TExprBase"} - ] + }, + { + "Name": "TCoEnsurePersistable", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "EnsurePersistable"}, + "Children": [ + {"Index": 0, "Name": "Input", "Type": "TExprBase"} + ] }, { "Name": "TCoSecureParam", diff --git a/ydb/library/yql/core/issue/protos/issue_id.proto b/ydb/library/yql/core/issue/protos/issue_id.proto index a2ed91d640..7e26d76eea 100644 --- a/ydb/library/yql/core/issue/protos/issue_id.proto +++ b/ydb/library/yql/core/issue/protos/issue_id.proto @@ -163,10 +163,10 @@ message TIssuesIds { JSONPATH_INVALID_IS_UNKNOWN_ARGUMENT = 4722; JSONPATH_INVALID_STARTS_WITH_ARGUMENT = 4723; -// stat - STAT_DEPRECATED_STRING_TREE = 5000; - STAT_ACCESS_DENIED = 5001; - +// stat + STAT_DEPRECATED_STRING_TREE = 5000; + STAT_ACCESS_DENIED = 5001; + // dq DQ_GATEWAY_ERROR = 6000; DQ_GATEWAY_NEED_FALLBACK_ERROR = 6001; diff --git a/ydb/library/yql/core/issue/yql_issue.txt b/ydb/library/yql/core/issue/yql_issue.txt index 897d5d6f9a..855845cba9 100644 --- a/ydb/library/yql/core/issue/yql_issue.txt +++ b/ydb/library/yql/core/issue/yql_issue.txt @@ -376,7 +376,7 @@ ids { code: YT_ENTRY_NOT_TABLE_OR_VIEW severity: S_ERROR } -ids { +ids { code: YT_CONCURRENT_TABLE_MODIF severity: S_ERROR } @@ -409,13 +409,13 @@ ids { severity: S_WARNING } ids { - code: STAT_DEPRECATED_STRING_TREE - severity: S_WARNING -} -ids { - code: STAT_ACCESS_DENIED - severity: S_ERROR -} + code: STAT_DEPRECATED_STRING_TREE + severity: S_WARNING +} +ids { + code: STAT_ACCESS_DENIED + severity: S_ERROR +} ids { code: YQL_DEPRECATED_FUNCTION_OR_SIGNATURE severity: S_WARNING diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index 5846e6cb10..1bcc141204 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -6936,8 +6936,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } auto moduleName = input->Head().Content(); - auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(moduleName); - if (scriptType == NKikimr::NMiniKQL::EScriptType::Unknown) { + auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(moduleName); + if (scriptType == NKikimr::NMiniKQL::EScriptType::Unknown) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Unknown script type: " << moduleName)); return IGraphTransformer::TStatus::Error; } @@ -6946,7 +6946,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto canonizedModuleName = NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType); bool isCustomPython = NKikimr::NMiniKQL::IsCustomPython(scriptType); bool foundModule = false; - + // resolve script udf from external resources (files / urls) // (main usage of CustomPython) { @@ -6958,12 +6958,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot> foundModule = ctx.Types.UdfModules.find(canonizedModuleName) != ctx.Types.UdfModules.end(); } - - // fallback for preinstalled CustomPython case + + // fallback for preinstalled CustomPython case if (!foundModule) { foundModule = static_cast<bool>(ctx.Types.UdfResolver->GetSystemModulePath(canonizedModuleName)); - } - + } + if (!foundModule) { if (isCustomPython) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() @@ -6973,9 +6973,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot> ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Module not loaded for script type: " << canonizedModuleName)); } - return IGraphTransformer::TStatus::Error; - } - + return IGraphTransformer::TStatus::Error; + } + if (input->ChildrenSize() == 5) { if (!EnsureTuple(*input->Child(4), ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -12817,7 +12817,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["Fold1Map"] = &Fold1MapWrapper; Functions["Chain1Map"] = &Chain1MapWrapper; Functions["LMap"] = &LMapWrapper; - Functions["OrderedLMap"] = &LMapWrapper; + Functions["OrderedLMap"] = &LMapWrapper; Functions["Struct"] = &StructWrapper; Functions["AddMember"] = &AddMemberWrapper; Functions["RemoveMember"] = &RemoveMemberWrapper<false>; diff --git a/ydb/library/yql/core/yql_type_helpers.cpp b/ydb/library/yql/core/yql_type_helpers.cpp index 9ae7cab56f..1cf0fcc81a 100644 --- a/ydb/library/yql/core/yql_type_helpers.cpp +++ b/ydb/library/yql/core/yql_type_helpers.cpp @@ -1,10 +1,10 @@ -#include "yql_type_helpers.h" - -#include <util/string/builder.h> - - -namespace NYql { - +#include "yql_type_helpers.h" + +#include <util/string/builder.h> + + +namespace NYql { + const TTypeAnnotationNode* GetItemType(const TTypeAnnotationNode& type) { switch (type.GetKind()) { case ETypeAnnotationKind::List: @@ -48,38 +48,38 @@ bool SilentGetSequenceItemType(TPosition pos, const TTypeAnnotationNode& inputTy if (!itemType) { error = TIssue(pos, TStringBuilder() << "Expected list, stream, flow or optional, but got: " << inputType); return false; - } - + } + if (allowMultiIO && itemType->GetKind() != ETypeAnnotationKind::Struct && itemType->GetKind() != ETypeAnnotationKind::Multi) { if (itemType->GetKind() != ETypeAnnotationKind::Variant) { error = TIssue(pos, TStringBuilder() << "Expected Struct or Variant as row type, but got: " << *itemType); return false; } - auto varType = itemType->Cast<TVariantExprType>()->GetUnderlyingType(); - TTypeAnnotationNode::TListType varItemTypes; - if (varType->GetKind() == ETypeAnnotationKind::Struct) { - for (auto item: varType->Cast<TStructExprType>()->GetItems()) { - varItemTypes.push_back(item->GetItemType()); - } - } else { - varItemTypes = varType->Cast<TTupleExprType>()->GetItems(); - } + auto varType = itemType->Cast<TVariantExprType>()->GetUnderlyingType(); + TTypeAnnotationNode::TListType varItemTypes; + if (varType->GetKind() == ETypeAnnotationKind::Struct) { + for (auto item: varType->Cast<TStructExprType>()->GetItems()) { + varItemTypes.push_back(item->GetItemType()); + } + } else { + varItemTypes = varType->Cast<TTupleExprType>()->GetItems(); + } if (varItemTypes.size() < 2) { error = TIssue(pos, TStringBuilder() << "Expected at least two items in Variant row type, but got: " << varItemTypes.size()); return false; } - for (auto varItemType: varItemTypes) { + for (auto varItemType: varItemTypes) { if (varItemType->GetKind() != ETypeAnnotationKind::Struct) { error = TIssue(pos, TStringBuilder() << "Expected Struct in Variant item type, but got: " << *varItemType); return false; } - } + } } else { if (itemType->GetKind() != ETypeAnnotationKind::Struct && itemType->GetKind() != ETypeAnnotationKind::Multi) { error = TIssue(pos, TStringBuilder() << "Expected Struct or Multi as row type, but got: " << *itemType); return false; } - } + } result = itemType; YQL_ENSURE(result); @@ -94,31 +94,31 @@ const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool if (!SilentGetSequenceItemType({}, *listNode.Ref().GetTypeAnn(), allowMultiIO, itemType, error)) { // position is not used YQL_ENSURE(false, "" << error.Message); - } - return itemType; -} - + } + return itemType; +} + const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO, TExprContext& ctx) { return GetSequenceItemType(listNode.Pos(), listNode.Ref().GetTypeAnn(), allowMultiIO, ctx); -} - +} + const TTypeAnnotationNode* GetSequenceItemType(TPositionHandle pos, const TTypeAnnotationNode* inputType, bool allowMultiIO, TExprContext& ctx) { - const TTypeAnnotationNode* itemType = nullptr; + const TTypeAnnotationNode* itemType = nullptr; TIssue error; if (!SilentGetSequenceItemType(ctx.GetPosition(pos), *inputType, allowMultiIO, itemType, error)) { ctx.AddError(error); - return nullptr; - } - return itemType; -} - + return nullptr; + } + return itemType; +} + bool GetSequenceItemType(const TExprNode& list, const TTypeAnnotationNode*& itemType, TExprContext& ctx) { itemType = GetSequenceItemType(list.Pos(), list.GetTypeAnn(), false, ctx); - return itemType != nullptr; -} - + return itemType != nullptr; +} + const TTypeAnnotationNode* SilentGetSequenceItemType(const TExprNode& list, bool allowMultiIO) { TIssue error; const TTypeAnnotationNode* itemType = nullptr; @@ -128,4 +128,4 @@ const TTypeAnnotationNode* SilentGetSequenceItemType(const TExprNode& list, bool return itemType; } -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/core/yql_type_helpers.h b/ydb/library/yql/core/yql_type_helpers.h index 593c77592b..9ab94cf212 100644 --- a/ydb/library/yql/core/yql_type_helpers.h +++ b/ydb/library/yql/core/yql_type_helpers.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> - + #include <util/generic/set.h> - -namespace NYql { - + +namespace NYql { + const TTypeAnnotationNode* GetItemType(const TTypeAnnotationNode& type); TSet<TStringBuf> GetColumnsOfStructOrSequenceOfStruct(const TTypeAnnotationNode& type); @@ -16,5 +16,5 @@ const TTypeAnnotationNode* GetSequenceItemType(TPositionHandle pos, const TTypeA bool allowMultiIO, TExprContext& ctx); bool GetSequenceItemType(const TExprNode& list, const TTypeAnnotationNode*& itemType, TExprContext& ctx); const TTypeAnnotationNode* SilentGetSequenceItemType(const TExprNode& list, bool allowMultiIO); - -} // namespace NYql + +} // namespace NYql diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp index a20f016b57..7df6691ec1 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp @@ -21,7 +21,7 @@ public: TCodegenValue(TMemoryUsageInfo* memInfo, TNextPtr next, TComputationContext* ctx, NUdf::TUnboxedValue&& dict) : TComputationValue<TCodegenValue>(memInfo) - , NextFunc(next) + , NextFunc(next) , Ctx(ctx) , Dict(std::move(dict)) {} @@ -117,9 +117,9 @@ public: const TSelf* const Self; }; - TDictItemsWrapper(TComputationMutables& mutables, IComputationNode* dict) + TDictItemsWrapper(TComputationMutables& mutables, IComputationNode* dict) : TBaseComputation(mutables) - , Dict(dict) + , Dict(dict) , ResPair(mutables) {} @@ -230,7 +230,7 @@ public: TValue( TMemoryUsageInfo* memInfo, const NUdf::TUnboxedValue&& dict, - TComputationContext&, const TSelf*) + TComputationContext&, const TSelf*) : TComputationValue<TValue>(memInfo) , Dict(std::move(dict)) {} @@ -284,11 +284,11 @@ IComputationNode* WrapDictItems(TCallable& callable, const TComputationNodeFacto const auto mode = AS_VALUE(TDataLiteral, callable.GetInput(1))->AsValue().Get<ui32>(); switch (static_cast<EDictItems>(mode)) { case EDictItems::Both: - return new TDictItemsWrapper(ctx.Mutables, node); + return new TDictItemsWrapper(ctx.Mutables, node); case EDictItems::Keys: - return new TDictHalfsWrapper<true>(ctx.Mutables, node); + return new TDictHalfsWrapper<true>(ctx.Mutables, node); case EDictItems::Payloads: - return new TDictHalfsWrapper<false>(ctx.Mutables, node); + return new TDictHalfsWrapper<false>(ctx.Mutables, node); default: Y_FAIL("Unknown mode: %" PRIu32, mode); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp index 7090aff8cd..11f1146ef0 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp @@ -8,7 +8,7 @@ #include <ydb/library/yql/public/udf/udf_terminator.h> -#ifndef MKQL_DISABLE_CODEGEN +#ifndef MKQL_DISABLE_CODEGEN extern "C" NKikimr::NUdf::TUnboxedValuePod DataFromString(const NKikimr::NUdf::TUnboxedValuePod data, NKikimr::NUdf::EDataSlot slot) { return NKikimr::NMiniKQL::ValueFromString(slot, data.AsStringRef()); } @@ -16,7 +16,7 @@ extern "C" NKikimr::NUdf::TUnboxedValuePod DataFromString(const NKikimr::NUdf::T extern "C" NYql::NDecimal::TInt128 DecimalFromString(const NKikimr::NUdf::TUnboxedValuePod decimal, ui8 precision, ui8 scale) { return NYql::NDecimal::FromStringEx(decimal.AsStringRef(), precision, scale); } -#endif +#endif namespace NKikimr { namespace NMiniKQL { diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp index d2182bbc23..136066521f 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp @@ -361,12 +361,12 @@ public: } private: - // copypaste to resolve -Woverloaded-virtual - bool Next(NUdf::TUnboxedValue&) override { + // copypaste to resolve -Woverloaded-virtual + bool Next(NUdf::TUnboxedValue&) override { this->ThrowNotSupported(__func__); - return false; - } - + return false; + } + NUdf::TUnboxedValue Next(IComputationNode* flow, TComputationContext& ctx) { while (EatInput) { if (!InitialUsage) { diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp index 2f67b24406..631aefec30 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp @@ -14,7 +14,7 @@ namespace NMiniKQL { namespace { std::vector<NUdf::EDataSlot> PrepareKeyTypesByScheme(const std::vector<std::tuple<NUdf::EDataSlot, bool, TType*>>& keySchemeTypes) { - MKQL_ENSURE(!keySchemeTypes.empty(), "No key types provided"); + MKQL_ENSURE(!keySchemeTypes.empty(), "No key types provided"); std::vector<NUdf::EDataSlot> keyTypes; keyTypes.reserve(keySchemeTypes.size()); for (const auto& schemeType: keySchemeTypes) { @@ -205,34 +205,34 @@ struct TCompareDescr { template<class Container> std::function<bool(const typename Container::value_type&, const typename Container::value_type&)> - MakeComparator(const NUdf::TUnboxedValue& ascending) const { - if (KeyTypes.size() > 1U) { - // sort tuples - return [this, &ascending](const typename Container::value_type& x, const typename Container::value_type& y) { - const auto& left = Get(x); - const auto& right = Get(y); - - for (ui32 i = 0; i < KeyTypes.size(); ++i) { - const auto& keyType = KeyTypes[i]; - const auto& leftElem = left.GetElement(i); - const auto& rightElem = right.GetElement(i); - const bool asc = ascending.GetElement(i).Get<bool>(); - - if (const auto cmp = CompareValues(keyType, asc, std::get<1>(KeySchemeTypes[i]), leftElem, rightElem)) { - return cmp < 0; + MakeComparator(const NUdf::TUnboxedValue& ascending) const { + if (KeyTypes.size() > 1U) { + // sort tuples + return [this, &ascending](const typename Container::value_type& x, const typename Container::value_type& y) { + const auto& left = Get(x); + const auto& right = Get(y); + + for (ui32 i = 0; i < KeyTypes.size(); ++i) { + const auto& keyType = KeyTypes[i]; + const auto& leftElem = left.GetElement(i); + const auto& rightElem = right.GetElement(i); + const bool asc = ascending.GetElement(i).Get<bool>(); + + if (const auto cmp = CompareValues(keyType, asc, std::get<1>(KeySchemeTypes[i]), leftElem, rightElem)) { + return cmp < 0; } - } - - return false; - }; - } else { - // sort one column - const bool isOptional = std::get<1>(KeySchemeTypes.front()); - const bool asc = ascending.Get<bool>(); - - return [this, asc, isOptional](const typename Container::value_type& x, const typename Container::value_type& y) { - return CompareValues(KeyTypes.front(), asc, isOptional, Get(x), Get(y)) < 0; - }; + } + + return false; + }; + } else { + // sort one column + const bool isOptional = std::get<1>(KeySchemeTypes.front()); + const bool asc = ascending.Get<bool>(); + + return [this, asc, isOptional](const typename Container::value_type& x, const typename Container::value_type& y) { + return CompareValues(KeyTypes.front(), asc, isOptional, Get(x), Get(y)) < 0; + }; } } @@ -519,7 +519,7 @@ public: }); const auto& ascending = Ascending->GetValue(ctx); - const auto max = std::max_element(keys.begin(), keys.end(), Description.MakeComparator<TUnboxedValueVector>(ascending)); + const auto max = std::max_element(keys.begin(), keys.end(), Description.MakeComparator<TUnboxedValueVector>(ascending)); hotkey = *max; HotKey->SetValue(ctx, std::move(*max)); } @@ -530,7 +530,7 @@ public: const auto& key = Key->GetValue(ctx); const auto& ascending = Ascending->GetValue(ctx); - if (Description.MakeComparator<TUnboxedValueVector>(ascending)(key, hotkey)) { + if (Description.MakeComparator<TUnboxedValueVector>(ascending)(key, hotkey)) { const auto reserve = std::max<ui64>(count << 1ULL, 1ULL << 8ULL); if (size < reserve) { return ctx.HolderFactory.Append(list.Release(), Arg->GetValue(ctx).Release()); @@ -546,7 +546,7 @@ public: }); Description.Prepare(ctx, items); - std::nth_element(items.begin(), items.begin() + count - 1U, items.end(), Description.MakeComparator<TKeyPayloadPairVector>(ascending)); + std::nth_element(items.begin(), items.begin() + count - 1U, items.end(), Description.MakeComparator<TKeyPayloadPairVector>(ascending)); items.resize(count); NUdf::TUnboxedValue *inplace = nullptr; diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp index 66e6c73c87..af5c0612fd 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp @@ -10,7 +10,7 @@ #include <ydb/library/yql/public/udf/udf_terminator.h> -#ifndef MKQL_DISABLE_CODEGEN +#ifndef MKQL_DISABLE_CODEGEN extern "C" NYql::NUdf::TUnboxedValuePod DataToString(NYql::NUdf::TUnboxedValuePod data, NYql::NUdf::EDataSlot slot) { return NKikimr::NMiniKQL::ValueToString(slot, data); } @@ -21,7 +21,7 @@ extern "C" NYql::NUdf::TUnboxedValuePod DecimalToString(NYql::NDecimal::TInt128 } return NYql::NUdf::TUnboxedValuePod(); } -#endif +#endif namespace NKikimr { namespace NMiniKQL { diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make index 55838c4012..27c18b3fd8 100644 --- a/ydb/library/yql/minikql/comp_nodes/ya.make +++ b/ydb/library/yql/minikql/comp_nodes/ya.make @@ -220,7 +220,7 @@ PEERDIR( NO_COMPILER_WARNINGS() IF (NOT MKQL_DISABLE_CODEGEN) - PEERDIR( + PEERDIR( ydb/library/yql/minikql/codegen contrib/libs/llvm12/lib/IR contrib/libs/llvm12/lib/ExecutionEngine/MCJIT @@ -228,8 +228,8 @@ IF (NOT MKQL_DISABLE_CODEGEN) contrib/libs/llvm12/lib/Target/X86 contrib/libs/llvm12/lib/Target/X86/AsmParser contrib/libs/llvm12/lib/Transforms/IPO - ) -ELSE() + ) +ELSE() CFLAGS( -DMKQL_DISABLE_CODEGEN ) diff --git a/ydb/library/yql/minikql/computation/ya.make b/ydb/library/yql/minikql/computation/ya.make index ae419b3b23..d55010d585 100644 --- a/ydb/library/yql/minikql/computation/ya.make +++ b/ydb/library/yql/minikql/computation/ya.make @@ -68,8 +68,8 @@ PEERDIR( ) IF (NOT MKQL_DISABLE_CODEGEN) - NO_COMPILER_WARNINGS() - PEERDIR( + NO_COMPILER_WARNINGS() + PEERDIR( ydb/library/yql/minikql/codegen contrib/libs/llvm12/lib/IR contrib/libs/llvm12/lib/ExecutionEngine/MCJIT @@ -77,12 +77,12 @@ IF (NOT MKQL_DISABLE_CODEGEN) contrib/libs/llvm12/lib/Target/X86 contrib/libs/llvm12/lib/Target/X86/AsmParser contrib/libs/llvm12/lib/Transforms/IPO - ) -ELSE() + ) +ELSE() CFLAGS( -DMKQL_DISABLE_CODEGEN ) -ENDIF() +ENDIF() YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp index ba308a35c6..a789de4858 100644 --- a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp +++ b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp @@ -59,7 +59,7 @@ struct TDecimalAbs { } #ifndef MKQL_DISABLE_CODEGEN - static Value* Generate(Value* arg, const TCodegenContext&, BasicBlock*& block) + static Value* Generate(Value* arg, const TCodegenContext&, BasicBlock*& block) { const auto val = GetterForInt128(arg, block); const auto zero = ConstantInt::get(val->getType(), 0); diff --git a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp index 613aff688c..bec44b7283 100644 --- a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp +++ b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp @@ -39,16 +39,16 @@ struct TByteAt { const auto type = Type::getInt8Ty(context); const auto embType = FixedVectorType::get(type, 16); const auto cast = CastInst::Create(Instruction::BitCast, left, embType, "cast", block); - const auto mark = ExtractElementInst::Create(cast, ConstantInt::get(type, 15), "mark", block); + const auto mark = ExtractElementInst::Create(cast, ConstantInt::get(type, 15), "mark", block); const auto index = GetterFor<ui32>(right, context, block); - const auto bsize = ExtractElementInst::Create(cast, ConstantInt::get(type, 14), "bsize", block); + const auto bsize = ExtractElementInst::Create(cast, ConstantInt::get(type, 14), "bsize", block); const auto esize = CastInst::Create(Instruction::ZExt, bsize, index->getType(), "esize", block); const auto sizeType = Type::getInt32Ty(context); const auto strType = FixedVectorType::get(sizeType, 4); const auto four = CastInst::Create(Instruction::BitCast, left, strType, "four", block); - const auto ssize = ExtractElementInst::Create(four, ConstantInt::get(type, 2), "ssize", block); + const auto ssize = ExtractElementInst::Create(four, ConstantInt::get(type, 2), "ssize", block); const auto cemb = CastInst::Create(Instruction::Trunc, mark, Type::getInt1Ty(context), "cemb", block); const auto size = SelectInst::Create(cemb, esize, ssize, "size", block); @@ -74,7 +74,7 @@ struct TByteAt { { block = emb; - const auto byte = ExtractElementInst::Create(cast, index, "byte", block); + const auto byte = ExtractElementInst::Create(cast, index, "byte", block); const auto full = SetterFor<ui8>(byte, context, block); result->addIncoming(full, block); BranchInst::Create(done, block); @@ -83,7 +83,7 @@ struct TByteAt { { block = str; - const auto foffs = ExtractElementInst::Create(four, ConstantInt::get(type, 3), "foffs", block); + const auto foffs = ExtractElementInst::Create(four, ConstantInt::get(type, 3), "foffs", block); const auto offs = BinaryOperator::CreateAnd(foffs, ConstantInt::get(foffs->getType(), 0xFFFFFF), "offs", block); const auto skip = BinaryOperator::CreateAdd(offs, ConstantInt::get(offs->getType(), 16), "skip", block); const auto pos = BinaryOperator::CreateAdd(index, skip, "pos", block); diff --git a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp index 43f7287be2..d321a057c1 100644 --- a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp +++ b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp @@ -372,7 +372,7 @@ struct TStringConvert { } #ifndef MKQL_DISABLE_CODEGEN - static Value* Generate(Value* arg, const TCodegenContext&, BasicBlock*&) + static Value* Generate(Value* arg, const TCodegenContext&, BasicBlock*&) { return arg; } diff --git a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp index 49979a435a..e8280d38c7 100644 --- a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp +++ b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp @@ -13,7 +13,7 @@ struct TPlus : public TSimpleArithmeticUnary<TInput, TOutput, TPlus<TInput, TOut } #ifndef MKQL_DISABLE_CODEGEN - static Value* Gen(Value* arg, const TCodegenContext&, BasicBlock*&) + static Value* Gen(Value* arg, const TCodegenContext&, BasicBlock*&) { return arg; } @@ -26,7 +26,7 @@ struct TDecimalPlus { } #ifndef MKQL_DISABLE_CODEGEN - static Value* Generate(Value* arg, const TCodegenContext&, BasicBlock*&) + static Value* Generate(Value* arg, const TCodegenContext&, BasicBlock*&) { return arg; } diff --git a/ydb/library/yql/minikql/invoke_builtins/ya.make b/ydb/library/yql/minikql/invoke_builtins/ya.make index 8ccc53ba2d..f2efb007fa 100644 --- a/ydb/library/yql/minikql/invoke_builtins/ya.make +++ b/ydb/library/yql/minikql/invoke_builtins/ya.make @@ -59,7 +59,7 @@ PEERDIR( ) IF (NOT MKQL_DISABLE_CODEGEN) - PEERDIR( + PEERDIR( ydb/library/yql/minikql/codegen contrib/libs/llvm12/lib/IR contrib/libs/llvm12/lib/ExecutionEngine/MCJIT @@ -67,8 +67,8 @@ IF (NOT MKQL_DISABLE_CODEGEN) contrib/libs/llvm12/lib/Target/X86 contrib/libs/llvm12/lib/Target/X86/AsmParser contrib/libs/llvm12/lib/Transforms/IPO - ) -ELSE() + ) +ELSE() CFLAGS( -DMKQL_DISABLE_CODEGEN ) diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index 47ae6e4c8f..7014901284 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -77,14 +77,14 @@ void EnsureScriptSpecificTypes( case EScriptType::ArcPython: case EScriptType::ArcPython2: case EScriptType::ArcPython3: - case EScriptType::CustomPython: + case EScriptType::CustomPython: case EScriptType::CustomPython2: case EScriptType::CustomPython3: return TPythonTypeChecker().Walk(funcType, env); case EScriptType::Javascript: return TJavascriptTypeChecker().Walk(funcType, env); default: - MKQL_ENSURE(false, "Unknown script type " << static_cast<ui32>(scriptType)); + MKQL_ENSURE(false, "Unknown script type " << static_cast<ui32>(scriptType)); } } @@ -229,7 +229,7 @@ bool ReduceOptionalElements(const TType* type, const TArrayRef<const ui32>& test std::string_view ScriptTypeAsStr(EScriptType type) { switch (type) { -#define MKQL_SCRIPT_TYPE_CASE(name, value, ...) \ +#define MKQL_SCRIPT_TYPE_CASE(name, value, ...) \ case EScriptType::name: return std::string_view(#name); MKQL_SCRIPT_TYPES(MKQL_SCRIPT_TYPE_CASE) @@ -242,9 +242,9 @@ std::string_view ScriptTypeAsStr(EScriptType type) { EScriptType ScriptTypeFromStr(std::string_view str) { TString lowerStr = TString(str); - lowerStr.to_lower(); -#define MKQL_SCRIPT_TYPE_FROM_STR(name, value, lowerName) \ - if (lowerStr == #lowerName) return EScriptType::name; + lowerStr.to_lower(); +#define MKQL_SCRIPT_TYPE_FROM_STR(name, value, lowerName) \ + if (lowerStr == #lowerName) return EScriptType::name; MKQL_SCRIPT_TYPES(MKQL_SCRIPT_TYPE_FROM_STR) #undef MKQL_SCRIPT_TYPE_FROM_STR diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h index 10f1ad7ccf..5b157c441a 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.h +++ b/ydb/library/yql/minikql/mkql_program_builder.h @@ -85,11 +85,11 @@ inline void AddAnyJoinSide(EAnyJoinSettings& combined, EAnyJoinSettings value) { } #define MKQL_SCRIPT_TYPES(xx) \ - xx(Unknown, 0, unknown) \ - xx(Python, 1, python) \ - xx(Lua, 2, lua) \ - xx(ArcPython, 3, arcpython) \ - xx(CustomPython, 4, custompython) \ + xx(Unknown, 0, unknown) \ + xx(Python, 1, python) \ + xx(Lua, 2, lua) \ + xx(ArcPython, 3, arcpython) \ + xx(CustomPython, 4, custompython) \ xx(Javascript, 5, javascript) \ xx(Python2, 6, python2) \ xx(ArcPython2, 7, arcpython2) \ diff --git a/ydb/library/yql/providers/common/codec/yql_codec.cpp b/ydb/library/yql/providers/common/codec/yql_codec.cpp index 7587e188fa..8db02923ae 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.cpp +++ b/ydb/library/yql/providers/common/codec/yql_codec.cpp @@ -554,8 +554,8 @@ TExprNode::TPtr NodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode void CopyYsonWithAttrs(char cmd, TInputBuf& buf, TVector<char>& yson) { if (cmd == BeginAttributesSymbol) { yson.push_back(cmd); - cmd = buf.Read(); - + cmd = buf.Read(); + for (;;) { if (cmd == EndAttributesSymbol) { yson.push_back(cmd); @@ -574,7 +574,7 @@ void CopyYsonWithAttrs(char cmd, TInputBuf& buf, TVector<char>& yson) { cmd = buf.Read(); if (cmd == KeyedItemSeparatorSymbol) { yson.push_back(cmd); - cmd = buf.Read(); + cmd = buf.Read(); } } } @@ -610,8 +610,8 @@ void CopyYson(char cmd, TInputBuf& buf, TVector<char>& yson) { } case BeginListSymbol: { yson.push_back(cmd); - cmd = buf.Read(); - + cmd = buf.Read(); + for (;;) { if (cmd == EndListSymbol) { yson.push_back(cmd); @@ -622,15 +622,15 @@ void CopyYson(char cmd, TInputBuf& buf, TVector<char>& yson) { cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { yson.push_back(cmd); - cmd = buf.Read(); + cmd = buf.Read(); } } break; } case BeginMapSymbol: { yson.push_back(cmd); - cmd = buf.Read(); - + cmd = buf.Read(); + for (;;) { if (cmd == EndMapSymbol) { yson.push_back(cmd); @@ -648,7 +648,7 @@ void CopyYson(char cmd, TInputBuf& buf, TVector<char>& yson) { cmd = buf.Read(); if (cmd == KeyedItemSeparatorSymbol) { yson.push_back(cmd); - cmd = buf.Read(); + cmd = buf.Read(); } } break; @@ -985,11 +985,11 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, NUdf::TUnboxedValue* items; NUdf::TUnboxedValue ret = holderFactory.CreateDirectArrayHolder(structType->GetMembersCount(), items); if (cmd == BeginListSymbol) { - cmd = buf.Read(); - + cmd = buf.Read(); + for (ui32 i = 0; i < structType->GetMembersCount(); ++i) { items[i] = ReadYsonValue(structType->GetMemberType(i), holderFactory, cmd, buf, isTableFormat); - + cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { cmd = buf.Read(); @@ -1000,8 +1000,8 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, return ret; } else { - cmd = buf.Read(); - + cmd = buf.Read(); + for (;;) { if (cmd == EndMapSymbol) { break; @@ -1022,11 +1022,11 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, } else { SkipYson(cmd, buf); } - - cmd = buf.Read(); - if (cmd == KeyedItemSeparatorSymbol) { - cmd = buf.Read(); - } + + cmd = buf.Read(); + if (cmd == KeyedItemSeparatorSymbol) { + cmd = buf.Read(); + } } for (ui32 i = 0; i < structType->GetMembersCount(); ++i) { @@ -1045,18 +1045,18 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, auto itemType = static_cast<TListType*>(type)->GetItemType(); TDefaultListRepresentation items; CHECK_EXPECTED(cmd, BeginListSymbol); - cmd = buf.Read(); - + cmd = buf.Read(); + for (;;) { if (cmd == EndListSymbol) { break; } items = items.Append(ReadYsonValue(itemType, holderFactory, cmd, buf, isTableFormat)); - - cmd = buf.Read(); + + cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { - cmd = buf.Read(); + cmd = buf.Read(); } } @@ -1112,8 +1112,8 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, "Expected String type as dictionary key type"); auto filler = [&](TValuesDictHashMap& map) { - cmd = buf.Read(); - + cmd = buf.Read(); + for (;;) { if (cmd == EndMapSymbol) { break; @@ -1125,11 +1125,11 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, cmd = buf.Read(); auto payload = ReadYsonValue(payloadType, holderFactory, cmd, buf, isTableFormat); map.emplace(std::move(keyStr), std::move(payload)); - - cmd = buf.Read(); - if (cmd == KeyedItemSeparatorSymbol) { - cmd = buf.Read(); - } + + cmd = buf.Read(); + if (cmd == KeyedItemSeparatorSymbol) { + cmd = buf.Read(); + } } }; @@ -1137,8 +1137,8 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, } else { auto filler = [&](TValuesDictHashMap& map) { - cmd = buf.Read(); - + cmd = buf.Read(); + for (;;) { if (cmd == EndListSymbol) { break; @@ -1161,11 +1161,11 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, } map.emplace(std::move(key), std::move(payload)); - - cmd = buf.Read(); - if (cmd == ListItemSeparatorSymbol) { - cmd = buf.Read(); - } + + cmd = buf.Read(); + if (cmd == ListItemSeparatorSymbol) { + cmd = buf.Read(); + } } }; @@ -1178,11 +1178,11 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, NUdf::TUnboxedValue* items; NUdf::TUnboxedValue ret = holderFactory.CreateDirectArrayHolder(tupleType->GetElementsCount(), items); CHECK_EXPECTED(cmd, BeginListSymbol); - cmd = buf.Read(); - + cmd = buf.Read(); + for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { items[i] = ReadYsonValue(tupleType->GetElementType(i), holderFactory, cmd, buf, isTableFormat); - + cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { cmd = buf.Read(); diff --git a/ydb/library/yql/providers/common/comp_nodes/ya.make b/ydb/library/yql/providers/common/comp_nodes/ya.make index 0498885783..548d5560f7 100644 --- a/ydb/library/yql/providers/common/comp_nodes/ya.make +++ b/ydb/library/yql/providers/common/comp_nodes/ya.make @@ -46,7 +46,7 @@ PEERDIR( ) IF (NOT MKQL_DISABLE_CODEGEN) - PEERDIR( + PEERDIR( ydb/library/yql/minikql/codegen contrib/libs/llvm12/lib/IR contrib/libs/llvm12/lib/ExecutionEngine/MCJIT @@ -54,8 +54,8 @@ IF (NOT MKQL_DISABLE_CODEGEN) contrib/libs/llvm12/lib/Target/X86 contrib/libs/llvm12/lib/Target/X86/AsmParser contrib/libs/llvm12/lib/Transforms/IPO - ) -ELSE() + ) +ELSE() CFLAGS( -DMKQL_DISABLE_CODEGEN ) diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index 8561fe2536..b4ff5a4279 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -280,25 +280,25 @@ message TPqGatewayConfig { repeated TAttr DefaultSettings = 100; } -///////////////////////////// Stat ///////////////////////////// - -message TStatClusterConfig { - optional string Name = 1; // Short cluster name - optional string Cluster = 2; // Real cluster server name with port - optional bool Default = 3 [default = false]; // Default cluster - optional string StatToken = 4; // token to use in communication with Statface - optional string StatName = 5; // Stat cluster name - - repeated TAttr Settings = 100; -} - -message TStatGatewayConfig { - optional uint32 GatewayThreads = 1 [default = 0]; // Number of gateway MtpQueue threads. The adaptive MtpQueue will be used for 0 value - - repeated TStatClusterConfig ClusterMapping = 101; // Accepted cluster shortcuts. All names not from this list will be rejected - repeated TAttr DefaultSettings = 102; -} - +///////////////////////////// Stat ///////////////////////////// + +message TStatClusterConfig { + optional string Name = 1; // Short cluster name + optional string Cluster = 2; // Real cluster server name with port + optional bool Default = 3 [default = false]; // Default cluster + optional string StatToken = 4; // token to use in communication with Statface + optional string StatName = 5; // Stat cluster name + + repeated TAttr Settings = 100; +} + +message TStatGatewayConfig { + optional uint32 GatewayThreads = 1 [default = 0]; // Number of gateway MtpQueue threads. The adaptive MtpQueue will be used for 0 value + + repeated TStatClusterConfig ClusterMapping = 101; // Accepted cluster shortcuts. All names not from this list will be rejected + repeated TAttr DefaultSettings = 102; +} + ///////////////////////////// CHYT ///////////////////////////// message TChytClusterConfig { @@ -441,7 +441,7 @@ message TGatewaysConfig { optional TClickHouseGatewayConfig ClickHouse = 3; optional TRtmrGatewayConfig Rtmr = 4; optional TKikimrMvpGatewayConfig KikimrMvp = 5; - optional TStatGatewayConfig Stat = 6; + optional TStatGatewayConfig Stat = 6; optional TChytGatewayConfig Chyt = 7; optional TSolomonGatewayConfig Solomon = 8; optional TFileStorageAdditionalConfig Fs = 9; diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index aa782ee93f..923f639b95 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -464,11 +464,11 @@ bool FillUsedFilesImpl( YQL_ENSURE(SplitUdfName(node.Head().Content(), moduleName, funcName)); } - auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(moduleName); + auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(moduleName); if (node.IsCallable("ScriptUdf")) { moduleName = NKikimr::NMiniKQL::ScriptTypeAsStr(NKikimr::NMiniKQL::CanonizeScriptType(scriptType)); } - + bool addSysModule = true; TString fileAlias; if (node.IsCallable("Udf")) { @@ -478,9 +478,9 @@ bool FillUsedFilesImpl( // we have external UdfModule (not in preinstalled udfs) if (iterator != types.UdfModules.end()) { fileAlias = iterator->second; - } + } } - + if (!fileAlias.empty()) { addSysModule = false; const auto block = types.UserDataStorage->FindUserDataBlock(fileAlias); diff --git a/ydb/library/yql/providers/common/provider/yql_provider_names.h b/ydb/library/yql/providers/common/provider/yql_provider_names.h index 7cf29b4c1d..e23337e3f6 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider_names.h +++ b/ydb/library/yql/providers/common/provider/yql_provider_names.h @@ -22,6 +22,6 @@ constexpr std::array<const TStringBuf, 11> Providers = { {ConfigProviderName, YtProviderName, KikimrProviderName, RtmrProviderName, S3ProviderName, StatProviderName, SolomonProviderName, DqProviderName, ClickHouseProviderName, YdbProviderName, PqProviderName} -}; +}; } // namespace NYql diff --git a/ydb/library/yql/sql/v0/SQL.g b/ydb/library/yql/sql/v0/SQL.g index 3b616b9c9f..c1d7f874ee 100644 --- a/ydb/library/yql/sql/v0/SQL.g +++ b/ydb/library/yql/sql/v0/SQL.g @@ -284,7 +284,7 @@ join_constraint: | USING pure_column_or_named_list ; -into_table_stmt: (INSERT | INSERT OR ABORT | INSERT OR REVERT | INSERT OR IGNORE | UPSERT | REPLACE) INTO into_simple_table_ref into_values_source; +into_table_stmt: (INSERT | INSERT OR ABORT | INSERT OR REVERT | INSERT OR IGNORE | UPSERT | REPLACE) INTO into_simple_table_ref into_values_source; into_values_source: pure_column_list? values_source @@ -327,7 +327,7 @@ table_arg: AT? expr (COLON id_or_string)?; table_hints: WITH (id_or_string | pure_column_list); simple_table_ref: ((opt_id_prefix id_or_at) | AT? bind_parameter) table_hints?; -into_simple_table_ref: simple_table_ref (ERASE BY pure_column_list)?; +into_simple_table_ref: simple_table_ref (ERASE BY pure_column_list)?; delete_stmt: DELETE FROM simple_table_ref (WHERE expr | ON into_values_source)?; update_stmt: UPDATE simple_table_ref (SET set_clause_choice (WHERE expr)? | ON into_values_source); @@ -448,8 +448,8 @@ keyword_expr_uncompat: ; keyword_table_uncompat: - ERASE - | STREAM + ERASE + | STREAM ; keyword_select_uncompat: @@ -751,7 +751,7 @@ EACH: E A C H; ELSE: E L S E; EMPTY_ACTION: E M P T Y '_' A C T I O N; END: E N D; -ERASE: E R A S E; +ERASE: E R A S E; ESCAPE: E S C A P E; EVALUATE: E V A L U A T E; EXCEPT: E X C E P T; diff --git a/ydb/library/yql/sql/v0/builtin.cpp b/ydb/library/yql/sql/v0/builtin.cpp index a0945f9d05..ff4164efe4 100644 --- a/ydb/library/yql/sql/v0/builtin.cpp +++ b/ydb/library/yql/sql/v0/builtin.cpp @@ -2217,15 +2217,15 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec moduleResource = ctx.Settings.ModuleMapping.at(ns); } - if (ns == "js") { - ns = "javascript"; - } - - auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(ns); + if (ns == "js") { + ns = "javascript"; + } + + auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(ns); if (scriptType == NKikimr::NMiniKQL::EScriptType::SystemPython2) { scriptType = NKikimr::NMiniKQL::EScriptType::Python2; } - + if (ns == "yql") { return new TCallNodeImpl(pos, name, -1, -1, args); } else if (ns == "string" && name == "SplitToList") { @@ -2337,8 +2337,8 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec } else if (ns == "datetime2" && (name == "Format" || name == "Parse")) { return BuildUdf(ctx, pos, nameSpace, name, args); - } else if (scriptType != NKikimr::NMiniKQL::EScriptType::Unknown) { - auto scriptName = NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType); + } else if (scriptType != NKikimr::NMiniKQL::EScriptType::Unknown) { + auto scriptName = NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType); return new TScriptUdf(pos, TString(scriptName), name, args); } else if (ns.empty()) { auto type = NormalizeTypeString(normalizedName); diff --git a/ydb/library/yql/sql/v0/insert.cpp b/ydb/library/yql/sql/v0/insert.cpp index 1d8aec7134..ccfd3ccb3f 100644 --- a/ydb/library/yql/sql/v0/insert.cpp +++ b/ydb/library/yql/sql/v0/insert.cpp @@ -268,12 +268,12 @@ TSourcePtr BuildUpdateValues(TPosition pos, const TVector<TString>& columnsHint, class TWriteColumnsNode: public TAstListNode { public: - TWriteColumnsNode(TPosition pos, const TTableRef& table, EWriteColumnMode mode, TSourcePtr values = nullptr, TNodePtr options = nullptr) + TWriteColumnsNode(TPosition pos, const TTableRef& table, EWriteColumnMode mode, TSourcePtr values = nullptr, TNodePtr options = nullptr) : TAstListNode(pos) , Table(table) , Mode(mode) , Values(std::move(values)) - , Options(std::move(options)) + , Options(std::move(options)) { FakeSource = BuildFakeSource(pos); } @@ -293,14 +293,14 @@ public: TTableList tableList; TNodePtr values; - auto options = Y(); - if (Options) { - if (!Options->Init(ctx, src)) { - return false; - } - options = L(Options); - } - + auto options = Y(); + if (Options) { + if (!Options->Init(ctx, src)) { + return false; + } + options = L(Options); + } + ISource* underlyingSrc = src; if (TableSource) { @@ -308,7 +308,7 @@ public: if (!TableSource->Init(ctx, src) || !TableSource->InitFilters(ctx)) { return false; } - options = L(options, Q(Y(Q("filter"), TableSource->BuildFilterLambda(ctx.GroundBlockShortcuts(Pos))))); + options = L(options, Q(Y(Q("filter"), TableSource->BuildFilterLambda(ctx.GroundBlockShortcuts(Pos))))); } bool unordered = false; @@ -336,10 +336,10 @@ public: if (!Update->Init(ctx, TableSource.Get()) || !Update->InitFilters(ctx)) { return false; } - options = L(options, Q(Y(Q("update"), Update->Build(ctx)))); + options = L(options, Q(Y(Q("update"), Update->Build(ctx)))); } - auto write = BuildWriteTable(Pos, "values", Table, Mode, std::move(options)); + auto write = BuildWriteTable(Pos, "values", Table, Mode, std::move(options)); if (!write->Init(ctx, FakeSource.Get())) { return false; } @@ -370,16 +370,16 @@ protected: TSourcePtr Values; TSourcePtr Update; TSourcePtr FakeSource; - TNodePtr Options; + TNodePtr Options; }; EWriteColumnMode ToWriteColumnsMode(ESQLWriteColumnMode sqlWriteColumnMode) { return sqlIntoMode2WriteColumn.at(sqlWriteColumnMode); } -TNodePtr BuildWriteColumns(TPosition pos, const TTableRef& table, EWriteColumnMode mode, TSourcePtr values, TNodePtr options) { +TNodePtr BuildWriteColumns(TPosition pos, const TTableRef& table, EWriteColumnMode mode, TSourcePtr values, TNodePtr options) { YQL_ENSURE(values, "Invalid values node"); - return new TWriteColumnsNode(pos, table, mode, std::move(values), std::move(options)); + return new TWriteColumnsNode(pos, table, mode, std::move(values), std::move(options)); } TNodePtr BuildUpdateColumns(TPosition pos, const TTableRef& table, TSourcePtr values, TSourcePtr source) { @@ -396,40 +396,40 @@ TNodePtr BuildDelete(TPosition pos, const TTableRef& table, TSourcePtr source) { return writeNode; } - -class TEraseColumnsNode: public TAstListNode { -public: - TEraseColumnsNode(TPosition pos, const TVector<TString>& columns) - : TAstListNode(pos) - , Columns(columns) - { - } - - bool DoInit(TContext& ctx, ISource* src) override { - Y_UNUSED(ctx); - Y_UNUSED(src); - - TNodePtr columnList = Y(); - for (const auto& column: Columns) { - columnList->Add(Q(column)); - } - - Add(Q(Y(Q("erase_columns"), Q(columnList)))); - - return true; - } - - TNodePtr DoClone() const final { - return {}; - } - -private: - TVector<TString> Columns; -}; - - -TNodePtr BuildEraseColumns(TPosition pos, const TVector<TString>& columns) { - return new TEraseColumnsNode(pos, columns); -} - + +class TEraseColumnsNode: public TAstListNode { +public: + TEraseColumnsNode(TPosition pos, const TVector<TString>& columns) + : TAstListNode(pos) + , Columns(columns) + { + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(ctx); + Y_UNUSED(src); + + TNodePtr columnList = Y(); + for (const auto& column: Columns) { + columnList->Add(Q(column)); + } + + Add(Q(Y(Q("erase_columns"), Q(columnList)))); + + return true; + } + + TNodePtr DoClone() const final { + return {}; + } + +private: + TVector<TString> Columns; +}; + + +TNodePtr BuildEraseColumns(TPosition pos, const TVector<TString>& columns) { + return new TEraseColumnsNode(pos, columns); +} + } // namespace NSQLTranslationV0 diff --git a/ydb/library/yql/sql/v0/node.h b/ydb/library/yql/sql/v0/node.h index 4c7cfc273b..ad2c3f278a 100644 --- a/ydb/library/yql/sql/v0/node.h +++ b/ydb/library/yql/sql/v0/node.h @@ -995,8 +995,8 @@ namespace NSQLTranslationV0 { TSourcePtr BuildUpdateValues(TPosition pos, const TVector<TString>& columnsHint, const TVector<TNodePtr>& values); EWriteColumnMode ToWriteColumnsMode(ESQLWriteColumnMode sqlWriteColumnMode); - TNodePtr BuildEraseColumns(TPosition pos, const TVector<TString>& columns); - TNodePtr BuildWriteColumns(TPosition pos, const TTableRef& table, EWriteColumnMode mode, TSourcePtr values, TNodePtr options = nullptr); + TNodePtr BuildEraseColumns(TPosition pos, const TVector<TString>& columns); + TNodePtr BuildWriteColumns(TPosition pos, const TTableRef& table, EWriteColumnMode mode, TSourcePtr values, TNodePtr options = nullptr); TNodePtr BuildUpdateColumns(TPosition pos, const TTableRef& table, TSourcePtr values, TSourcePtr source); TNodePtr BuildDelete(TPosition pos, const TTableRef& table, TSourcePtr source); @@ -1010,7 +1010,7 @@ namespace NSQLTranslationV0 { const TVector<std::pair<TIdentifier, bool>>& orderByColumns); TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TVector<TColumnSchema>& columns, EAlterTableIntentnt mode); TNodePtr BuildDropTable(TPosition pos, const TTableRef& table); - TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options = nullptr); + TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options = nullptr); TNodePtr BuildWriteResult(TPosition pos, const TString& label, TNodePtr settings, const TSet<TString>& clusters); TNodePtr BuildCommitClusters(TPosition pos, const TSet<TString>& clusters = TSet<TString>()); TNodePtr BuildRollbackClusters(TPosition pos, const TSet<TString>& clusters = TSet<TString>()); diff --git a/ydb/library/yql/sql/v0/query.cpp b/ydb/library/yql/sql/v0/query.cpp index 7752b28b01..841eac96e0 100644 --- a/ydb/library/yql/sql/v0/query.cpp +++ b/ydb/library/yql/sql/v0/query.cpp @@ -712,10 +712,10 @@ static const TMap<EWriteColumnMode, TString> columnModeToStrMapMR { {EWriteColumnMode::Renew, "renew"} }; -static const TMap<EWriteColumnMode, TString> columnModeToStrMapStat { - {EWriteColumnMode::Upsert, "upsert"} -}; - +static const TMap<EWriteColumnMode, TString> columnModeToStrMapStat { + {EWriteColumnMode::Upsert, "upsert"} +}; + static const TMap<EWriteColumnMode, TString> columnModeToStrMapKikimr { {EWriteColumnMode::Default, ""}, {EWriteColumnMode::Insert, "insert_abort"}, @@ -733,7 +733,7 @@ static const TMap<EWriteColumnMode, TString> columnModeToStrMapKikimr { class TWriteTableNode final: public TAstListNode { public: TWriteTableNode(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, - TNodePtr options) + TNodePtr options) : TAstListNode(pos) , Label(label) , Table(table) @@ -756,22 +756,22 @@ public: auto getModesMap = [] (const TString& serviceName) -> const TMap<EWriteColumnMode, TString>& { if (serviceName == KikimrProviderName) { return columnModeToStrMapKikimr; - } else if (serviceName == StatProviderName) { - return columnModeToStrMapStat; + } else if (serviceName == StatProviderName) { + return columnModeToStrMapStat; } else { return columnModeToStrMapMR; } }; auto options = Y(); - if (Options) { - if (!Options->Init(ctx, src)) { - return false; - } - - options = L(Options); - } - + if (Options) { + if (!Options->Init(ctx, src)) { + return false; + } + + options = L(Options); + } + if (Mode != EWriteColumnMode::Default) { auto modeStr = getModesMap(serviceName).FindPtr(Mode); @@ -795,12 +795,12 @@ private: TString Label; TTableRef Table; EWriteColumnMode Mode; - TNodePtr Options; + TNodePtr Options; }; -TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options) +TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options) { - return new TWriteTableNode(pos, label, table, mode, std::move(options)); + return new TWriteTableNode(pos, label, table, mode, std::move(options)); } class TClustersSinkOperationBase: public TAstListNode { diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp index b028fc0439..0860d96f7e 100644 --- a/ydb/library/yql/sql/v0/sql.cpp +++ b/ydb/library/yql/sql/v0/sql.cpp @@ -1093,17 +1093,17 @@ void TSqlCallExpr::IncCounters() { Ctx.IncrementMonCounter("sql_features", "NamedNodeUseApply"); } else if (!Module.empty()) { if (ValidateForCounters(Module)) { - Ctx.IncrementMonCounter("udf_modules", Module); + Ctx.IncrementMonCounter("udf_modules", Module); Ctx.IncrementMonCounter("sql_features", "CallUdf"); if (ValidateForCounters(Func)) { - auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(Module); - if (scriptType == NKikimr::NMiniKQL::EScriptType::Unknown) { - Ctx.IncrementMonCounter("udf_functions", Module + "." + Func); + auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(Module); + if (scriptType == NKikimr::NMiniKQL::EScriptType::Unknown) { + Ctx.IncrementMonCounter("udf_functions", Module + "." + Func); } } } } else if (ValidateForCounters(Func)) { - Ctx.IncrementMonCounter("sql_builtins", Func); + Ctx.IncrementMonCounter("sql_builtins", Func); Ctx.IncrementMonCounter("sql_features", "CallBuiltin"); } } @@ -4040,9 +4040,9 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { SqlIntoModeStr = JoinRange("", modeStrings.begin(), modeStrings.end()); SqlIntoUserModeStr = JoinRange(" ", userModeStrings.begin(), userModeStrings.end()); - auto intoTableRef = node.GetRule_into_simple_table_ref3(); - auto tableRef = intoTableRef.GetRule_simple_table_ref1(); - + auto intoTableRef = node.GetRule_into_simple_table_ref3(); + auto tableRef = intoTableRef.GetRule_simple_table_ref1(); + auto cluster = Ctx.CurrCluster; std::pair<bool, TDeferredAtom> nameOrAt; if (tableRef.HasBlock1()) { @@ -4091,24 +4091,24 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { } } - TVector<TString> eraseColumns; - if (intoTableRef.HasBlock2()) { - auto service = Ctx.GetClusterProvider(cluster); - if (!service) { - Ctx.Error() << "Unknown cluster name: " << cluster; - return nullptr; - } - - if (*service != StatProviderName) { - Ctx.Error() << "ERASE BY is unsupported for " << *service; - return nullptr; - } - - PureColumnListStr( - intoTableRef.GetBlock2().GetRule_pure_column_list3(), *this, eraseColumns - ); - } - + TVector<TString> eraseColumns; + if (intoTableRef.HasBlock2()) { + auto service = Ctx.GetClusterProvider(cluster); + if (!service) { + Ctx.Error() << "Unknown cluster name: " << cluster; + return nullptr; + } + + if (*service != StatProviderName) { + Ctx.Error() << "ERASE BY is unsupported for " << *service; + return nullptr; + } + + PureColumnListStr( + intoTableRef.GetBlock2().GetRule_pure_column_list3(), *this, eraseColumns + ); + } + if (withTruncate) { if (SqlIntoModeStr != "InsertInto") { Error() << "Unable " << SqlIntoUserModeStr << " with truncate mode"; @@ -4135,13 +4135,13 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { return nullptr; } Ctx.IncrementMonCounter("sql_features", SqlIntoModeStr); - - TNodePtr options; - if (eraseColumns) { - options = BuildEraseColumns(pos, std::move(eraseColumns)); - } - - return BuildWriteColumns(pos, table, ToWriteColumnsMode(SqlIntoMode), std::move(values), std::move(options)); + + TNodePtr options; + if (eraseColumns) { + options = BuildEraseColumns(pos, std::move(eraseColumns)); + } + + return BuildWriteColumns(pos, table, ToWriteColumnsMode(SqlIntoMode), std::move(values), std::move(options)); } bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const TTableRef& table, @@ -4154,13 +4154,13 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const const bool isMapReduce = serviceName == YtProviderName; const bool isKikimr = serviceName == KikimrProviderName; const bool isRtmr = serviceName == RtmrProviderName; - const bool isStat = serviceName == StatProviderName; + const bool isStat = serviceName == StatProviderName; if (!isKikimr) { if (mode == ESQLWriteColumnMode::InsertOrAbortInto || mode == ESQLWriteColumnMode::InsertOrIgnoreInto || mode == ESQLWriteColumnMode::InsertOrRevertInto || - mode == ESQLWriteColumnMode::UpsertInto && !isStat) + mode == ESQLWriteColumnMode::UpsertInto && !isStat) { Ctx.Error(pos) << SqlIntoUserModeStr << " is not supported for " << serviceName << " tables"; Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); @@ -4186,12 +4186,12 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); return false; } - } else if (isStat) { - if (mode != ESQLWriteColumnMode::UpsertInto) { - Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; - Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); - return false; - } + } else if (isStat) { + if (mode != ESQLWriteColumnMode::UpsertInto) { + Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; + Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); + return false; + } } return true; diff --git a/ydb/library/yql/sql/v0/sql_ut.cpp b/ydb/library/yql/sql/v0/sql_ut.cpp index 3a70aef93c..b0e336bef2 100644 --- a/ydb/library/yql/sql/v0/sql_ut.cpp +++ b/ydb/library/yql/sql/v0/sql_ut.cpp @@ -633,23 +633,23 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { NYql::TAstParseResult res = SqlToYql("update plato.Input set (key, value, subkey) = (select key, value, subkey from plato.Input where key = 911) where key = 200;", 10, "kikimr"); UNIT_ASSERT(res.Root); - int lineIndex = 0; - int writeLineIndex = -1; - bool found = false; - - TVerifyLineFunc verifyLine = [&lineIndex, &writeLineIndex, &found](const TString& word, const TString& line) { + int lineIndex = 0; + int writeLineIndex = -1; + bool found = false; + + TVerifyLineFunc verifyLine = [&lineIndex, &writeLineIndex, &found](const TString& word, const TString& line) { if (word == "Write") { - writeLineIndex = lineIndex; - found = line.find("('mode 'update)") != TString::npos; - } else if (word == "mode") { - found |= lineIndex == writeLineIndex + 1 && line.find("('mode 'update)") != TString::npos; - UNIT_ASSERT(found); + writeLineIndex = lineIndex; + found = line.find("('mode 'update)") != TString::npos; + } else if (word == "mode") { + found |= lineIndex == writeLineIndex + 1 && line.find("('mode 'update)") != TString::npos; + UNIT_ASSERT(found); } - - ++lineIndex; + + ++lineIndex; }; - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("mode"), 0}}; + TWordCountHive elementStat = {{TString("Write"), 0}, {TString("mode"), 0}}; VerifyProgram(res, elementStat, verifyLine); UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index ef394c7a00..32470599f7 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -433,7 +433,7 @@ join_constraint: | USING pure_column_or_named_list ; -into_table_stmt: (INSERT | INSERT OR ABORT | INSERT OR REVERT | INSERT OR IGNORE | UPSERT | REPLACE) INTO into_simple_table_ref into_values_source; +into_table_stmt: (INSERT | INSERT OR ABORT | INSERT OR REVERT | INSERT OR IGNORE | UPSERT | REPLACE) INTO into_simple_table_ref into_values_source; into_values_source: pure_column_list? values_source @@ -581,7 +581,7 @@ table_hint: ; simple_table_ref: (((cluster_expr DOT)? id_or_at) | AT? bind_parameter) table_hints?; -into_simple_table_ref: simple_table_ref (ERASE BY pure_column_list)?; +into_simple_table_ref: simple_table_ref (ERASE BY pure_column_list)?; delete_stmt: DELETE FROM simple_table_ref (WHERE expr | ON into_values_source)?; update_stmt: UPDATE simple_table_ref (SET set_clause_choice (WHERE expr)? | ON into_values_source); @@ -814,7 +814,7 @@ keyword_expr_uncompat: keyword_table_uncompat: ANY | ERASE - | STREAM + | STREAM ; keyword_select_uncompat: @@ -1181,7 +1181,7 @@ EMPTY_ACTION: E M P T Y '_' A C T I O N; ENCRYPTED: E N C R Y P T E D; END: E N D; ENUM: E N U M; -ERASE: E R A S E; +ERASE: E R A S E; ESCAPE: E S C A P E; EVALUATE: E V A L U A T E; EXCEPT: E X C E P T; diff --git a/ydb/library/yql/sql/v1/builtin.cpp b/ydb/library/yql/sql/v1/builtin.cpp index 43ef1243f1..ff67fe0ea8 100644 --- a/ydb/library/yql/sql/v1/builtin.cpp +++ b/ydb/library/yql/sql/v1/builtin.cpp @@ -2913,11 +2913,11 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec moduleResource = ctx.Settings.ModuleMapping.at(ns); } - if (ns == "js") { - ns = "javascript"; + if (ns == "js") { + ns = "javascript"; nameSpace = "JavaScript"; - } - + } + if (ns == "datetime2") { ctx.Warning(pos, TIssuesIds::YQL_DEPRECATED_DATETIME2) << "DateTime2:: is a temporary alias for DateTime:: which will be removed in the future, use DateTime:: instead"; } @@ -2927,7 +2927,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec nameSpace = "DateTime2"; } - auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(ns); + auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(ns); switch (scriptType) { case NKikimr::NMiniKQL::EScriptType::Python: case NKikimr::NMiniKQL::EScriptType::Python3: @@ -2943,7 +2943,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec default: break; } - + if (ns == "yql" || ns == "@yql") { if (warnOnYqlNameSpace && GetEnv("YQL_DETERMINISTIC_MODE").empty()) { ctx.Warning(pos, TIssuesIds::YQL_S_EXPRESSIONS_CALL) @@ -3070,8 +3070,8 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec } return BuildUdf(ctx, pos, nameSpace, name, makeUdfArgs()); - } else if (scriptType != NKikimr::NMiniKQL::EScriptType::Unknown) { - auto scriptName = NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType); + } else if (scriptType != NKikimr::NMiniKQL::EScriptType::Unknown) { + auto scriptName = NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType); return new TScriptUdf(pos, TString(scriptName), name, args); } else if (ns.empty()) { if (auto simpleType = LookupSimpleTypeBySqlAlias(normalizedName, ctx.FlexibleTypes)) { diff --git a/ydb/library/yql/sql/v1/insert.cpp b/ydb/library/yql/sql/v1/insert.cpp index 4f4c863018..cdbd25ae0d 100644 --- a/ydb/library/yql/sql/v1/insert.cpp +++ b/ydb/library/yql/sql/v1/insert.cpp @@ -270,7 +270,7 @@ public: , Table(table) , Mode(mode) , Values(std::move(values)) - , Options(std::move(options)) + , Options(std::move(options)) { FakeSource = BuildFakeSource(pos); } @@ -286,14 +286,14 @@ public: bool DoInit(TContext& ctx, ISource* src) override { TTableList tableList; TNodePtr values; - auto options = Y(); - if (Options) { - if (!Options->Init(ctx, src)) { - return false; - } - options = L(Options); - } - + auto options = Y(); + if (Options) { + if (!Options->Init(ctx, src)) { + return false; + } + options = L(Options); + } + ISource* underlyingSrc = src; if (TableSource) { @@ -327,7 +327,7 @@ public: if (!Update->Init(ctx, TableSource.Get()) || !Update->InitFilters(ctx)) { return false; } - options = L(options, Q(Y(Q("update"), Update->Build(ctx)))); + options = L(options, Q(Y(Q("update"), Update->Build(ctx)))); } auto write = BuildWriteTable(Pos, "values", Table, Mode, std::move(options), Scoped); @@ -361,7 +361,7 @@ protected: TSourcePtr Values; TSourcePtr Update; TSourcePtr FakeSource; - TNodePtr Options; + TNodePtr Options; }; EWriteColumnMode ToWriteColumnsMode(ESQLWriteColumnMode sqlWriteColumnMode) { @@ -387,40 +387,40 @@ TNodePtr BuildDelete(TPosition pos, TScopedStatePtr scoped, const TTableRef& tab return writeNode; } - -class TEraseColumnsNode: public TAstListNode { -public: - TEraseColumnsNode(TPosition pos, const TVector<TString>& columns) - : TAstListNode(pos) - , Columns(columns) - { - } - - bool DoInit(TContext& ctx, ISource* src) override { - Y_UNUSED(ctx); - Y_UNUSED(src); - - TNodePtr columnList = Y(); - for (const auto& column: Columns) { - columnList->Add(Q(column)); - } - - Add(Q(Y(Q("erase_columns"), Q(columnList)))); - - return true; - } - - TNodePtr DoClone() const final { - return {}; - } - -private: - TVector<TString> Columns; -}; - - -TNodePtr BuildEraseColumns(TPosition pos, const TVector<TString>& columns) { - return new TEraseColumnsNode(pos, columns); -} - + +class TEraseColumnsNode: public TAstListNode { +public: + TEraseColumnsNode(TPosition pos, const TVector<TString>& columns) + : TAstListNode(pos) + , Columns(columns) + { + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(ctx); + Y_UNUSED(src); + + TNodePtr columnList = Y(); + for (const auto& column: Columns) { + columnList->Add(Q(column)); + } + + Add(Q(Y(Q("erase_columns"), Q(columnList)))); + + return true; + } + + TNodePtr DoClone() const final { + return {}; + } + +private: + TVector<TString> Columns; +}; + + +TNodePtr BuildEraseColumns(TPosition pos, const TVector<TString>& columns) { + return new TEraseColumnsNode(pos, columns); +} + } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index de82b45a23..258bd3062b 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -1311,7 +1311,7 @@ namespace NSQLTranslationV1 { TSourcePtr BuildUpdateValues(TPosition pos, const TVector<TString>& columnsHint, const TVector<TNodePtr>& values); EWriteColumnMode ToWriteColumnsMode(ESQLWriteColumnMode sqlWriteColumnMode); - TNodePtr BuildEraseColumns(TPosition pos, const TVector<TString>& columns); + TNodePtr BuildEraseColumns(TPosition pos, const TVector<TString>& columns); TNodePtr BuildWriteColumns(TPosition pos, TScopedStatePtr scoped, const TTableRef& table, EWriteColumnMode mode, TSourcePtr values, TNodePtr options = nullptr); TNodePtr BuildUpdateColumns(TPosition pos, TScopedStatePtr scoped, const TTableRef& table, TSourcePtr values, TSourcePtr source); TNodePtr BuildDelete(TPosition pos, TScopedStatePtr scoped, const TTableRef& table, TSourcePtr source); diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 725356b9f5..266ddece64 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -1388,10 +1388,10 @@ static const TMap<EWriteColumnMode, TString> columnModeToStrMapMR { {EWriteColumnMode::Renew, "renew"} }; -static const TMap<EWriteColumnMode, TString> columnModeToStrMapStat { - {EWriteColumnMode::Upsert, "upsert"} -}; - +static const TMap<EWriteColumnMode, TString> columnModeToStrMapStat { + {EWriteColumnMode::Upsert, "upsert"} +}; + static const TMap<EWriteColumnMode, TString> columnModeToStrMapKikimr { {EWriteColumnMode::Default, ""}, {EWriteColumnMode::Insert, "insert_abort"}, @@ -1429,22 +1429,22 @@ public: auto getModesMap = [] (const TString& serviceName) -> const TMap<EWriteColumnMode, TString>& { if (serviceName == KikimrProviderName || serviceName == YdbProviderName) { return columnModeToStrMapKikimr; - } else if (serviceName == StatProviderName) { - return columnModeToStrMapStat; + } else if (serviceName == StatProviderName) { + return columnModeToStrMapStat; } else { return columnModeToStrMapMR; } }; auto options = Y(); - if (Options) { - if (!Options->Init(ctx, src)) { - return false; - } - - options = L(Options); - } - + if (Options) { + if (!Options->Init(ctx, src)) { + return false; + } + + options = L(Options); + } + if (Mode != EWriteColumnMode::Default) { auto modeStr = getModesMap(Table.Service).FindPtr(Mode); @@ -1467,7 +1467,7 @@ private: TString Label; TTableRef Table; EWriteColumnMode Mode; - TNodePtr Options; + TNodePtr Options; TScopedStatePtr Scoped; }; diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index ab5a864788..c2832cffc9 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -3458,17 +3458,17 @@ void TSqlCallExpr::IncCounters() { Ctx.IncrementMonCounter("sql_features", "NamedNodeUseApply"); } else if (!Module.empty()) { if (ValidateForCounters(Module)) { - Ctx.IncrementMonCounter("udf_modules", Module); + Ctx.IncrementMonCounter("udf_modules", Module); Ctx.IncrementMonCounter("sql_features", "CallUdf"); if (ValidateForCounters(Func)) { - auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(Module); - if (scriptType == NKikimr::NMiniKQL::EScriptType::Unknown) { - Ctx.IncrementMonCounter("udf_functions", Module + "." + Func); + auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(Module); + if (scriptType == NKikimr::NMiniKQL::EScriptType::Unknown) { + Ctx.IncrementMonCounter("udf_functions", Module + "." + Func); } } } } else if (ValidateForCounters(Func)) { - Ctx.IncrementMonCounter("sql_builtins", Func); + Ctx.IncrementMonCounter("sql_builtins", Func); Ctx.IncrementMonCounter("sql_features", "CallBuiltin"); } } @@ -7969,9 +7969,9 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { SqlIntoModeStr = JoinRange("", modeStrings.begin(), modeStrings.end()); SqlIntoUserModeStr = JoinRange(" ", userModeStrings.begin(), userModeStrings.end()); - auto intoTableRef = node.GetRule_into_simple_table_ref3(); - auto tableRef = intoTableRef.GetRule_simple_table_ref1(); - + auto intoTableRef = node.GetRule_into_simple_table_ref3(); + auto tableRef = intoTableRef.GetRule_simple_table_ref1(); + auto service = Ctx.Scoped->CurrService; auto cluster = Ctx.Scoped->CurrCluster; std::pair<bool, TDeferredAtom> nameOrAt; @@ -8037,18 +8037,18 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { } } - TVector<TString> eraseColumns; - if (intoTableRef.HasBlock2()) { + TVector<TString> eraseColumns; + if (intoTableRef.HasBlock2()) { if (service != StatProviderName) { Ctx.Error() << "ERASE BY is unsupported for " << service; - return nullptr; - } - - PureColumnListStr( - intoTableRef.GetBlock2().GetRule_pure_column_list3(), *this, eraseColumns - ); - } - + return nullptr; + } + + PureColumnListStr( + intoTableRef.GetBlock2().GetRule_pure_column_list3(), *this, eraseColumns + ); + } + if (withTruncate) { if (SqlIntoModeStr != "InsertInto") { Error() << "Unable " << SqlIntoUserModeStr << " with truncate mode"; @@ -8075,12 +8075,12 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { return nullptr; } Ctx.IncrementMonCounter("sql_features", SqlIntoModeStr); - - TNodePtr options; - if (eraseColumns) { - options = BuildEraseColumns(pos, std::move(eraseColumns)); - } - + + TNodePtr options; + if (eraseColumns) { + options = BuildEraseColumns(pos, std::move(eraseColumns)); + } + return BuildWriteColumns(pos, Ctx.Scoped, table, ToWriteColumnsMode(SqlIntoMode), std::move(values), std::move(options)); } @@ -8091,13 +8091,13 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const const bool isMapReduce = serviceName == YtProviderName; const bool isKikimr = serviceName == KikimrProviderName || serviceName == YdbProviderName; const bool isRtmr = serviceName == RtmrProviderName; - const bool isStat = serviceName == StatProviderName; + const bool isStat = serviceName == StatProviderName; if (!isKikimr) { if (mode == ESQLWriteColumnMode::InsertOrAbortInto || mode == ESQLWriteColumnMode::InsertOrIgnoreInto || mode == ESQLWriteColumnMode::InsertOrRevertInto || - mode == ESQLWriteColumnMode::UpsertInto && !isStat) + mode == ESQLWriteColumnMode::UpsertInto && !isStat) { Ctx.Error(pos) << SqlIntoUserModeStr << " is not supported for " << serviceName << " tables"; Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); @@ -8123,12 +8123,12 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); return false; } - } else if (isStat) { - if (mode != ESQLWriteColumnMode::UpsertInto) { - Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; - Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); - return false; - } + } else if (isStat) { + if (mode != ESQLWriteColumnMode::UpsertInto) { + Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName; + Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName); + return false; + } } return true; diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 85b6e0dfcd..04afc0ba64 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -706,23 +706,23 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { NYql::TAstParseResult res = SqlToYql("update plato.Input set (key, value, subkey) = (select key, value, subkey from plato.Input where key = 911) where key = 200;", 10, "kikimr"); UNIT_ASSERT(res.Root); - int lineIndex = 0; - int writeLineIndex = -1; - bool found = false; - - TVerifyLineFunc verifyLine = [&lineIndex, &writeLineIndex, &found](const TString& word, const TString& line) { + int lineIndex = 0; + int writeLineIndex = -1; + bool found = false; + + TVerifyLineFunc verifyLine = [&lineIndex, &writeLineIndex, &found](const TString& word, const TString& line) { if (word == "Write") { - writeLineIndex = lineIndex; - found = line.find("('mode 'update)") != TString::npos; - } else if (word == "mode") { - found |= lineIndex == writeLineIndex + 1 && line.find("('mode 'update)") != TString::npos; - UNIT_ASSERT(found); + writeLineIndex = lineIndex; + found = line.find("('mode 'update)") != TString::npos; + } else if (word == "mode") { + found |= lineIndex == writeLineIndex + 1 && line.find("('mode 'update)") != TString::npos; + UNIT_ASSERT(found); } - - ++lineIndex; + + ++lineIndex; }; - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("mode"), 0}}; + TWordCountHive elementStat = {{TString("Write"), 0}, {TString("mode"), 0}}; VerifyProgram(res, elementStat, verifyLine); UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); diff --git a/ydb/library/yql/utils/log/log_component.h b/ydb/library/yql/utils/log/log_component.h index d7a1e10ca1..ac9747eceb 100644 --- a/ydb/library/yql/utils/log/log_component.h +++ b/ydb/library/yql/utils/log/log_component.h @@ -22,7 +22,7 @@ enum class EComponent { ProviderRtmr, Performance, Perf = Performance, Net, - ProviderStat, + ProviderStat, ProviderSolomon, CoreEval, CorePeepHole, |