diff options
17 files changed, 164 insertions, 0 deletions
diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp index 8320c08b502..2a3147c21a6 100644 --- a/yql/essentials/core/type_ann/type_ann_core.cpp +++ b/yql/essentials/core/type_ann/type_ann_core.cpp @@ -3298,6 +3298,49 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } + IGraphTransformer::TStatus SqlConcatWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { + if (!IsBackwardCompatibleFeatureAvailable(ctx.Types.LangVer, MakeLangVersion(2025, 04), ctx.Types.BackportMode)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), "Concat function is not available before version 2025.04")); + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + bool hasOptionals = false; + bool hasBinary = false; + for (ui32 i = 0; i < input->ChildrenSize(); ++i) { + if (IsNull(*input->Child(i))) { + output = input->ChildPtr(i); + return IGraphTransformer::TStatus::Repeat; + } + + bool isOptional; + const TDataExprType* dataType ; + if (!EnsureDataOrOptionalOfData(*input->Child(i), isOptional, dataType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (dataType->GetSlot() != EDataSlot::String && dataType->GetSlot() != EDataSlot::Utf8) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), + TStringBuilder() << "Expected (optional) String or Utf8, but got: " << *input->Child(i)->GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + + hasOptionals = hasOptionals || isOptional; + hasBinary = hasBinary || (dataType->GetSlot() == EDataSlot::String); + } + + const TTypeAnnotationNode* retType = ctx.Expr.MakeType<TDataExprType>(hasBinary ? EDataSlot::String : EDataSlot::Utf8); + if (hasOptionals) { + retType = ctx.Expr.MakeType<TOptionalExprType>(retType); + } + + input->SetTypeAnn(retType); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus SubstringWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -13012,6 +13055,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["WithWorld"] = &WithWorldWrapper; Functions["Concat"] = &ConcatWrapper; Functions["AggrConcat"] = &AggrConcatWrapper; + ExtFunctions["SqlConcat"] = &SqlConcatWrapper; ExtFunctions["Substring"] = &SubstringWrapper; ExtFunctions["Find"] = &FindWrapper; ExtFunctions["RFind"] = &FindWrapper; diff --git a/yql/essentials/data/language/sql_functions.json b/yql/essentials/data/language/sql_functions.json index 074cb24e656..cebadbbef92 100644 --- a/yql/essentials/data/language/sql_functions.json +++ b/yql/essentials/data/language/sql_functions.json @@ -220,6 +220,10 @@ "kind": "Normal" }, { + "name": "Concat", + "kind": "Normal" + }, + { "name": "Corr", "kind": "Agg" }, diff --git a/yql/essentials/docs/en/builtins/basic.md b/yql/essentials/docs/en/builtins/basic.md index 994387eaf57..7619c7e9b7f 100644 --- a/yql/essentials/docs/en/builtins/basic.md +++ b/yql/essentials/docs/en/builtins/basic.md @@ -90,6 +90,26 @@ SELECT SUBSTRING("abcdefg", 3); -- defg SELECT SUBSTRING("abcdefg", NULL, 3); -- abc ``` +## Concat {#concat} + +Concatenate one or more strings. + +#### Signature + +```yql +Concat((String|Utf8)[?], ...)->(String|Utf8)[?] +``` + +This function is available since version [2025.04](../changelog/2025.04.md). +If at least one input string is of type `Optional`, then the result is also of type `Optional`. +If all input strings are of type `Utf8`, then the result is also of type `Utf8`; otherwise, it is `String`. +If at least one input string is `NULL`, then the result is also of type `NULL`. + +#### Examples + +```yql +SELECT Concat("abc", "de", "f"); -- "abcdef" +``` ## FIND {#find} diff --git a/yql/essentials/docs/en/changelog/2025.04.md b/yql/essentials/docs/en/changelog/2025.04.md index f6e20c7092d..42652f7a3bf 100644 --- a/yql/essentials/docs/en/changelog/2025.04.md +++ b/yql/essentials/docs/en/changelog/2025.04.md @@ -11,6 +11,7 @@ Added linear types. * Added `ToDynamicLinear`/`FromDynamicLinear` functions for converting between linear types. * Added `LinearType`/`DynamicLinearType` functions for constructing linear types. * Added the `LinearTypeHandle`/`DynamicLinearTypeHandle` functions for constructing linear types during code generation. * Added the `LinearItemType` function for retrieving the parameter type from a linear type. +* Added `Concat` function for string concatenation. ## Changes in NOT NULL expression diff --git a/yql/essentials/docs/en/syntax/expressions.md b/yql/essentials/docs/en/syntax/expressions.md index 7d6ceceb9fd..cc8f9bdeb82 100644 --- a/yql/essentials/docs/en/syntax/expressions.md +++ b/yql/essentials/docs/en/syntax/expressions.md @@ -10,6 +10,8 @@ As with other binary operators, if the data on either side is `NULL`, the result Don't confuse this operator with a logical "or": in SQL, it's denoted by the `OR` keyword. It's also not worth doing concatenation using `+`. +There is a similar function for this operator [Concat](../builtins/basic.md#concat), which supports an arbitrary number of arguments. + #### Examples ```yql diff --git a/yql/essentials/docs/ru/builtins/basic.md b/yql/essentials/docs/ru/builtins/basic.md index 5286146d226..ff56f6d994e 100644 --- a/yql/essentials/docs/ru/builtins/basic.md +++ b/yql/essentials/docs/ru/builtins/basic.md @@ -109,6 +109,26 @@ SELECT SUBSTRING("abcdefg", 3); -- defg SELECT SUBSTRING("abcdefg", NULL, 3); -- abc ``` +## Concat {#concat} + +Конкатенация одной и более строк. + +#### Сигнатура + +```yql +Concat((String|Utf8)[?], ...)->(String|Utf8)[?] +``` + +Функция доступна начиная с версии [2025.04](../changelog/2025.04.md). +Если хотя бы у одной входной строки тип `Optional`, то таким же является и результат. +Если все входные строки имеют тип `Utf8`, то таким же является и результат, в противном случае - `String`. +Если хотя бы одна входная строка является `NULL`, то таким же является и результат. + +#### Примеры + +```yql +SELECT Concat("abc", "de", "f"); -- "abcdef" +``` ## FIND {#find} diff --git a/yql/essentials/docs/ru/changelog/2025.04.md b/yql/essentials/docs/ru/changelog/2025.04.md index 49cf7e82b2c..813940dd483 100644 --- a/yql/essentials/docs/ru/changelog/2025.04.md +++ b/yql/essentials/docs/ru/changelog/2025.04.md @@ -12,6 +12,7 @@ * Добавлены функции `LinearType`/`DynamicLinearType` для построения линейных типов. * Добавлены функции `LinearTypeHandle`/`DynamicLinearTypeHandle` для построения линейных типов во время кодогенерации. * Добавлена функция `LinearItemType` для получения типа параметра из линейного типа. +* Добавлена функция `Concat` для конкатенации строк. ## Изменения в выражении NOT NULL diff --git a/yql/essentials/docs/ru/syntax/expressions.md b/yql/essentials/docs/ru/syntax/expressions.md index 89687484760..c7825fc340c 100644 --- a/yql/essentials/docs/ru/syntax/expressions.md +++ b/yql/essentials/docs/ru/syntax/expressions.md @@ -10,6 +10,8 @@ Не следует путать этот оператор с логическим «или», в SQL оно обозначается ключевым словом `OR`. Также не стоит пытаться делать конкатенацию через `+`. +Для этого оператора есть аналогичная функция [Concat](../builtins/basic.md#concat), которая поддерживает произвольное число аргументов. + #### Примеры ```yql diff --git a/yql/essentials/minikql/mkql_program_builder.cpp b/yql/essentials/minikql/mkql_program_builder.cpp index 52e201899b2..2bef8570841 100644 --- a/yql/essentials/minikql/mkql_program_builder.cpp +++ b/yql/essentials/minikql/mkql_program_builder.cpp @@ -4807,6 +4807,24 @@ TRuntimeNode TProgramBuilder::Concat(TRuntimeNode data1, TRuntimeNode data2) { return InvokeBinary(__func__, isOpt1 || isOpt2 ? NewOptionalType(resultType) : resultType, data1, data2); } +TRuntimeNode TProgramBuilder::ConcatMany(const TArrayRef<const TRuntimeNode>& args) { + MKQL_ENSURE(args.size() >= 1, "Expected at least one argument"); + if (args.size() == 1) { + return args[0]; + } + + if (args.size() == 2) { + return Concat(args[0], args[1]); + } + + // TODO make a dedicated callable + // split into pairs + ui32 midPoint = args.size() / 2; + return Concat( + ConcatMany(TArrayRef<const TRuntimeNode>(args.data(), args.data() + midPoint)), + ConcatMany(TArrayRef<const TRuntimeNode>(args.data() + midPoint, args.end()))); +} + TRuntimeNode TProgramBuilder::AggrConcat(TRuntimeNode data1, TRuntimeNode data2) { MKQL_ENSURE(data1.GetStaticType()->IsSameType(*data2.GetStaticType()), "Operands type mismatch."); const std::array<TRuntimeNode, 2> args = {{data1, data2}}; diff --git a/yql/essentials/minikql/mkql_program_builder.h b/yql/essentials/minikql/mkql_program_builder.h index bc02b06ac68..2644b049c91 100644 --- a/yql/essentials/minikql/mkql_program_builder.h +++ b/yql/essentials/minikql/mkql_program_builder.h @@ -222,6 +222,7 @@ public: TRuntimeNode ToDecimal(TRuntimeNode data, ui8 precision, ui8 scale); TRuntimeNode Concat(TRuntimeNode data1, TRuntimeNode data2); TRuntimeNode AggrConcat(TRuntimeNode data1, TRuntimeNode data2); + TRuntimeNode ConcatMany(const TArrayRef<const TRuntimeNode>& args); TRuntimeNode Substring(TRuntimeNode data, TRuntimeNode start, TRuntimeNode count); TRuntimeNode Find(TRuntimeNode haystack, TRuntimeNode needle, TRuntimeNode pos); TRuntimeNode RFind(TRuntimeNode haystack, TRuntimeNode needle, TRuntimeNode pos); diff --git a/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp b/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp index 54a922941f6..760147b6b2c 100644 --- a/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp +++ b/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp @@ -656,6 +656,15 @@ TMkqlCommonCallableCompiler::TShared::TShared() { {"NextValue", &TProgramBuilder::NextValue}, }); + AddCallable("SqlConcat", [](const TExprNode& node, TMkqlBuildContext& ctx) { + TVector<TRuntimeNode> args; + for (const auto& child : node.Children()) { + args.push_back(MkqlBuildExpr(*child, ctx)); + } + + return ctx.ProgramBuilder.ConcatMany(args); + }); + AddCallable({"MultiMap", "OrderedMultiMap"}, [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto arg = MkqlBuildExpr(node.Head(), ctx); const auto lambda = [&](TRuntimeNode item) { return MkqlBuildWideLambda(node.Tail(), ctx, {item}); }; diff --git a/yql/essentials/sql/v1/builtin.cpp b/yql/essentials/sql/v1/builtin.cpp index e5ed591beec..17eae94ee4f 100644 --- a/yql/essentials/sql/v1/builtin.cpp +++ b/yql/essentials/sql/v1/builtin.cpp @@ -3035,6 +3035,7 @@ struct TBuiltinFuncData { {"byteat", {"ByteAt", "Normal", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("ByteAt", 2, 2)}}, {"startswith", {"StartsWith", "Normal", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StartsWith", 2, 2)}}, {"endswith", {"EndsWith", "Normal", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("EndsWith", 2, 2)}}, + {"concat", {"Concat", "Normal", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("SqlConcat", 1, -1)}}, // Numeric builtins {"abs", {"Abs", "Normal", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("Abs", 1, 1)}}, diff --git a/yql/essentials/tests/sql/minirun/part1/canondata/result.json b/yql/essentials/tests/sql/minirun/part1/canondata/result.json index bf94623311a..5cb02eb55e3 100644 --- a/yql/essentials/tests/sql/minirun/part1/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part1/canondata/result.json @@ -559,6 +559,20 @@ "uri": "https://{canondata_backend}/1942525/d23808e75f3586fe0f94591ae8678878a50edd66/resource.tar.gz#test.test_expr-as_variant_enum-default.txt-Results_/results.txt" } ], + "test.test[expr-concat_many--Debug]": [ + { + "checksum": "7a4c31257f61a51d15236580c89e6122", + "size": 581, + "uri": "https://{canondata_backend}/212715/6d9ed45819070f2bb9ee6b84a7d7ce8c50efaa7c/resource.tar.gz#test.test_expr-concat_many--Debug_/opt.yql" + } + ], + "test.test[expr-concat_many--Results]": [ + { + "checksum": "66b26bd358dcff46379f0033958b0d96", + "size": 2515, + "uri": "https://{canondata_backend}/1784117/a4a7ed2a9293d986efe2868cc4011856a43b2563/resource.tar.gz#test.test_expr-concat_many--Results_/results.txt" + } + ], "test.test[expr-expr_named_yql_lambda_quotes-default.txt-Debug]": [ { "checksum": "9da8a64a8f8feb07730d84fe360e3096", diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index c30e4b85e42..17df9327bdd 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -2918,6 +2918,13 @@ "uri": "https://{canondata_backend}/1937150/ec0019724df75083b0e89cab22f57e10ef36744e/resource.tar.gz#test_sql2yql.test_expr-common_type_for_resource_and_data_/sql.yql" } ], + "test_sql2yql.test[expr-concat_many]": [ + { + "checksum": "d9e698da5ede3c7520b7f0d122ab8566", + "size": 1935, + "uri": "https://{canondata_backend}/1936842/92e2458bff4de81de9f2c44dae4a80b1a84e7c0c/resource.tar.gz#test_sql2yql.test_expr-concat_many_/sql.yql" + } + ], "test_sql2yql.test[expr-container_comp]": [ { "checksum": "6aaee0040ccee9661847c7edf6f681bb", @@ -10918,6 +10925,11 @@ "uri": "file://test_sql_format.test_expr-common_type_for_resource_and_data_/formatted.sql" } ], + "test_sql_format.test[expr-concat_many]": [ + { + "uri": "file://test_sql_format.test_expr-concat_many_/formatted.sql" + } + ], "test_sql_format.test[expr-container_comp]": [ { "uri": "file://test_sql_format.test_expr-container_comp_/formatted.sql" diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-concat_many_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-concat_many_/formatted.sql new file mode 100644 index 00000000000..0b837768499 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-concat_many_/formatted.sql @@ -0,0 +1,8 @@ +SELECT + concat(NULL), + concat('aa'), + concat('aa', 'bb', 'cc'), + concat(just('aa'u), just('bb')), + concat('aa', 'bb'u), + concat('aa'u, just('bb'u)) +; diff --git a/yql/essentials/tests/sql/suites/expr/concat_many.cfg b/yql/essentials/tests/sql/suites/expr/concat_many.cfg new file mode 100644 index 00000000000..7e882f1da85 --- /dev/null +++ b/yql/essentials/tests/sql/suites/expr/concat_many.cfg @@ -0,0 +1,2 @@ +langver 2025.04 + diff --git a/yql/essentials/tests/sql/suites/expr/concat_many.yql b/yql/essentials/tests/sql/suites/expr/concat_many.yql new file mode 100644 index 00000000000..58ef1cc243a --- /dev/null +++ b/yql/essentials/tests/sql/suites/expr/concat_many.yql @@ -0,0 +1,5 @@ +select concat(null),concat('aa'), + concat('aa','bb','cc'),concat(just('aa'u),just('bb')), + concat('aa','bb'u),concat('aa'u,just('bb'u)); + + |