aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authornsofya <nsofya@yandex-team.com>2023-06-21 10:39:12 +0300
committernsofya <nsofya@yandex-team.com>2023-06-21 10:39:12 +0300
commite24a4fb8bb193bb7132c660bfe6dd24a95ee5bfc (patch)
tree3b7aff98082ec5dfeb1f7ad69a03d2ecc3bcf30e
parentf877fbb30a7febe8d500083401821e75d0f6fd5e (diff)
downloadydb-e24a4fb8bb193bb7132c660bfe6dd24a95ee5bfc.tar.gz
Test json_exists
-rw-r--r--ydb/core/tx/columnshard/engines/ut/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/core/tx/columnshard/engines/ut/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/core/tx/columnshard/engines/ut/ya.make2
-rw-r--r--ydb/core/tx/columnshard/engines/ut_program.cpp102
-rw-r--r--ydb/core/tx/program/registry.cpp3
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp23
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt2
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt2
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt2
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt2
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/request.cpp8
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/ya.make2
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_block_impl.cpp5
-rw-r--r--ydb/library/yql/minikql/mkql_type_builder.cpp1
-rw-r--r--ydb/library/yql/public/udf/arrow/block_builder.h1
-rw-r--r--ydb/library/yql/public/udf/arrow/block_reader.h1
18 files changed, 146 insertions, 14 deletions
diff --git a/ydb/core/tx/columnshard/engines/ut/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/engines/ut/CMakeLists.darwin-x86_64.txt
index d59807c727..5810287df7 100644
--- a/ydb/core/tx/columnshard/engines/ut/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/tx/columnshard/engines/ut/CMakeLists.darwin-x86_64.txt
@@ -28,6 +28,7 @@ target_link_libraries(ydb-core-tx-columnshard-engines-ut PUBLIC
yql-sql-pg_dummy
core-arrow_kernels-request
core-testlib-default
+ json2_udf
)
target_link_options(ydb-core-tx-columnshard-engines-ut PRIVATE
-Wl,-platform_version,macos,11.0,11.0
diff --git a/ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-aarch64.txt
index 50f51e0ad2..3637b62e83 100644
--- a/ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-aarch64.txt
@@ -28,6 +28,7 @@ target_link_libraries(ydb-core-tx-columnshard-engines-ut PUBLIC
yql-sql-pg_dummy
core-arrow_kernels-request
core-testlib-default
+ json2_udf
)
target_link_options(ydb-core-tx-columnshard-engines-ut PRIVATE
-ldl
diff --git a/ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-x86_64.txt
index 47a14cdffd..62b2cbc742 100644
--- a/ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/tx/columnshard/engines/ut/CMakeLists.linux-x86_64.txt
@@ -29,6 +29,7 @@ target_link_libraries(ydb-core-tx-columnshard-engines-ut PUBLIC
yql-sql-pg_dummy
core-arrow_kernels-request
core-testlib-default
+ json2_udf
)
target_link_options(ydb-core-tx-columnshard-engines-ut PRIVATE
-ldl
diff --git a/ydb/core/tx/columnshard/engines/ut/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/engines/ut/CMakeLists.windows-x86_64.txt
index c2cce5aea1..fc4d07a486 100644
--- a/ydb/core/tx/columnshard/engines/ut/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/tx/columnshard/engines/ut/CMakeLists.windows-x86_64.txt
@@ -28,6 +28,7 @@ target_link_libraries(ydb-core-tx-columnshard-engines-ut PUBLIC
yql-sql-pg_dummy
core-arrow_kernels-request
core-testlib-default
+ json2_udf
)
target_sources(ydb-core-tx-columnshard-engines-ut PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/engines/ut_insert_table.cpp
diff --git a/ydb/core/tx/columnshard/engines/ut/ya.make b/ydb/core/tx/columnshard/engines/ut/ya.make
index 0d7ee35e64..04f31654b9 100644
--- a/ydb/core/tx/columnshard/engines/ut/ya.make
+++ b/ydb/core/tx/columnshard/engines/ut/ya.make
@@ -23,6 +23,8 @@ PEERDIR(
ydb/library/yql/sql/pg_dummy
ydb/library/yql/core/arrow_kernels/request
ydb/core/testlib/default
+
+ ydb/library/yql/udfs/common/json2
)
YQL_LAST_ABI_VERSION()
diff --git a/ydb/core/tx/columnshard/engines/ut_program.cpp b/ydb/core/tx/columnshard/engines/ut_program.cpp
index f5f8561907..f6be9dcdc6 100644
--- a/ydb/core/tx/columnshard/engines/ut_program.cpp
+++ b/ydb/core/tx/columnshard/engines/ut_program.cpp
@@ -4,6 +4,7 @@
#include <ydb/core/tx/columnshard/columnshard__index_scan.h>
#include <ydb/core/tx/columnshard/columnshard_ut_common.h>
#include <ydb/core/tx/program/program.h>
+#include <ydb/core/formats/arrow/converter.h>
#include <ydb/library/yql/core/arrow_kernels/request/request.h>
#include <ydb/library/yql/core/arrow_kernels/registry/registry.h>
@@ -37,12 +38,13 @@ Y_UNIT_TEST_SUITE(TestProgram) {
class TKernelsWrapper {
TIntrusivePtr<NMiniKQL::IFunctionRegistry> Reg;
- NYql::TKernelRequestBuilder ReqBuilder;
+ std::unique_ptr<NYql::TKernelRequestBuilder> ReqBuilder;
public:
- TKernelsWrapper()
- : Reg(CreateFunctionRegistry(NMiniKQL::CreateBuiltinRegistry()))
- , ReqBuilder((*Reg)) {
-
+ TKernelsWrapper() {
+ auto reg = CreateFunctionRegistry(NMiniKQL::CreateBuiltinRegistry())->Clone();
+ NMiniKQL::FillStaticModules(*reg);
+ Reg.Reset(reg.Release());
+ ReqBuilder = std::make_unique<NYql::TKernelRequestBuilder>(*Reg);
}
ui32 Add(NYql::TKernelRequestBuilder::EBinaryOp operation) {
@@ -51,21 +53,21 @@ Y_UNIT_TEST_SUITE(TestProgram) {
{
NYql::TExprContext ctx;
auto blockInt32Type = ctx.template MakeType<NYql::TBlockExprType>(ctx.template MakeType<NYql::TDataExprType>(NYql::EDataSlot::Int32));
- return ReqBuilder.AddBinaryOp(NYql::TKernelRequestBuilder::EBinaryOp::Add, blockInt32Type, blockInt32Type, blockInt32Type);
+ return ReqBuilder->AddBinaryOp(NYql::TKernelRequestBuilder::EBinaryOp::Add, blockInt32Type, blockInt32Type, blockInt32Type);
}
case NYql::TKernelRequestBuilder::EBinaryOp::StartsWith:
{
NYql::TExprContext ctx;
auto blockStringType = ctx.template MakeType<NYql::TBlockExprType>(ctx.template MakeType<NYql::TDataExprType>(NYql::EDataSlot::Utf8));
auto blockBoolType = ctx.template MakeType<NYql::TBlockExprType>(ctx.template MakeType<NYql::TDataExprType>(NYql::EDataSlot::Bool));
- return ReqBuilder.AddBinaryOp(NYql::TKernelRequestBuilder::EBinaryOp::StartsWith, blockStringType, blockStringType, blockBoolType);
+ return ReqBuilder->AddBinaryOp(NYql::TKernelRequestBuilder::EBinaryOp::StartsWith, blockStringType, blockStringType, blockBoolType);
}
case NYql::TKernelRequestBuilder::EBinaryOp::StringContains:
{
NYql::TExprContext ctx;
auto blockStringType = ctx.template MakeType<NYql::TBlockExprType>(ctx.template MakeType<NYql::TDataExprType>(NYql::EDataSlot::String));
auto blockBoolType = ctx.template MakeType<NYql::TBlockExprType>(ctx.template MakeType<NYql::TDataExprType>(NYql::EDataSlot::Bool));
- return ReqBuilder.AddBinaryOp(NYql::TKernelRequestBuilder::EBinaryOp::StringContains, blockStringType, blockStringType, blockBoolType);
+ return ReqBuilder->AddBinaryOp(NYql::TKernelRequestBuilder::EBinaryOp::StringContains, blockStringType, blockStringType, blockBoolType);
}
default:
Y_FAIL("Not implemented");
@@ -73,8 +75,21 @@ Y_UNIT_TEST_SUITE(TestProgram) {
}
}
+ ui32 AddJsonExists(bool isBinaryType = true) {
+ NYql::TExprContext ctx;
+ auto blockOptJsonType = ctx.template MakeType<NYql::TBlockExprType>(
+ ctx.template MakeType<NYql::TOptionalExprType>(
+ ctx.template MakeType<NYql::TDataExprType>(isBinaryType ? NYql::EDataSlot::JsonDocument : NYql::EDataSlot::Json)));
+ auto scalarStringType = ctx.template MakeType<NYql::TScalarExprType>(ctx.template MakeType<NYql::TDataExprType>(NYql::EDataSlot::Utf8));
+ auto blockBoolType = ctx.template MakeType<NYql::TBlockExprType>(
+ ctx.template MakeType<NYql::TOptionalExprType>(
+ ctx.template MakeType<NYql::TDataExprType>(NYql::EDataSlot::Bool)));
+
+ return ReqBuilder->JsonExists(blockOptJsonType, scalarStringType, blockBoolType);
+ }
+
TString Serialize() {
- return ReqBuilder.Serialize();
+ return ReqBuilder->Serialize();
}
};
@@ -216,6 +231,75 @@ Y_UNIT_TEST_SUITE(TestProgram) {
}
}
+ void JsonExistsImpl(bool isBinaryType) {
+ TIndexInfo indexInfo = BuildTableInfo(testColumns, testKey);
+ TIndexColumnResolver columnResolver(indexInfo);
+
+ NKikimrSSA::TProgram programProto;
+ {
+ auto* command = programProto.AddCommand();
+ auto* constantProto = command->MutableAssign()->MutableConstant();
+ constantProto->SetText("$.key");
+ command->MutableAssign()->MutableColumn()->SetName("json_path");
+ }
+ {
+ auto* command = programProto.AddCommand();
+ auto* functionProto = command->MutableAssign()->MutableFunction();
+ functionProto->SetFunctionType(NKikimrSSA::TProgram::EFunctionType::TProgram_EFunctionType_YQL_KERNEL);
+ functionProto->SetKernelIdx(0);
+ functionProto->AddArguments()->SetName("json_data");
+ functionProto->AddArguments()->SetName("json_path");
+ functionProto->SetId(NKikimrSSA::TProgram::TAssignment::EFunction::TProgram_TAssignment_EFunction_FUNC_STR_LENGTH);
+ }
+ {
+ auto* command = programProto.AddCommand();
+ auto* prjectionProto = command->MutableProjection();
+ auto* column = prjectionProto->AddColumns();
+ column->SetName("0");
+ }
+
+ TKernelsWrapper kernels;
+ kernels.AddJsonExists(isBinaryType);
+ const auto programSerialized = SerializeProgram(programProto, kernels.Serialize());
+
+ TProgramContainer program;
+ TString errors;
+ UNIT_ASSERT_C(program.Init(columnResolver, NKikimrSchemeOp::EOlapProgramType::OLAP_PROGRAM_SSA_PROGRAM_WITH_PARAMETERS, programSerialized, errors), errors);
+
+ TTableUpdatesBuilder updates(NArrow::MakeArrowSchema({{"json_data", TTypeInfo(isBinaryType ? NTypeIds::JsonDocument : NTypeIds::Utf8) }}));
+ NJson::TJsonValue testJson;
+ testJson["key"] = "value";
+ updates.AddRow().Add<std::string>(testJson.GetStringRobust());
+ updates.AddRow().Add<std::string>(NJson::TJsonValue(NJson::JSON_ARRAY).GetStringRobust());
+
+ auto batch = updates.BuildArrow();
+ Cerr << batch->ToString() << Endl;
+
+ if (isBinaryType) {
+ THashMap<TString, NScheme::TTypeInfo> cc;
+ cc["json_data"] = TTypeInfo(NTypeIds::JsonDocument);
+ batch = NArrow::ConvertColumns(batch, cc);
+ Cerr << batch->ToString() << Endl;
+ }
+ auto res = program.ApplyProgram(batch);
+ UNIT_ASSERT_C(res.ok(), res.ToString());
+
+ TTableUpdatesBuilder result(NArrow::MakeArrowSchema( { std::make_pair("0", TTypeInfo(NTypeIds::Uint8)) }));
+ result.AddRow().Add<ui8>(1);
+ result.AddRow().Add<ui8>(0);
+
+ auto expected = result.BuildArrow();
+ UNIT_ASSERT_VALUES_EQUAL(batch->ToString(), expected->ToString());
+ }
+
+ Y_UNIT_TEST(JsonExists) {
+ JsonExistsImpl(false);
+ }
+
+ Y_UNIT_TEST(JsonExistsBinary) {
+ JsonExistsImpl(true);
+ }
+
Y_UNIT_TEST(SimpleFunction) {
TIndexInfo indexInfo = BuildTableInfo(testColumns, testKey);;
TIndexColumnResolver columnResolver(indexInfo);
diff --git a/ydb/core/tx/program/registry.cpp b/ydb/core/tx/program/registry.cpp
index d7e31ff8c4..159dbc456f 100644
--- a/ydb/core/tx/program/registry.cpp
+++ b/ydb/core/tx/program/registry.cpp
@@ -7,7 +7,8 @@
namespace NKikimr::NOlap {
bool TKernelsRegistry::Parse(const TString& serialized) {
- auto functionRegistry = NMiniKQL::CreateFunctionRegistry(NMiniKQL::CreateBuiltinRegistry());
+ auto functionRegistry = NMiniKQL::CreateFunctionRegistry(NMiniKQL::CreateBuiltinRegistry())->Clone();
+ NMiniKQL::FillStaticModules(*functionRegistry);
auto nodeFactory = NMiniKQL::GetBuiltinFactory();
auto kernels = NYql::LoadKernels(serialized, *functionRegistry, nodeFactory);
Kernels.swap(kernels);
diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp b/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp
index 026878ecb9..797343859a 100644
--- a/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp
+++ b/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp
@@ -139,5 +139,28 @@ Y_UNIT_TEST_SUITE(TKernelRegistryTest) {
ctx.template MakeType<TDataExprType>(EDataSlot::Bool)));
return b.JsonExists(blockJsonType, scalarUtf8Type, blockOptBoolType);
});
+
+ TestOne([](auto& b,auto& ctx) {
+ auto blockOptJsonType = ctx.template MakeType<TBlockExprType>(
+ ctx.template MakeType<TOptionalExprType>(
+ ctx.template MakeType<TDataExprType>(EDataSlot::JsonDocument)));
+ auto scalarUtf8Type = ctx.template MakeType<TScalarExprType>(
+ ctx.template MakeType<TDataExprType>(EDataSlot::Utf8));
+ auto blockOptBoolType = ctx.template MakeType<TBlockExprType>(
+ ctx.template MakeType<TOptionalExprType>(
+ ctx.template MakeType<TDataExprType>(EDataSlot::Bool)));
+ return b.JsonExists(blockOptJsonType, scalarUtf8Type, blockOptBoolType);
+ });
+
+ TestOne([](auto& b,auto& ctx) {
+ auto blockJsonType = ctx.template MakeType<TBlockExprType>(
+ ctx.template MakeType<TDataExprType>(EDataSlot::JsonDocument));
+ auto scalarUtf8Type = ctx.template MakeType<TScalarExprType>(
+ ctx.template MakeType<TDataExprType>(EDataSlot::Utf8));
+ auto blockOptBoolType = ctx.template MakeType<TBlockExprType>(
+ ctx.template MakeType<TOptionalExprType>(
+ ctx.template MakeType<TDataExprType>(EDataSlot::Bool)));
+ return b.JsonExists(blockJsonType, scalarUtf8Type, blockOptBoolType);
+ });
}
}
diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt
index 5877718c2a..ec3f3c6f6f 100644
--- a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt
@@ -17,6 +17,8 @@ target_link_libraries(core-arrow_kernels-request PUBLIC
library-yql-ast
library-yql-minikql
providers-common-mkql
+ library-yql-core
+ library-yql-sql
)
target_sources(core-arrow_kernels-request PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp
diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt
index a38846dc53..9627c10463 100644
--- a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt
@@ -18,6 +18,8 @@ target_link_libraries(core-arrow_kernels-request PUBLIC
library-yql-ast
library-yql-minikql
providers-common-mkql
+ library-yql-core
+ library-yql-sql
)
target_sources(core-arrow_kernels-request PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp
diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt
index a38846dc53..9627c10463 100644
--- a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt
@@ -18,6 +18,8 @@ target_link_libraries(core-arrow_kernels-request PUBLIC
library-yql-ast
library-yql-minikql
providers-common-mkql
+ library-yql-core
+ library-yql-sql
)
target_sources(core-arrow_kernels-request PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp
diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt
index 5877718c2a..ec3f3c6f6f 100644
--- a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt
@@ -17,6 +17,8 @@ target_link_libraries(core-arrow_kernels-request PUBLIC
library-yql-ast
library-yql-minikql
providers-common-mkql
+ library-yql-core
+ library-yql-sql
)
target_sources(core-arrow_kernels-request PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp
diff --git a/ydb/library/yql/core/arrow_kernels/request/request.cpp b/ydb/library/yql/core/arrow_kernels/request/request.cpp
index eb5c1c5c4e..8cd544e80f 100644
--- a/ydb/library/yql/core/arrow_kernels/request/request.cpp
+++ b/ydb/library/yql/core/arrow_kernels/request/request.cpp
@@ -2,6 +2,7 @@
#include <ydb/library/yql/providers/common/mkql/yql_type_mkql.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_node_serialization.h>
+#include <ydb/library/yql/core/yql_opt_utils.h>
namespace NYql {
@@ -91,7 +92,10 @@ ui32 TKernelRequestBuilder::Udf(const TString& name, bool isPolymorphic, const s
ui32 TKernelRequestBuilder::JsonExists(const TTypeAnnotationNode* arg1Type, const TTypeAnnotationNode* arg2Type, const TTypeAnnotationNode* retType) {
TGuard<NKikimr::NMiniKQL::TScopedAlloc> allocGuard(Alloc_);
- auto exists = Pb_.Udf("Json2.SqlExists");
+
+ bool isBinaryJson = (RemoveOptionalType(arg1Type->Cast<TBlockExprType>()->GetItemType())->Cast<TDataExprType>()->GetSlot() == EDataSlot::JsonDocument);
+
+ auto exists = Pb_.Udf(isBinaryJson ? "Json2.JsonDocumentSqlExists" : "Json2.SqlExists");
auto parse = Pb_.Udf("Json2.Parse");
auto compilePath = Pb_.Udf("Json2.CompilePath");
auto outType = MakeType(retType);
@@ -100,7 +104,7 @@ ui32 TKernelRequestBuilder::JsonExists(const TTypeAnnotationNode* arg1Type, cons
auto scalarApply = Pb_.ScalarApply({arg1, arg2}, [&](const auto& args) {
auto json = args[0];
auto processJson = [&](auto unpacked) {
- auto input = Pb_.NewOptional(Pb_.Apply(parse, { unpacked }));
+ auto input = Pb_.NewOptional(isBinaryJson ? unpacked : Pb_.Apply(parse, { unpacked }));
auto path = Pb_.Apply(compilePath, { args[1] });
auto dictType = Pb_.NewDictType(Pb_.NewDataType(NUdf::EDataSlot::Utf8), Pb_.NewResourceType("JsonNode"), false);
return Pb_.Apply(exists, { input, path, Pb_.NewDict(dictType, {}), Pb_.NewOptional(Pb_.NewDataLiteral(false)) });
diff --git a/ydb/library/yql/core/arrow_kernels/request/ya.make b/ydb/library/yql/core/arrow_kernels/request/ya.make
index 03d179915a..45b85ecf06 100644
--- a/ydb/library/yql/core/arrow_kernels/request/ya.make
+++ b/ydb/library/yql/core/arrow_kernels/request/ya.make
@@ -8,6 +8,8 @@ PEERDIR(
ydb/library/yql/ast
ydb/library/yql/minikql
ydb/library/yql/providers/common/mkql
+ ydb/library/yql/core
+ ydb/library/yql/sql
)
YQL_LAST_ABI_VERSION()
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_block_impl.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_block_impl.cpp
index 70ba311261..ed7637addf 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_block_impl.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_block_impl.cpp
@@ -69,11 +69,12 @@ arrow::Datum ConvertScalar(TType* type, const NUdf::TUnboxedValuePod& value, arr
case NUdf::EDataSlot::String:
case NUdf::EDataSlot::Utf8:
case NUdf::EDataSlot::Yson:
- case NUdf::EDataSlot::Json: {
+ case NUdf::EDataSlot::Json:
+ case NUdf::EDataSlot::JsonDocument: {
const auto& str = value.AsStringRef();
std::shared_ptr<arrow::Buffer> buffer(ARROW_RESULT(arrow::AllocateBuffer(str.Size(), &pool)));
std::memcpy(buffer->mutable_data(), str.Data(), str.Size());
- auto type = (slot == NUdf::EDataSlot::String || slot == NUdf::EDataSlot::Yson) ? arrow::binary() : arrow::utf8();
+ auto type = (slot == NUdf::EDataSlot::String || slot == NUdf::EDataSlot::Yson || slot == NUdf::EDataSlot::JsonDocument) ? arrow::binary() : arrow::utf8();
std::shared_ptr<arrow::Scalar> scalar = std::make_shared<arrow::BinaryScalar>(buffer, type);
return arrow::Datum(scalar);
}
diff --git a/ydb/library/yql/minikql/mkql_type_builder.cpp b/ydb/library/yql/minikql/mkql_type_builder.cpp
index 45e13342cc..c6dd847f91 100644
--- a/ydb/library/yql/minikql/mkql_type_builder.cpp
+++ b/ydb/library/yql/minikql/mkql_type_builder.cpp
@@ -1478,6 +1478,7 @@ bool ConvertArrowType(NUdf::EDataSlot slot, std::shared_ptr<arrow::DataType>& ty
return true;
case NUdf::EDataSlot::String:
case NUdf::EDataSlot::Yson:
+ case NUdf::EDataSlot::JsonDocument:
type = arrow::binary();
return true;
case NUdf::EDataSlot::Utf8:
diff --git a/ydb/library/yql/public/udf/arrow/block_builder.h b/ydb/library/yql/public/udf/arrow/block_builder.h
index bf6e54bc89..a9d06df0c3 100644
--- a/ydb/library/yql/public/udf/arrow/block_builder.h
+++ b/ydb/library/yql/public/udf/arrow/block_builder.h
@@ -1121,6 +1121,7 @@ inline std::unique_ptr<TArrayBuilderBase> MakeArrayBuilderImpl(
return std::make_unique<TFixedSizeArrayBuilder<double, Nullable>>(typeInfoHelper, type, pool, maxLen);
case NUdf::EDataSlot::String:
case NUdf::EDataSlot::Yson:
+ case NUdf::EDataSlot::JsonDocument:
return std::make_unique<TStringArrayBuilder<arrow::BinaryType, Nullable>>(typeInfoHelper, type, pool, maxLen);
case NUdf::EDataSlot::Utf8:
case NUdf::EDataSlot::Json:
diff --git a/ydb/library/yql/public/udf/arrow/block_reader.h b/ydb/library/yql/public/udf/arrow/block_reader.h
index 5d0d1fa78e..a451486854 100644
--- a/ydb/library/yql/public/udf/arrow/block_reader.h
+++ b/ydb/library/yql/public/udf/arrow/block_reader.h
@@ -385,6 +385,7 @@ std::unique_ptr<typename TTraits::TResult> MakeBlockReaderImpl(const ITypeInfoHe
return MakeFixedSizeBlockReaderImpl<TTraits, double>(isOptional);
case NUdf::EDataSlot::String:
case NUdf::EDataSlot::Yson:
+ case NUdf::EDataSlot::JsonDocument:
return MakeStringBlockReaderImpl<TTraits, arrow::BinaryType>(isOptional);
case NUdf::EDataSlot::Utf8:
case NUdf::EDataSlot::Json: