diff options
author | Oleg Evseev <git@x3n.me> | 2022-02-10 16:52:14 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:52:14 +0300 |
commit | 440c9dae6ba32011dee69689f9d005290e8a7fa7 (patch) | |
tree | ab7fbbf3253d4c0e2793218f09378908beb025fb | |
parent | e6d600f0dc52ccbce6fb5a3c49d48432b7110359 (diff) | |
download | ydb-440c9dae6ba32011dee69689f9d005290e8a7fa7.tar.gz |
Restoring authorship annotation for Oleg Evseev <git@x3n.me>. Commit 2 of 2.
32 files changed, 1706 insertions, 1706 deletions
diff --git a/ydb/core/engine/mkql_proto_ut.cpp b/ydb/core/engine/mkql_proto_ut.cpp index f785ae733a5..c6e00e947c5 100644 --- a/ydb/core/engine/mkql_proto_ut.cpp +++ b/ydb/core/engine/mkql_proto_ut.cpp @@ -1,7 +1,7 @@ #include "mkql_proto.h" - + #include <ydb/library/mkql_proto/ut/helpers/helpers.h> - + #include <ydb/library/yql/minikql/mkql_program_builder.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> @@ -16,7 +16,7 @@ using namespace std::string_view_literals; namespace NKikimr { namespace NMiniKQL { -Y_UNIT_TEST_SUITE(TMiniKQLProtoTestYdb) { +Y_UNIT_TEST_SUITE(TMiniKQLProtoTestYdb) { Y_UNIT_TEST(TestExportVoidTypeYdb) { TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { auto pgmReturn = pgmBuilder.NewVoid(); diff --git a/ydb/core/kqp/common/kqp_transform.h b/ydb/core/kqp/common/kqp_transform.h index 36cae9077df..457c4248011 100644 --- a/ydb/core/kqp/common/kqp_transform.h +++ b/ydb/core/kqp/common/kqp_transform.h @@ -43,7 +43,7 @@ public: TKey GetKey() const { return std::make_tuple(GetLockId(), GetDataShard(), GetSchemeShard(), GetPathId()); } NKikimrMiniKQL::TValue GetValue() const { return LockValue; } - NYql::NDq::TMkqlValueRef GetValueRef(const NKikimrMiniKQL::TType& type) const { return NYql::NDq::TMkqlValueRef(type, LockValue); } + NYql::NDq::TMkqlValueRef GetValueRef(const NKikimrMiniKQL::TType& type) const { return NYql::NDq::TMkqlValueRef(type, LockValue); } bool Invalidated(const TKqpTxLock& newLock) const { YQL_ENSURE(GetKey() == newLock.GetKey()); diff --git a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp index 6e15e22225f..3148a2ec59b 100644 --- a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp +++ b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp @@ -71,7 +71,7 @@ TKqpParamsRefMap GetParamRefsMap(const TKikimrParamsMap& paramsMap) { TKqpParamsRefMap refsMap; for (auto& pair : paramsMap) { - refsMap.emplace(std::make_pair(pair.first, NDq::TMkqlValueRef(pair.second))); + refsMap.emplace(std::make_pair(pair.first, NDq::TMkqlValueRef(pair.second))); } return refsMap; diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp index aa3bb5610d6..47489a9bd54 100644 --- a/ydb/core/kqp/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/kqp_ic_gateway.cpp @@ -603,7 +603,7 @@ private: structBuilder.Reserve(paramsMap.Values.size()); for (auto& pair : paramsMap.Values) { const TString& name = pair.first; - const NYql::NDq::TMkqlValueRef& param = pair.second; + const NYql::NDq::TMkqlValueRef& param = pair.second; auto valueNode = NMiniKQL::ImportValueFromProto(param.GetType(), param.GetValue(), env); structBuilder.Add(name, valueNode); diff --git a/ydb/core/kqp/prepare/kqp_query_exec.cpp b/ydb/core/kqp/prepare/kqp_query_exec.cpp index 4f71aee221d..bc2d4e94ce9 100644 --- a/ydb/core/kqp/prepare/kqp_query_exec.cpp +++ b/ydb/core/kqp/prepare/kqp_query_exec.cpp @@ -53,7 +53,7 @@ NKikimrKqp::TParameterBinding GetParameterBinding(TCoParameter param) { return binding; } -NDq::TMkqlValueRef GetParamFromResult(const NKikimrKqp::TParameterBinding& binding, +NDq::TMkqlValueRef GetParamFromResult(const NKikimrKqp::TParameterBinding& binding, const TKqlTransformContext& transformCtx) { YQL_ENSURE(binding.HasMkqlIndex()); @@ -67,7 +67,7 @@ NDq::TMkqlValueRef GetParamFromResult(const NKikimrKqp::TParameterBinding& bindi YQL_ENSURE(type); YQL_ENSURE(value); - return NDq::TMkqlValueRef(*type, *value); + return NDq::TMkqlValueRef(*type, *value); } NKikimrMiniKQL::TParams BuildParamFromResult(const NKikimrKqp::TParameterBinding& binding, @@ -580,19 +580,19 @@ TKqpParamsMap BuildParamsMap(const TVector<NKikimrKqp::TParameterBinding>& bindi for (auto& binding : bindings) { auto name = binding.GetName(); - TMaybe<NDq::TMkqlValueRef> paramRef; + TMaybe<NDq::TMkqlValueRef> paramRef; if (binding.GetName() == LocksAcquireParamName) { auto& param = txState->Tx().ParamsState->Values[LocksAcquireParamName]; param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<bool>::Id); param.MutableValue()->SetBool(acquireLocks); - paramRef = NDq::TMkqlValueRef(param); + paramRef = NDq::TMkqlValueRef(param); } else if (binding.GetName() == LocksTxIdParamName) { auto& param = txState->Tx().ParamsState->Values[LocksTxIdParamName]; param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); param.MutableValue()->SetUint64(txState->Tx().Locks.GetLockTxId()); - paramRef = NDq::TMkqlValueRef(param); + paramRef = NDq::TMkqlValueRef(param); } else if (binding.GetName() == ReadTargetParamName) { YQL_ENSURE(txState->Tx().EffectiveIsolationLevel); @@ -612,7 +612,7 @@ TKqpParamsMap BuildParamsMap(const TVector<NKikimrKqp::TParameterBinding>& bindi param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui32>::Id); param.MutableValue()->SetUint32(readTarget); - paramRef = NDq::TMkqlValueRef(param); + paramRef = NDq::TMkqlValueRef(param); } else if (binding.GetName() == NowParamName) { auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); @@ -665,12 +665,12 @@ TKqpParamsMap BuildParamsMap(const TVector<NKikimrKqp::TParameterBinding>& bindi } else { auto clientParam = transformCtx->QueryCtx->Parameters.FindPtr(binding.GetName()); if (clientParam) { - paramRef = NDq::TMkqlValueRef(*clientParam); + paramRef = NDq::TMkqlValueRef(*clientParam); } else { auto paramValue = txState->Tx().ParamsState->Values.FindPtr(binding.GetName()); YQL_ENSURE(paramValue, "Parameter not found: " << binding.GetName()); - paramRef = NDq::TMkqlValueRef(*paramValue); + paramRef = NDq::TMkqlValueRef(*paramValue); } } diff --git a/ydb/core/kqp/runtime/kqp_compute.h b/ydb/core/kqp/runtime/kqp_compute.h index 80ee11d22df..c55a4e2a21b 100644 --- a/ydb/core/kqp/runtime/kqp_compute.h +++ b/ydb/core/kqp/runtime/kqp_compute.h @@ -10,7 +10,7 @@ namespace NKikimr { namespace NMiniKQL { -class TKqpComputeContextBase : public NYql::NDq::TDqComputeContextBase { +class TKqpComputeContextBase : public NYql::NDq::TDqComputeContextBase { public: struct TColumn { NTable::TTag Tag; diff --git a/ydb/core/kqp/runtime/kqp_tasks_runner.cpp b/ydb/core/kqp/runtime/kqp_tasks_runner.cpp index a7499292214..70b1038e6cd 100644 --- a/ydb/core/kqp/runtime/kqp_tasks_runner.cpp +++ b/ydb/core/kqp/runtime/kqp_tasks_runner.cpp @@ -19,7 +19,7 @@ namespace NKqp { using namespace NMiniKQL; using namespace NYql; -using namespace NDq; +using namespace NDq; IDqOutputConsumer::TPtr KqpBuildOutputConsumer(const NDqProto::TTaskOutput& outputDesc, const TType* type, NUdf::IApplyContext* applyCtx, const TTypeEnvironment& typeEnv, TVector<IDqOutput::TPtr>&& outputs) diff --git a/ydb/core/kqp/runtime/kqp_transport.cpp b/ydb/core/kqp/runtime/kqp_transport.cpp index f24ea8e03ce..8c1b3262d0d 100644 --- a/ydb/core/kqp/runtime/kqp_transport.cpp +++ b/ydb/core/kqp/runtime/kqp_transport.cpp @@ -244,7 +244,7 @@ void TKqpProtoBuilder::BuildStream(TUnboxedValueVector& rows, const NKikimrMiniK } } -Ydb::ResultSet TKqpProtoBuilder::BuildYdbResultSet(const TVector<NDqProto::TData>& data, +Ydb::ResultSet TKqpProtoBuilder::BuildYdbResultSet(const TVector<NDqProto::TData>& data, const NKikimrMiniKQL::TType& srcRowType, const NKikimrMiniKQL::TType* dstRowType) { YQL_ENSURE(srcRowType.GetKind() == NKikimrMiniKQL::Struct); diff --git a/ydb/core/kqp/runtime/kqp_transport.h b/ydb/core/kqp/runtime/kqp_transport.h index 46de0b3c222..9fa6adec139 100644 --- a/ydb/core/kqp/runtime/kqp_transport.h +++ b/ydb/core/kqp/runtime/kqp_transport.h @@ -34,7 +34,7 @@ public: void BuildStream(NMiniKQL::TUnboxedValueVector& rows, const NKikimrMiniKQL::TType& srcRowType, const NKikimrMiniKQL::TType* dstRowType, NKikimrMiniKQL::TResult* result); - Ydb::ResultSet BuildYdbResultSet(const TVector<NYql::NDqProto::TData>& data, + Ydb::ResultSet BuildYdbResultSet(const TVector<NYql::NDqProto::TData>& data, const NKikimrMiniKQL::TType& srcRowType, const NKikimrMiniKQL::TType* dstRowType = nullptr); private: diff --git a/ydb/core/tx/datashard/datashard__engine_host.h b/ydb/core/tx/datashard/datashard__engine_host.h index 17e0a20df54..8e3064c83d8 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.h +++ b/ydb/core/tx/datashard/datashard__engine_host.h @@ -111,7 +111,7 @@ private: TEngineHostCounters EngineHostCounters; ui64 LockTxId; TString TraceMessage; - NYql::NDq::TLogFunc KqpLogFunc; + NYql::NDq::TLogFunc KqpLogFunc; THolder<NUdf::IApplyContext> KqpApplyCtx; THolder<NMiniKQL::TKqpDatashardComputeContext> ComputeCtx; THolder<NMiniKQL::TScopedAlloc> KqpAlloc; diff --git a/ydb/core/tx/datashard/datashard_active_transaction.h b/ydb/core/tx/datashard/datashard_active_transaction.h index acd7e421e80..43abd525e15 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.h +++ b/ydb/core/tx/datashard/datashard_active_transaction.h @@ -9,7 +9,7 @@ #include <ydb/core/tablet_flat/flat_cxx_database.h> #include <ydb/library/yql/public/issue/yql_issue.h> - + namespace NKikimr { class TBalanceCoverageBuilder; diff --git a/ydb/core/tx/datashard/datashard_kqp.cpp b/ydb/core/tx/datashard/datashard_kqp.cpp index 5e566339e84..a262c98de5a 100644 --- a/ydb/core/tx/datashard/datashard_kqp.cpp +++ b/ydb/core/tx/datashard/datashard_kqp.cpp @@ -10,7 +10,7 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> #include <ydb/library/yql/dq/runtime/dq_transport.h> - + #include <util/generic/size_literals.h> namespace NKikimr { diff --git a/ydb/library/mkql_proto/mkql_proto.cpp b/ydb/library/mkql_proto/mkql_proto.cpp index 29f5b8afb41..53d7722dee8 100644 --- a/ydb/library/mkql_proto/mkql_proto.cpp +++ b/ydb/library/mkql_proto/mkql_proto.cpp @@ -1,20 +1,20 @@ -#include "mkql_proto.h" - +#include "mkql_proto.h" + #include <ydb/library/yql/minikql/defs.h> #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/mkql_type_ops.h> #include <ydb/library/yql/public/decimal/yql_decimal.h> - + #include <library/cpp/containers/stack_vector/stack_vec.h> - -namespace NKikimr::NMiniKQL { - -namespace { - -void ExportTypeToProtoImpl(TType* type, NKikimrMiniKQL::TType& res); - + +namespace NKikimr::NMiniKQL { + +namespace { + +void ExportTypeToProtoImpl(TType* type, NKikimrMiniKQL::TType& res); + Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res) { auto dataType = static_cast<const TDataType*>(type); switch (dataType->GetSchemeType()) { @@ -107,115 +107,115 @@ Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxed } } -template<typename TOut> -Y_FORCE_INLINE void ExportStructTypeToProto(TStructType* structType, TOut& res) { - for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { - auto newMember = res.AddMember(); - newMember->SetName(TString(structType->GetMemberName(index))); - ExportTypeToProtoImpl(structType->GetMemberType(index), *newMember->MutableType()); - } -} - -template<typename TOut> -Y_FORCE_INLINE void ExportTupleTypeToProto(TTupleType* tupleType, TOut& res) { - for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { - ExportTypeToProtoImpl(tupleType->GetElementType(index), *res.AddElement()); - } -} - -void ExportTypeToProtoImpl(TType* type, NKikimrMiniKQL::TType& res) { - switch (type->GetKind()) { - case TType::EKind::Void: - res.SetKind(NKikimrMiniKQL::ETypeKind::Void); - break; - +template<typename TOut> +Y_FORCE_INLINE void ExportStructTypeToProto(TStructType* structType, TOut& res) { + for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { + auto newMember = res.AddMember(); + newMember->SetName(TString(structType->GetMemberName(index))); + ExportTypeToProtoImpl(structType->GetMemberType(index), *newMember->MutableType()); + } +} + +template<typename TOut> +Y_FORCE_INLINE void ExportTupleTypeToProto(TTupleType* tupleType, TOut& res) { + for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { + ExportTypeToProtoImpl(tupleType->GetElementType(index), *res.AddElement()); + } +} + +void ExportTypeToProtoImpl(TType* type, NKikimrMiniKQL::TType& res) { + switch (type->GetKind()) { + case TType::EKind::Void: + res.SetKind(NKikimrMiniKQL::ETypeKind::Void); + break; + case TType::EKind::Null: res.SetKind(NKikimrMiniKQL::ETypeKind::Null); break; - case TType::EKind::Data: { - auto dataType = static_cast<TDataType *>(type); - auto schemeType = dataType->GetSchemeType(); - res.SetKind(NKikimrMiniKQL::ETypeKind::Data); - res.MutableData()->SetScheme(schemeType); - if (schemeType == NYql::NProto::TypeIds::Decimal) { - auto decimalType = static_cast<TDataDecimalType *>(dataType); - auto params = decimalType->GetParams(); - auto decimalParams = res.MutableData()->MutableDecimalParams(); - decimalParams->SetPrecision(params.first); - decimalParams->SetScale(params.second); - } - break; - } - - case TType::EKind::Optional: { - auto optionalType = static_cast<TOptionalType *>(type); - res.SetKind(NKikimrMiniKQL::ETypeKind::Optional); - ExportTypeToProtoImpl(optionalType->GetItemType(), *res.MutableOptional()->MutableItem()); - break; - } - - case TType::EKind::List: { - auto listType = static_cast<TListType *>(type); - res.SetKind(NKikimrMiniKQL::ETypeKind::List); - ExportTypeToProtoImpl(listType->GetItemType(), *res.MutableList()->MutableItem()); - break; - } - - case TType::EKind::Struct: { - auto structType = static_cast<TStructType *>(type); - res.SetKind(NKikimrMiniKQL::ETypeKind::Struct); - if (structType->GetMembersCount()) { - ExportStructTypeToProto(structType, *res.MutableStruct()); - } - break; - } - - case TType::EKind::Tuple: { - auto tupleType = static_cast<TTupleType *>(type); - res.SetKind(NKikimrMiniKQL::ETypeKind::Tuple); - if (tupleType->GetElementsCount()) { - ExportTupleTypeToProto(tupleType, *res.MutableTuple()); - } - break; - } - - case TType::EKind::Dict: { - auto dictType = static_cast<TDictType *>(type); - res.SetKind(NKikimrMiniKQL::ETypeKind::Dict); - ExportTypeToProtoImpl(dictType->GetKeyType(), *res.MutableDict()->MutableKey()); - ExportTypeToProtoImpl(dictType->GetPayloadType(), *res.MutableDict()->MutablePayload()); - break; - } - - case TType::EKind::Variant: { - auto variantType = static_cast<TVariantType *>(type); - TType *innerType = variantType->GetUnderlyingType(); - res.SetKind(NKikimrMiniKQL::ETypeKind::Variant); - if (innerType->IsStruct()) { - TStructType *structType = static_cast<TStructType *>(innerType); - auto resItems = res.MutableVariant()->MutableStructItems(); - if (structType->GetMembersCount()) { - ExportStructTypeToProto(structType, *resItems); - } - } else if (innerType->IsTuple()) { - auto resItems = res.MutableVariant()->MutableTupleItems(); - TTupleType *tupleType = static_cast<TTupleType *>(innerType); - if (tupleType->GetElementsCount()) { - ExportTupleTypeToProto(tupleType, *resItems); - } - } else { - MKQL_ENSURE(false, - TStringBuilder() << "Unknown underlying variant type: " << innerType->GetKindAsStr()); - } - break; - } - - default: - MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); - } -} - + case TType::EKind::Data: { + auto dataType = static_cast<TDataType *>(type); + auto schemeType = dataType->GetSchemeType(); + res.SetKind(NKikimrMiniKQL::ETypeKind::Data); + res.MutableData()->SetScheme(schemeType); + if (schemeType == NYql::NProto::TypeIds::Decimal) { + auto decimalType = static_cast<TDataDecimalType *>(dataType); + auto params = decimalType->GetParams(); + auto decimalParams = res.MutableData()->MutableDecimalParams(); + decimalParams->SetPrecision(params.first); + decimalParams->SetScale(params.second); + } + break; + } + + case TType::EKind::Optional: { + auto optionalType = static_cast<TOptionalType *>(type); + res.SetKind(NKikimrMiniKQL::ETypeKind::Optional); + ExportTypeToProtoImpl(optionalType->GetItemType(), *res.MutableOptional()->MutableItem()); + break; + } + + case TType::EKind::List: { + auto listType = static_cast<TListType *>(type); + res.SetKind(NKikimrMiniKQL::ETypeKind::List); + ExportTypeToProtoImpl(listType->GetItemType(), *res.MutableList()->MutableItem()); + break; + } + + case TType::EKind::Struct: { + auto structType = static_cast<TStructType *>(type); + res.SetKind(NKikimrMiniKQL::ETypeKind::Struct); + if (structType->GetMembersCount()) { + ExportStructTypeToProto(structType, *res.MutableStruct()); + } + break; + } + + case TType::EKind::Tuple: { + auto tupleType = static_cast<TTupleType *>(type); + res.SetKind(NKikimrMiniKQL::ETypeKind::Tuple); + if (tupleType->GetElementsCount()) { + ExportTupleTypeToProto(tupleType, *res.MutableTuple()); + } + break; + } + + case TType::EKind::Dict: { + auto dictType = static_cast<TDictType *>(type); + res.SetKind(NKikimrMiniKQL::ETypeKind::Dict); + ExportTypeToProtoImpl(dictType->GetKeyType(), *res.MutableDict()->MutableKey()); + ExportTypeToProtoImpl(dictType->GetPayloadType(), *res.MutableDict()->MutablePayload()); + break; + } + + case TType::EKind::Variant: { + auto variantType = static_cast<TVariantType *>(type); + TType *innerType = variantType->GetUnderlyingType(); + res.SetKind(NKikimrMiniKQL::ETypeKind::Variant); + if (innerType->IsStruct()) { + TStructType *structType = static_cast<TStructType *>(innerType); + auto resItems = res.MutableVariant()->MutableStructItems(); + if (structType->GetMembersCount()) { + ExportStructTypeToProto(structType, *resItems); + } + } else if (innerType->IsTuple()) { + auto resItems = res.MutableVariant()->MutableTupleItems(); + TTupleType *tupleType = static_cast<TTupleType *>(innerType); + if (tupleType->GetElementsCount()) { + ExportTupleTypeToProto(tupleType, *resItems); + } + } else { + MKQL_ENSURE(false, + TStringBuilder() << "Unknown underlying variant type: " << innerType->GetKindAsStr()); + } + break; + } + + default: + MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); + } +} + void ExportTypeToProtoImpl(TType* type, Ydb::Type& res) { switch (type->GetKind()) { case TType::EKind::Void: @@ -332,80 +332,80 @@ void ExportTypeToProtoImpl(TType* type, Ydb::Type& res) { } -Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res) { - auto dataType = static_cast<const TDataType*>(type); - switch (dataType->GetSchemeType()) { - case NUdf::TDataType<bool>::Id: - res.SetBool(value.Get<bool>()); - break; - case NUdf::TDataType<ui8>::Id: - res.SetUint32(value.Get<ui8>()); - break; - case NUdf::TDataType<i8>::Id: - res.SetInt32(value.Get<i8>()); - break; - case NUdf::TDataType<ui16>::Id: - res.SetUint32(value.Get<ui16>()); - break; - case NUdf::TDataType<i16>::Id: - res.SetInt32(value.Get<i16>()); - break; - case NUdf::TDataType<i32>::Id: - res.SetInt32(value.Get<i32>()); - break; - case NUdf::TDataType<ui32>::Id: - res.SetUint32(value.Get<ui32>()); - break; - case NUdf::TDataType<i64>::Id: - res.SetInt64(value.Get<i64>()); - break; - case NUdf::TDataType<ui64>::Id: - res.SetUint64(value.Get<ui64>()); - break; - case NUdf::TDataType<float>::Id: - res.SetFloat(value.Get<float>()); - break; - case NUdf::TDataType<double>::Id: - res.SetDouble(value.Get<double>()); - break; - case NUdf::TDataType<NUdf::TJson>::Id: - case NUdf::TDataType<NUdf::TUtf8>::Id: { - auto stringRef = value.AsStringRef(); - res.SetText(stringRef.Data(), stringRef.Size()); - break; - } - case NUdf::TDataType<NUdf::TTzDate>::Id: - case NUdf::TDataType<NUdf::TTzDatetime>::Id: - case NUdf::TDataType<NUdf::TTzTimestamp>::Id: { - const NUdf::TUnboxedValue out(ValueToString(NUdf::GetDataSlot(dataType->GetSchemeType()), value)); - const auto& stringRef = out.AsStringRef(); - res.SetText(stringRef.Data(), stringRef.Size()); - break; - } - case NUdf::TDataType<NUdf::TDecimal>::Id: { - auto decimal = value.GetInt128(); - const auto p = reinterpret_cast<ui8*>(&decimal); - res.SetLow128(*reinterpret_cast<ui64*>(p)); - res.SetHi128(*reinterpret_cast<ui64*>(p+8)); - break; - } - case NUdf::TDataType<NUdf::TDate>::Id: - res.SetUint32(value.Get<ui16>()); - break; - case NUdf::TDataType<NUdf::TDatetime>::Id: - res.SetUint32(value.Get<ui32>()); - break; - case NUdf::TDataType<NUdf::TTimestamp>::Id: - res.SetUint64(value.Get<ui64>()); - break; - case NUdf::TDataType<NUdf::TInterval>::Id: - res.SetInt64(value.Get<i64>()); - break; - case NUdf::TDataType<NUdf::TUuid>::Id: { - const auto& stringRef = value.AsStringRef(); - UuidToMkqlProto(stringRef.Data(), stringRef.Size(), res); - break; - } +Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res) { + auto dataType = static_cast<const TDataType*>(type); + switch (dataType->GetSchemeType()) { + case NUdf::TDataType<bool>::Id: + res.SetBool(value.Get<bool>()); + break; + case NUdf::TDataType<ui8>::Id: + res.SetUint32(value.Get<ui8>()); + break; + case NUdf::TDataType<i8>::Id: + res.SetInt32(value.Get<i8>()); + break; + case NUdf::TDataType<ui16>::Id: + res.SetUint32(value.Get<ui16>()); + break; + case NUdf::TDataType<i16>::Id: + res.SetInt32(value.Get<i16>()); + break; + case NUdf::TDataType<i32>::Id: + res.SetInt32(value.Get<i32>()); + break; + case NUdf::TDataType<ui32>::Id: + res.SetUint32(value.Get<ui32>()); + break; + case NUdf::TDataType<i64>::Id: + res.SetInt64(value.Get<i64>()); + break; + case NUdf::TDataType<ui64>::Id: + res.SetUint64(value.Get<ui64>()); + break; + case NUdf::TDataType<float>::Id: + res.SetFloat(value.Get<float>()); + break; + case NUdf::TDataType<double>::Id: + res.SetDouble(value.Get<double>()); + break; + case NUdf::TDataType<NUdf::TJson>::Id: + case NUdf::TDataType<NUdf::TUtf8>::Id: { + auto stringRef = value.AsStringRef(); + res.SetText(stringRef.Data(), stringRef.Size()); + break; + } + case NUdf::TDataType<NUdf::TTzDate>::Id: + case NUdf::TDataType<NUdf::TTzDatetime>::Id: + case NUdf::TDataType<NUdf::TTzTimestamp>::Id: { + const NUdf::TUnboxedValue out(ValueToString(NUdf::GetDataSlot(dataType->GetSchemeType()), value)); + const auto& stringRef = out.AsStringRef(); + res.SetText(stringRef.Data(), stringRef.Size()); + break; + } + case NUdf::TDataType<NUdf::TDecimal>::Id: { + auto decimal = value.GetInt128(); + const auto p = reinterpret_cast<ui8*>(&decimal); + res.SetLow128(*reinterpret_cast<ui64*>(p)); + res.SetHi128(*reinterpret_cast<ui64*>(p+8)); + break; + } + case NUdf::TDataType<NUdf::TDate>::Id: + res.SetUint32(value.Get<ui16>()); + break; + case NUdf::TDataType<NUdf::TDatetime>::Id: + res.SetUint32(value.Get<ui32>()); + break; + case NUdf::TDataType<NUdf::TTimestamp>::Id: + res.SetUint64(value.Get<ui64>()); + break; + case NUdf::TDataType<NUdf::TInterval>::Id: + res.SetInt64(value.Get<i64>()); + break; + case NUdf::TDataType<NUdf::TUuid>::Id: { + const auto& stringRef = value.AsStringRef(); + UuidToMkqlProto(stringRef.Data(), stringRef.Size(), res); + break; + } case NUdf::TDataType<NUdf::TJsonDocument>::Id: { auto stringRef = value.AsStringRef(); res.SetBytes(stringRef.Data(), stringRef.Size()); @@ -416,108 +416,108 @@ Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxed res.SetBytes(stringRef.Data(), stringRef.Size()); break; } - default: - auto stringRef = value.AsStringRef(); - res.SetBytes(stringRef.Data(), stringRef.Size()); - } -} - + default: + auto stringRef = value.AsStringRef(); + res.SetBytes(stringRef.Data(), stringRef.Size()); + } +} + void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder = nullptr) { - switch (type->GetKind()) { - case TType::EKind::Void: + switch (type->GetKind()) { + case TType::EKind::Void: case TType::EKind::EmptyList: case TType::EKind::EmptyDict: - break; - + break; + case TType::EKind::Null: { res.SetNullFlagValue(::google::protobuf::NULL_VALUE); break; } - case TType::EKind::Data: { - HandleKindDataExport(type, value, res); - break; - } - - case TType::EKind::Optional: { - auto optionalType = static_cast<TOptionalType*>(type); - if (value) { - ExportValueToProtoImpl(optionalType->GetItemType(), value.GetOptionalValue(), *res.MutableOptional()); - } - - break; - } - - case TType::EKind::List: { - auto listType = static_cast<TListType*>(type); - auto itemType = listType->GetItemType(); - if (value.HasFastListLength()) - res.MutableList()->Reserve(value.GetListLength()); - const auto iter = value.GetListIterator(); - for (NUdf::TUnboxedValue item; iter.Next(item);) { - ExportValueToProtoImpl(itemType, item, *res.MutableList()->Add()); - } - - break; - } - - case TType::EKind::Struct: { - auto structType = static_cast<TStructType*>(type); - res.MutableStruct()->Reserve(structType->GetMembersCount()); - for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { + case TType::EKind::Data: { + HandleKindDataExport(type, value, res); + break; + } + + case TType::EKind::Optional: { + auto optionalType = static_cast<TOptionalType*>(type); + if (value) { + ExportValueToProtoImpl(optionalType->GetItemType(), value.GetOptionalValue(), *res.MutableOptional()); + } + + break; + } + + case TType::EKind::List: { + auto listType = static_cast<TListType*>(type); + auto itemType = listType->GetItemType(); + if (value.HasFastListLength()) + res.MutableList()->Reserve(value.GetListLength()); + const auto iter = value.GetListIterator(); + for (NUdf::TUnboxedValue item; iter.Next(item);) { + ExportValueToProtoImpl(itemType, item, *res.MutableList()->Add()); + } + + break; + } + + case TType::EKind::Struct: { + auto structType = static_cast<TStructType*>(type); + res.MutableStruct()->Reserve(structType->GetMembersCount()); + for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { auto memberIndex = (!columnOrder || columnOrder->empty()) ? index : (*columnOrder)[index]; auto memberType = structType->GetMemberType(memberIndex); ExportValueToProtoImpl(memberType, value.GetElement(memberIndex), *res.MutableStruct()->Add()); - } - - break; - } - - case TType::EKind::Tuple: { - auto tupleType = static_cast<TTupleType*>(type); - res.MutableTuple()->Reserve(tupleType->GetElementsCount()); - for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { - auto elementType = tupleType->GetElementType(index); - ExportValueToProtoImpl(elementType, value.GetElement(index), *res.MutableTuple()->Add()); - } - - break; - } - - case TType::EKind::Dict: { - auto dictType = static_cast<TDictType*>(type); - auto keyType = dictType->GetKeyType(); - auto payloadType = dictType->GetPayloadType(); - const auto iter = value.GetDictIterator(); - for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) { - auto dictItem = res.MutableDict()->Add(); - ExportValueToProtoImpl(keyType, key, *dictItem->MutableKey()); - ExportValueToProtoImpl(payloadType, payload, *dictItem->MutablePayload()); - } - - break; - } - - case TType::EKind::Variant: { - auto variantType = static_cast<TVariantType*>(type); - auto variantIndex = value.GetVariantIndex(); - res.SetVariantIndex(variantIndex); - ExportValueToProtoImpl(variantType->GetAlternativeType(variantIndex), value.GetVariantItem(), *res.MutableOptional()); - - break; - } - + } + + break; + } + + case TType::EKind::Tuple: { + auto tupleType = static_cast<TTupleType*>(type); + res.MutableTuple()->Reserve(tupleType->GetElementsCount()); + for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { + auto elementType = tupleType->GetElementType(index); + ExportValueToProtoImpl(elementType, value.GetElement(index), *res.MutableTuple()->Add()); + } + + break; + } + + case TType::EKind::Dict: { + auto dictType = static_cast<TDictType*>(type); + auto keyType = dictType->GetKeyType(); + auto payloadType = dictType->GetPayloadType(); + const auto iter = value.GetDictIterator(); + for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) { + auto dictItem = res.MutableDict()->Add(); + ExportValueToProtoImpl(keyType, key, *dictItem->MutableKey()); + ExportValueToProtoImpl(payloadType, payload, *dictItem->MutablePayload()); + } + + break; + } + + case TType::EKind::Variant: { + auto variantType = static_cast<TVariantType*>(type); + auto variantIndex = value.GetVariantIndex(); + res.SetVariantIndex(variantIndex); + ExportValueToProtoImpl(variantType->GetAlternativeType(variantIndex), value.GetVariantItem(), *res.MutableOptional()); + + break; + } + case TType::EKind::Tagged: { auto taggedType = static_cast<TTaggedType*>(type); ExportValueToProtoImpl(taggedType->GetBaseType(), value, res); break; } - default: - MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); - } -} - + default: + MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); + } +} + void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr) { switch (type->GetKind()) { case TType::EKind::Void: @@ -630,62 +630,62 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Yd Y_FORCE_INLINE NUdf::TUnboxedValue HandleKindDataImport(const TType* type, const NKikimrMiniKQL::TValue& value) { auto dataType = static_cast<const TDataType*>(type); auto oneOfCase = value.value_value_case(); - switch (dataType->GetSchemeType()) { - case NUdf::TDataType<bool>::Id: + switch (dataType->GetSchemeType()) { + case NUdf::TDataType<bool>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBool); - return NUdf::TUnboxedValuePod(value.GetBool()); - case NUdf::TDataType<ui8>::Id: + return NUdf::TUnboxedValuePod(value.GetBool()); + case NUdf::TDataType<ui8>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); - return NUdf::TUnboxedValuePod(ui8(value.GetUint32())); - case NUdf::TDataType<i8>::Id: + return NUdf::TUnboxedValuePod(ui8(value.GetUint32())); + case NUdf::TDataType<i8>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt32); - return NUdf::TUnboxedValuePod(i8(value.GetInt32())); - case NUdf::TDataType<ui16>::Id: + return NUdf::TUnboxedValuePod(i8(value.GetInt32())); + case NUdf::TDataType<ui16>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); - return NUdf::TUnboxedValuePod(ui16(value.GetUint32())); - case NUdf::TDataType<i16>::Id: + return NUdf::TUnboxedValuePod(ui16(value.GetUint32())); + case NUdf::TDataType<i16>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt32); - return NUdf::TUnboxedValuePod(i16(value.GetInt32())); - case NUdf::TDataType<i32>::Id: + return NUdf::TUnboxedValuePod(i16(value.GetInt32())); + case NUdf::TDataType<i32>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt32); - return NUdf::TUnboxedValuePod(value.GetInt32()); - case NUdf::TDataType<ui32>::Id: + return NUdf::TUnboxedValuePod(value.GetInt32()); + case NUdf::TDataType<ui32>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); - return NUdf::TUnboxedValuePod(value.GetUint32()); - case NUdf::TDataType<i64>::Id: + return NUdf::TUnboxedValuePod(value.GetUint32()); + case NUdf::TDataType<i64>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt64); - return NUdf::TUnboxedValuePod(value.GetInt64()); - case NUdf::TDataType<ui64>::Id: + return NUdf::TUnboxedValuePod(value.GetInt64()); + case NUdf::TDataType<ui64>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint64); - return NUdf::TUnboxedValuePod(value.GetUint64()); - case NUdf::TDataType<float>::Id: + return NUdf::TUnboxedValuePod(value.GetUint64()); + case NUdf::TDataType<float>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kFloat); - return NUdf::TUnboxedValuePod(value.GetFloat()); - case NUdf::TDataType<double>::Id: + return NUdf::TUnboxedValuePod(value.GetFloat()); + case NUdf::TDataType<double>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kDouble); - return NUdf::TUnboxedValuePod(value.GetDouble()); - case NUdf::TDataType<NUdf::TJson>::Id: - case NUdf::TDataType<NUdf::TUtf8>::Id: + return NUdf::TUnboxedValuePod(value.GetDouble()); + case NUdf::TDataType<NUdf::TJson>::Id: + case NUdf::TDataType<NUdf::TUtf8>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kText); - return MakeString(value.GetText()); + return MakeString(value.GetText()); case NUdf::TDataType<NUdf::TTzDate>::Id: case NUdf::TDataType<NUdf::TTzDatetime>::Id: case NUdf::TDataType<NUdf::TTzTimestamp>::Id: { MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kText); return NUdf::TUnboxedValuePod(ValueFromString(NUdf::GetDataSlot(dataType->GetSchemeType()), value.GetText())); } - case NUdf::TDataType<NUdf::TDate>::Id: + case NUdf::TDataType<NUdf::TDate>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); - return NUdf::TUnboxedValuePod(ui16(value.GetUint32())); - case NUdf::TDataType<NUdf::TDatetime>::Id: + return NUdf::TUnboxedValuePod(ui16(value.GetUint32())); + case NUdf::TDataType<NUdf::TDatetime>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); - return NUdf::TUnboxedValuePod(value.GetUint32()); - case NUdf::TDataType<NUdf::TTimestamp>::Id: + return NUdf::TUnboxedValuePod(value.GetUint32()); + case NUdf::TDataType<NUdf::TTimestamp>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint64); - return NUdf::TUnboxedValuePod(value.GetUint64()); - case NUdf::TDataType<NUdf::TInterval>::Id: + return NUdf::TUnboxedValuePod(value.GetUint64()); + case NUdf::TDataType<NUdf::TInterval>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt64); - return NUdf::TUnboxedValuePod(value.GetInt64()); + return NUdf::TUnboxedValuePod(value.GetInt64()); case NUdf::TDataType<NUdf::TJsonDocument>::Id: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes); return MakeString(value.GetBytes()); @@ -704,15 +704,15 @@ Y_FORCE_INLINE NUdf::TUnboxedValue HandleKindDataImport(const TType* type, const buf.half[1] = value.GetHi128(); return MakeString(NUdf::TStringRef(buf.bytes, 16)); } - default: + default: MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes, "got: " << (int) oneOfCase << ", type: " << (int) dataType->GetSchemeType()); - return MakeString(value.GetBytes()); - } -} - -} - + return MakeString(value.GetBytes()); + } +} + +} + void ExportPrimitiveTypeToProto(ui32 schemeType, Ydb::Type& output) { switch (schemeType) { case NYql::NProto::TypeIds::Bool: @@ -756,516 +756,516 @@ void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::V ExportValueToProtoImpl(type, value, res, columnOrder); } -template <typename ValueType> class TBufferArray { -protected: - using size_type = ui32; - using cookie_type = decltype(((TStructType*)nullptr)->GetCookie()); - - struct header_type { - size_type size; - size_type capacity; - }; - - void* Buffer; - - explicit TBufferArray(size_type capacity, void* buffer) - : Buffer(buffer) - { - static_assert(sizeof(cookie_type) == sizeof(Buffer), "cookie_type should be equal to uintptr_t"); - new (buffer) header_type({0, capacity}); - } - -public: - TBufferArray(void* buffer) - : Buffer(buffer) - {} - - void* buffer() { - return Buffer; - } - - const void* buffer() const { - return Buffer; - } - - header_type* header() { - return reinterpret_cast<header_type*>(buffer()); - } - - const header_type* header() const { - return reinterpret_cast<const header_type*>(buffer()); - } - - ValueType* data() { - return reinterpret_cast<ValueType*>((reinterpret_cast<char*>(buffer()) + sizeof(header_type))); - } - - const ValueType* data() const { - return reinterpret_cast<const ValueType*>((reinterpret_cast<char*>(buffer()) + sizeof(header_type))); - } - - size_type size() const { - return header()->size; - } - - size_type capacity() const { - return header()->capacity; - } - - bool empty() const { - return buffer() == nullptr || size() == 0; - } - - ValueType* begin() { - return data(); - } - - ValueType* end() { - return begin() + size(); - } - - void push_back(const ValueType& value) { - MKQL_ENSURE(size() < capacity(), "not enough size of array"); - new (data() + header()->size) ValueType(value); - ++(header()->size); - } - - template <typename... Types> void emplace_back(Types... types) { - MKQL_ENSURE(size() < capacity(), "not enough size of array"); - new (data() + header()->size) ValueType(types...); - ++(header()->size); - } - - const ValueType& operator [] (size_type pos) const { - MKQL_ENSURE(pos < size(), "out of range"); - return data()[pos]; - } - - ValueType& operator [] (size_type pos) { - MKQL_ENSURE(pos < size(), "out of range"); - return data()[pos]; - } - - static ui64 GetRequiredSize(size_type count) { - return sizeof(header_type) + sizeof(ValueType) * count; - } - - static TBufferArray Create(size_type count, const TTypeEnvironment& env) { - void* buffer = env.AllocateBuffer(GetRequiredSize(count)); - return TBufferArray(count, buffer); - } - - cookie_type ToCookie() const { - return reinterpret_cast<cookie_type>(Buffer); - } - - static TBufferArray FromCookie(cookie_type cookie) { - return TBufferArray(reinterpret_cast<void*>(cookie)); - } -}; - -using TRemapArray = TBufferArray<ui32>; - -class TProtoImporter { -public: - TProtoImporter(const TTypeEnvironment& env); - ~TProtoImporter(); - TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type); - TNode* ImportNodeFromProto(TType* type, const NKikimrMiniKQL::TValue& value); +template <typename ValueType> class TBufferArray { +protected: + using size_type = ui32; + using cookie_type = decltype(((TStructType*)nullptr)->GetCookie()); + + struct header_type { + size_type size; + size_type capacity; + }; + + void* Buffer; + + explicit TBufferArray(size_type capacity, void* buffer) + : Buffer(buffer) + { + static_assert(sizeof(cookie_type) == sizeof(Buffer), "cookie_type should be equal to uintptr_t"); + new (buffer) header_type({0, capacity}); + } + +public: + TBufferArray(void* buffer) + : Buffer(buffer) + {} + + void* buffer() { + return Buffer; + } + + const void* buffer() const { + return Buffer; + } + + header_type* header() { + return reinterpret_cast<header_type*>(buffer()); + } + + const header_type* header() const { + return reinterpret_cast<const header_type*>(buffer()); + } + + ValueType* data() { + return reinterpret_cast<ValueType*>((reinterpret_cast<char*>(buffer()) + sizeof(header_type))); + } + + const ValueType* data() const { + return reinterpret_cast<const ValueType*>((reinterpret_cast<char*>(buffer()) + sizeof(header_type))); + } + + size_type size() const { + return header()->size; + } + + size_type capacity() const { + return header()->capacity; + } + + bool empty() const { + return buffer() == nullptr || size() == 0; + } + + ValueType* begin() { + return data(); + } + + ValueType* end() { + return begin() + size(); + } + + void push_back(const ValueType& value) { + MKQL_ENSURE(size() < capacity(), "not enough size of array"); + new (data() + header()->size) ValueType(value); + ++(header()->size); + } + + template <typename... Types> void emplace_back(Types... types) { + MKQL_ENSURE(size() < capacity(), "not enough size of array"); + new (data() + header()->size) ValueType(types...); + ++(header()->size); + } + + const ValueType& operator [] (size_type pos) const { + MKQL_ENSURE(pos < size(), "out of range"); + return data()[pos]; + } + + ValueType& operator [] (size_type pos) { + MKQL_ENSURE(pos < size(), "out of range"); + return data()[pos]; + } + + static ui64 GetRequiredSize(size_type count) { + return sizeof(header_type) + sizeof(ValueType) * count; + } + + static TBufferArray Create(size_type count, const TTypeEnvironment& env) { + void* buffer = env.AllocateBuffer(GetRequiredSize(count)); + return TBufferArray(count, buffer); + } + + cookie_type ToCookie() const { + return reinterpret_cast<cookie_type>(Buffer); + } + + static TBufferArray FromCookie(cookie_type cookie) { + return TBufferArray(reinterpret_cast<void*>(cookie)); + } +}; + +using TRemapArray = TBufferArray<ui32>; + +class TProtoImporter { +public: + TProtoImporter(const TTypeEnvironment& env); + ~TProtoImporter(); + TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type); + TNode* ImportNodeFromProto(TType* type, const NKikimrMiniKQL::TValue& value); NUdf::TUnboxedValue ImportValueFromProto(const TType* type, const NKikimrMiniKQL::TValue& value, const THolderFactory& factory); private: - TTupleType* ImportTupleTypeFromProto(const NKikimrMiniKQL::TTupleType& protoTupleType); - TStructType* ImportStructTypeFromProto(const NKikimrMiniKQL::TStructType& protoStructType); - const TTypeEnvironment& env; - TStackVec<TStructType*, 16> SortedStructs; -}; - -TProtoImporter::TProtoImporter(const TTypeEnvironment& env) - : env(env) -{} - -TProtoImporter::~TProtoImporter() { - for (TStructType* structType : SortedStructs) { - structType->SetCookie(0); - } -} - -TTupleType* TProtoImporter::ImportTupleTypeFromProto(const NKikimrMiniKQL::TTupleType& protoTupleType) -{ - const ui32 elementsCount = static_cast<ui32>(protoTupleType.ElementSize()); - TSmallVec<TType*> elementTypes; - for (ui32 elementIdx = 0; elementIdx < elementsCount; ++elementIdx) { - const NKikimrMiniKQL::TType& elementProtoType = protoTupleType.GetElement(elementIdx); - TType* elementType = ImportTypeFromProto(elementProtoType); - elementTypes.emplace_back(elementType); - } - - return TTupleType::Create(elementsCount, elementTypes.data(), env); -} - -TStructType* TProtoImporter::ImportStructTypeFromProto(const NKikimrMiniKQL::TStructType& protoStructType) -{ - ui32 membersCount = static_cast<ui32>(protoStructType.MemberSize()); - TStackVec<std::pair<TString, TType*>, 16> members; - TRemapArray remap = TRemapArray::Create(membersCount, env); - members.reserve(membersCount); - for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { - const NKikimrMiniKQL::TMember& protoMember = protoStructType.GetMember(memberNum); - TType* child = ImportTypeFromProto(protoMember.GetType()); - members.emplace_back(protoMember.GetName(), child); - remap.emplace_back(memberNum); - } - Sort(remap, [&members](ui32 a, ui32 b) -> bool { return members[a].first < members[b].first; }); - TStackVec<std::pair<TString, TType*>, 16> sortedMembers(membersCount); - bool resorted = false; - for (ui32 pos = 0; pos < membersCount; ++pos) { - sortedMembers[pos] = members[remap[pos]]; - if (remap[pos] != pos) - resorted = true; - } - TStructType* structType = TStructType::Create(sortedMembers.data(), sortedMembers.size(), env); - if (resorted) { - structType->SetCookie(remap.ToCookie()); - SortedStructs.push_back(structType); - } - return structType; -} - -TType* TProtoImporter::ImportTypeFromProto(const NKikimrMiniKQL::TType& type) { - switch (type.GetKind()) { - case NKikimrMiniKQL::ETypeKind::Void: { - return env.GetVoid()->GetType(); - } + TTupleType* ImportTupleTypeFromProto(const NKikimrMiniKQL::TTupleType& protoTupleType); + TStructType* ImportStructTypeFromProto(const NKikimrMiniKQL::TStructType& protoStructType); + const TTypeEnvironment& env; + TStackVec<TStructType*, 16> SortedStructs; +}; + +TProtoImporter::TProtoImporter(const TTypeEnvironment& env) + : env(env) +{} + +TProtoImporter::~TProtoImporter() { + for (TStructType* structType : SortedStructs) { + structType->SetCookie(0); + } +} + +TTupleType* TProtoImporter::ImportTupleTypeFromProto(const NKikimrMiniKQL::TTupleType& protoTupleType) +{ + const ui32 elementsCount = static_cast<ui32>(protoTupleType.ElementSize()); + TSmallVec<TType*> elementTypes; + for (ui32 elementIdx = 0; elementIdx < elementsCount; ++elementIdx) { + const NKikimrMiniKQL::TType& elementProtoType = protoTupleType.GetElement(elementIdx); + TType* elementType = ImportTypeFromProto(elementProtoType); + elementTypes.emplace_back(elementType); + } + + return TTupleType::Create(elementsCount, elementTypes.data(), env); +} + +TStructType* TProtoImporter::ImportStructTypeFromProto(const NKikimrMiniKQL::TStructType& protoStructType) +{ + ui32 membersCount = static_cast<ui32>(protoStructType.MemberSize()); + TStackVec<std::pair<TString, TType*>, 16> members; + TRemapArray remap = TRemapArray::Create(membersCount, env); + members.reserve(membersCount); + for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { + const NKikimrMiniKQL::TMember& protoMember = protoStructType.GetMember(memberNum); + TType* child = ImportTypeFromProto(protoMember.GetType()); + members.emplace_back(protoMember.GetName(), child); + remap.emplace_back(memberNum); + } + Sort(remap, [&members](ui32 a, ui32 b) -> bool { return members[a].first < members[b].first; }); + TStackVec<std::pair<TString, TType*>, 16> sortedMembers(membersCount); + bool resorted = false; + for (ui32 pos = 0; pos < membersCount; ++pos) { + sortedMembers[pos] = members[remap[pos]]; + if (remap[pos] != pos) + resorted = true; + } + TStructType* structType = TStructType::Create(sortedMembers.data(), sortedMembers.size(), env); + if (resorted) { + structType->SetCookie(remap.ToCookie()); + SortedStructs.push_back(structType); + } + return structType; +} + +TType* TProtoImporter::ImportTypeFromProto(const NKikimrMiniKQL::TType& type) { + switch (type.GetKind()) { + case NKikimrMiniKQL::ETypeKind::Void: { + return env.GetVoid()->GetType(); + } case NKikimrMiniKQL::ETypeKind::Null: { return env.GetNull()->GetType(); } - case NKikimrMiniKQL::ETypeKind::Data: { - const NKikimrMiniKQL::TDataType& protoData = type.GetData(); - NUdf::TDataTypeId schemeType = protoData.GetScheme(); - MKQL_ENSURE(NUdf::FindDataSlot(schemeType), TStringBuilder() << "unknown type id: " << schemeType); - if (schemeType == NYql::NProto::TypeIds::Decimal) { - const NKikimrMiniKQL::TDecimalParams& decimalParams = protoData.GetDecimalParams(); - return TDataDecimalType::Create(decimalParams.GetPrecision(), decimalParams.GetScale(), env); - } else { - return TDataType::Create(schemeType, env); - } - } - case NKikimrMiniKQL::ETypeKind::Optional: { - const NKikimrMiniKQL::TOptionalType& protoOptionalType = type.GetOptional(); - TType* child = ImportTypeFromProto(protoOptionalType.GetItem()); - TOptionalType* optionalType = TOptionalType::Create(child, env); - return optionalType; - } - case NKikimrMiniKQL::ETypeKind::List: { - const NKikimrMiniKQL::TListType& protoListType = type.GetList(); - const NKikimrMiniKQL::TType& protoItemType = protoListType.GetItem(); - TType* itemType = ImportTypeFromProto(protoItemType); - TListType* listType = TListType::Create(itemType, env); - return listType; - } - case NKikimrMiniKQL::ETypeKind::Tuple: { - const NKikimrMiniKQL::TTupleType& protoTupleType = type.GetTuple(); - return ImportTupleTypeFromProto(protoTupleType); - } - case NKikimrMiniKQL::ETypeKind::Struct: { - const NKikimrMiniKQL::TStructType& protoStructType = type.GetStruct(); - return ImportStructTypeFromProto(protoStructType); - } - case NKikimrMiniKQL::ETypeKind::Dict: { - const NKikimrMiniKQL::TDictType& protoDictType = type.GetDict(); - - TType* keyType = ImportTypeFromProto(protoDictType.GetKey()); - TType* payloadType = ImportTypeFromProto(protoDictType.GetPayload()); - - TDictType* dictType = TDictType::Create(keyType, payloadType, env); - return dictType; - } - case NKikimrMiniKQL::ETypeKind::Variant: { - const NKikimrMiniKQL::TVariantType& protoVariantType = type.GetVariant(); - switch (protoVariantType.type_case()) { - case NKikimrMiniKQL::TVariantType::kTupleItems: { - const NKikimrMiniKQL::TTupleType& protoTupleType = protoVariantType.GetTupleItems(); - - TTupleType* tupleType = ImportTupleTypeFromProto(protoTupleType); - - return TVariantType::Create(tupleType, env); - } - case NKikimrMiniKQL::TVariantType::kStructItems: { - const NKikimrMiniKQL::TStructType& protoStructType = protoVariantType.GetStructItems(); - - TStructType* structType = ImportStructTypeFromProto(protoStructType); - - return TVariantType::Create(structType, env); - } - default: - MKQL_ENSURE(false, TStringBuilder() << "Unknown variant type representation: " << protoVariantType.DebugString()); - } - } - default: { - MKQL_ENSURE(false, TStringBuilder() << "Unknown protobuf type: " << type.DebugString()); - } - } -} - -TNode* TProtoImporter::ImportNodeFromProto(TType* type, const NKikimrMiniKQL::TValue& value) { - switch (type->GetKind()) { - case TCallableType::EKind::Void: { - return env.GetVoid(); - } + case NKikimrMiniKQL::ETypeKind::Data: { + const NKikimrMiniKQL::TDataType& protoData = type.GetData(); + NUdf::TDataTypeId schemeType = protoData.GetScheme(); + MKQL_ENSURE(NUdf::FindDataSlot(schemeType), TStringBuilder() << "unknown type id: " << schemeType); + if (schemeType == NYql::NProto::TypeIds::Decimal) { + const NKikimrMiniKQL::TDecimalParams& decimalParams = protoData.GetDecimalParams(); + return TDataDecimalType::Create(decimalParams.GetPrecision(), decimalParams.GetScale(), env); + } else { + return TDataType::Create(schemeType, env); + } + } + case NKikimrMiniKQL::ETypeKind::Optional: { + const NKikimrMiniKQL::TOptionalType& protoOptionalType = type.GetOptional(); + TType* child = ImportTypeFromProto(protoOptionalType.GetItem()); + TOptionalType* optionalType = TOptionalType::Create(child, env); + return optionalType; + } + case NKikimrMiniKQL::ETypeKind::List: { + const NKikimrMiniKQL::TListType& protoListType = type.GetList(); + const NKikimrMiniKQL::TType& protoItemType = protoListType.GetItem(); + TType* itemType = ImportTypeFromProto(protoItemType); + TListType* listType = TListType::Create(itemType, env); + return listType; + } + case NKikimrMiniKQL::ETypeKind::Tuple: { + const NKikimrMiniKQL::TTupleType& protoTupleType = type.GetTuple(); + return ImportTupleTypeFromProto(protoTupleType); + } + case NKikimrMiniKQL::ETypeKind::Struct: { + const NKikimrMiniKQL::TStructType& protoStructType = type.GetStruct(); + return ImportStructTypeFromProto(protoStructType); + } + case NKikimrMiniKQL::ETypeKind::Dict: { + const NKikimrMiniKQL::TDictType& protoDictType = type.GetDict(); + + TType* keyType = ImportTypeFromProto(protoDictType.GetKey()); + TType* payloadType = ImportTypeFromProto(protoDictType.GetPayload()); + + TDictType* dictType = TDictType::Create(keyType, payloadType, env); + return dictType; + } + case NKikimrMiniKQL::ETypeKind::Variant: { + const NKikimrMiniKQL::TVariantType& protoVariantType = type.GetVariant(); + switch (protoVariantType.type_case()) { + case NKikimrMiniKQL::TVariantType::kTupleItems: { + const NKikimrMiniKQL::TTupleType& protoTupleType = protoVariantType.GetTupleItems(); + + TTupleType* tupleType = ImportTupleTypeFromProto(protoTupleType); + + return TVariantType::Create(tupleType, env); + } + case NKikimrMiniKQL::TVariantType::kStructItems: { + const NKikimrMiniKQL::TStructType& protoStructType = protoVariantType.GetStructItems(); + + TStructType* structType = ImportStructTypeFromProto(protoStructType); + + return TVariantType::Create(structType, env); + } + default: + MKQL_ENSURE(false, TStringBuilder() << "Unknown variant type representation: " << protoVariantType.DebugString()); + } + } + default: { + MKQL_ENSURE(false, TStringBuilder() << "Unknown protobuf type: " << type.DebugString()); + } + } +} + +TNode* TProtoImporter::ImportNodeFromProto(TType* type, const NKikimrMiniKQL::TValue& value) { + switch (type->GetKind()) { + case TCallableType::EKind::Void: { + return env.GetVoid(); + } case TCallableType::EKind::Null: { return env.GetNull(); } - case TCallableType::EKind::Data: { - TDataType* dataType = static_cast<TDataType*>(type); - TDataLiteral* dataNode = nullptr; - switch (const auto schemeType = dataType->GetSchemeType()) { - case NUdf::TDataType<bool>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetBool()), dataType, env); - break; - case NUdf::TDataType<ui8>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(ui8(value.GetUint32())), dataType, env); - break; - case NUdf::TDataType<i32>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetInt32()), dataType, env); - break; - case NUdf::TDataType<ui32>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetUint32()), dataType, env); - break; - case NUdf::TDataType<i64>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetInt64()), dataType, env); - break; - case NUdf::TDataType<ui64>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetUint64()), dataType, env); - break; - case NUdf::TDataType<float>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetFloat()), dataType, env); - break; - case NUdf::TDataType<double>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetDouble()), dataType, env); - break; - case NUdf::TDataType<char*>::Id: - case NUdf::TDataType<NUdf::TYson>::Id: + case TCallableType::EKind::Data: { + TDataType* dataType = static_cast<TDataType*>(type); + TDataLiteral* dataNode = nullptr; + switch (const auto schemeType = dataType->GetSchemeType()) { + case NUdf::TDataType<bool>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetBool()), dataType, env); + break; + case NUdf::TDataType<ui8>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(ui8(value.GetUint32())), dataType, env); + break; + case NUdf::TDataType<i32>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetInt32()), dataType, env); + break; + case NUdf::TDataType<ui32>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetUint32()), dataType, env); + break; + case NUdf::TDataType<i64>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetInt64()), dataType, env); + break; + case NUdf::TDataType<ui64>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetUint64()), dataType, env); + break; + case NUdf::TDataType<float>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetFloat()), dataType, env); + break; + case NUdf::TDataType<double>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetDouble()), dataType, env); + break; + case NUdf::TDataType<char*>::Id: + case NUdf::TDataType<NUdf::TYson>::Id: case NUdf::TDataType<NUdf::TJsonDocument>::Id: case NUdf::TDataType<NUdf::TDyNumber>::Id: - dataNode = TDataLiteral::Create(env.NewStringValue(value.GetBytes()), dataType, env); - break; - case NUdf::TDataType<NUdf::TJson>::Id: - case NUdf::TDataType<NUdf::TUtf8>::Id: - dataNode = TDataLiteral::Create(env.NewStringValue(value.GetText()), dataType, env); - break; + dataNode = TDataLiteral::Create(env.NewStringValue(value.GetBytes()), dataType, env); + break; + case NUdf::TDataType<NUdf::TJson>::Id: + case NUdf::TDataType<NUdf::TUtf8>::Id: + dataNode = TDataLiteral::Create(env.NewStringValue(value.GetText()), dataType, env); + break; case NUdf::TDataType<NUdf::TTzDate>::Id: case NUdf::TDataType<NUdf::TTzDatetime>::Id: case NUdf::TDataType<NUdf::TTzTimestamp>::Id: dataNode = TDataLiteral::Create(ValueFromString(NUdf::GetDataSlot(dataType->GetSchemeType()), value.GetText()), dataType, env); break; - case NUdf::TDataType<NUdf::TDate>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(ui16(value.GetUint32())), dataType, env); - break; - case NUdf::TDataType<NUdf::TDatetime>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetUint32()), dataType, env); - break; - case NUdf::TDataType<NUdf::TTimestamp>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetUint64()), dataType, env); - break; - case NUdf::TDataType<NUdf::TInterval>::Id: - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetInt64()), dataType, env); - break; - case NUdf::TDataType<NUdf::TDecimal>::Id: { - using NYql::NDecimal::FromProto; - dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(FromProto(value)), dataType, env); - break; - } - case NUdf::TDataType<NUdf::TUuid>::Id: { - union { - ui64 half[2]; - char bytes[16]; - } buf; - buf.half[0] = value.GetLow128(); - buf.half[1] = value.GetHi128(); - dataNode = TDataLiteral::Create(env.NewStringValue(NUdf::TStringRef(buf.bytes, 16)), dataType, env); - break; - } - default: - MKQL_ENSURE(false, TStringBuilder() << "Unknown data type: " << schemeType); - } - return dataNode; - } - case TCallableType::EKind::Optional: { - TOptionalType* optionalType = static_cast<TOptionalType*>(type); - TOptionalLiteral* optionalNode; - if (value.HasOptional()) { - const NKikimrMiniKQL::TValue& protoValue = value.GetOptional(); - TNode* child = ImportNodeFromProto(optionalType->GetItemType(), protoValue); - optionalNode = TOptionalLiteral::Create(TRuntimeNode(child, true), optionalType, env); - } else { - optionalNode = TOptionalLiteral::Create(optionalType, env); - } - return optionalNode; - } - case TCallableType::EKind::List: { - TListType* listType = static_cast<TListType*>(type); - TType* itemType = listType->GetItemType(); - TVector<TRuntimeNode> items; - ui32 listSize = value.ListSize(); - items.reserve(listSize); - for (ui32 itemNum = 0; itemNum < listSize; ++itemNum) { - const NKikimrMiniKQL::TValue& protoValue = value.GetList(itemNum); - TNode* item = ImportNodeFromProto(itemType, protoValue); - items.push_back(TRuntimeNode(item, true)); - } - TListLiteral* listNode = TListLiteral::Create(items.data(), items.size(), listType, env); - return listNode; - } - case TCallableType::EKind::Tuple: { - TTupleType* tupleType = static_cast<TTupleType*>(type); - ui32 elementsCount = tupleType->GetElementsCount(); - MKQL_ENSURE(elementsCount == value.TupleSize(), "Invalid protobuf format, tuple size mismatch between Type and Value"); - TSmallVec<TRuntimeNode> elements(elementsCount); - for (ui32 elementIdx = 0; elementIdx < elementsCount; ++elementIdx) { - TType* elementType = tupleType->GetElementType(elementIdx); - auto& elementProtoValue = value.GetTuple(elementIdx); - TNode* elementNode = ImportNodeFromProto(elementType, elementProtoValue); - elements[elementIdx] = TRuntimeNode(elementNode, true); - } - TTupleLiteral* tupleNode = TTupleLiteral::Create(elements.size(), elements.data(), tupleType, env); - return tupleNode; - } - case TCallableType::EKind::Struct: { - TStructType* structType = static_cast<TStructType*>(type); - ui32 membersCount = structType->GetMembersCount(); - MKQL_ENSURE(membersCount == value.StructSize(), "Invalid protobuf format, struct size mismatch between Type and Value"); - TStackVec<TRuntimeNode, 16> members(membersCount); - TRemapArray remap = TRemapArray::FromCookie(structType->GetCookie()); - for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { - ui32 sortedMemberNum = remap.empty() ? memberNum : remap[memberNum]; - TType* memberType = structType->GetMemberType(memberNum); - const NKikimrMiniKQL::TValue& protoValue = value.GetStruct(sortedMemberNum); - TNode* child = ImportNodeFromProto(memberType, protoValue); - members[memberNum] = TRuntimeNode(child, true); - } - TStructLiteral* structNode = TStructLiteral::Create(members.size(), members.data(), structType, env); - return structNode; - } - case TCallableType::EKind::Dict: { - TDictType* dictType = static_cast<TDictType*>(type); - ui32 dictSize = value.DictSize(); - - TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; - for (ui32 itemNum = 0; itemNum < dictSize; ++itemNum) { - const NKikimrMiniKQL::TValuePair& protoPair = value.GetDict(itemNum); - TNode* keyNode = ImportNodeFromProto(dictType->GetKeyType(), protoPair.GetKey()); - TNode* payloadNode = ImportNodeFromProto(dictType->GetPayloadType(), protoPair.GetPayload()); - items.push_back(std::make_pair(TRuntimeNode(keyNode, true), TRuntimeNode(payloadNode, true))); - } - - TDictLiteral* dictNode = TDictLiteral::Create(items.size(), items.data(), dictType, env); - return dictNode; - } - case TCallableType::EKind::Variant: { - TVariantType* variantType = static_cast<TVariantType*>(type); - auto variantIndex = value.GetVariantIndex(); - TType* innerType = variantType->GetAlternativeType(variantIndex); - TNode* node = ImportNodeFromProto(innerType, value.GetOptional()); - return TVariantLiteral::Create(TRuntimeNode(node, true), variantIndex, variantType, env); - } - default: - break; - } - MKQL_ENSURE(false, TStringBuilder() << "Unknown protobuf type: " << type->GetKindAsStr()); -} - -void ExportTypeToProto(TType* type, NKikimrMiniKQL::TType& res) { - ExportTypeToProtoImpl(type, res); -} - + case NUdf::TDataType<NUdf::TDate>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(ui16(value.GetUint32())), dataType, env); + break; + case NUdf::TDataType<NUdf::TDatetime>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetUint32()), dataType, env); + break; + case NUdf::TDataType<NUdf::TTimestamp>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetUint64()), dataType, env); + break; + case NUdf::TDataType<NUdf::TInterval>::Id: + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(value.GetInt64()), dataType, env); + break; + case NUdf::TDataType<NUdf::TDecimal>::Id: { + using NYql::NDecimal::FromProto; + dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(FromProto(value)), dataType, env); + break; + } + case NUdf::TDataType<NUdf::TUuid>::Id: { + union { + ui64 half[2]; + char bytes[16]; + } buf; + buf.half[0] = value.GetLow128(); + buf.half[1] = value.GetHi128(); + dataNode = TDataLiteral::Create(env.NewStringValue(NUdf::TStringRef(buf.bytes, 16)), dataType, env); + break; + } + default: + MKQL_ENSURE(false, TStringBuilder() << "Unknown data type: " << schemeType); + } + return dataNode; + } + case TCallableType::EKind::Optional: { + TOptionalType* optionalType = static_cast<TOptionalType*>(type); + TOptionalLiteral* optionalNode; + if (value.HasOptional()) { + const NKikimrMiniKQL::TValue& protoValue = value.GetOptional(); + TNode* child = ImportNodeFromProto(optionalType->GetItemType(), protoValue); + optionalNode = TOptionalLiteral::Create(TRuntimeNode(child, true), optionalType, env); + } else { + optionalNode = TOptionalLiteral::Create(optionalType, env); + } + return optionalNode; + } + case TCallableType::EKind::List: { + TListType* listType = static_cast<TListType*>(type); + TType* itemType = listType->GetItemType(); + TVector<TRuntimeNode> items; + ui32 listSize = value.ListSize(); + items.reserve(listSize); + for (ui32 itemNum = 0; itemNum < listSize; ++itemNum) { + const NKikimrMiniKQL::TValue& protoValue = value.GetList(itemNum); + TNode* item = ImportNodeFromProto(itemType, protoValue); + items.push_back(TRuntimeNode(item, true)); + } + TListLiteral* listNode = TListLiteral::Create(items.data(), items.size(), listType, env); + return listNode; + } + case TCallableType::EKind::Tuple: { + TTupleType* tupleType = static_cast<TTupleType*>(type); + ui32 elementsCount = tupleType->GetElementsCount(); + MKQL_ENSURE(elementsCount == value.TupleSize(), "Invalid protobuf format, tuple size mismatch between Type and Value"); + TSmallVec<TRuntimeNode> elements(elementsCount); + for (ui32 elementIdx = 0; elementIdx < elementsCount; ++elementIdx) { + TType* elementType = tupleType->GetElementType(elementIdx); + auto& elementProtoValue = value.GetTuple(elementIdx); + TNode* elementNode = ImportNodeFromProto(elementType, elementProtoValue); + elements[elementIdx] = TRuntimeNode(elementNode, true); + } + TTupleLiteral* tupleNode = TTupleLiteral::Create(elements.size(), elements.data(), tupleType, env); + return tupleNode; + } + case TCallableType::EKind::Struct: { + TStructType* structType = static_cast<TStructType*>(type); + ui32 membersCount = structType->GetMembersCount(); + MKQL_ENSURE(membersCount == value.StructSize(), "Invalid protobuf format, struct size mismatch between Type and Value"); + TStackVec<TRuntimeNode, 16> members(membersCount); + TRemapArray remap = TRemapArray::FromCookie(structType->GetCookie()); + for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { + ui32 sortedMemberNum = remap.empty() ? memberNum : remap[memberNum]; + TType* memberType = structType->GetMemberType(memberNum); + const NKikimrMiniKQL::TValue& protoValue = value.GetStruct(sortedMemberNum); + TNode* child = ImportNodeFromProto(memberType, protoValue); + members[memberNum] = TRuntimeNode(child, true); + } + TStructLiteral* structNode = TStructLiteral::Create(members.size(), members.data(), structType, env); + return structNode; + } + case TCallableType::EKind::Dict: { + TDictType* dictType = static_cast<TDictType*>(type); + ui32 dictSize = value.DictSize(); + + TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; + for (ui32 itemNum = 0; itemNum < dictSize; ++itemNum) { + const NKikimrMiniKQL::TValuePair& protoPair = value.GetDict(itemNum); + TNode* keyNode = ImportNodeFromProto(dictType->GetKeyType(), protoPair.GetKey()); + TNode* payloadNode = ImportNodeFromProto(dictType->GetPayloadType(), protoPair.GetPayload()); + items.push_back(std::make_pair(TRuntimeNode(keyNode, true), TRuntimeNode(payloadNode, true))); + } + + TDictLiteral* dictNode = TDictLiteral::Create(items.size(), items.data(), dictType, env); + return dictNode; + } + case TCallableType::EKind::Variant: { + TVariantType* variantType = static_cast<TVariantType*>(type); + auto variantIndex = value.GetVariantIndex(); + TType* innerType = variantType->GetAlternativeType(variantIndex); + TNode* node = ImportNodeFromProto(innerType, value.GetOptional()); + return TVariantLiteral::Create(TRuntimeNode(node, true), variantIndex, variantType, env); + } + default: + break; + } + MKQL_ENSURE(false, TStringBuilder() << "Unknown protobuf type: " << type->GetKindAsStr()); +} + +void ExportTypeToProto(TType* type, NKikimrMiniKQL::TType& res) { + ExportTypeToProtoImpl(type, res); +} + void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder) { ExportValueToProtoImpl(type, value, res, columnOrder); -} - - +} + + NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, const NKikimrMiniKQL::TValue& value, const THolderFactory& factory) { - switch (type->GetKind()) { - case TType::EKind::Void: - return NUdf::TUnboxedValuePod::Void(); - + switch (type->GetKind()) { + case TType::EKind::Void: + return NUdf::TUnboxedValuePod::Void(); + case TType::EKind::Null: return NUdf::TUnboxedValuePod(); - case TType::EKind::Data: - return HandleKindDataImport(type, value); - - case TType::EKind::Optional: { + case TType::EKind::Data: + return HandleKindDataImport(type, value); + + case TType::EKind::Optional: { auto optionalType = static_cast<const TOptionalType*>(type); - if (value.HasOptional()) { + if (value.HasOptional()) { const TType* itemType = optionalType->GetItemType(); return ImportValueFromProto(itemType, value.GetOptional(), factory).MakeOptional(); - } - else { - return NUdf::TUnboxedValue(); - } - } - - case TType::EKind::List: { + } + else { + return NUdf::TUnboxedValue(); + } + } + + case TType::EKind::List: { auto listType = static_cast<const TListType*>(type); const TType* itemType = listType->GetItemType(); - const auto& list = value.GetList(); - NUdf::TUnboxedValue *items = nullptr; - auto array = factory.CreateDirectArrayHolder(list.size(), items); - for (const auto& x : list) { - *items++ = ImportValueFromProto(itemType, x, factory); - } - - return std::move(array); - } - - case TType::EKind::Struct: { + const auto& list = value.GetList(); + NUdf::TUnboxedValue *items = nullptr; + auto array = factory.CreateDirectArrayHolder(list.size(), items); + for (const auto& x : list) { + *items++ = ImportValueFromProto(itemType, x, factory); + } + + return std::move(array); + } + + case TType::EKind::Struct: { auto structType = static_cast<const TStructType*>(type); - NUdf::TUnboxedValue* itemsPtr = nullptr; - auto res = factory.CreateDirectArrayHolder(structType->GetMembersCount(), itemsPtr); + NUdf::TUnboxedValue* itemsPtr = nullptr; + auto res = factory.CreateDirectArrayHolder(structType->GetMembersCount(), itemsPtr); TRemapArray remap = TRemapArray::FromCookie(structType->GetCookie()); - for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { + for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { ui32 remapped = remap.empty() ? index : remap[index]; const TType* memberType = structType->GetMemberType(index); itemsPtr[index] = ImportValueFromProto(memberType, value.GetStruct(remapped), factory); - } - - return std::move(res); - } - - case TType::EKind::Tuple: { + } + + return std::move(res); + } + + case TType::EKind::Tuple: { auto tupleType = static_cast<const TTupleType*>(type); - NUdf::TUnboxedValue* itemsPtr = nullptr; - auto res = factory.CreateDirectArrayHolder(tupleType->GetElementsCount(), itemsPtr); - for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { + NUdf::TUnboxedValue* itemsPtr = nullptr; + auto res = factory.CreateDirectArrayHolder(tupleType->GetElementsCount(), itemsPtr); + for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { const TType* elementType = tupleType->GetElementType(index); - itemsPtr[index] = ImportValueFromProto(elementType, value.GetTuple(index), factory); - } - - return std::move(res); - } - - case TType::EKind::Dict: { + itemsPtr[index] = ImportValueFromProto(elementType, value.GetTuple(index), factory); + } + + return std::move(res); + } + + case TType::EKind::Dict: { auto dictType = static_cast<const TDictType*>(type); const TType* keyType = dictType->GetKeyType(); const TType* payloadType = dictType->GetPayloadType(); auto dictBuilder = factory.NewDict(dictType, NUdf::TDictFlags::EDictKind::Hashed); - - for (const auto& x : value.GetDict()) { - dictBuilder->Add( - ImportValueFromProto(keyType, x.GetKey(), factory), - ImportValueFromProto(payloadType, x.GetPayload(), factory) - ); - } - - return dictBuilder->Build(); - } - - default: - MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); - } -} - + + for (const auto& x : value.GetDict()) { + dictBuilder->Add( + ImportValueFromProto(keyType, x.GetKey(), factory), + ImportValueFromProto(payloadType, x.GetPayload(), factory) + ); + } + + return dictBuilder->Build(); + } + + default: + MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); + } +} + std::pair<TType*, NUdf::TUnboxedValue> ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, const TTypeEnvironment& env, const THolderFactory& factory) { @@ -1280,20 +1280,20 @@ TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type, const TTypeEnviron return importer.ImportTypeFromProto(type); } -TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, - const TTypeEnvironment& env) -{ - TProtoImporter importer(env); - TType* nodeType = importer.ImportTypeFromProto(type); - TNode* nodeValue = importer.ImportNodeFromProto(nodeType, value); - return TRuntimeNode(nodeValue, true); -} - -TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TParams& params, const TTypeEnvironment& env) { - if (params.HasType() && params.HasValue()) { - return ImportValueFromProto(params.GetType(), params.GetValue(), env); - } - return TRuntimeNode(env.GetEmptyStruct(), true); -} - -} +TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, + const TTypeEnvironment& env) +{ + TProtoImporter importer(env); + TType* nodeType = importer.ImportTypeFromProto(type); + TNode* nodeValue = importer.ImportNodeFromProto(nodeType, value); + return TRuntimeNode(nodeValue, true); +} + +TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TParams& params, const TTypeEnvironment& env) { + if (params.HasType() && params.HasValue()) { + return ImportValueFromProto(params.GetType(), params.GetValue(), env); + } + return TRuntimeNode(env.GetEmptyStruct(), true); +} + +} diff --git a/ydb/library/mkql_proto/mkql_proto.h b/ydb/library/mkql_proto/mkql_proto.h index 262ff035f2f..716f8530d3d 100644 --- a/ydb/library/mkql_proto/mkql_proto.h +++ b/ydb/library/mkql_proto/mkql_proto.h @@ -1,42 +1,42 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/minikql/defs.h> #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/library/mkql_proto/protos/minikql.pb.h> #include <ydb/public/api/protos/ydb_value.pb.h> - -namespace NKikimr::NMiniKQL { - -class THolderFactory; - -void ExportTypeToProto(TType* type, NKikimrMiniKQL::TType& res); + +namespace NKikimr::NMiniKQL { + +class THolderFactory; + +void ExportTypeToProto(TType* type, NKikimrMiniKQL::TType& res); void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder = nullptr); - + void ExportPrimitiveTypeToProto(ui32 schemeType, Ydb::Type& output); void ExportTypeToProto(TType* type, Ydb::Type& res); void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr); -TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type, const TTypeEnvironment& env); - +TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type, const TTypeEnvironment& env); + std::pair<TType*, NUdf::TUnboxedValue> ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, const TTypeEnvironment& env, const THolderFactory& factory); -TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, - const TTypeEnvironment& env); -TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TParams& params, const TTypeEnvironment& env); - -inline void UuidToMkqlProto(const char* str, size_t sz, NKikimrMiniKQL::TValue& res) { - union { - ui64 half[2]; - char bytes[sizeof(ui64) * 2]; - } buf; - Y_VERIFY(sizeof(buf) == sz); - memcpy(buf.bytes, str, sizeof(buf)); - res.SetLow128(buf.half[0]); - res.SetHi128(buf.half[1]); -} - +TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, + const TTypeEnvironment& env); +TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TParams& params, const TTypeEnvironment& env); + +inline void UuidToMkqlProto(const char* str, size_t sz, NKikimrMiniKQL::TValue& res) { + union { + ui64 half[2]; + char bytes[sizeof(ui64) * 2]; + } buf; + Y_VERIFY(sizeof(buf) == sz); + memcpy(buf.bytes, str, sizeof(buf)); + res.SetLow128(buf.half[0]); + res.SetHi128(buf.half[1]); +} + inline void UuidToYdbProto(const char* str, size_t sz, Ydb::Value& res) { union { ui64 half[2]; @@ -46,6 +46,6 @@ inline void UuidToYdbProto(const char* str, size_t sz, Ydb::Value& res) { memcpy(buf.bytes, str, sizeof(buf)); res.set_low_128(buf.half[0]); res.set_high_128(buf.half[1]); -} +} } diff --git a/ydb/library/mkql_proto/mkql_proto_ut.cpp b/ydb/library/mkql_proto/mkql_proto_ut.cpp index c0df31785b3..fdda085f18a 100644 --- a/ydb/library/mkql_proto/mkql_proto_ut.cpp +++ b/ydb/library/mkql_proto/mkql_proto_ut.cpp @@ -1,511 +1,511 @@ -#include "mkql_proto.h" - +#include "mkql_proto.h" + #include <ydb/library/mkql_proto/ut/helpers/helpers.h> - + using namespace std::string_view_literals; -namespace NKikimr::NMiniKQL { - -Y_UNIT_TEST_SUITE(TMiniKQLProtoTest) { - - Y_UNIT_TEST(TestCanExport) { - auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); - TScopedAlloc alloc; - TTypeEnvironment env(alloc); - TProgramBuilder pgmBuilder(env, *functionRegistry); - - UNIT_ASSERT(!CanExportType(pgmBuilder.NewVoid().GetStaticType()->GetType(), env)); - UNIT_ASSERT(CanExportType(pgmBuilder.NewVoid().GetStaticType(), env)); - auto dtype = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(CanExportType(dtype, env)); - UNIT_ASSERT(CanExportType(pgmBuilder.NewOptionalType(dtype), env)); - UNIT_ASSERT(CanExportType(pgmBuilder.NewListType(dtype), env)); - TVector<TType *> dtype1{dtype}; - UNIT_ASSERT(CanExportType(pgmBuilder.NewTupleType(dtype1), env)); - UNIT_ASSERT(CanExportType(pgmBuilder.NewStructType(pgmBuilder.NewEmptyStructType(), "x", dtype), env)); - UNIT_ASSERT(CanExportType(pgmBuilder.NewDictType(dtype, dtype, false), env)); - UNIT_ASSERT(!CanExportType(pgmBuilder.NewDictType(dtype, dtype->GetType(), false), env)); - } - - Y_UNIT_TEST(TestExportVoidType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder &pgmBuilder) { - auto pgmReturn = pgmBuilder.NewVoid(); - return pgmReturn; - }, "Kind: Void\n"); - } - - Y_UNIT_TEST(TestExportDataType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral<i32>(42); - return pgmReturn; - }, - "Kind: Data\n" - "Data {\n" - " Scheme: 1\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportDecimalType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { - NYql::NDecimal::TInt128 x; - ui64* p = (ui64*)&x; - p[0] = 1; - p[1] = 0; - auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, - "Kind: Data\n" - "Data {\n" - " Scheme: 4865\n" - " DecimalParams {\n" - " Precision: 21\n" - " Scale: 8\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportUuidType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { +namespace NKikimr::NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLProtoTest) { + + Y_UNIT_TEST(TestCanExport) { + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + TScopedAlloc alloc; + TTypeEnvironment env(alloc); + TProgramBuilder pgmBuilder(env, *functionRegistry); + + UNIT_ASSERT(!CanExportType(pgmBuilder.NewVoid().GetStaticType()->GetType(), env)); + UNIT_ASSERT(CanExportType(pgmBuilder.NewVoid().GetStaticType(), env)); + auto dtype = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(CanExportType(dtype, env)); + UNIT_ASSERT(CanExportType(pgmBuilder.NewOptionalType(dtype), env)); + UNIT_ASSERT(CanExportType(pgmBuilder.NewListType(dtype), env)); + TVector<TType *> dtype1{dtype}; + UNIT_ASSERT(CanExportType(pgmBuilder.NewTupleType(dtype1), env)); + UNIT_ASSERT(CanExportType(pgmBuilder.NewStructType(pgmBuilder.NewEmptyStructType(), "x", dtype), env)); + UNIT_ASSERT(CanExportType(pgmBuilder.NewDictType(dtype, dtype, false), env)); + UNIT_ASSERT(!CanExportType(pgmBuilder.NewDictType(dtype, dtype->GetType(), false), env)); + } + + Y_UNIT_TEST(TestExportVoidType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder &pgmBuilder) { + auto pgmReturn = pgmBuilder.NewVoid(); + return pgmReturn; + }, "Kind: Void\n"); + } + + Y_UNIT_TEST(TestExportDataType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewDataLiteral<i32>(42); + return pgmReturn; + }, + "Kind: Data\n" + "Data {\n" + " Scheme: 1\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportDecimalType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + NYql::NDecimal::TInt128 x; + ui64* p = (ui64*)&x; + p[0] = 1; + p[1] = 0; + auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); + return pgmReturn; + }, + "Kind: Data\n" + "Data {\n" + " Scheme: 4865\n" + " DecimalParams {\n" + " Precision: 21\n" + " Scale: 8\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportUuidType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { auto pgmReturn = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Uuid>(TStringBuf("\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"sv)); - return pgmReturn; - }, - "Kind: Data\n" - "Data {\n" - " Scheme: 4611\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportOptionalType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<i32>(42)); - return pgmReturn; - }, - "Kind: Optional\n" - "Optional {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportOptionalType2) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); - return pgmReturn; - }, - "Kind: Optional\n" - "Optional {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportOptionalOptionalType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewOptional( - pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id)); - return pgmReturn; - }, - "Kind: Optional\n" - "Optional {\n" - " Item {\n" - " Kind: Optional\n" - " Optional {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - " }\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportListType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.AsList(pgmBuilder.NewDataLiteral<i32>(42)); - return pgmReturn; - }, - "Kind: List\n" - "List {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportTupleType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { - TVector<TRuntimeNode> items; - items.push_back(pgmBuilder.NewDataLiteral<i32>(42)); - items.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); - auto pgmReturn = pgmBuilder.NewTuple(items); - return pgmReturn; - }, - "Kind: Tuple\n" - "Tuple {\n" - " Element {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - " Element {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportStructType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + return pgmReturn; + }, + "Kind: Data\n" + "Data {\n" + " Scheme: 4611\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportOptionalType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<i32>(42)); + return pgmReturn; + }, + "Kind: Optional\n" + "Optional {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportOptionalType2) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); + return pgmReturn; + }, + "Kind: Optional\n" + "Optional {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportOptionalOptionalType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewOptional( + pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id)); + return pgmReturn; + }, + "Kind: Optional\n" + "Optional {\n" + " Item {\n" + " Kind: Optional\n" + " Optional {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + " }\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportListType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.AsList(pgmBuilder.NewDataLiteral<i32>(42)); + return pgmReturn; + }, + "Kind: List\n" + "List {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportTupleType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + TVector<TRuntimeNode> items; + items.push_back(pgmBuilder.NewDataLiteral<i32>(42)); + items.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); + auto pgmReturn = pgmBuilder.NewTuple(items); + return pgmReturn; + }, + "Kind: Tuple\n" + "Tuple {\n" + " Element {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + " Element {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportStructType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { std::vector<std::pair<std::string_view, TRuntimeNode>> items; - items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); - items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto pgmReturn = pgmBuilder.NewStruct(items); - return pgmReturn; - }, - "Kind: Struct\n" - "Struct {\n" - " Member {\n" - " Name: \"x\"\n" - " Type {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - " }\n" - " Member {\n" - " Name: \"y\"\n" - " Type {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportDictType) { - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { - auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id), - pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); - TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; - items.push_back({ pgmBuilder.NewDataLiteral<i32>(42), - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto pgmReturn = pgmBuilder.NewDict(dictType, items); - return pgmReturn; - }, - "Kind: Dict\n" - "Dict {\n" - " Key {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - " Payload {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportVariantTupleType) { - const TString expected = R"___(Kind: Variant -Variant { - TupleItems { - Element { - Kind: Data - Data { - Scheme: 4 - } - } - Element { - Kind: Data - Data { - Scheme: 2 - } - } - } -} -)___"; - - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { - std::vector<TType*> tupleElemenTypes; - tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); - tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); - auto pgmReturn = pgmBuilder.NewVariant( - pgmBuilder.NewDataLiteral<ui32>(66), - 1, - pgmBuilder.NewVariantType(pgmBuilder.NewTupleType(tupleElemenTypes))); - return pgmReturn; - }, - expected); - } - - Y_UNIT_TEST(TestExportVariantStructType) { - const TString expected = R"___(Kind: Variant -Variant { - StructItems { - Member { - Name: "a" - Type { - Kind: Data - Data { - Scheme: 4 - } - } - } - Member { - Name: "b" - Type { - Kind: Data - Data { - Scheme: 2 - } - } - } - } -} -)___"; - - TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); + items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto pgmReturn = pgmBuilder.NewStruct(items); + return pgmReturn; + }, + "Kind: Struct\n" + "Struct {\n" + " Member {\n" + " Name: \"x\"\n" + " Type {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + " }\n" + " Member {\n" + " Name: \"y\"\n" + " Type {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportDictType) { + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id), + pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); + TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; + items.push_back({ pgmBuilder.NewDataLiteral<i32>(42), + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto pgmReturn = pgmBuilder.NewDict(dictType, items); + return pgmReturn; + }, + "Kind: Dict\n" + "Dict {\n" + " Key {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + " Payload {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportVariantTupleType) { + const TString expected = R"___(Kind: Variant +Variant { + TupleItems { + Element { + Kind: Data + Data { + Scheme: 4 + } + } + Element { + Kind: Data + Data { + Scheme: 2 + } + } + } +} +)___"; + + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { + std::vector<TType*> tupleElemenTypes; + tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); + tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto pgmReturn = pgmBuilder.NewVariant( + pgmBuilder.NewDataLiteral<ui32>(66), + 1, + pgmBuilder.NewVariantType(pgmBuilder.NewTupleType(tupleElemenTypes))); + return pgmReturn; + }, + expected); + } + + Y_UNIT_TEST(TestExportVariantStructType) { + const TString expected = R"___(Kind: Variant +Variant { + StructItems { + Member { + Name: "a" + Type { + Kind: Data + Data { + Scheme: 4 + } + } + } + Member { + Name: "b" + Type { + Kind: Data + Data { + Scheme: 2 + } + } + } + } +} +)___"; + + TestExportType<NKikimrMiniKQL::TType>([](TProgramBuilder& pgmBuilder) { std::vector<std::pair<std::string_view, TType*>> structElemenTypes; structElemenTypes.push_back({"a", pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}); structElemenTypes.push_back({"b", pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)}); - TType* structType = pgmBuilder.NewStructType(structElemenTypes); - auto pgmReturn = pgmBuilder.NewVariant( - pgmBuilder.NewDataLiteral<ui32>(66), - "b", - pgmBuilder.NewVariantType(structType)); - return pgmReturn; - }, - expected); - } - - Y_UNIT_TEST(TestExportVoid) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewVoid(); - return pgmReturn; - }, ""); - } - - Y_UNIT_TEST(TestExportBool) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral(true); - return pgmReturn; - }, "Bool: true\n"); - } - - Y_UNIT_TEST(TestExportIntegral) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral<i32>(42); - return pgmReturn; - }, "Int32: 42\n"); - } - - Y_UNIT_TEST(TestExportDouble) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral(3.5); - return pgmReturn; - }, "Double: 3.5\n"); - } - - Y_UNIT_TEST(TestExportString) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc"); - return pgmReturn; - }, "Bytes: \"abc\"\n"); - } - - Y_UNIT_TEST(TestExportUuid) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + TType* structType = pgmBuilder.NewStructType(structElemenTypes); + auto pgmReturn = pgmBuilder.NewVariant( + pgmBuilder.NewDataLiteral<ui32>(66), + "b", + pgmBuilder.NewVariantType(structType)); + return pgmReturn; + }, + expected); + } + + Y_UNIT_TEST(TestExportVoid) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewVoid(); + return pgmReturn; + }, ""); + } + + Y_UNIT_TEST(TestExportBool) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewDataLiteral(true); + return pgmReturn; + }, "Bool: true\n"); + } + + Y_UNIT_TEST(TestExportIntegral) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewDataLiteral<i32>(42); + return pgmReturn; + }, "Int32: 42\n"); + } + + Y_UNIT_TEST(TestExportDouble) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewDataLiteral(3.5); + return pgmReturn; + }, "Double: 3.5\n"); + } + + Y_UNIT_TEST(TestExportString) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + return pgmReturn; + }, "Bytes: \"abc\"\n"); + } + + Y_UNIT_TEST(TestExportUuid) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { auto pgmReturn = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Uuid>(TStringBuf("\1\0\0\0\0\0\0\0\2\0\0\0\0\0\0\0"sv)); - return pgmReturn; - }, "Low128: 1\n" - "Hi128: 2\n"); - } - - Y_UNIT_TEST(TestExportDecimal) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - NYql::NDecimal::TInt128 x; - ui64* p = (ui64*)&x; - p[0] = 1; - p[1] = 0; - auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, "Low128: 1\n" - "Hi128: 0\n"); - } - - Y_UNIT_TEST(TestImportDecimal) { - const TString type = - "Kind: Data\n" - "Data {\n" - " Scheme: 4865\n" - " DecimalParams {\n" - " Precision: 21\n" - " Scale: 8\n" - " }\n" - "}\n"; - const TString value = - "Low128: 1\n" - "Hi128: 0\n"; - - TestImportParams<NKikimrMiniKQL::TParams, NKikimrMiniKQL::TType, NKikimrMiniKQL::TValue>(type, value); - } - - Y_UNIT_TEST(TestExportDecimalNegative) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - NYql::NDecimal::TInt128 x; - ui64* p = (ui64*)&x; - p[0] = Max<ui64>(); - p[1] = Max<ui64>(); - auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, "Low128: 18446744073709551615\n" - "Hi128: 18446744073709551615\n"); - } - - Y_UNIT_TEST(TestExportDecimalMax64bit) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - NYql::NDecimal::TInt128 x; - ui64* p = (ui64*)&x; - p[0] = Max<ui64>(); - p[1] = 0; - auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, "Low128: 18446744073709551615\n" - "Hi128: 0\n"); - } - - Y_UNIT_TEST(TestExportDecimalHuge) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - NYql::NDecimal::TInt128 x; - ui64* p = (ui64*)&x; - p[0] = 0; - p[1] = 1; - auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, "Low128: 0\n" - "Hi128: 1\n"); - } - - Y_UNIT_TEST(TestExportDecimalHugePlusOne) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - NYql::NDecimal::TInt128 x; - ui64* p = (ui64*)&x; - p[0] = 1; - p[1] = 1; - auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, "Low128: 1\n" - "Hi128: 1\n"); - } - - Y_UNIT_TEST(TestExportDecimalNan) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDecimalLiteral(NYql::NDecimal::Nan(), 21, 8); - return pgmReturn; - }, "Low128: 3136633892082024449\n" - "Hi128: 5421010862427522\n"); - } - - Y_UNIT_TEST(TestExportDecimalMunusInf) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDecimalLiteral(-NYql::NDecimal::Inf(), 21, 8); - return pgmReturn; - }, "Low128: 15310110181627527168\n" - "Hi128: 18441323062847124093\n"); - } - - Y_UNIT_TEST(TestExportEmptyOptional) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); - return pgmReturn; - }, ""); - } - - Y_UNIT_TEST(TestExportEmptyOptionalOptional) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewOptional( - pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id)); - return pgmReturn; - }, - "Optional {\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportOptional) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); - return pgmReturn; - }, - "Optional {\n" - " Bytes: \"abc\"\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportList) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.AsList(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); - return pgmReturn; - }, - "List {\n" - " Bytes: \"abc\"\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportTuple) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + return pgmReturn; + }, "Low128: 1\n" + "Hi128: 2\n"); + } + + Y_UNIT_TEST(TestExportDecimal) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + NYql::NDecimal::TInt128 x; + ui64* p = (ui64*)&x; + p[0] = 1; + p[1] = 0; + auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); + return pgmReturn; + }, "Low128: 1\n" + "Hi128: 0\n"); + } + + Y_UNIT_TEST(TestImportDecimal) { + const TString type = + "Kind: Data\n" + "Data {\n" + " Scheme: 4865\n" + " DecimalParams {\n" + " Precision: 21\n" + " Scale: 8\n" + " }\n" + "}\n"; + const TString value = + "Low128: 1\n" + "Hi128: 0\n"; + + TestImportParams<NKikimrMiniKQL::TParams, NKikimrMiniKQL::TType, NKikimrMiniKQL::TValue>(type, value); + } + + Y_UNIT_TEST(TestExportDecimalNegative) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + NYql::NDecimal::TInt128 x; + ui64* p = (ui64*)&x; + p[0] = Max<ui64>(); + p[1] = Max<ui64>(); + auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); + return pgmReturn; + }, "Low128: 18446744073709551615\n" + "Hi128: 18446744073709551615\n"); + } + + Y_UNIT_TEST(TestExportDecimalMax64bit) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + NYql::NDecimal::TInt128 x; + ui64* p = (ui64*)&x; + p[0] = Max<ui64>(); + p[1] = 0; + auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); + return pgmReturn; + }, "Low128: 18446744073709551615\n" + "Hi128: 0\n"); + } + + Y_UNIT_TEST(TestExportDecimalHuge) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + NYql::NDecimal::TInt128 x; + ui64* p = (ui64*)&x; + p[0] = 0; + p[1] = 1; + auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); + return pgmReturn; + }, "Low128: 0\n" + "Hi128: 1\n"); + } + + Y_UNIT_TEST(TestExportDecimalHugePlusOne) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + NYql::NDecimal::TInt128 x; + ui64* p = (ui64*)&x; + p[0] = 1; + p[1] = 1; + auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); + return pgmReturn; + }, "Low128: 1\n" + "Hi128: 1\n"); + } + + Y_UNIT_TEST(TestExportDecimalNan) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewDecimalLiteral(NYql::NDecimal::Nan(), 21, 8); + return pgmReturn; + }, "Low128: 3136633892082024449\n" + "Hi128: 5421010862427522\n"); + } + + Y_UNIT_TEST(TestExportDecimalMunusInf) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewDecimalLiteral(-NYql::NDecimal::Inf(), 21, 8); + return pgmReturn; + }, "Low128: 15310110181627527168\n" + "Hi128: 18441323062847124093\n"); + } + + Y_UNIT_TEST(TestExportEmptyOptional) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); + return pgmReturn; + }, ""); + } + + Y_UNIT_TEST(TestExportEmptyOptionalOptional) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewOptional( + pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id)); + return pgmReturn; + }, + "Optional {\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportOptional) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); + return pgmReturn; + }, + "Optional {\n" + " Bytes: \"abc\"\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportList) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.AsList(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); + return pgmReturn; + }, + "List {\n" + " Bytes: \"abc\"\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportTuple) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { TRuntimeNode::TList items; - items.push_back(pgmBuilder.NewDataLiteral<i32>(42)); - items.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); - auto pgmReturn = pgmBuilder.NewTuple(items); - return pgmReturn; - }, - "Tuple {\n" - " Int32: 42\n" - "}\n" - "Tuple {\n" - " Bytes: \"abc\"\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportStruct) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + items.push_back(pgmBuilder.NewDataLiteral<i32>(42)); + items.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); + auto pgmReturn = pgmBuilder.NewTuple(items); + return pgmReturn; + }, + "Tuple {\n" + " Int32: 42\n" + "}\n" + "Tuple {\n" + " Bytes: \"abc\"\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportStruct) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { std::vector<std::pair<std::string_view, TRuntimeNode>> items; - items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); - items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto pgmReturn = pgmBuilder.NewStruct(items); - return pgmReturn; - }, - "Struct {\n" - " Int32: 42\n" - "}\n" - "Struct {\n" - " Bytes: \"abc\"\n" - "}\n"); - } - + items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); + items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto pgmReturn = pgmBuilder.NewStruct(items); + return pgmReturn; + }, + "Struct {\n" + " Int32: 42\n" + "}\n" + "Struct {\n" + " Bytes: \"abc\"\n" + "}\n"); + } + Y_UNIT_TEST(TestExportStructEmptyColumnOrder) { const TVector<ui32> emptyColumnOrder; TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { @@ -571,87 +571,87 @@ Variant { &columnOrder); } - Y_UNIT_TEST(TestExportDict) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id), - pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); - TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; - items.push_back({ pgmBuilder.NewDataLiteral<i32>(42), - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto pgmReturn = pgmBuilder.NewDict(dictType, items); - return pgmReturn; - }, - "Dict {\n" - " Key {\n" - " Int32: 42\n" - " }\n" - " Payload {\n" - " Bytes: \"abc\"\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportVariant) { - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - std::vector<TType*> tupleElemenTypes; - tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); - tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); - auto pgmReturn = pgmBuilder.NewVariant( - pgmBuilder.NewDataLiteral<ui32>(66), - 1, - pgmBuilder.NewVariantType(pgmBuilder.NewTupleType(tupleElemenTypes))); - return pgmReturn; - }, - "Optional {\n" - " Uint32: 66\n" - "}\n" - "VariantIndex: 1\n"); - } - - Y_UNIT_TEST(TestImportVariant) { - const TString type = R"___(Kind: Variant -Variant { - TupleItems { - Element { - Kind: Data - Data { - Scheme: 4 - } - } - Element { - Kind: Data - Data { - Scheme: 2 - } - } - } -} -)___"; - - const TString value = R"___(Optional { - Uint32: 66 -} -VariantIndex: 1 -)___"; - - TestImportParams<NKikimrMiniKQL::TParams, NKikimrMiniKQL::TType, NKikimrMiniKQL::TValue>(type, value); - } - - Y_UNIT_TEST(TestImportUuid) { - const TString type = R"___(Kind: Data -Data { - Scheme: 4611 -} -)___"; - - - const TString value = R"___(Low128: 1 -Hi128: 2 -)___"; - - TestImportParams<NKikimrMiniKQL::TParams, NKikimrMiniKQL::TType, NKikimrMiniKQL::TValue>(type, value); - } - -} - -} + Y_UNIT_TEST(TestExportDict) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id), + pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); + TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; + items.push_back({ pgmBuilder.NewDataLiteral<i32>(42), + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto pgmReturn = pgmBuilder.NewDict(dictType, items); + return pgmReturn; + }, + "Dict {\n" + " Key {\n" + " Int32: 42\n" + " }\n" + " Payload {\n" + " Bytes: \"abc\"\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportVariant) { + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + std::vector<TType*> tupleElemenTypes; + tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); + tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto pgmReturn = pgmBuilder.NewVariant( + pgmBuilder.NewDataLiteral<ui32>(66), + 1, + pgmBuilder.NewVariantType(pgmBuilder.NewTupleType(tupleElemenTypes))); + return pgmReturn; + }, + "Optional {\n" + " Uint32: 66\n" + "}\n" + "VariantIndex: 1\n"); + } + + Y_UNIT_TEST(TestImportVariant) { + const TString type = R"___(Kind: Variant +Variant { + TupleItems { + Element { + Kind: Data + Data { + Scheme: 4 + } + } + Element { + Kind: Data + Data { + Scheme: 2 + } + } + } +} +)___"; + + const TString value = R"___(Optional { + Uint32: 66 +} +VariantIndex: 1 +)___"; + + TestImportParams<NKikimrMiniKQL::TParams, NKikimrMiniKQL::TType, NKikimrMiniKQL::TValue>(type, value); + } + + Y_UNIT_TEST(TestImportUuid) { + const TString type = R"___(Kind: Data +Data { + Scheme: 4611 +} +)___"; + + + const TString value = R"___(Low128: 1 +Hi128: 2 +)___"; + + TestImportParams<NKikimrMiniKQL::TParams, NKikimrMiniKQL::TType, NKikimrMiniKQL::TValue>(type, value); + } + +} + +} diff --git a/ydb/library/mkql_proto/ut/helpers/helpers.cpp b/ydb/library/mkql_proto/ut/helpers/helpers.cpp index 2c3c5b3661a..3e6ab73e9ec 100644 --- a/ydb/library/mkql_proto/ut/helpers/helpers.cpp +++ b/ydb/library/mkql_proto/ut/helpers/helpers.cpp @@ -1 +1 @@ -#include "helpers.h" +#include "helpers.h" diff --git a/ydb/library/mkql_proto/ut/helpers/helpers.h b/ydb/library/mkql_proto/ut/helpers/helpers.h index 56f261f7f4a..e5094e8085d 100644 --- a/ydb/library/mkql_proto/ut/helpers/helpers.h +++ b/ydb/library/mkql_proto/ut/helpers/helpers.h @@ -1,96 +1,96 @@ #pragma once #include <ydb/library/yql/minikql/mkql_program_builder.h> - + #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> #include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> - + #include <google/protobuf/text_format.h> #include <library/cpp/testing/unittest/registar.h> - -#include <functional> - -namespace NKikimr::NMiniKQL { - -template <typename TProtoType> -void TestExportType(std::function<TRuntimeNode(TProgramBuilder& pgmBuilder)> setup, const TString& expectedString) { - auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); - TScopedAlloc alloc; - TTypeEnvironment env(alloc); - TProgramBuilder pgmBuilder(env, *functionRegistry); - - auto pgmReturn = setup(pgmBuilder); - - TProtoType res; - ExportTypeToProto(pgmReturn.GetStaticType(), res); - using ::google::protobuf::TextFormat; - TString str; - TextFormat::PrintToString(res, &str); - UNIT_ASSERT_NO_DIFF(str, expectedString); -} - -template <typename TProtoValue> + +#include <functional> + +namespace NKikimr::NMiniKQL { + +template <typename TProtoType> +void TestExportType(std::function<TRuntimeNode(TProgramBuilder& pgmBuilder)> setup, const TString& expectedString) { + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + TScopedAlloc alloc; + TTypeEnvironment env(alloc); + TProgramBuilder pgmBuilder(env, *functionRegistry); + + auto pgmReturn = setup(pgmBuilder); + + TProtoType res; + ExportTypeToProto(pgmReturn.GetStaticType(), res); + using ::google::protobuf::TextFormat; + TString str; + TextFormat::PrintToString(res, &str); + UNIT_ASSERT_NO_DIFF(str, expectedString); +} + +template <typename TProtoValue> void TestExportValue(std::function<TRuntimeNode(TProgramBuilder& pgmBuilder)> setup, const TString& expectedString, const TVector<ui32>* columnOrder = nullptr) { - auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); - auto randomProvider = CreateDeterministicRandomProvider(1); - auto timeProvider = CreateDeterministicTimeProvider(1); - TScopedAlloc alloc; - TTypeEnvironment env(alloc); - TProgramBuilder pgmBuilder(env, *functionRegistry); - - auto pgmReturn = setup(pgmBuilder); - - TExploringNodeVisitor explorer; - explorer.Walk(pgmReturn.GetNode(), env); - TComputationPatternOpts opts(alloc.Ref(), env, GetBuiltinFactory(), - functionRegistry.Get(), NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, - "OFF", EGraphPerProcess::Multi); - auto pattern = MakeComputationPattern(explorer, pgmReturn, {}, opts); - auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider)); - TProtoValue res; + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + auto randomProvider = CreateDeterministicRandomProvider(1); + auto timeProvider = CreateDeterministicTimeProvider(1); + TScopedAlloc alloc; + TTypeEnvironment env(alloc); + TProgramBuilder pgmBuilder(env, *functionRegistry); + + auto pgmReturn = setup(pgmBuilder); + + TExploringNodeVisitor explorer; + explorer.Walk(pgmReturn.GetNode(), env); + TComputationPatternOpts opts(alloc.Ref(), env, GetBuiltinFactory(), + functionRegistry.Get(), NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, + "OFF", EGraphPerProcess::Multi); + auto pattern = MakeComputationPattern(explorer, pgmReturn, {}, opts); + auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider)); + TProtoValue res; ExportValueToProto(pgmReturn.GetStaticType(), graph->GetValue(), res, columnOrder); - using ::google::protobuf::TextFormat; - TString str; - TextFormat::PrintToString(res, &str); - UNIT_ASSERT_NO_DIFF(str, expectedString); -} - -template <typename TParam, typename TType, typename TValue> -void TestImportParams(const TString& type, const TString& value) { - auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); - auto randomProvider = CreateDeterministicRandomProvider(1); - auto timeProvider = CreateDeterministicTimeProvider(1); - - TParam protoParam; - TType protoType; - TValue protoValue; - ::google::protobuf::TextFormat::ParseFromString(type, &protoType); - ::google::protobuf::TextFormat::ParseFromString(value, &protoValue); - protoParam.MutableType()->CopyFrom(protoType); - protoParam.MutableValue()->CopyFrom(protoValue); - - TScopedAlloc alloc; - TTypeEnvironment env(alloc); - auto runtimeNode = ImportValueFromProto(protoParam, env); - UNIT_ASSERT(runtimeNode.HasValue()); - - TValue res; - - TExploringNodeVisitor explorer; - explorer.Walk(runtimeNode.GetValue(), env); - TComputationPatternOpts opts(alloc.Ref(), env, GetBuiltinFactory(), - functionRegistry.Get(), NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, - "OFF", EGraphPerProcess::Multi); - auto pattern = MakeComputationPattern(explorer, runtimeNode, {}, opts); - - auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider)); - ExportValueToProto(runtimeNode.GetStaticType(), graph->GetValue(), res); - using ::google::protobuf::TextFormat; - TString str; - TextFormat::PrintToString(res, &str); - - UNIT_ASSERT_NO_DIFF(str, value); -} - -} + using ::google::protobuf::TextFormat; + TString str; + TextFormat::PrintToString(res, &str); + UNIT_ASSERT_NO_DIFF(str, expectedString); +} + +template <typename TParam, typename TType, typename TValue> +void TestImportParams(const TString& type, const TString& value) { + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + auto randomProvider = CreateDeterministicRandomProvider(1); + auto timeProvider = CreateDeterministicTimeProvider(1); + + TParam protoParam; + TType protoType; + TValue protoValue; + ::google::protobuf::TextFormat::ParseFromString(type, &protoType); + ::google::protobuf::TextFormat::ParseFromString(value, &protoValue); + protoParam.MutableType()->CopyFrom(protoType); + protoParam.MutableValue()->CopyFrom(protoValue); + + TScopedAlloc alloc; + TTypeEnvironment env(alloc); + auto runtimeNode = ImportValueFromProto(protoParam, env); + UNIT_ASSERT(runtimeNode.HasValue()); + + TValue res; + + TExploringNodeVisitor explorer; + explorer.Walk(runtimeNode.GetValue(), env); + TComputationPatternOpts opts(alloc.Ref(), env, GetBuiltinFactory(), + functionRegistry.Get(), NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, + "OFF", EGraphPerProcess::Multi); + auto pattern = MakeComputationPattern(explorer, runtimeNode, {}, opts); + + auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider)); + ExportValueToProto(runtimeNode.GetStaticType(), graph->GetValue(), res); + using ::google::protobuf::TextFormat; + TString str; + TextFormat::PrintToString(res, &str); + + UNIT_ASSERT_NO_DIFF(str, value); +} + +} diff --git a/ydb/library/mkql_proto/ut/helpers/ya.make b/ydb/library/mkql_proto/ut/helpers/ya.make index 4ad55af1f51..8bb5d97871d 100644 --- a/ydb/library/mkql_proto/ut/helpers/ya.make +++ b/ydb/library/mkql_proto/ut/helpers/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -PEERDIR( + +PEERDIR( ydb/library/yql/minikql ydb/library/yql/minikql/comp_nodes ydb/library/yql/minikql/computation ydb/library/yql/minikql/invoke_builtins library/cpp/testing/unittest - contrib/libs/protobuf -) - + contrib/libs/protobuf +) + YQL_LAST_ABI_VERSION() -SRCS( - helpers.cpp -) - -END() +SRCS( + helpers.cpp +) + +END() diff --git a/ydb/library/mkql_proto/ut/ya.make b/ydb/library/mkql_proto/ut/ya.make index b38918687f1..3edb42746e1 100644 --- a/ydb/library/mkql_proto/ut/ya.make +++ b/ydb/library/mkql_proto/ut/ya.make @@ -1,25 +1,25 @@ UNITTEST_FOR(ydb/library/mkql_proto) - + OWNER(g:kikimr) - -ALLOCATOR(J) - -FORK_SUBTESTS() - -TIMEOUT(150) - -SIZE(MEDIUM) - -SRCS( - mkql_proto_ut.cpp -) - -PEERDIR( + +ALLOCATOR(J) + +FORK_SUBTESTS() + +TIMEOUT(150) + +SIZE(MEDIUM) + +SRCS( + mkql_proto_ut.cpp +) + +PEERDIR( ydb/library/mkql_proto/ut/helpers ydb/library/yql/public/udf/service/exception_policy ydb/core/yql_testlib -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/library/mkql_proto/ya.make b/ydb/library/mkql_proto/ya.make index 08caeebde48..ff651fa6635 100644 --- a/ydb/library/mkql_proto/ya.make +++ b/ydb/library/mkql_proto/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -PEERDIR( + +PEERDIR( ydb/library/mkql_proto/protos ydb/library/mkql_proto/ut/helpers ydb/public/api/protos ydb/library/yql/minikql ydb/library/yql/minikql/computation -) - -SRCS( - mkql_proto.cpp -) - +) + +SRCS( + mkql_proto.cpp +) + YQL_LAST_ABI_VERSION() -END() +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h index 29ef8272dca..84eeda2edbf 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h @@ -8,7 +8,7 @@ #include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h> #include <ydb/library/yql/dq/runtime/dq_tasks_runner.h> #include <ydb/library/yql/dq/runtime/dq_transport.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/log.h> diff --git a/ydb/library/yql/dq/actors/protos/dq_events.proto b/ydb/library/yql/dq/actors/protos/dq_events.proto index 11e1dcf9665..0e38ab3bbc1 100644 --- a/ydb/library/yql/dq/actors/protos/dq_events.proto +++ b/ydb/library/yql/dq/actors/protos/dq_events.proto @@ -22,7 +22,7 @@ message TCheckpoint { */ message TChannelData { optional uint64 ChannelId = 1; - optional NYql.NDqProto.TData Data = 2; + optional NYql.NDqProto.TData Data = 2; optional bool Finished = 3; optional TCheckpoint Checkpoint = 4; }; diff --git a/ydb/library/yql/dq/common/dq_value.h b/ydb/library/yql/dq/common/dq_value.h index 7a308aea4c2..6699f304113 100644 --- a/ydb/library/yql/dq/common/dq_value.h +++ b/ydb/library/yql/dq/common/dq_value.h @@ -1,28 +1,28 @@ -#pragma once - +#pragma once + #include <ydb/library/mkql_proto/protos/minikql.pb.h> - -namespace NYql::NDq { - -class TMkqlValueRef { -public: + +namespace NYql::NDq { + +class TMkqlValueRef { +public: TMkqlValueRef(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value) : Type(&type) , Value(&value) {} - + explicit TMkqlValueRef(const NKikimrMiniKQL::TParams& params) : TMkqlValueRef(params.GetType(), params.GetValue()) {} - + explicit TMkqlValueRef(const NKikimrMiniKQL::TResult& result) : TMkqlValueRef(result.GetType(), result.GetValue()) {} - + const NKikimrMiniKQL::TType& GetType() const { return *Type; } - + const NKikimrMiniKQL::TValue& GetValue() const { return *Value; } - -private: - const NKikimrMiniKQL::TType *Type; - const NKikimrMiniKQL::TValue *Value; -}; - -} + +private: + const NKikimrMiniKQL::TType *Type; + const NKikimrMiniKQL::TValue *Value; +}; + +} diff --git a/ydb/library/yql/dq/runtime/dq_columns_resolve.cpp b/ydb/library/yql/dq/runtime/dq_columns_resolve.cpp index a8574cb498d..850f6108d9e 100644 --- a/ydb/library/yql/dq/runtime/dq_columns_resolve.cpp +++ b/ydb/library/yql/dq/runtime/dq_columns_resolve.cpp @@ -1,37 +1,37 @@ #include "dq_columns_resolve.h" - + #include <ydb/library/yql/utils/yql_panic.h> - -namespace NYql::NDq { - -using namespace NKikimr::NMiniKQL; - + +namespace NYql::NDq { + +using namespace NKikimr::NMiniKQL; + TMaybe<TColumnInfo> FindColumnInfo(const NKikimr::NMiniKQL::TType* type, TStringBuf columnName) { - YQL_ENSURE(type->GetKind() == TType::EKind::Struct); + YQL_ENSURE(type->GetKind() == TType::EKind::Struct); const auto& structType = static_cast<const TStructType&>(*type); - + auto columnIndex = structType.FindMemberIndex(columnName); if (!columnIndex) { return {}; } - + auto memberType = structType.GetMemberType(*columnIndex); - - if (memberType->GetKind() == TType::EKind::Optional) { - memberType = static_cast<TOptionalType&>(*memberType).GetItemType(); - } - - YQL_ENSURE(memberType->GetKind() == TType::EKind::Data); - auto columnType = static_cast<TDataType&>(*memberType).GetSchemeType(); - + + if (memberType->GetKind() == TType::EKind::Optional) { + memberType = static_cast<TOptionalType&>(*memberType).GetItemType(); + } + + YQL_ENSURE(memberType->GetKind() == TType::EKind::Data); + auto columnType = static_cast<TDataType&>(*memberType).GetSchemeType(); + return TColumnInfo{TString(columnName), *columnIndex, columnType}; -} - +} + TColumnInfo GetColumnInfo(const TType* type, TStringBuf columnName) { auto columnInfo = FindColumnInfo(type, columnName); YQL_ENSURE(columnInfo); return *columnInfo; -} +} } diff --git a/ydb/library/yql/dq/runtime/dq_columns_resolve.h b/ydb/library/yql/dq/runtime/dq_columns_resolve.h index 10297718cfc..b61d14f4778 100644 --- a/ydb/library/yql/dq/runtime/dq_columns_resolve.h +++ b/ydb/library/yql/dq/runtime/dq_columns_resolve.h @@ -1,17 +1,17 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/public/udf/udf_data_type.h> #include <ydb/library/yql/minikql/mkql_node.h> // #include <ydb/library/yql/dq/proto/dq_tasks.pb.h> - -namespace NYql::NDq { - -struct TColumnInfo { + +namespace NYql::NDq { + +struct TColumnInfo { TString Name; ui32 Index; - NUdf::TDataTypeId TypeId; -}; - + NUdf::TDataTypeId TypeId; +}; + struct TSortColumnInfo : public TColumnInfo { bool Ascending; @@ -23,24 +23,24 @@ struct TSortColumnInfo : public TColumnInfo { TMaybe<TColumnInfo> FindColumnInfo(const NKikimr::NMiniKQL::TType* type, TStringBuf column); TColumnInfo GetColumnInfo(const NKikimr::NMiniKQL::TType* type, TStringBuf column); - -template<typename TList> + +template<typename TList> void GetColumnsInfo(const NKikimr::NMiniKQL::TType* type, const TList& columns, - TVector<NUdf::TDataTypeId>& columnTypes, TVector<ui32>& columnIndices) -{ - columnTypes.clear(); - columnIndices.clear(); - + TVector<NUdf::TDataTypeId>& columnTypes, TVector<ui32>& columnIndices) +{ + columnTypes.clear(); + columnIndices.clear(); + columnTypes.reserve(columns.size()); columnIndices.reserve(columns.size()); - for (auto& column : columns) { - auto columnInfo = GetColumnInfo(type, column); - columnTypes.push_back(columnInfo.TypeId); - columnIndices.push_back(columnInfo.Index); - } -} - + for (auto& column : columns) { + auto columnInfo = GetColumnInfo(type, column); + columnTypes.push_back(columnInfo.TypeId); + columnIndices.push_back(columnInfo.Index); + } +} + template<typename TList> void GetColumnsInfo(const NKikimr::NMiniKQL::TType* type, const TList& protoSortCols, TVector<TSortColumnInfo>& sortCols) @@ -53,6 +53,6 @@ void GetColumnsInfo(const NKikimr::NMiniKQL::TType* type, const TList& protoSort colInfo.Ascending = protoSortCol.GetAscending(); sortCols.emplace_back(std::move(colInfo)); } -} +} } diff --git a/ydb/library/yql/dq/runtime/dq_compute.cpp b/ydb/library/yql/dq/runtime/dq_compute.cpp index 43e3d1198c4..0da2ed9630d 100644 --- a/ydb/library/yql/dq/runtime/dq_compute.cpp +++ b/ydb/library/yql/dq/runtime/dq_compute.cpp @@ -1,26 +1,26 @@ #include "dq_compute.h" - + #include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> #include <ydb/library/yql/minikql/mkql_node.h> #include "ydb/library/yql/utils/yql_panic.h" - -namespace NYql::NDq { - -using namespace NKikimr; -using namespace NMiniKQL; - -TComputationNodeFactory GetDqBaseComputeFactory(const TDqComputeContextBase* computeCtx) { - YQL_ENSURE(computeCtx); - auto builtinFactory = GetBuiltinFactory(); - - return [builtinFactory] - (TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { - if (auto builtin = builtinFactory(callable, ctx)) { - return builtin; - } - - return nullptr; - }; -} - -} + +namespace NYql::NDq { + +using namespace NKikimr; +using namespace NMiniKQL; + +TComputationNodeFactory GetDqBaseComputeFactory(const TDqComputeContextBase* computeCtx) { + YQL_ENSURE(computeCtx); + auto builtinFactory = GetBuiltinFactory(); + + return [builtinFactory] + (TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (auto builtin = builtinFactory(callable, ctx)) { + return builtin; + } + + return nullptr; + }; +} + +} diff --git a/ydb/library/yql/dq/runtime/dq_compute.h b/ydb/library/yql/dq/runtime/dq_compute.h index ded0e90c3d4..6aca63b8c42 100644 --- a/ydb/library/yql/dq/runtime/dq_compute.h +++ b/ydb/library/yql/dq/runtime/dq_compute.h @@ -1,14 +1,14 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> - -namespace NYql::NDq { - + +namespace NYql::NDq { + class TDqComputeContextBase : private TNonCopyable { public: virtual ~TDqComputeContextBase() = default; }; - -NKikimr::NMiniKQL::TComputationNodeFactory GetDqBaseComputeFactory(const TDqComputeContextBase* computeCtx); - -} + +NKikimr::NMiniKQL::TComputationNodeFactory GetDqBaseComputeFactory(const TDqComputeContextBase* computeCtx); + +} diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp index 6c272f843bf..10900352c3b 100644 --- a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp +++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp @@ -6,7 +6,7 @@ #include <ydb/library/yql/dq/runtime/dq_input_producer.h> #include <ydb/library/yql/dq/runtime/dq_source.h> #include <ydb/library/yql/dq/runtime/dq_transport.h> - + #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> #include <ydb/library/yql/public/udf/udf_value.h> @@ -757,6 +757,6 @@ TIntrusivePtr<IDqTaskRunner> MakeDqTaskRunner(const TDqTaskRunnerContext& ctx, c const TLogFunc& logFunc) { return new TDqTaskRunner(ctx, settings, logFunc); -} - +} + } // namespace NYql::NDq diff --git a/ydb/library/yql/dq/runtime/dq_transport.cpp b/ydb/library/yql/dq/runtime/dq_transport.cpp index 6e54197bf47..93b85b0c6f8 100644 --- a/ydb/library/yql/dq/runtime/dq_transport.cpp +++ b/ydb/library/yql/dq/runtime/dq_transport.cpp @@ -15,8 +15,8 @@ using namespace NKikimr; using namespace NMiniKQL; using namespace NYql; -namespace { - +namespace { + NDqProto::TData SerializeBufferArrowV1(TUnboxedValueVector& buffer, const TType* itemType); void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType, @@ -24,16 +24,16 @@ void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType NDqProto::TData SerializeValuePickleV1(const TType* type, const NUdf::TUnboxedValuePod& value) { TValuePacker packer(/* stable */ false, type); - TStringBuf packResult = packer.Pack(value); - - NDqProto::TData data; + TStringBuf packResult = packer.Pack(value); + + NDqProto::TData data; data.SetTransportVersion(NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0); - data.SetRaw(packResult.data(), packResult.size()); + data.SetRaw(packResult.data(), packResult.size()); data.SetRows(1); - - return data; -} - + + return data; +} + NDqProto::TData SerializeValueArrowV1(const TType* type, const NUdf::TUnboxedValuePod& value) { TUnboxedValueVector buffer; buffer.push_back(value); @@ -45,9 +45,9 @@ void DeserializeValuePickleV1(const TType* type, const NDqProto::TData& data, NU { YQL_ENSURE(data.GetTransportVersion() == (ui32) NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0); TValuePacker packer(/* stable */ false, type); - value = packer.Unpack(data.GetRaw(), holderFactory); -} - + value = packer.Unpack(data.GetRaw(), holderFactory); +} + void DeserializeValueArrowV1(const TType* type, const NDqProto::TData& data, NUdf::TUnboxedValue& value, const THolderFactory& holderFactory) { @@ -60,13 +60,13 @@ NDqProto::TData SerializeBufferPickleV1(TUnboxedValueVector& buffer, const TType const TTypeEnvironment& typeEnv, const THolderFactory& holderFactory) { const auto listType = TListType::Create(const_cast<TType*>(itemType), typeEnv); - const NUdf::TUnboxedValue listValue = holderFactory.VectorAsArray(buffer); - + const NUdf::TUnboxedValue listValue = holderFactory.VectorAsArray(buffer); + auto data = SerializeValuePickleV1(listType, listValue); data.SetRows(buffer.size()); return data; -} - +} + NDqProto::TData SerializeBufferArrowV1(TUnboxedValueVector& buffer, const TType* itemType) { auto array = NArrow::MakeArray(buffer, itemType); @@ -83,16 +83,16 @@ void DeserializeBufferPickleV1(const NDqProto::TData& data, const TType* itemTyp const THolderFactory& holderFactory, TUnboxedValueVector& buffer) { auto listType = TListType::Create(const_cast<TType*>(itemType), typeEnv); - - NUdf::TUnboxedValue value; + + NUdf::TUnboxedValue value; DeserializeValuePickleV1(listType, data, value, holderFactory); - - const auto iter = value.GetListIterator(); - for (NUdf::TUnboxedValue item; iter.Next(item);) { - buffer.emplace_back(std::move(item)); - } -} - + + const auto iter = value.GetListIterator(); + for (NUdf::TUnboxedValue item; iter.Next(item);) { + buffer.emplace_back(std::move(item)); + } +} + void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType, const THolderFactory& holderFactory, TUnboxedValueVector& buffer) { @@ -106,26 +106,26 @@ void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType } } -NDqProto::TData SerializeParamV1(const TMkqlValueRef& param, const TTypeEnvironment& typeEnv, +NDqProto::TData SerializeParamV1(const TMkqlValueRef& param, const TTypeEnvironment& typeEnv, const THolderFactory& holderFactory) { auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory); - + return SerializeValuePickleV1(type, value); -} - +} + void DeserializeParamV1(const NDqProto::TData& data, const TType* type, const THolderFactory& holderFactory, NUdf::TUnboxedValue& value) { DeserializeValuePickleV1(type, data, value, holderFactory); -} - -} // namespace - +} + +} // namespace + NDqProto::EDataTransportVersion TDqDataSerializer::GetTransportVersion() const { return TransportVersion; -} - +} + NDqProto::TData TDqDataSerializer::Serialize(const NUdf::TUnboxedValue& value, const TType* itemType) const { switch (TransportVersion) { case NDqProto::DATA_TRANSPORT_VERSION_UNSPECIFIED: @@ -137,7 +137,7 @@ NDqProto::TData TDqDataSerializer::Serialize(const NUdf::TUnboxedValue& value, c YQL_ENSURE(false, "Unsupported TransportVersion"); } } - + NDqProto::TData TDqDataSerializer::Serialize(TUnboxedValueVector& buffer, const TType* itemType) const { switch (TransportVersion) { case NDqProto::DATA_TRANSPORT_VERSION_UNSPECIFIED: @@ -166,8 +166,8 @@ void TDqDataSerializer::Deserialize(const NDqProto::TData& data, const TType* it default: YQL_ENSURE(false, "Unsupported TransportVersion"); } -} - +} + void TDqDataSerializer::Deserialize(const NDqProto::TData& data, const TType* itemType, NUdf::TUnboxedValue& value) const { @@ -191,16 +191,16 @@ NDqProto::TData TDqDataSerializer::SerializeParam(const TMkqlValueRef& param, co const NKikimr::NMiniKQL::THolderFactory& holderFactory) { return SerializeParamV1(param, typeEnv, holderFactory); -} - +} + void TDqDataSerializer::DeserializeParam(const NDqProto::TData& data, const TType* type, const NKikimr::NMiniKQL::THolderFactory& holderFactory, NUdf::TUnboxedValue& value) { YQL_ENSURE(data.GetTransportVersion() == (ui32) NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0); - + return DeserializeParamV1(data, type, holderFactory, value); -} - +} + NDqProto::TData TDqDataSerializer::SerializeParamValue(const TType* type, const NUdf::TUnboxedValuePod& value) { return SerializeValuePickleV1(type, value); } diff --git a/ydb/library/yql/dq/runtime/dq_transport.h b/ydb/library/yql/dq/runtime/dq_transport.h index c912c220403..8f6b6f947ff 100644 --- a/ydb/library/yql/dq/runtime/dq_transport.h +++ b/ydb/library/yql/dq/runtime/dq_transport.h @@ -24,7 +24,7 @@ public: NDqProto::TData Serialize(const NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* itemType) const; NDqProto::TData Serialize(NKikimr::NMiniKQL::TUnboxedValueVector& buffer, const NKikimr::NMiniKQL::TType* itemType) const; - + template <class TForwardIterator> NDqProto::TData Serialize(TForwardIterator first, TForwardIterator last, const NKikimr::NMiniKQL::TType* itemType) const { switch (TransportVersion) { diff --git a/ydb/library/yql/dq/ya.make b/ydb/library/yql/dq/ya.make index 5ad644c6b47..c206ab90ef1 100644 --- a/ydb/library/yql/dq/ya.make +++ b/ydb/library/yql/dq/ya.make @@ -5,7 +5,7 @@ RECURSE( expr_nodes opt proto - runtime + runtime state tasks type_ann diff --git a/ydb/library/yql/providers/dq/actors/events.cpp b/ydb/library/yql/providers/dq/actors/events.cpp index d73d324d281..892c5cccd70 100644 --- a/ydb/library/yql/providers/dq/actors/events.cpp +++ b/ydb/library/yql/providers/dq/actors/events.cpp @@ -16,7 +16,7 @@ namespace NYql::NDqs { TEvQueryResponse::TEvQueryResponse(NDqProto::TQueryResponse&& queryResult) { Record = std::move(queryResult); } - + TEvGraphRequest::TEvGraphRequest(const Yql::DqsProto::ExecuteGraphRequest& request, NActors::TActorId controlId, NActors::TActorId resultId, NActors::TActorId checkPointCoordinatorId) { *Record.MutableRequest() = request; |