aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorhrustyashko <hrustyashko@yandex-team.ru>2022-02-17 15:24:07 +0300
committerhrustyashko <hrustyashko@yandex-team.ru>2022-02-17 15:24:07 +0300
commit329e7bdd49970bba890ca97109ef98486b41a7cd (patch)
treecb5781f57320495efdce89f0fd5b53d9dede7328
parentf22b29e7eda4d2c622480b2f181257562353272b (diff)
downloadydb-329e7bdd49970bba890ca97109ef98486b41a7cd.tar.gz
YQ-418 Библиотека конвертации YQ типов в JSON
ref:8eced38335e97fd1e3738ef6cf0b647b26209181
-rw-r--r--ydb/library/yql/minikql/mkql_type_ops.cpp27
-rw-r--r--ydb/library/yql/providers/common/codec/ut/ya.make22
-rw-r--r--ydb/library/yql/providers/common/codec/ya.make6
-rw-r--r--ydb/library/yql/providers/common/codec/yql_json_codec.cpp458
-rw-r--r--ydb/library/yql/providers/common/codec/yql_json_codec.h30
-rw-r--r--ydb/library/yql/providers/common/codec/yql_json_codec_ut.cpp636
6 files changed, 1152 insertions, 27 deletions
diff --git a/ydb/library/yql/minikql/mkql_type_ops.cpp b/ydb/library/yql/minikql/mkql_type_ops.cpp
index d559bb108c..3801de6c81 100644
--- a/ydb/library/yql/minikql/mkql_type_ops.cpp
+++ b/ydb/library/yql/minikql/mkql_type_ops.cpp
@@ -582,33 +582,6 @@ NUdf::TUnboxedValuePod NumberFromString(NUdf::TStringRef buf) {
return NUdf::TUnboxedValuePod(value);
}
-bool MakeDateUncached(ui32 year, ui32 month, ui32 day, ui16& value) {
- if (year < NUdf::MIN_YEAR || year >= NUdf::MAX_YEAR) {
- return false;
- }
-
- if (month < 1 || month > 12) {
- return false;
- }
-
- const bool isLeap = IsLeapYear(year);
- auto monthLength = GetMonthLength(month, isLeap);
- if (day < 1 || day > monthLength) {
- return false;
- }
-
- year -= NUdf::MIN_YEAR;
- ui32 leapDaysCount = LeapDaysSinceEpoch(year);
- value = year * 365 + leapDaysCount;
- while (month > 1) {
- --month;
- value += GetMonthLength(month, isLeap);
- }
-
- value += day - 1;
- return true;
-}
-
bool MakeTime(ui32 hour, ui32 minute, ui32 second, ui32& value) {
if (hour >= 24 || minute >= 60 || second >= 60) {
return false;
diff --git a/ydb/library/yql/providers/common/codec/ut/ya.make b/ydb/library/yql/providers/common/codec/ut/ya.make
new file mode 100644
index 0000000000..0c5ad6d824
--- /dev/null
+++ b/ydb/library/yql/providers/common/codec/ut/ya.make
@@ -0,0 +1,22 @@
+UNITTEST_FOR(ydb/library/yql/providers/common/codec)
+
+OWNER(g:yql)
+
+FORK_SUBTESTS()
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
+ yql_json_codec_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/testing/unittest
+ ydb/library/yql/public/udf/service/exception_policy
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/providers/common/codec/ya.make b/ydb/library/yql/providers/common/codec/ya.make
index b3f3a92db5..0c44e3fdc2 100644
--- a/ydb/library/yql/providers/common/codec/ya.make
+++ b/ydb/library/yql/providers/common/codec/ya.make
@@ -13,6 +13,7 @@ SRCS(
yql_restricted_yson.h
yql_codec_type_flags.cpp
yql_codec_type_flags.h
+ yql_json_codec.cpp
)
PEERDIR(
@@ -21,6 +22,7 @@ PEERDIR(
ydb/library/yql/providers/common/mkql
library/cpp/yson/node
library/cpp/yson
+ library/cpp/json
)
YQL_LAST_ABI_VERSION()
@@ -28,3 +30,7 @@ YQL_LAST_ABI_VERSION()
GENERATE_ENUM_SERIALIZATION(yql_codec_type_flags.h)
END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/providers/common/codec/yql_json_codec.cpp b/ydb/library/yql/providers/common/codec/yql_json_codec.cpp
new file mode 100644
index 0000000000..cad700b449
--- /dev/null
+++ b/ydb/library/yql/providers/common/codec/yql_json_codec.cpp
@@ -0,0 +1,458 @@
+#include "yql_json_codec.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_type_ops.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+#include <ydb/library/yql/utils/yql_panic.h>
+
+#include <library/cpp/string_utils/base64/base64.h>
+#include <util/string/join.h>
+
+namespace NYql {
+namespace NCommon {
+
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+using namespace NJson;
+
+namespace {
+
+constexpr i64 MAX_JS_SAFE_INTEGER = 9007199254740991; // 2^53 - 1; JavaScript Number.MAX_SAFE_INTEGER
+constexpr i64 MIN_JS_SAFE_INTEGER = -9007199254740991; // -(2^53 - 1); JavaScript Number.MIN_SAFE_INTEGER
+
+constexpr i8 DOUBLE_N_DIGITS = std::numeric_limits<double>::max_digits10;
+constexpr i8 FLOAT_N_DIGITS = std::numeric_limits<float>::max_digits10;
+constexpr EFloatToStringMode FLOAT_MODE = EFloatToStringMode::PREC_NDIGITS;
+}
+
+TJsonWriterConfig MakeJsonConfig() {
+ TJsonWriterConfig config;
+ config.DoubleNDigits = DOUBLE_N_DIGITS;
+ config.FloatNDigits = FLOAT_N_DIGITS;
+ config.FloatToStringMode = FLOAT_MODE;
+ config.FormatOutput = false;
+ config.SortKeys = false;
+ config.ValidateUtf8 = false;
+ config.DontEscapeStrings = true;
+ config.WriteNanAsString = false;
+
+ return config;
+}
+
+void WriteValueToJson(TJsonWriter& writer, const NKikimr::NUdf::TUnboxedValuePod& value,
+ NKikimr::NMiniKQL::TType* type, std::set<EValueConvertPolicy> convertPolicy) {
+
+ switch (type->GetKind()) {
+ case TType::EKind::Void:
+ case TType::EKind::Null:
+ writer.WriteNull();
+ break;
+ case TType::EKind::EmptyList:
+ case TType::EKind::EmptyDict:
+ writer.OpenArray();
+ writer.CloseArray();
+ break;
+ case TType::EKind::Data:
+ {
+ bool numberToStr = convertPolicy.contains(EValueConvertPolicy::WriteNumberString);
+ auto dataType = AS_TYPE(TDataType, type);
+ switch (dataType->GetSchemeType()) {
+ case NUdf::TDataType<bool>::Id:
+ writer.Write(value.Get<bool>());
+ break;
+ case NUdf::TDataType<i32>::Id: {
+ auto number = value.Get<i32>();
+ if (numberToStr) {
+ writer.Write(ToString(number));
+ } else {
+ writer.Write(number);
+ }
+ break;
+ }
+ case NUdf::TDataType<ui32>::Id: {
+ auto number = value.Get<ui32>();
+ if (numberToStr) {
+ writer.Write(ToString(number));
+ } else {
+ writer.Write(number);
+ }
+ break;
+ }
+ case NUdf::TDataType<i64>::Id: {
+ auto number = value.Get<i64>();
+ if (numberToStr) {
+ writer.Write(ToString(number));
+ } else if (convertPolicy.contains(EValueConvertPolicy::WriteUnsafeNumberString)) {
+ if (number > MAX_JS_SAFE_INTEGER || number < MIN_JS_SAFE_INTEGER) {
+ writer.Write(ToString(number));
+ } else {
+ writer.Write(number);
+ };
+ } else {
+ writer.Write(number);
+ }
+ break;
+ }
+ case NUdf::TDataType<ui64>::Id: {
+ auto number = value.Get<ui64>();
+ if (numberToStr) {
+ writer.Write(ToString(number));
+ } else if (convertPolicy.contains(EValueConvertPolicy::WriteUnsafeNumberString)) {
+ if (number > MAX_JS_SAFE_INTEGER) {
+ writer.Write(ToString(number));
+ } else {
+ writer.Write(number);
+ };
+ } else {
+ writer.Write(number);
+ }
+ break;
+ }
+ case NUdf::TDataType<ui8>::Id: {
+ auto number = value.Get<ui8>();
+ if (numberToStr) {
+ writer.Write(ToString(number));
+ } else {
+ writer.Write(number);
+ }
+ break;
+ }
+ case NUdf::TDataType<i8>::Id: {
+ auto number = value.Get<i8>();
+ if (numberToStr) {
+ writer.Write(ToString(number));
+ } else {
+ writer.Write(number);
+ }
+ break;
+ }
+ case NUdf::TDataType<ui16>::Id: {
+ auto number = value.Get<ui16>();
+ if (numberToStr) {
+ writer.Write(ToString(number));
+ } else {
+ writer.Write(number);
+ }
+ break;
+ }
+ case NUdf::TDataType<i16>::Id: {
+ auto number = value.Get<i16>();
+ if (numberToStr) {
+ writer.Write(ToString(number));
+ } else {
+ writer.Write(number);
+ }
+ break;
+ }
+ case NUdf::TDataType<float>::Id:
+ if (numberToStr) {
+ TString number = FloatToString(value.Get<float>(), FLOAT_MODE, FLOAT_N_DIGITS);
+ writer.Write(number);
+ } else {
+ writer.Write(value.Get<float>());
+ }
+ break;
+ case NUdf::TDataType<double>::Id:
+ if (numberToStr) {
+ TString number = FloatToString(value.Get<double>(), FLOAT_MODE, DOUBLE_N_DIGITS);
+ writer.Write(number);
+ } else {
+ writer.Write(value.Get<double>());
+ }
+ break;
+ case NUdf::TDataType<NUdf::TJson>::Id:
+ writer.UnsafeWrite(value.AsStringRef());
+ break;
+ case NUdf::TDataType<NUdf::TUtf8>::Id:
+ writer.Write(value.AsStringRef());
+ break;
+ case NUdf::TDataType<char*>::Id: {
+ TString encoded = Base64Encode(value.AsStringRef());
+ writer.Write(encoded);
+ break;
+ }
+ case NUdf::TDataType<NUdf::TDecimal>::Id: {
+ const auto params = static_cast<TDataDecimalType*>(type)->GetParams();
+ const auto str = NDecimal::ToString(value.GetInt128(), params.first, params.second);
+ const auto size = str ? std::strlen(str) : 0;
+ writer.Write(TStringBuf(str, size));
+ break;
+ }
+ case NUdf::TDataType<NUdf::TUuid>::Id:
+ writer.Write(value.AsStringRef());
+ break;
+ case NUdf::TDataType<NUdf::TYson>::Id:
+ case NUdf::TDataType<NUdf::TDyNumber>::Id:
+ case NUdf::TDataType<NUdf::TDate>::Id:
+ case NUdf::TDataType<NUdf::TDatetime>::Id:
+ case NUdf::TDataType<NUdf::TTimestamp>::Id:
+ case NUdf::TDataType<NUdf::TInterval>::Id:
+ case NUdf::TDataType<NUdf::TTzDate>::Id:
+ case NUdf::TDataType<NUdf::TTzDatetime>::Id:
+ case NUdf::TDataType<NUdf::TTzTimestamp>::Id:
+ case NUdf::TDataType<NUdf::TJsonDocument>::Id: {
+ const NUdf::TUnboxedValue out(ValueToString(*dataType->GetDataSlot(), value));
+ writer.Write(out.AsStringRef());
+ break;
+ }
+ default:
+ throw yexception() << "Unknown data type: " << dataType->GetSchemeType();
+ }
+ }
+ break;
+ case TType::EKind::Struct:
+ {
+ writer.OpenMap();
+ auto structType = AS_TYPE(TStructType, type);
+ for (ui32 i = 0, e = structType->GetMembersCount(); i < e; ++i) {
+ writer.WriteKey(structType->GetMemberName(i));
+ WriteValueToJson(writer, value.GetElement(i), structType->GetMemberType(i), convertPolicy);
+ }
+ writer.CloseMap();
+ break;
+ }
+ case TType::EKind::List:
+ {
+ writer.OpenArray();
+ auto listType = AS_TYPE(TListType, type);
+ const auto it = value.GetListIterator();
+ for (NUdf::TUnboxedValue item; it.Next(item);) {
+ WriteValueToJson(writer, item, listType->GetItemType(), convertPolicy);
+ }
+ writer.CloseArray();
+ break;
+ }
+ case TType::EKind::Optional:
+ {
+ writer.OpenArray();
+ if (!value.GetOptionalValue()) {
+ writer.WriteNull();
+ } else {
+ auto optionalType = AS_TYPE(TOptionalType, type);
+ WriteValueToJson(writer, value.GetOptionalValue(), optionalType->GetItemType(), convertPolicy);
+ }
+ writer.CloseArray();
+ break;
+ }
+ case TType::EKind::Dict:
+ {
+ writer.OpenArray();
+ auto dictType = AS_TYPE(TDictType, type);
+ const auto it = value.GetDictIterator();
+ for (NUdf::TUnboxedValue key, payload; it.NextPair(key, payload);) {
+ writer.OpenArray();
+ WriteValueToJson(writer, key, dictType->GetKeyType(), convertPolicy);
+ WriteValueToJson(writer, payload, dictType->GetPayloadType(), convertPolicy);
+ writer.CloseArray();
+ }
+ writer.CloseArray();
+ break;
+ }
+ case TType::EKind::Tuple:
+ {
+ writer.OpenArray();
+ auto tupleType = AS_TYPE(TTupleType, type);
+ for (ui32 i = 0, e = tupleType->GetElementsCount(); i < e; ++i) {
+ WriteValueToJson(writer, value.GetElement(i), tupleType->GetElementType(i), convertPolicy);
+ }
+ writer.CloseArray();
+ break;
+ }
+ case TType::EKind::Variant:
+ {
+ writer.OpenArray();
+ auto index = value.GetVariantIndex();
+ writer.Write(index);
+
+ auto underlyingType = AS_TYPE(TVariantType, type)->GetUnderlyingType();
+ if (underlyingType->IsTuple()) {
+ WriteValueToJson(writer, value.GetVariantItem(),
+ AS_TYPE(TTupleType, underlyingType)->GetElementType(index), convertPolicy);
+ } else {
+ WriteValueToJson(writer, value.GetVariantItem(),
+ AS_TYPE(TStructType, underlyingType)->GetMemberType(index), convertPolicy);
+ }
+ writer.CloseArray();
+ break;
+ }
+ case TType::EKind::Tagged:
+ {
+ auto underlyingType = AS_TYPE(TTaggedType, type)->GetBaseType();
+ WriteValueToJson(writer, value, underlyingType, convertPolicy);
+ break;
+ }
+ default:
+ YQL_ENSURE(false, "unknown type " << type->GetKindAsStr());
+ }
+}
+
+NKikimr::NUdf::TUnboxedValue ReadJsonValue(TJsonValue& json, NKikimr::NMiniKQL::TType* type,
+ const NMiniKQL::THolderFactory& holderFactory)
+{
+ auto jsonType = json.GetType();
+ switch (type->GetKind()) {
+ case TType::EKind::Void:
+ case TType::EKind::Null:
+ YQL_ENSURE(json.IsNull(), "Unexpected json type (expected null value, but got type " << jsonType << ")");
+ return NKikimr::NUdf::TUnboxedValuePod();
+ case TType::EKind::EmptyList:
+ case TType::EKind::EmptyDict:
+ YQL_ENSURE(json.IsArray(), "Unexpected json type (expected array, but got " << jsonType << ")");
+ YQL_ENSURE(json.GetArray().size() == 0, "Expected empty array, but got array with " << json.GetArray().size() << " elements");
+ return holderFactory.GetEmptyContainer();
+ case TType::EKind::Tuple:
+ {
+ YQL_ENSURE(json.IsArray(), "Unexpected json type (expected array, but got " << jsonType << ")");
+ auto tupleType = AS_TYPE(TTupleType, type);
+ auto array = json.GetArray();
+ YQL_ENSURE(array.size() == tupleType->GetElementsCount(),
+ "Expected " << tupleType->GetElementsCount() << " elements in tuple, but got " << array.size());
+ NUdf::TUnboxedValue* items;
+ NUdf::TUnboxedValue tuple = holderFactory.CreateDirectArrayHolder(tupleType->GetElementsCount(), items);
+ for (ui32 i = 0, e = array.size(); i < e; i++) {
+ items[i] = ReadJsonValue(array[i], tupleType->GetElementType(i), holderFactory);
+ }
+ return tuple;
+ }
+ case TType::EKind::List:
+ {
+ YQL_ENSURE(json.IsArray(), "Unexpected json type (expected array, but got " << jsonType << ")");
+ auto listType = AS_TYPE(TListType, type);
+ TDefaultListRepresentation items;
+ auto array = json.GetArray();
+ for (ui32 i = 0, e = array.size(); i < e; i++) {
+ items = items.Append(ReadJsonValue(array[i], listType->GetItemType(), holderFactory));
+ }
+ return holderFactory.CreateDirectListHolder(std::move(items));
+ }
+ case TType::EKind::Struct:
+ {
+ YQL_ENSURE(json.IsMap(), "Unexpected json type (expected map, but got " << jsonType << ")");
+ auto structType = AS_TYPE(TStructType, type);
+ NUdf::TUnboxedValue* items;
+ NUdf::TUnboxedValue structValue = holderFactory.CreateDirectArrayHolder(structType->GetMembersCount(), items);
+ auto jsonMap = json.GetMap();
+ std::unordered_set<TString> unprocessed(jsonMap.size());
+ for (auto const& map : jsonMap) {
+ unprocessed.insert(map.first);
+ }
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ const auto keyName = TString(structType->GetMemberName(i));
+ if (jsonMap.contains(keyName)) {
+ items[i] = ReadJsonValue(jsonMap[keyName], structType->GetMemberType(i), holderFactory);
+ } else {
+ YQL_ENSURE(structType->GetMemberType(i)->IsOptional(), "Absent non optional field " << keyName << " at struct");
+ items[i] = NKikimr::NUdf::TUnboxedValue();
+ }
+ unprocessed.erase(keyName);
+ }
+ YQL_ENSURE(unprocessed.empty(), "Extra fields into json map detected (" << JoinSeq(',', unprocessed) << ")");
+ return structValue;
+ }
+ case TType::EKind::Optional:
+ {
+ if (json.IsNull()) {
+ return NUdf::TUnboxedValuePod();
+ }
+ auto optionalType = AS_TYPE(TOptionalType, type);
+ auto value = ReadJsonValue(json, optionalType->GetItemType(), holderFactory);
+ return value.Release().MakeOptional();
+ }
+ case TType::EKind::Data:
+ {
+ auto dataType = AS_TYPE(TDataType, type);
+ switch (dataType->GetSchemeType()) {
+ case NUdf::TDataType<bool>::Id:
+ YQL_ENSURE(json.IsBoolean(), "Unexpected json type (expected bool, but got " << jsonType << ")");
+ return NUdf::TUnboxedValuePod(json.GetBoolean());
+
+#define INTEGER_CONVERTOR(type, wideType) \
+ case NUdf::TDataType<type>::Id: { \
+ YQL_ENSURE(jsonType == EJsonValueType::JSON_INTEGER \
+ || jsonType == EJsonValueType::JSON_UINTEGER, \
+ "Unexpected json type (expected " << #type << ", but got " << jsonType << ")"); \
+ wideType intValue = jsonType == EJsonValueType::JSON_INTEGER ? json.GetInteger() : json.GetUInteger(); \
+ YQL_ENSURE(intValue >= std::numeric_limits<type>::min() \
+ && intValue <= std::numeric_limits<type>::max(), \
+ "Exceeded the range of acceptable values for " << #type); \
+ return NUdf::TUnboxedValuePod(type(intValue)); \
+ }
+
+ INTEGER_CONVERTOR(ui8, ui64)
+ INTEGER_CONVERTOR(ui16, ui64)
+ INTEGER_CONVERTOR(ui32, ui64)
+ INTEGER_CONVERTOR(ui64, ui64)
+
+ INTEGER_CONVERTOR(i8, i64)
+ INTEGER_CONVERTOR(i16, i64)
+ INTEGER_CONVERTOR(i32, i64)
+ INTEGER_CONVERTOR(i64, i64)
+
+#undef INTEGER_CONVERTOR
+
+ case NUdf::TDataType<float>::Id: {
+ YQL_ENSURE(json.IsDouble() || json.IsInteger() || json.IsUInteger(),
+ "Unexpected json type (expected double or integer, but got " << jsonType << ")");
+ double value = jsonType == EJsonValueType::JSON_DOUBLE
+ ? json.GetDouble()
+ : (jsonType == EJsonValueType::JSON_INTEGER ? double(json.GetInteger()) : double(json.GetUInteger()));
+ YQL_ENSURE(value >= std::numeric_limits<float>::min() && value <= std::numeric_limits<float>::max(),
+ "Exceeded the range of acceptable values for float");
+ return NUdf::TUnboxedValuePod(float(value));
+ }
+ case NUdf::TDataType<double>::Id: {
+ YQL_ENSURE(json.IsDouble() || json.IsInteger() || json.IsUInteger(),
+ "Unexpected json type (expected double or integer, but got " << jsonType << ")");
+ double value = jsonType == EJsonValueType::JSON_DOUBLE
+ ? json.GetDouble()
+ : (jsonType == EJsonValueType::JSON_INTEGER ? double(json.GetInteger()) : double(json.GetUInteger()));
+ return NUdf::TUnboxedValuePod(value);
+ }
+ case NUdf::TDataType<NUdf::TUtf8>::Id:
+ case NUdf::TDataType<char*>::Id: {
+ YQL_ENSURE(json.IsString(), "Unexpected json type (expected string, but got " << jsonType << ")");
+ auto value = json.GetString();
+ return NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(value)));
+ }
+ case NUdf::TDataType<NUdf::TDecimal>::Id: {
+ YQL_ENSURE(json.IsString(), "Unexpected json type (expected string, but got " << jsonType << ")");
+ const auto params = static_cast<TDataDecimalType*>(type)->GetParams();
+ const auto value = NDecimal::FromString(json.GetString(), params.first, params.second);
+ YQL_ENSURE(!NDecimal::IsError(value));
+ return NUdf::TUnboxedValuePod(value);
+ }
+ case NUdf::TDataType<NUdf::TDate>::Id:
+ case NUdf::TDataType<NUdf::TDatetime>::Id:
+ case NUdf::TDataType<NUdf::TTimestamp>::Id:
+ case NUdf::TDataType<NUdf::TInterval>::Id:
+ case NUdf::TDataType<NUdf::TTzDate>::Id:
+ case NUdf::TDataType<NUdf::TTzDatetime>::Id:
+ case NUdf::TDataType<NUdf::TTzTimestamp>::Id: {
+ YQL_ENSURE(json.IsString(), "Unexpected json type (expected string, but got " << jsonType << ")");
+ YQL_ENSURE(IsValidStringValue(*dataType->GetDataSlot(), json.GetString()), "Invalid date format (expected ISO-8601)");
+ return ValueFromString(*dataType->GetDataSlot(), json.GetString());
+ }
+ default:
+ YQL_ENSURE(false, "Can't convert from JSON (unsupported YQL type " << dataType->GetSchemeType() << ")");
+ }
+ }
+ break;
+ default:
+ YQL_ENSURE(false, "Can't convert from JSON (unsupported YQL type " << type->GetKindAsStr() << ")");
+ }
+
+ return NKikimr::NUdf::TUnboxedValuePod();
+}
+
+NKikimr::NUdf::TUnboxedValue ReadJsonValue(IInputStream* in, NKikimr::NMiniKQL::TType* type,
+ const NMiniKQL::THolderFactory& holderFactory)
+{
+ TJsonValue json;
+ if (!ReadJsonTree(in, &json, false)) {
+ YQL_ENSURE(false, "Error parse json");
+ }
+ return ReadJsonValue(json, type, holderFactory);
+}
+
+}
+}
diff --git a/ydb/library/yql/providers/common/codec/yql_json_codec.h b/ydb/library/yql/providers/common/codec/yql_json_codec.h
new file mode 100644
index 0000000000..e541c3eef2
--- /dev/null
+++ b/ydb/library/yql/providers/common/codec/yql_json_codec.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include <library/cpp/json/json_writer.h>
+#include <library/cpp/json/json_reader.h>
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+using namespace NKikimr;
+
+namespace NYql {
+namespace NCommon {
+
+enum class EValueConvertPolicy : ui8 {
+ WriteNumberString = 1,
+ WriteUnsafeNumberString = 2,
+ //WriteNumberAsString = 4,
+ //WriteNumberAsString = 8,
+};
+
+NJson::TJsonWriterConfig MakeJsonConfig();
+
+void WriteValueToJson(NJson::TJsonWriter& writer, const NUdf::TUnboxedValuePod& value,
+ NMiniKQL::TType* type, std::set<EValueConvertPolicy> convertPolicy = {});
+
+NUdf::TUnboxedValue ReadJsonValue(NJson::TJsonValue& json, NMiniKQL::TType* type, const NMiniKQL::THolderFactory& holderFactory);
+
+NUdf::TUnboxedValue ReadJsonValue(IInputStream* in, NMiniKQL::TType* type, const NMiniKQL::THolderFactory& holderFactory);
+
+}
+}
diff --git a/ydb/library/yql/providers/common/codec/yql_json_codec_ut.cpp b/ydb/library/yql/providers/common/codec/yql_json_codec_ut.cpp
new file mode 100644
index 0000000000..3a6fed2953
--- /dev/null
+++ b/ydb/library/yql/providers/common/codec/yql_json_codec_ut.cpp
@@ -0,0 +1,636 @@
+#include "yql_json_codec.h"
+
+#include <ydb/library/yql/minikql/computation/mkql_value_builder.h>
+#include <ydb/library/yql/minikql/mkql_type_ops.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/string/cast.h>
+
+namespace NYql {
+namespace NCommon {
+
+using namespace NYql::NCommon;
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+
+
+namespace {
+struct TTestContext {
+ TScopedAlloc Alloc;
+ TTypeEnvironment TypeEnv;
+ TMemoryUsageInfo MemInfo;
+ THolderFactory HolderFactory;
+ TDefaultValueBuilder Vb;
+
+ TTestContext()
+ : Alloc()
+ , TypeEnv(Alloc)
+ , MemInfo("Mem")
+ , HolderFactory(Alloc.Ref(), MemInfo)
+ , Vb(HolderFactory)
+ {
+
+ }
+};
+
+TString WriteValueToExportJsonStr(const NUdf::TUnboxedValuePod& value, NMiniKQL::TType* type) {
+ TStringStream out;
+ NJson::TJsonWriter jsonWriter(&out, MakeJsonConfig());
+ WriteValueToJson(jsonWriter,value, type, {});
+ jsonWriter.Flush();
+ return out.Str();
+}
+
+TString WriteValueToFuncJsonStr(const NUdf::TUnboxedValuePod& value, NMiniKQL::TType* type) {
+ TStringStream out;
+ NJson::TJsonWriter jsonWriter(&out, MakeJsonConfig());
+ WriteValueToJson(jsonWriter,value, type, {EValueConvertPolicy::WriteNumberString});
+ jsonWriter.Flush();
+ return out.Str();
+}
+}
+
+Y_UNIT_TEST_SUITE(SerializeVoid) {
+ Y_UNIT_TEST(Null) {
+ TTestContext ctx;
+ auto value = NUdf::TUnboxedValuePod();
+ auto nullJson = WriteValueToFuncJsonStr(value, ctx.TypeEnv.GetTypeOfNull());
+ UNIT_ASSERT_VALUES_EQUAL(nullJson, "null");
+
+ auto voidJson = WriteValueToFuncJsonStr(value, ctx.TypeEnv.GetTypeOfVoid());
+ UNIT_ASSERT_VALUES_EQUAL(voidJson, "null");
+ }
+}
+
+Y_UNIT_TEST_SUITE(SerializeBool) {
+ Y_UNIT_TEST(Bool) {
+ TTestContext ctx;
+ auto type = TDataType::Create(NUdf::TDataType<bool>::Id, ctx.TypeEnv);
+ auto json1 = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(true), type);
+ UNIT_ASSERT_VALUES_EQUAL(json1, "true");
+
+ auto json2 = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(false), type);
+ UNIT_ASSERT_VALUES_EQUAL(json2, "false");
+ }
+}
+
+Y_UNIT_TEST_SUITE(SerializeUuid) {
+ Y_UNIT_TEST(Uuid) {
+ TTestContext ctx;
+ auto type = TDataType::Create(NUdf::TDataType<NUdf::TUuid>::Id, ctx.TypeEnv);
+ auto value = ctx.Vb.NewString("80070214-6ad1-4077-add8-74b68f105e3c");
+ auto json = WriteValueToFuncJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"80070214-6ad1-4077-add8-74b68f105e3c\"");
+ }
+}
+
+Y_UNIT_TEST_SUITE(SerializeJson) {
+ Y_UNIT_TEST(ScalarJson) {
+ TTestContext ctx;
+ auto type = TDataType::Create(NUdf::TDataType<NUdf::TJson>::Id, ctx.TypeEnv);
+ auto value = ctx.Vb.NewString("\"some string с русскими йЁ\"");
+ auto json = WriteValueToFuncJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"some string с русскими йЁ\"");
+ }
+
+ Y_UNIT_TEST(ComplexJson) {
+ TTestContext ctx;
+ TStructMember members[] = {
+ TStructMember("X", TDataType::Create(NUdf::TDataType<NUdf::TJson>::Id, ctx.TypeEnv)),
+ TStructMember("Y", TDataType::Create(NUdf::TDataType<ui32>::Id, ctx.TypeEnv))
+ };
+ auto type = TStructType::Create(2, members, ctx.TypeEnv);
+
+ NUdf::TUnboxedValue* items;
+ auto value = ctx.Vb.NewArray(2, items);
+ items[0] = ctx.Vb.NewString("{\"a\":500,\"b\":[1,2,3]}");
+ items[1] = NUdf::TUnboxedValuePod(ui32(73));
+
+ auto json = WriteValueToExportJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "{\"X\":{\"a\":500,\"b\":[1,2,3]},\"Y\":73}");
+ }
+}
+
+Y_UNIT_TEST_SUITE(SerializeContainers) {
+ Y_UNIT_TEST(EmptyContainer) {
+ TTestContext ctx;
+ auto value = ctx.HolderFactory.GetEmptyContainer();
+ auto dictJson = WriteValueToFuncJsonStr(value, ctx.TypeEnv.GetTypeOfEmptyDict());
+ UNIT_ASSERT_VALUES_EQUAL(dictJson, "[]");
+
+ auto listJson = WriteValueToFuncJsonStr(value, ctx.TypeEnv.GetTypeOfEmptyList());
+ UNIT_ASSERT_VALUES_EQUAL(listJson, "[]");
+ }
+
+ Y_UNIT_TEST(NormalList) {
+ TTestContext ctx;
+ NUdf::TUnboxedValue* items;
+ auto value = ctx.Vb.NewArray(4, items);
+ items[0] = NUdf::TUnboxedValuePod(1);
+ items[1] = NUdf::TUnboxedValuePod(5);
+ items[2] = NUdf::TUnboxedValuePod(-18);
+ items[3] = NUdf::TUnboxedValuePod(4);
+ auto type = TListType::Create(TDataType::Create(NUdf::TDataType<i32>::Id, ctx.TypeEnv), ctx.TypeEnv);
+ auto json = WriteValueToFuncJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "[\"1\",\"5\",\"-18\",\"4\"]");
+ }
+
+ Y_UNIT_TEST(EmptyList) {
+ TTestContext ctx;
+ NUdf::TUnboxedValue* items;
+ auto value = ctx.Vb.NewArray(0, items);
+ auto type = TListType::Create(TDataType::Create(NUdf::TDataType<i32>::Id, ctx.TypeEnv), ctx.TypeEnv);
+ auto json = WriteValueToFuncJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "[]");
+ }
+
+ Y_UNIT_TEST(FullTuple) {
+ TTestContext ctx;
+
+ TType* members[] = {
+ TDataType::Create(NUdf::TDataType<ui8>::Id, ctx.TypeEnv),
+ TDataType::Create(NUdf::TDataType<NUdf::TUtf8>::Id, ctx.TypeEnv),
+ TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv),
+ };
+ auto type = TTupleType::Create(3, members, ctx.TypeEnv);
+
+ NUdf::TUnboxedValue* items;
+ NUdf::TUnboxedValue value = ctx.HolderFactory.CreateDirectArrayHolder(type->GetElementsCount(), items);
+ items[0] = NUdf::TUnboxedValuePod(ui8(65));
+ items[1] = ctx.Vb.NewString("абсдева");
+ ui16 date;
+ ctx.Vb.MakeDate(2021, 5, 5, date);
+ items[2] = NUdf::TUnboxedValuePod(date);
+
+ auto json = WriteValueToExportJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "[65,\"абсдева\",\"2021-05-05\"]");
+ }
+
+ Y_UNIT_TEST(EmptyTuple) {
+ TTestContext ctx;
+ TType* members[] = {};
+ auto type = TTupleType::Create(0, members, ctx.TypeEnv);
+ NUdf::TUnboxedValue* items;
+ NUdf::TUnboxedValue value = ctx.HolderFactory.CreateDirectArrayHolder(type->GetElementsCount(), items);
+ auto json = WriteValueToFuncJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "[]");
+ }
+
+ Y_UNIT_TEST(DictType) {
+ TTestContext ctx;
+ auto type = TDictType::Create(
+ TDataType::Create(NUdf::TDataType<NUdf::TUtf8>::Id, ctx.TypeEnv),
+ TDataType::Create(NUdf::TDataType<i32>::Id, ctx.TypeEnv),
+ ctx.TypeEnv
+ );
+ auto dictBuilder = ctx.Vb.NewDict(type, NUdf::TDictFlags::EDictKind::Hashed);
+ dictBuilder->Add(
+ ctx.Vb.NewString("key_a"),
+ NUdf::TUnboxedValuePod(781)
+ );
+ dictBuilder->Add(
+ ctx.Vb.NewString("key_b"),
+ NUdf::TUnboxedValuePod(-500)
+ );
+ auto json = WriteValueToExportJsonStr(dictBuilder->Build(), type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "[[\"key_b\",-500],[\"key_a\",781]]");
+ }
+
+ Y_UNIT_TEST(Tagged) {
+ TTestContext ctx;
+ auto type = TTaggedType::Create(TDataType::Create(NUdf::TDataType<NUdf::TDatetime>::Id, ctx.TypeEnv),
+ "test-tag", ctx.TypeEnv);
+ ui32 datetime;
+ ctx.Vb.MakeDatetime(2021, 1, 1, 14, 5, 43, datetime);
+ auto json = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(datetime), type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"2021-01-01T14:05:43Z\"");
+ }
+
+ Y_UNIT_TEST(TupleVariant) {
+ TTestContext ctx;
+ TType* tupleTypes[] = {
+ TDataType::Create(NUdf::TDataType<bool>::Id, ctx.TypeEnv),
+ TDataType::Create(NUdf::TDataType<ui32>::Id, ctx.TypeEnv)
+ };
+ auto underlying = TTupleType::Create(2, tupleTypes, ctx.TypeEnv);
+ auto type = TVariantType::Create(underlying, ctx.TypeEnv);
+
+ auto value0 = ctx.HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(false), 0);
+ auto json0 = WriteValueToFuncJsonStr(value0, type);
+ UNIT_ASSERT_VALUES_EQUAL(json0, "[0,false]");
+
+ auto value1 = ctx.HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(200), 1);
+ auto json1 = WriteValueToExportJsonStr(value1, type);
+ UNIT_ASSERT_VALUES_EQUAL(json1, "[1,200]");
+ }
+
+ Y_UNIT_TEST(StructVariant) {
+ TTestContext ctx;
+
+ TStructMember members[] = {
+ TStructMember("A", TDataType::Create(NUdf::TDataType<bool>::Id, ctx.TypeEnv)),
+ TStructMember("B", TDataType::Create(NUdf::TDataType<ui32>::Id, ctx.TypeEnv))
+ };
+ auto underlying = TStructType::Create(2, members, ctx.TypeEnv);
+ auto type = TVariantType::Create(underlying, ctx.TypeEnv);
+
+ auto value0 = ctx.HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(true), 0);
+ auto json0 = WriteValueToExportJsonStr(value0, type);
+ UNIT_ASSERT_VALUES_EQUAL(json0, "[0,true]");
+
+ auto value1 = ctx.HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(200), 1);
+ auto json1 = WriteValueToExportJsonStr(value1, type);
+ UNIT_ASSERT_VALUES_EQUAL(json1, "[1,200]");
+ }
+
+ Y_UNIT_TEST(StructType) {
+ TTestContext ctx;
+
+ TStructMember members[] = {
+ {"A", TDataType::Create(NUdf::TDataType<bool>::Id, ctx.TypeEnv)},
+ {"B", TDataType::Create(NUdf::TDataType<ui32>::Id, ctx.TypeEnv)}
+ };
+ auto type = TStructType::Create(2, members, ctx.TypeEnv);
+
+ NUdf::TUnboxedValue* items;
+ auto value = ctx.Vb.NewArray(2, items);
+ items[0] = NUdf::TUnboxedValuePod(false);
+ items[1] = NUdf::TUnboxedValuePod(ui32(555));
+
+ auto json = WriteValueToExportJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "{\"A\":false,\"B\":555}");
+ }
+}
+
+Y_UNIT_TEST_SUITE(SerializeOptional) {
+ Y_UNIT_TEST(JustOptional) {
+ TTestContext ctx;
+ NUdf::TUnboxedValue* items;
+ auto value = ctx.HolderFactory.CreateDirectArrayHolder(3, items).MakeOptional();
+ items[0] = NUdf::TUnboxedValuePod(ui8(0));
+ items[1] = NUdf::TUnboxedValuePod(ui8(67));
+ items[2] = NUdf::TUnboxedValuePod(ui8(4));
+ auto elementType = TListType::Create(TDataType::Create(NUdf::TDataType<ui8>::Id, ctx.TypeEnv), ctx.TypeEnv);
+ auto type = TOptionalType::Create(elementType, ctx.TypeEnv);
+ auto json = WriteValueToExportJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "[[0,67,4]]");
+ }
+
+ Y_UNIT_TEST(NothingOptional) {
+ TTestContext ctx;
+ auto value = NUdf::TUnboxedValuePod().MakeOptional();
+ auto elementType = TListType::Create(TDataType::Create(NUdf::TDataType<ui8>::Id, ctx.TypeEnv), ctx.TypeEnv);
+ auto type = TOptionalType::Create(elementType, ctx.TypeEnv);
+ auto json = WriteValueToFuncJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "[null]");
+ }
+
+ Y_UNIT_TEST(SeveralOptionals) {
+ TTestContext ctx;
+ auto elementType = TDataType::Create(NUdf::TDataType<ui8>::Id, ctx.TypeEnv);
+ TType* type = TOptionalType::Create(elementType, ctx.TypeEnv);
+ type = TOptionalType::Create(type, ctx.TypeEnv);
+ type = TOptionalType::Create(type, ctx.TypeEnv);
+
+ auto value1 = NUdf::TUnboxedValuePod().MakeOptional().MakeOptional().MakeOptional();
+ auto json1 = WriteValueToFuncJsonStr(value1, type);
+ UNIT_ASSERT_VALUES_EQUAL(json1, "[[[null]]]");
+
+ auto value2 = NUdf::TUnboxedValuePod(ui8(120)).MakeOptional().MakeOptional().MakeOptional();
+ auto json2 = WriteValueToExportJsonStr(value2, type);
+ UNIT_ASSERT_VALUES_EQUAL(json2, "[[[120]]]");
+ }
+}
+
+Y_UNIT_TEST_SUITE(SerializeNumbers) {
+
+#define TEST_SERIALIZE_NUMBER_TYPE(type) \
+ Y_UNIT_TEST(type) { \
+ TTestContext ctx; \
+ auto maxValue = NUdf::TUnboxedValuePod(std::numeric_limits<type>::max()); \
+ auto maxJson = WriteValueToFuncJsonStr(maxValue, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv)); \
+ UNIT_ASSERT_VALUES_EQUAL(maxJson, "\"" + ToString(std::numeric_limits<type>::max()) + "\""); \
+ \
+ auto minValue = NUdf::TUnboxedValuePod(std::numeric_limits<type>::min()); \
+ auto minJson = WriteValueToFuncJsonStr(minValue, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv)); \
+ UNIT_ASSERT_VALUES_EQUAL(minJson, "\"" + ToString(std::numeric_limits<type>::min()) + "\""); \
+ }
+
+ TEST_SERIALIZE_NUMBER_TYPE(i8)
+ TEST_SERIALIZE_NUMBER_TYPE(i16)
+ TEST_SERIALIZE_NUMBER_TYPE(i32)
+
+ TEST_SERIALIZE_NUMBER_TYPE(ui8)
+ TEST_SERIALIZE_NUMBER_TYPE(ui16)
+ TEST_SERIALIZE_NUMBER_TYPE(ui32)
+
+#undef TEST_SERIALIZE_NUMBER_TYPE
+
+ Y_UNIT_TEST(float) {
+ TTestContext ctx;
+ auto value = NUdf::TUnboxedValuePod(std::numeric_limits<float>::max());
+ auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<float>::Id, ctx.TypeEnv));
+ auto expected = FloatToString(std::numeric_limits<float>::max(), EFloatToStringMode::PREC_NDIGITS, std::numeric_limits<float>::max_digits10);
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"" + expected + "\"");
+ }
+
+ Y_UNIT_TEST(double) {
+ TTestContext ctx;
+ auto value = NUdf::TUnboxedValuePod(std::numeric_limits<double>::min());
+ auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<double>::Id, ctx.TypeEnv));
+ auto expected = FloatToString(std::numeric_limits<double>::min(), EFloatToStringMode::PREC_NDIGITS, std::numeric_limits<double>::max_digits10);
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"" + expected + "\"");
+ }
+
+ Y_UNIT_TEST(DoubleSignificantNumbers) {
+ TTestContext ctx;
+ auto value = NUdf::TUnboxedValuePod(double(3.1415926535897932384626433832795));
+ auto type = TDataType::Create(NUdf::TDataType<double>::Id, ctx.TypeEnv);
+ auto json = WriteValueToFuncJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"3.1415926535897931\"");
+
+ auto json2 = WriteValueToExportJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json2, "3.1415926535897931");
+ }
+
+ Y_UNIT_TEST(i64) {
+ TTestContext ctx;
+ auto maxValue = NUdf::TUnboxedValuePod(std::numeric_limits<i64>::max());
+ auto maxJson = WriteValueToFuncJsonStr(maxValue, TDataType::Create(NUdf::TDataType<i64>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(maxJson, "\"" + ToString(std::numeric_limits<i64>::max()) + "\"");
+
+ auto minValue = NUdf::TUnboxedValuePod(std::numeric_limits<i64>::min());
+ auto minJson = WriteValueToFuncJsonStr(minValue, TDataType::Create(NUdf::TDataType<i64>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(minJson, "\"" + ToString(std::numeric_limits<i64>::min()) + "\"");
+
+ auto fitInJS = NUdf::TUnboxedValuePod(i64(-9007199254740991)); // == Number.MIN_SAFE_INTEGER
+ auto fitInJSJson = WriteValueToFuncJsonStr(fitInJS, TDataType::Create(NUdf::TDataType<i64>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(fitInJSJson, "\"-9007199254740991\"");
+ }
+
+ Y_UNIT_TEST(ui64) {
+ TTestContext ctx;
+ auto maxValue = NUdf::TUnboxedValuePod(std::numeric_limits<ui64>::max());
+ auto maxJson = WriteValueToFuncJsonStr(maxValue, TDataType::Create(NUdf::TDataType<ui64>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(maxJson, "\"" + ToString(std::numeric_limits<ui64>::max()) + "\"");
+
+ auto minValue = NUdf::TUnboxedValuePod(std::numeric_limits<ui64>::min());
+ auto minJson = WriteValueToFuncJsonStr(minValue, TDataType::Create(NUdf::TDataType<ui64>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(minJson, "\"" + ToString(std::numeric_limits<ui64>::min()) + "\"");
+
+ auto fitInJS = NUdf::TUnboxedValuePod(ui64(2419787883133419));
+ auto fitInJSJson = WriteValueToFuncJsonStr(fitInJS, TDataType::Create(NUdf::TDataType<ui64>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(fitInJSJson, "\"2419787883133419\"");
+ }
+
+ Y_UNIT_TEST(Decimal) {
+ TTestContext ctx;
+ auto value = NUdf::TUnboxedValuePod(NYql::NDecimal::TInt128(78876543LL));
+ auto type = TDataDecimalType::Create(10, 2, ctx.TypeEnv);
+ auto json1 = WriteValueToFuncJsonStr(value, type);
+ UNIT_ASSERT_VALUES_EQUAL(json1, "\"788765.43\"");
+ }
+}
+
+Y_UNIT_TEST_SUITE(DeserializeNumbers) {
+
+#define TEST_DESERIALIZE_NUMBER_TYPE(type, wideType) \
+ Y_UNIT_TEST(type) { \
+ TTestContext ctx; \
+ \
+ TStringStream maxJson; \
+ maxJson << ToString(std::numeric_limits<type>::max()); \
+ auto maxValue = ReadJsonValue(&maxJson, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv), ctx.HolderFactory); \
+ UNIT_ASSERT_VALUES_EQUAL(maxValue.Get<type>(), std::numeric_limits<type>::max()); \
+ \
+ TStringStream minJson; \
+ minJson << ToString(std::numeric_limits<type>::min()); \
+ auto minValue = ReadJsonValue(&minJson, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv), ctx.HolderFactory); \
+ UNIT_ASSERT_VALUES_EQUAL(minValue.Get<type>(), std::numeric_limits<type>::min()); \
+ \
+ TStringStream exceededJson; \
+ exceededJson << ToString(wideType(std::numeric_limits<type>::max() + wideType(1))); \
+ UNIT_ASSERT_EXCEPTION_CONTAINS( \
+ ReadJsonValue(&exceededJson, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv), ctx.HolderFactory), \
+ yexception, \
+ "Exceeded the range" \
+ ); \
+ }
+
+ TEST_DESERIALIZE_NUMBER_TYPE(i8, i64)
+ TEST_DESERIALIZE_NUMBER_TYPE(i16, i64)
+ TEST_DESERIALIZE_NUMBER_TYPE(i32, i64)
+
+ TEST_DESERIALIZE_NUMBER_TYPE(ui8, ui64)
+ TEST_DESERIALIZE_NUMBER_TYPE(ui16, ui64)
+ TEST_DESERIALIZE_NUMBER_TYPE(ui32, ui64)
+
+#undef TEST_DESERIALIZE_NUMBER_TYPE
+
+ Y_UNIT_TEST(float) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "0.0431";
+ auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<float>::Id, ctx.TypeEnv), ctx.HolderFactory);
+ UNIT_ASSERT_VALUES_EQUAL(value.Get<float>(), 0.0431f);
+
+ TStringStream exceededJson;
+ exceededJson << ToString(std::numeric_limits<double>::max());
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ ReadJsonValue(&exceededJson, TDataType::Create(NUdf::TDataType<float>::Id, ctx.TypeEnv), ctx.HolderFactory),
+ yexception,
+ "Exceeded the range"
+ );
+ }
+
+ Y_UNIT_TEST(FloatFromInt) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "1766718243";
+ auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<float>::Id, ctx.TypeEnv), ctx.HolderFactory);
+ UNIT_ASSERT_VALUES_EQUAL(value.Get<float>(), 1766718243.0f);
+ }
+
+ Y_UNIT_TEST(double) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "7773.13";
+ auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<double>::Id, ctx.TypeEnv), ctx.HolderFactory);
+ UNIT_ASSERT_VALUES_EQUAL(value.Get<double>(), 7773.13);
+ }
+
+ Y_UNIT_TEST(DoubleFromInt) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "-667319001";
+ auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<double>::Id, ctx.TypeEnv), ctx.HolderFactory);
+ UNIT_ASSERT_VALUES_EQUAL(value.Get<double>(), -667319001.0l);
+ }
+
+}
+
+Y_UNIT_TEST_SUITE(SerializeStringTypes) {
+ Y_UNIT_TEST(Utf8) {
+ TTestContext ctx;
+ auto value = ctx.Vb.NewString("aaaaabbbbbcccccc");
+ auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<NUdf::TUtf8>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"aaaaabbbbbcccccc\"");
+ }
+
+ Y_UNIT_TEST(String) {
+ TTestContext ctx;
+
+ auto type = TDataType::Create(NUdf::TDataType<char*>::Id, ctx.TypeEnv);
+
+ auto value1 = ctx.Vb.NewString("aaaaabbbbbcccccc");
+ auto json1 = WriteValueToFuncJsonStr(value1, type);
+ UNIT_ASSERT_VALUES_EQUAL(json1, "\"YWFhYWFiYmJiYmNjY2NjYw==\"");
+
+ auto value2 = ctx.Vb.NewString("абсёЙabc");
+ auto json2 = WriteValueToFuncJsonStr(value2, type);
+ UNIT_ASSERT_VALUES_EQUAL(json2, "\"0LDQsdGB0ZHQmWFiYw==\"");
+ }
+}
+
+Y_UNIT_TEST_SUITE(DeserializeStringTypes) {
+ Y_UNIT_TEST(String) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "\"fffaaae423\"";
+ auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TUtf8>::Id, ctx.TypeEnv), ctx.HolderFactory);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf("fffaaae423"), TStringBuf(value.AsStringRef()));
+ }
+}
+
+Y_UNIT_TEST_SUITE(SerializeDateTypes) {
+ Y_UNIT_TEST(Date) {
+ TTestContext ctx;
+ ui16 date;
+ ctx.Vb.MakeDate(2022, 2, 9, date);
+ auto json = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(date), TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"2022-02-09\"");
+ }
+
+ Y_UNIT_TEST(Datetime) {
+ TTestContext ctx;
+ ui32 datetime;
+ ctx.Vb.MakeDatetime(2021, 1, 1, 14, 5, 43, datetime);
+ auto json = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(datetime), TDataType::Create(NUdf::TDataType<NUdf::TDatetime>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"2021-01-01T14:05:43Z\"");
+ }
+
+ Y_UNIT_TEST(Timestamp) {
+ TTestContext ctx;
+ auto value = ui64(1644755212879622);
+ auto json = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(value), TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"2022-02-13T12:26:52.879622Z\"");
+ }
+
+ Y_UNIT_TEST(TzDate) {
+ TTestContext ctx;
+ ui16 rawDate;
+ ctx.Vb.MakeDate(2023, 4, 14, rawDate);
+ auto date = NUdf::TUnboxedValuePod(rawDate);
+ date.SetTimezoneId(530);
+ auto json = WriteValueToFuncJsonStr(date, TDataType::Create(NUdf::TDataType<NUdf::TTzDate>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"2023-04-14,Pacific/Easter\"");
+ }
+
+ Y_UNIT_TEST(TzDateTime) {
+ TTestContext ctx;
+ ui16 timeZone = 530;
+ ui32 rawDate;
+ ctx.Vb.MakeDatetime(2023, 4, 14, 0, 15, 0, rawDate, timeZone);
+ auto date = NUdf::TUnboxedValuePod(rawDate);
+ date.SetTimezoneId(timeZone);
+ auto json = WriteValueToFuncJsonStr(date, TDataType::Create(NUdf::TDataType<NUdf::TTzDatetime>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"2023-04-14T00:15:00,Pacific/Easter\"");
+ }
+
+ Y_UNIT_TEST(TzTimestamp) {
+ TTestContext ctx;
+ auto value = NUdf::TUnboxedValuePod(ui64(1644755564087924));
+ value.SetTimezoneId(327);
+ auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<NUdf::TTzTimestamp>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"2022-02-13T19:32:44.087924,Asia/Vientiane\"");
+ }
+
+ Y_UNIT_TEST(Interval) {
+ TTestContext ctx;
+ // 2 days 2 hours in ms
+ auto value = NUdf::TUnboxedValuePod(i64(180000000000));
+ auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<NUdf::TInterval>::Id, ctx.TypeEnv));
+ UNIT_ASSERT_VALUES_EQUAL(json, "\"P2DT2H\"");
+ }
+}
+
+Y_UNIT_TEST_SUITE(DeserializeDateTypes) {
+
+ Y_UNIT_TEST(Date) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "\"2020-09-11\"";
+ auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv), ctx.HolderFactory);
+ ui16 date;
+ ctx.Vb.MakeDate(2020, 9, 11, date);
+ UNIT_ASSERT_VALUES_EQUAL(value.Get<ui16>(), date);
+ }
+
+ Y_UNIT_TEST(Datetime) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "\"2021-07-14T00:00:43Z\"";
+ auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TDatetime>::Id, ctx.TypeEnv), ctx.HolderFactory);
+ ui32 datetime;
+ ctx.Vb.MakeDatetime(2021, 7, 14, 0, 0, 43, datetime);
+ UNIT_ASSERT_VALUES_EQUAL(value.Get<ui32>(), datetime);
+ }
+
+ Y_UNIT_TEST(TzDate) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "\"2020-09-11,Europe/Moscow\""; // timeZoneId == 1
+ auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TTzDate>::Id, ctx.TypeEnv), ctx.HolderFactory);
+ ui16 date;
+ ctx.Vb.MakeDate(2020, 9, 10, date);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetTimezoneId(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(value.Get<ui16>(), date);
+ }
+
+ Y_UNIT_TEST(TzDatetime) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "\"2020-09-11T01:11:05,Europe/Moscow\""; // timeZoneId == 1
+ auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TTzDatetime>::Id, ctx.TypeEnv), ctx.HolderFactory);
+ ui32 datetime;
+ ctx.Vb.MakeDatetime(2020, 9, 11, 1, 11, 5, datetime, 1);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetTimezoneId(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(value.Get<ui32>(), datetime);
+ }
+
+ Y_UNIT_TEST(WrongTypeEx) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "[\"2020-22-12\"]";
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv), ctx.HolderFactory),
+ yexception,
+ "Unexpected json type (expected string"
+ );
+ }
+
+ Y_UNIT_TEST(WrongFormatEx) {
+ TTestContext ctx;
+ TStringStream json;
+ json << "\"2020-18-43\"";
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv), ctx.HolderFactory),
+ yexception,
+ "Invalid date format"
+ );
+ }
+}
+
+} // namespace
+} // namespace
+