diff options
author | hrustyashko <hrustyashko@yandex-team.ru> | 2022-02-17 15:24:07 +0300 |
---|---|---|
committer | hrustyashko <hrustyashko@yandex-team.ru> | 2022-02-17 15:24:07 +0300 |
commit | 329e7bdd49970bba890ca97109ef98486b41a7cd (patch) | |
tree | cb5781f57320495efdce89f0fd5b53d9dede7328 | |
parent | f22b29e7eda4d2c622480b2f181257562353272b (diff) | |
download | ydb-329e7bdd49970bba890ca97109ef98486b41a7cd.tar.gz |
YQ-418 Библиотека конвертации YQ типов в JSON
ref:8eced38335e97fd1e3738ef6cf0b647b26209181
-rw-r--r-- | ydb/library/yql/minikql/mkql_type_ops.cpp | 27 | ||||
-rw-r--r-- | ydb/library/yql/providers/common/codec/ut/ya.make | 22 | ||||
-rw-r--r-- | ydb/library/yql/providers/common/codec/ya.make | 6 | ||||
-rw-r--r-- | ydb/library/yql/providers/common/codec/yql_json_codec.cpp | 458 | ||||
-rw-r--r-- | ydb/library/yql/providers/common/codec/yql_json_codec.h | 30 | ||||
-rw-r--r-- | ydb/library/yql/providers/common/codec/yql_json_codec_ut.cpp | 636 |
6 files changed, 1152 insertions, 27 deletions
diff --git a/ydb/library/yql/minikql/mkql_type_ops.cpp b/ydb/library/yql/minikql/mkql_type_ops.cpp index d559bb108c..3801de6c81 100644 --- a/ydb/library/yql/minikql/mkql_type_ops.cpp +++ b/ydb/library/yql/minikql/mkql_type_ops.cpp @@ -582,33 +582,6 @@ NUdf::TUnboxedValuePod NumberFromString(NUdf::TStringRef buf) { return NUdf::TUnboxedValuePod(value); } -bool MakeDateUncached(ui32 year, ui32 month, ui32 day, ui16& value) { - if (year < NUdf::MIN_YEAR || year >= NUdf::MAX_YEAR) { - return false; - } - - if (month < 1 || month > 12) { - return false; - } - - const bool isLeap = IsLeapYear(year); - auto monthLength = GetMonthLength(month, isLeap); - if (day < 1 || day > monthLength) { - return false; - } - - year -= NUdf::MIN_YEAR; - ui32 leapDaysCount = LeapDaysSinceEpoch(year); - value = year * 365 + leapDaysCount; - while (month > 1) { - --month; - value += GetMonthLength(month, isLeap); - } - - value += day - 1; - return true; -} - bool MakeTime(ui32 hour, ui32 minute, ui32 second, ui32& value) { if (hour >= 24 || minute >= 60 || second >= 60) { return false; diff --git a/ydb/library/yql/providers/common/codec/ut/ya.make b/ydb/library/yql/providers/common/codec/ut/ya.make new file mode 100644 index 0000000000..0c5ad6d824 --- /dev/null +++ b/ydb/library/yql/providers/common/codec/ut/ya.make @@ -0,0 +1,22 @@ +UNITTEST_FOR(ydb/library/yql/providers/common/codec) + +OWNER(g:yql) + +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + SIZE(MEDIUM) +ENDIF() + +SRCS( + yql_json_codec_ut.cpp +) + +PEERDIR( + library/cpp/testing/unittest + ydb/library/yql/public/udf/service/exception_policy +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/providers/common/codec/ya.make b/ydb/library/yql/providers/common/codec/ya.make index b3f3a92db5..0c44e3fdc2 100644 --- a/ydb/library/yql/providers/common/codec/ya.make +++ b/ydb/library/yql/providers/common/codec/ya.make @@ -13,6 +13,7 @@ SRCS( yql_restricted_yson.h yql_codec_type_flags.cpp yql_codec_type_flags.h + yql_json_codec.cpp ) PEERDIR( @@ -21,6 +22,7 @@ PEERDIR( ydb/library/yql/providers/common/mkql library/cpp/yson/node library/cpp/yson + library/cpp/json ) YQL_LAST_ABI_VERSION() @@ -28,3 +30,7 @@ YQL_LAST_ABI_VERSION() GENERATE_ENUM_SERIALIZATION(yql_codec_type_flags.h) END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/providers/common/codec/yql_json_codec.cpp b/ydb/library/yql/providers/common/codec/yql_json_codec.cpp new file mode 100644 index 0000000000..cad700b449 --- /dev/null +++ b/ydb/library/yql/providers/common/codec/yql_json_codec.cpp @@ -0,0 +1,458 @@ +#include "yql_json_codec.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_type_ops.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> +#include <ydb/library/yql/utils/yql_panic.h> + +#include <library/cpp/string_utils/base64/base64.h> +#include <util/string/join.h> + +namespace NYql { +namespace NCommon { + +using namespace NKikimr; +using namespace NKikimr::NMiniKQL; +using namespace NJson; + +namespace { + +constexpr i64 MAX_JS_SAFE_INTEGER = 9007199254740991; // 2^53 - 1; JavaScript Number.MAX_SAFE_INTEGER +constexpr i64 MIN_JS_SAFE_INTEGER = -9007199254740991; // -(2^53 - 1); JavaScript Number.MIN_SAFE_INTEGER + +constexpr i8 DOUBLE_N_DIGITS = std::numeric_limits<double>::max_digits10; +constexpr i8 FLOAT_N_DIGITS = std::numeric_limits<float>::max_digits10; +constexpr EFloatToStringMode FLOAT_MODE = EFloatToStringMode::PREC_NDIGITS; +} + +TJsonWriterConfig MakeJsonConfig() { + TJsonWriterConfig config; + config.DoubleNDigits = DOUBLE_N_DIGITS; + config.FloatNDigits = FLOAT_N_DIGITS; + config.FloatToStringMode = FLOAT_MODE; + config.FormatOutput = false; + config.SortKeys = false; + config.ValidateUtf8 = false; + config.DontEscapeStrings = true; + config.WriteNanAsString = false; + + return config; +} + +void WriteValueToJson(TJsonWriter& writer, const NKikimr::NUdf::TUnboxedValuePod& value, + NKikimr::NMiniKQL::TType* type, std::set<EValueConvertPolicy> convertPolicy) { + + switch (type->GetKind()) { + case TType::EKind::Void: + case TType::EKind::Null: + writer.WriteNull(); + break; + case TType::EKind::EmptyList: + case TType::EKind::EmptyDict: + writer.OpenArray(); + writer.CloseArray(); + break; + case TType::EKind::Data: + { + bool numberToStr = convertPolicy.contains(EValueConvertPolicy::WriteNumberString); + auto dataType = AS_TYPE(TDataType, type); + switch (dataType->GetSchemeType()) { + case NUdf::TDataType<bool>::Id: + writer.Write(value.Get<bool>()); + break; + case NUdf::TDataType<i32>::Id: { + auto number = value.Get<i32>(); + if (numberToStr) { + writer.Write(ToString(number)); + } else { + writer.Write(number); + } + break; + } + case NUdf::TDataType<ui32>::Id: { + auto number = value.Get<ui32>(); + if (numberToStr) { + writer.Write(ToString(number)); + } else { + writer.Write(number); + } + break; + } + case NUdf::TDataType<i64>::Id: { + auto number = value.Get<i64>(); + if (numberToStr) { + writer.Write(ToString(number)); + } else if (convertPolicy.contains(EValueConvertPolicy::WriteUnsafeNumberString)) { + if (number > MAX_JS_SAFE_INTEGER || number < MIN_JS_SAFE_INTEGER) { + writer.Write(ToString(number)); + } else { + writer.Write(number); + }; + } else { + writer.Write(number); + } + break; + } + case NUdf::TDataType<ui64>::Id: { + auto number = value.Get<ui64>(); + if (numberToStr) { + writer.Write(ToString(number)); + } else if (convertPolicy.contains(EValueConvertPolicy::WriteUnsafeNumberString)) { + if (number > MAX_JS_SAFE_INTEGER) { + writer.Write(ToString(number)); + } else { + writer.Write(number); + }; + } else { + writer.Write(number); + } + break; + } + case NUdf::TDataType<ui8>::Id: { + auto number = value.Get<ui8>(); + if (numberToStr) { + writer.Write(ToString(number)); + } else { + writer.Write(number); + } + break; + } + case NUdf::TDataType<i8>::Id: { + auto number = value.Get<i8>(); + if (numberToStr) { + writer.Write(ToString(number)); + } else { + writer.Write(number); + } + break; + } + case NUdf::TDataType<ui16>::Id: { + auto number = value.Get<ui16>(); + if (numberToStr) { + writer.Write(ToString(number)); + } else { + writer.Write(number); + } + break; + } + case NUdf::TDataType<i16>::Id: { + auto number = value.Get<i16>(); + if (numberToStr) { + writer.Write(ToString(number)); + } else { + writer.Write(number); + } + break; + } + case NUdf::TDataType<float>::Id: + if (numberToStr) { + TString number = FloatToString(value.Get<float>(), FLOAT_MODE, FLOAT_N_DIGITS); + writer.Write(number); + } else { + writer.Write(value.Get<float>()); + } + break; + case NUdf::TDataType<double>::Id: + if (numberToStr) { + TString number = FloatToString(value.Get<double>(), FLOAT_MODE, DOUBLE_N_DIGITS); + writer.Write(number); + } else { + writer.Write(value.Get<double>()); + } + break; + case NUdf::TDataType<NUdf::TJson>::Id: + writer.UnsafeWrite(value.AsStringRef()); + break; + case NUdf::TDataType<NUdf::TUtf8>::Id: + writer.Write(value.AsStringRef()); + break; + case NUdf::TDataType<char*>::Id: { + TString encoded = Base64Encode(value.AsStringRef()); + writer.Write(encoded); + break; + } + case NUdf::TDataType<NUdf::TDecimal>::Id: { + const auto params = static_cast<TDataDecimalType*>(type)->GetParams(); + const auto str = NDecimal::ToString(value.GetInt128(), params.first, params.second); + const auto size = str ? std::strlen(str) : 0; + writer.Write(TStringBuf(str, size)); + break; + } + case NUdf::TDataType<NUdf::TUuid>::Id: + writer.Write(value.AsStringRef()); + break; + case NUdf::TDataType<NUdf::TYson>::Id: + case NUdf::TDataType<NUdf::TDyNumber>::Id: + case NUdf::TDataType<NUdf::TDate>::Id: + case NUdf::TDataType<NUdf::TDatetime>::Id: + case NUdf::TDataType<NUdf::TTimestamp>::Id: + case NUdf::TDataType<NUdf::TInterval>::Id: + case NUdf::TDataType<NUdf::TTzDate>::Id: + case NUdf::TDataType<NUdf::TTzDatetime>::Id: + case NUdf::TDataType<NUdf::TTzTimestamp>::Id: + case NUdf::TDataType<NUdf::TJsonDocument>::Id: { + const NUdf::TUnboxedValue out(ValueToString(*dataType->GetDataSlot(), value)); + writer.Write(out.AsStringRef()); + break; + } + default: + throw yexception() << "Unknown data type: " << dataType->GetSchemeType(); + } + } + break; + case TType::EKind::Struct: + { + writer.OpenMap(); + auto structType = AS_TYPE(TStructType, type); + for (ui32 i = 0, e = structType->GetMembersCount(); i < e; ++i) { + writer.WriteKey(structType->GetMemberName(i)); + WriteValueToJson(writer, value.GetElement(i), structType->GetMemberType(i), convertPolicy); + } + writer.CloseMap(); + break; + } + case TType::EKind::List: + { + writer.OpenArray(); + auto listType = AS_TYPE(TListType, type); + const auto it = value.GetListIterator(); + for (NUdf::TUnboxedValue item; it.Next(item);) { + WriteValueToJson(writer, item, listType->GetItemType(), convertPolicy); + } + writer.CloseArray(); + break; + } + case TType::EKind::Optional: + { + writer.OpenArray(); + if (!value.GetOptionalValue()) { + writer.WriteNull(); + } else { + auto optionalType = AS_TYPE(TOptionalType, type); + WriteValueToJson(writer, value.GetOptionalValue(), optionalType->GetItemType(), convertPolicy); + } + writer.CloseArray(); + break; + } + case TType::EKind::Dict: + { + writer.OpenArray(); + auto dictType = AS_TYPE(TDictType, type); + const auto it = value.GetDictIterator(); + for (NUdf::TUnboxedValue key, payload; it.NextPair(key, payload);) { + writer.OpenArray(); + WriteValueToJson(writer, key, dictType->GetKeyType(), convertPolicy); + WriteValueToJson(writer, payload, dictType->GetPayloadType(), convertPolicy); + writer.CloseArray(); + } + writer.CloseArray(); + break; + } + case TType::EKind::Tuple: + { + writer.OpenArray(); + auto tupleType = AS_TYPE(TTupleType, type); + for (ui32 i = 0, e = tupleType->GetElementsCount(); i < e; ++i) { + WriteValueToJson(writer, value.GetElement(i), tupleType->GetElementType(i), convertPolicy); + } + writer.CloseArray(); + break; + } + case TType::EKind::Variant: + { + writer.OpenArray(); + auto index = value.GetVariantIndex(); + writer.Write(index); + + auto underlyingType = AS_TYPE(TVariantType, type)->GetUnderlyingType(); + if (underlyingType->IsTuple()) { + WriteValueToJson(writer, value.GetVariantItem(), + AS_TYPE(TTupleType, underlyingType)->GetElementType(index), convertPolicy); + } else { + WriteValueToJson(writer, value.GetVariantItem(), + AS_TYPE(TStructType, underlyingType)->GetMemberType(index), convertPolicy); + } + writer.CloseArray(); + break; + } + case TType::EKind::Tagged: + { + auto underlyingType = AS_TYPE(TTaggedType, type)->GetBaseType(); + WriteValueToJson(writer, value, underlyingType, convertPolicy); + break; + } + default: + YQL_ENSURE(false, "unknown type " << type->GetKindAsStr()); + } +} + +NKikimr::NUdf::TUnboxedValue ReadJsonValue(TJsonValue& json, NKikimr::NMiniKQL::TType* type, + const NMiniKQL::THolderFactory& holderFactory) +{ + auto jsonType = json.GetType(); + switch (type->GetKind()) { + case TType::EKind::Void: + case TType::EKind::Null: + YQL_ENSURE(json.IsNull(), "Unexpected json type (expected null value, but got type " << jsonType << ")"); + return NKikimr::NUdf::TUnboxedValuePod(); + case TType::EKind::EmptyList: + case TType::EKind::EmptyDict: + YQL_ENSURE(json.IsArray(), "Unexpected json type (expected array, but got " << jsonType << ")"); + YQL_ENSURE(json.GetArray().size() == 0, "Expected empty array, but got array with " << json.GetArray().size() << " elements"); + return holderFactory.GetEmptyContainer(); + case TType::EKind::Tuple: + { + YQL_ENSURE(json.IsArray(), "Unexpected json type (expected array, but got " << jsonType << ")"); + auto tupleType = AS_TYPE(TTupleType, type); + auto array = json.GetArray(); + YQL_ENSURE(array.size() == tupleType->GetElementsCount(), + "Expected " << tupleType->GetElementsCount() << " elements in tuple, but got " << array.size()); + NUdf::TUnboxedValue* items; + NUdf::TUnboxedValue tuple = holderFactory.CreateDirectArrayHolder(tupleType->GetElementsCount(), items); + for (ui32 i = 0, e = array.size(); i < e; i++) { + items[i] = ReadJsonValue(array[i], tupleType->GetElementType(i), holderFactory); + } + return tuple; + } + case TType::EKind::List: + { + YQL_ENSURE(json.IsArray(), "Unexpected json type (expected array, but got " << jsonType << ")"); + auto listType = AS_TYPE(TListType, type); + TDefaultListRepresentation items; + auto array = json.GetArray(); + for (ui32 i = 0, e = array.size(); i < e; i++) { + items = items.Append(ReadJsonValue(array[i], listType->GetItemType(), holderFactory)); + } + return holderFactory.CreateDirectListHolder(std::move(items)); + } + case TType::EKind::Struct: + { + YQL_ENSURE(json.IsMap(), "Unexpected json type (expected map, but got " << jsonType << ")"); + auto structType = AS_TYPE(TStructType, type); + NUdf::TUnboxedValue* items; + NUdf::TUnboxedValue structValue = holderFactory.CreateDirectArrayHolder(structType->GetMembersCount(), items); + auto jsonMap = json.GetMap(); + std::unordered_set<TString> unprocessed(jsonMap.size()); + for (auto const& map : jsonMap) { + unprocessed.insert(map.first); + } + for (ui32 i = 0; i < structType->GetMembersCount(); ++i) { + const auto keyName = TString(structType->GetMemberName(i)); + if (jsonMap.contains(keyName)) { + items[i] = ReadJsonValue(jsonMap[keyName], structType->GetMemberType(i), holderFactory); + } else { + YQL_ENSURE(structType->GetMemberType(i)->IsOptional(), "Absent non optional field " << keyName << " at struct"); + items[i] = NKikimr::NUdf::TUnboxedValue(); + } + unprocessed.erase(keyName); + } + YQL_ENSURE(unprocessed.empty(), "Extra fields into json map detected (" << JoinSeq(',', unprocessed) << ")"); + return structValue; + } + case TType::EKind::Optional: + { + if (json.IsNull()) { + return NUdf::TUnboxedValuePod(); + } + auto optionalType = AS_TYPE(TOptionalType, type); + auto value = ReadJsonValue(json, optionalType->GetItemType(), holderFactory); + return value.Release().MakeOptional(); + } + case TType::EKind::Data: + { + auto dataType = AS_TYPE(TDataType, type); + switch (dataType->GetSchemeType()) { + case NUdf::TDataType<bool>::Id: + YQL_ENSURE(json.IsBoolean(), "Unexpected json type (expected bool, but got " << jsonType << ")"); + return NUdf::TUnboxedValuePod(json.GetBoolean()); + +#define INTEGER_CONVERTOR(type, wideType) \ + case NUdf::TDataType<type>::Id: { \ + YQL_ENSURE(jsonType == EJsonValueType::JSON_INTEGER \ + || jsonType == EJsonValueType::JSON_UINTEGER, \ + "Unexpected json type (expected " << #type << ", but got " << jsonType << ")"); \ + wideType intValue = jsonType == EJsonValueType::JSON_INTEGER ? json.GetInteger() : json.GetUInteger(); \ + YQL_ENSURE(intValue >= std::numeric_limits<type>::min() \ + && intValue <= std::numeric_limits<type>::max(), \ + "Exceeded the range of acceptable values for " << #type); \ + return NUdf::TUnboxedValuePod(type(intValue)); \ + } + + INTEGER_CONVERTOR(ui8, ui64) + INTEGER_CONVERTOR(ui16, ui64) + INTEGER_CONVERTOR(ui32, ui64) + INTEGER_CONVERTOR(ui64, ui64) + + INTEGER_CONVERTOR(i8, i64) + INTEGER_CONVERTOR(i16, i64) + INTEGER_CONVERTOR(i32, i64) + INTEGER_CONVERTOR(i64, i64) + +#undef INTEGER_CONVERTOR + + case NUdf::TDataType<float>::Id: { + YQL_ENSURE(json.IsDouble() || json.IsInteger() || json.IsUInteger(), + "Unexpected json type (expected double or integer, but got " << jsonType << ")"); + double value = jsonType == EJsonValueType::JSON_DOUBLE + ? json.GetDouble() + : (jsonType == EJsonValueType::JSON_INTEGER ? double(json.GetInteger()) : double(json.GetUInteger())); + YQL_ENSURE(value >= std::numeric_limits<float>::min() && value <= std::numeric_limits<float>::max(), + "Exceeded the range of acceptable values for float"); + return NUdf::TUnboxedValuePod(float(value)); + } + case NUdf::TDataType<double>::Id: { + YQL_ENSURE(json.IsDouble() || json.IsInteger() || json.IsUInteger(), + "Unexpected json type (expected double or integer, but got " << jsonType << ")"); + double value = jsonType == EJsonValueType::JSON_DOUBLE + ? json.GetDouble() + : (jsonType == EJsonValueType::JSON_INTEGER ? double(json.GetInteger()) : double(json.GetUInteger())); + return NUdf::TUnboxedValuePod(value); + } + case NUdf::TDataType<NUdf::TUtf8>::Id: + case NUdf::TDataType<char*>::Id: { + YQL_ENSURE(json.IsString(), "Unexpected json type (expected string, but got " << jsonType << ")"); + auto value = json.GetString(); + return NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(value))); + } + case NUdf::TDataType<NUdf::TDecimal>::Id: { + YQL_ENSURE(json.IsString(), "Unexpected json type (expected string, but got " << jsonType << ")"); + const auto params = static_cast<TDataDecimalType*>(type)->GetParams(); + const auto value = NDecimal::FromString(json.GetString(), params.first, params.second); + YQL_ENSURE(!NDecimal::IsError(value)); + return NUdf::TUnboxedValuePod(value); + } + case NUdf::TDataType<NUdf::TDate>::Id: + case NUdf::TDataType<NUdf::TDatetime>::Id: + case NUdf::TDataType<NUdf::TTimestamp>::Id: + case NUdf::TDataType<NUdf::TInterval>::Id: + case NUdf::TDataType<NUdf::TTzDate>::Id: + case NUdf::TDataType<NUdf::TTzDatetime>::Id: + case NUdf::TDataType<NUdf::TTzTimestamp>::Id: { + YQL_ENSURE(json.IsString(), "Unexpected json type (expected string, but got " << jsonType << ")"); + YQL_ENSURE(IsValidStringValue(*dataType->GetDataSlot(), json.GetString()), "Invalid date format (expected ISO-8601)"); + return ValueFromString(*dataType->GetDataSlot(), json.GetString()); + } + default: + YQL_ENSURE(false, "Can't convert from JSON (unsupported YQL type " << dataType->GetSchemeType() << ")"); + } + } + break; + default: + YQL_ENSURE(false, "Can't convert from JSON (unsupported YQL type " << type->GetKindAsStr() << ")"); + } + + return NKikimr::NUdf::TUnboxedValuePod(); +} + +NKikimr::NUdf::TUnboxedValue ReadJsonValue(IInputStream* in, NKikimr::NMiniKQL::TType* type, + const NMiniKQL::THolderFactory& holderFactory) +{ + TJsonValue json; + if (!ReadJsonTree(in, &json, false)) { + YQL_ENSURE(false, "Error parse json"); + } + return ReadJsonValue(json, type, holderFactory); +} + +} +} diff --git a/ydb/library/yql/providers/common/codec/yql_json_codec.h b/ydb/library/yql/providers/common/codec/yql_json_codec.h new file mode 100644 index 0000000000..e541c3eef2 --- /dev/null +++ b/ydb/library/yql/providers/common/codec/yql_json_codec.h @@ -0,0 +1,30 @@ +#pragma once + +#include <library/cpp/json/json_writer.h> +#include <library/cpp/json/json_reader.h> +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +using namespace NKikimr; + +namespace NYql { +namespace NCommon { + +enum class EValueConvertPolicy : ui8 { + WriteNumberString = 1, + WriteUnsafeNumberString = 2, + //WriteNumberAsString = 4, + //WriteNumberAsString = 8, +}; + +NJson::TJsonWriterConfig MakeJsonConfig(); + +void WriteValueToJson(NJson::TJsonWriter& writer, const NUdf::TUnboxedValuePod& value, + NMiniKQL::TType* type, std::set<EValueConvertPolicy> convertPolicy = {}); + +NUdf::TUnboxedValue ReadJsonValue(NJson::TJsonValue& json, NMiniKQL::TType* type, const NMiniKQL::THolderFactory& holderFactory); + +NUdf::TUnboxedValue ReadJsonValue(IInputStream* in, NMiniKQL::TType* type, const NMiniKQL::THolderFactory& holderFactory); + +} +} diff --git a/ydb/library/yql/providers/common/codec/yql_json_codec_ut.cpp b/ydb/library/yql/providers/common/codec/yql_json_codec_ut.cpp new file mode 100644 index 0000000000..3a6fed2953 --- /dev/null +++ b/ydb/library/yql/providers/common/codec/yql_json_codec_ut.cpp @@ -0,0 +1,636 @@ +#include "yql_json_codec.h" + +#include <ydb/library/yql/minikql/computation/mkql_value_builder.h> +#include <ydb/library/yql/minikql/mkql_type_ops.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/string/cast.h> + +namespace NYql { +namespace NCommon { + +using namespace NYql::NCommon; +using namespace NKikimr; +using namespace NKikimr::NMiniKQL; + + +namespace { +struct TTestContext { + TScopedAlloc Alloc; + TTypeEnvironment TypeEnv; + TMemoryUsageInfo MemInfo; + THolderFactory HolderFactory; + TDefaultValueBuilder Vb; + + TTestContext() + : Alloc() + , TypeEnv(Alloc) + , MemInfo("Mem") + , HolderFactory(Alloc.Ref(), MemInfo) + , Vb(HolderFactory) + { + + } +}; + +TString WriteValueToExportJsonStr(const NUdf::TUnboxedValuePod& value, NMiniKQL::TType* type) { + TStringStream out; + NJson::TJsonWriter jsonWriter(&out, MakeJsonConfig()); + WriteValueToJson(jsonWriter,value, type, {}); + jsonWriter.Flush(); + return out.Str(); +} + +TString WriteValueToFuncJsonStr(const NUdf::TUnboxedValuePod& value, NMiniKQL::TType* type) { + TStringStream out; + NJson::TJsonWriter jsonWriter(&out, MakeJsonConfig()); + WriteValueToJson(jsonWriter,value, type, {EValueConvertPolicy::WriteNumberString}); + jsonWriter.Flush(); + return out.Str(); +} +} + +Y_UNIT_TEST_SUITE(SerializeVoid) { + Y_UNIT_TEST(Null) { + TTestContext ctx; + auto value = NUdf::TUnboxedValuePod(); + auto nullJson = WriteValueToFuncJsonStr(value, ctx.TypeEnv.GetTypeOfNull()); + UNIT_ASSERT_VALUES_EQUAL(nullJson, "null"); + + auto voidJson = WriteValueToFuncJsonStr(value, ctx.TypeEnv.GetTypeOfVoid()); + UNIT_ASSERT_VALUES_EQUAL(voidJson, "null"); + } +} + +Y_UNIT_TEST_SUITE(SerializeBool) { + Y_UNIT_TEST(Bool) { + TTestContext ctx; + auto type = TDataType::Create(NUdf::TDataType<bool>::Id, ctx.TypeEnv); + auto json1 = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(true), type); + UNIT_ASSERT_VALUES_EQUAL(json1, "true"); + + auto json2 = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(false), type); + UNIT_ASSERT_VALUES_EQUAL(json2, "false"); + } +} + +Y_UNIT_TEST_SUITE(SerializeUuid) { + Y_UNIT_TEST(Uuid) { + TTestContext ctx; + auto type = TDataType::Create(NUdf::TDataType<NUdf::TUuid>::Id, ctx.TypeEnv); + auto value = ctx.Vb.NewString("80070214-6ad1-4077-add8-74b68f105e3c"); + auto json = WriteValueToFuncJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "\"80070214-6ad1-4077-add8-74b68f105e3c\""); + } +} + +Y_UNIT_TEST_SUITE(SerializeJson) { + Y_UNIT_TEST(ScalarJson) { + TTestContext ctx; + auto type = TDataType::Create(NUdf::TDataType<NUdf::TJson>::Id, ctx.TypeEnv); + auto value = ctx.Vb.NewString("\"some string с русскими йЁ\""); + auto json = WriteValueToFuncJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "\"some string с русскими йЁ\""); + } + + Y_UNIT_TEST(ComplexJson) { + TTestContext ctx; + TStructMember members[] = { + TStructMember("X", TDataType::Create(NUdf::TDataType<NUdf::TJson>::Id, ctx.TypeEnv)), + TStructMember("Y", TDataType::Create(NUdf::TDataType<ui32>::Id, ctx.TypeEnv)) + }; + auto type = TStructType::Create(2, members, ctx.TypeEnv); + + NUdf::TUnboxedValue* items; + auto value = ctx.Vb.NewArray(2, items); + items[0] = ctx.Vb.NewString("{\"a\":500,\"b\":[1,2,3]}"); + items[1] = NUdf::TUnboxedValuePod(ui32(73)); + + auto json = WriteValueToExportJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "{\"X\":{\"a\":500,\"b\":[1,2,3]},\"Y\":73}"); + } +} + +Y_UNIT_TEST_SUITE(SerializeContainers) { + Y_UNIT_TEST(EmptyContainer) { + TTestContext ctx; + auto value = ctx.HolderFactory.GetEmptyContainer(); + auto dictJson = WriteValueToFuncJsonStr(value, ctx.TypeEnv.GetTypeOfEmptyDict()); + UNIT_ASSERT_VALUES_EQUAL(dictJson, "[]"); + + auto listJson = WriteValueToFuncJsonStr(value, ctx.TypeEnv.GetTypeOfEmptyList()); + UNIT_ASSERT_VALUES_EQUAL(listJson, "[]"); + } + + Y_UNIT_TEST(NormalList) { + TTestContext ctx; + NUdf::TUnboxedValue* items; + auto value = ctx.Vb.NewArray(4, items); + items[0] = NUdf::TUnboxedValuePod(1); + items[1] = NUdf::TUnboxedValuePod(5); + items[2] = NUdf::TUnboxedValuePod(-18); + items[3] = NUdf::TUnboxedValuePod(4); + auto type = TListType::Create(TDataType::Create(NUdf::TDataType<i32>::Id, ctx.TypeEnv), ctx.TypeEnv); + auto json = WriteValueToFuncJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "[\"1\",\"5\",\"-18\",\"4\"]"); + } + + Y_UNIT_TEST(EmptyList) { + TTestContext ctx; + NUdf::TUnboxedValue* items; + auto value = ctx.Vb.NewArray(0, items); + auto type = TListType::Create(TDataType::Create(NUdf::TDataType<i32>::Id, ctx.TypeEnv), ctx.TypeEnv); + auto json = WriteValueToFuncJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "[]"); + } + + Y_UNIT_TEST(FullTuple) { + TTestContext ctx; + + TType* members[] = { + TDataType::Create(NUdf::TDataType<ui8>::Id, ctx.TypeEnv), + TDataType::Create(NUdf::TDataType<NUdf::TUtf8>::Id, ctx.TypeEnv), + TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv), + }; + auto type = TTupleType::Create(3, members, ctx.TypeEnv); + + NUdf::TUnboxedValue* items; + NUdf::TUnboxedValue value = ctx.HolderFactory.CreateDirectArrayHolder(type->GetElementsCount(), items); + items[0] = NUdf::TUnboxedValuePod(ui8(65)); + items[1] = ctx.Vb.NewString("абсдева"); + ui16 date; + ctx.Vb.MakeDate(2021, 5, 5, date); + items[2] = NUdf::TUnboxedValuePod(date); + + auto json = WriteValueToExportJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "[65,\"абсдева\",\"2021-05-05\"]"); + } + + Y_UNIT_TEST(EmptyTuple) { + TTestContext ctx; + TType* members[] = {}; + auto type = TTupleType::Create(0, members, ctx.TypeEnv); + NUdf::TUnboxedValue* items; + NUdf::TUnboxedValue value = ctx.HolderFactory.CreateDirectArrayHolder(type->GetElementsCount(), items); + auto json = WriteValueToFuncJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "[]"); + } + + Y_UNIT_TEST(DictType) { + TTestContext ctx; + auto type = TDictType::Create( + TDataType::Create(NUdf::TDataType<NUdf::TUtf8>::Id, ctx.TypeEnv), + TDataType::Create(NUdf::TDataType<i32>::Id, ctx.TypeEnv), + ctx.TypeEnv + ); + auto dictBuilder = ctx.Vb.NewDict(type, NUdf::TDictFlags::EDictKind::Hashed); + dictBuilder->Add( + ctx.Vb.NewString("key_a"), + NUdf::TUnboxedValuePod(781) + ); + dictBuilder->Add( + ctx.Vb.NewString("key_b"), + NUdf::TUnboxedValuePod(-500) + ); + auto json = WriteValueToExportJsonStr(dictBuilder->Build(), type); + UNIT_ASSERT_VALUES_EQUAL(json, "[[\"key_b\",-500],[\"key_a\",781]]"); + } + + Y_UNIT_TEST(Tagged) { + TTestContext ctx; + auto type = TTaggedType::Create(TDataType::Create(NUdf::TDataType<NUdf::TDatetime>::Id, ctx.TypeEnv), + "test-tag", ctx.TypeEnv); + ui32 datetime; + ctx.Vb.MakeDatetime(2021, 1, 1, 14, 5, 43, datetime); + auto json = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(datetime), type); + UNIT_ASSERT_VALUES_EQUAL(json, "\"2021-01-01T14:05:43Z\""); + } + + Y_UNIT_TEST(TupleVariant) { + TTestContext ctx; + TType* tupleTypes[] = { + TDataType::Create(NUdf::TDataType<bool>::Id, ctx.TypeEnv), + TDataType::Create(NUdf::TDataType<ui32>::Id, ctx.TypeEnv) + }; + auto underlying = TTupleType::Create(2, tupleTypes, ctx.TypeEnv); + auto type = TVariantType::Create(underlying, ctx.TypeEnv); + + auto value0 = ctx.HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(false), 0); + auto json0 = WriteValueToFuncJsonStr(value0, type); + UNIT_ASSERT_VALUES_EQUAL(json0, "[0,false]"); + + auto value1 = ctx.HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(200), 1); + auto json1 = WriteValueToExportJsonStr(value1, type); + UNIT_ASSERT_VALUES_EQUAL(json1, "[1,200]"); + } + + Y_UNIT_TEST(StructVariant) { + TTestContext ctx; + + TStructMember members[] = { + TStructMember("A", TDataType::Create(NUdf::TDataType<bool>::Id, ctx.TypeEnv)), + TStructMember("B", TDataType::Create(NUdf::TDataType<ui32>::Id, ctx.TypeEnv)) + }; + auto underlying = TStructType::Create(2, members, ctx.TypeEnv); + auto type = TVariantType::Create(underlying, ctx.TypeEnv); + + auto value0 = ctx.HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(true), 0); + auto json0 = WriteValueToExportJsonStr(value0, type); + UNIT_ASSERT_VALUES_EQUAL(json0, "[0,true]"); + + auto value1 = ctx.HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(200), 1); + auto json1 = WriteValueToExportJsonStr(value1, type); + UNIT_ASSERT_VALUES_EQUAL(json1, "[1,200]"); + } + + Y_UNIT_TEST(StructType) { + TTestContext ctx; + + TStructMember members[] = { + {"A", TDataType::Create(NUdf::TDataType<bool>::Id, ctx.TypeEnv)}, + {"B", TDataType::Create(NUdf::TDataType<ui32>::Id, ctx.TypeEnv)} + }; + auto type = TStructType::Create(2, members, ctx.TypeEnv); + + NUdf::TUnboxedValue* items; + auto value = ctx.Vb.NewArray(2, items); + items[0] = NUdf::TUnboxedValuePod(false); + items[1] = NUdf::TUnboxedValuePod(ui32(555)); + + auto json = WriteValueToExportJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "{\"A\":false,\"B\":555}"); + } +} + +Y_UNIT_TEST_SUITE(SerializeOptional) { + Y_UNIT_TEST(JustOptional) { + TTestContext ctx; + NUdf::TUnboxedValue* items; + auto value = ctx.HolderFactory.CreateDirectArrayHolder(3, items).MakeOptional(); + items[0] = NUdf::TUnboxedValuePod(ui8(0)); + items[1] = NUdf::TUnboxedValuePod(ui8(67)); + items[2] = NUdf::TUnboxedValuePod(ui8(4)); + auto elementType = TListType::Create(TDataType::Create(NUdf::TDataType<ui8>::Id, ctx.TypeEnv), ctx.TypeEnv); + auto type = TOptionalType::Create(elementType, ctx.TypeEnv); + auto json = WriteValueToExportJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "[[0,67,4]]"); + } + + Y_UNIT_TEST(NothingOptional) { + TTestContext ctx; + auto value = NUdf::TUnboxedValuePod().MakeOptional(); + auto elementType = TListType::Create(TDataType::Create(NUdf::TDataType<ui8>::Id, ctx.TypeEnv), ctx.TypeEnv); + auto type = TOptionalType::Create(elementType, ctx.TypeEnv); + auto json = WriteValueToFuncJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "[null]"); + } + + Y_UNIT_TEST(SeveralOptionals) { + TTestContext ctx; + auto elementType = TDataType::Create(NUdf::TDataType<ui8>::Id, ctx.TypeEnv); + TType* type = TOptionalType::Create(elementType, ctx.TypeEnv); + type = TOptionalType::Create(type, ctx.TypeEnv); + type = TOptionalType::Create(type, ctx.TypeEnv); + + auto value1 = NUdf::TUnboxedValuePod().MakeOptional().MakeOptional().MakeOptional(); + auto json1 = WriteValueToFuncJsonStr(value1, type); + UNIT_ASSERT_VALUES_EQUAL(json1, "[[[null]]]"); + + auto value2 = NUdf::TUnboxedValuePod(ui8(120)).MakeOptional().MakeOptional().MakeOptional(); + auto json2 = WriteValueToExportJsonStr(value2, type); + UNIT_ASSERT_VALUES_EQUAL(json2, "[[[120]]]"); + } +} + +Y_UNIT_TEST_SUITE(SerializeNumbers) { + +#define TEST_SERIALIZE_NUMBER_TYPE(type) \ + Y_UNIT_TEST(type) { \ + TTestContext ctx; \ + auto maxValue = NUdf::TUnboxedValuePod(std::numeric_limits<type>::max()); \ + auto maxJson = WriteValueToFuncJsonStr(maxValue, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv)); \ + UNIT_ASSERT_VALUES_EQUAL(maxJson, "\"" + ToString(std::numeric_limits<type>::max()) + "\""); \ + \ + auto minValue = NUdf::TUnboxedValuePod(std::numeric_limits<type>::min()); \ + auto minJson = WriteValueToFuncJsonStr(minValue, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv)); \ + UNIT_ASSERT_VALUES_EQUAL(minJson, "\"" + ToString(std::numeric_limits<type>::min()) + "\""); \ + } + + TEST_SERIALIZE_NUMBER_TYPE(i8) + TEST_SERIALIZE_NUMBER_TYPE(i16) + TEST_SERIALIZE_NUMBER_TYPE(i32) + + TEST_SERIALIZE_NUMBER_TYPE(ui8) + TEST_SERIALIZE_NUMBER_TYPE(ui16) + TEST_SERIALIZE_NUMBER_TYPE(ui32) + +#undef TEST_SERIALIZE_NUMBER_TYPE + + Y_UNIT_TEST(float) { + TTestContext ctx; + auto value = NUdf::TUnboxedValuePod(std::numeric_limits<float>::max()); + auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<float>::Id, ctx.TypeEnv)); + auto expected = FloatToString(std::numeric_limits<float>::max(), EFloatToStringMode::PREC_NDIGITS, std::numeric_limits<float>::max_digits10); + UNIT_ASSERT_VALUES_EQUAL(json, "\"" + expected + "\""); + } + + Y_UNIT_TEST(double) { + TTestContext ctx; + auto value = NUdf::TUnboxedValuePod(std::numeric_limits<double>::min()); + auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<double>::Id, ctx.TypeEnv)); + auto expected = FloatToString(std::numeric_limits<double>::min(), EFloatToStringMode::PREC_NDIGITS, std::numeric_limits<double>::max_digits10); + UNIT_ASSERT_VALUES_EQUAL(json, "\"" + expected + "\""); + } + + Y_UNIT_TEST(DoubleSignificantNumbers) { + TTestContext ctx; + auto value = NUdf::TUnboxedValuePod(double(3.1415926535897932384626433832795)); + auto type = TDataType::Create(NUdf::TDataType<double>::Id, ctx.TypeEnv); + auto json = WriteValueToFuncJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json, "\"3.1415926535897931\""); + + auto json2 = WriteValueToExportJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json2, "3.1415926535897931"); + } + + Y_UNIT_TEST(i64) { + TTestContext ctx; + auto maxValue = NUdf::TUnboxedValuePod(std::numeric_limits<i64>::max()); + auto maxJson = WriteValueToFuncJsonStr(maxValue, TDataType::Create(NUdf::TDataType<i64>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(maxJson, "\"" + ToString(std::numeric_limits<i64>::max()) + "\""); + + auto minValue = NUdf::TUnboxedValuePod(std::numeric_limits<i64>::min()); + auto minJson = WriteValueToFuncJsonStr(minValue, TDataType::Create(NUdf::TDataType<i64>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(minJson, "\"" + ToString(std::numeric_limits<i64>::min()) + "\""); + + auto fitInJS = NUdf::TUnboxedValuePod(i64(-9007199254740991)); // == Number.MIN_SAFE_INTEGER + auto fitInJSJson = WriteValueToFuncJsonStr(fitInJS, TDataType::Create(NUdf::TDataType<i64>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(fitInJSJson, "\"-9007199254740991\""); + } + + Y_UNIT_TEST(ui64) { + TTestContext ctx; + auto maxValue = NUdf::TUnboxedValuePod(std::numeric_limits<ui64>::max()); + auto maxJson = WriteValueToFuncJsonStr(maxValue, TDataType::Create(NUdf::TDataType<ui64>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(maxJson, "\"" + ToString(std::numeric_limits<ui64>::max()) + "\""); + + auto minValue = NUdf::TUnboxedValuePod(std::numeric_limits<ui64>::min()); + auto minJson = WriteValueToFuncJsonStr(minValue, TDataType::Create(NUdf::TDataType<ui64>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(minJson, "\"" + ToString(std::numeric_limits<ui64>::min()) + "\""); + + auto fitInJS = NUdf::TUnboxedValuePod(ui64(2419787883133419)); + auto fitInJSJson = WriteValueToFuncJsonStr(fitInJS, TDataType::Create(NUdf::TDataType<ui64>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(fitInJSJson, "\"2419787883133419\""); + } + + Y_UNIT_TEST(Decimal) { + TTestContext ctx; + auto value = NUdf::TUnboxedValuePod(NYql::NDecimal::TInt128(78876543LL)); + auto type = TDataDecimalType::Create(10, 2, ctx.TypeEnv); + auto json1 = WriteValueToFuncJsonStr(value, type); + UNIT_ASSERT_VALUES_EQUAL(json1, "\"788765.43\""); + } +} + +Y_UNIT_TEST_SUITE(DeserializeNumbers) { + +#define TEST_DESERIALIZE_NUMBER_TYPE(type, wideType) \ + Y_UNIT_TEST(type) { \ + TTestContext ctx; \ + \ + TStringStream maxJson; \ + maxJson << ToString(std::numeric_limits<type>::max()); \ + auto maxValue = ReadJsonValue(&maxJson, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv), ctx.HolderFactory); \ + UNIT_ASSERT_VALUES_EQUAL(maxValue.Get<type>(), std::numeric_limits<type>::max()); \ + \ + TStringStream minJson; \ + minJson << ToString(std::numeric_limits<type>::min()); \ + auto minValue = ReadJsonValue(&minJson, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv), ctx.HolderFactory); \ + UNIT_ASSERT_VALUES_EQUAL(minValue.Get<type>(), std::numeric_limits<type>::min()); \ + \ + TStringStream exceededJson; \ + exceededJson << ToString(wideType(std::numeric_limits<type>::max() + wideType(1))); \ + UNIT_ASSERT_EXCEPTION_CONTAINS( \ + ReadJsonValue(&exceededJson, TDataType::Create(NUdf::TDataType<type>::Id, ctx.TypeEnv), ctx.HolderFactory), \ + yexception, \ + "Exceeded the range" \ + ); \ + } + + TEST_DESERIALIZE_NUMBER_TYPE(i8, i64) + TEST_DESERIALIZE_NUMBER_TYPE(i16, i64) + TEST_DESERIALIZE_NUMBER_TYPE(i32, i64) + + TEST_DESERIALIZE_NUMBER_TYPE(ui8, ui64) + TEST_DESERIALIZE_NUMBER_TYPE(ui16, ui64) + TEST_DESERIALIZE_NUMBER_TYPE(ui32, ui64) + +#undef TEST_DESERIALIZE_NUMBER_TYPE + + Y_UNIT_TEST(float) { + TTestContext ctx; + TStringStream json; + json << "0.0431"; + auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<float>::Id, ctx.TypeEnv), ctx.HolderFactory); + UNIT_ASSERT_VALUES_EQUAL(value.Get<float>(), 0.0431f); + + TStringStream exceededJson; + exceededJson << ToString(std::numeric_limits<double>::max()); + UNIT_ASSERT_EXCEPTION_CONTAINS( + ReadJsonValue(&exceededJson, TDataType::Create(NUdf::TDataType<float>::Id, ctx.TypeEnv), ctx.HolderFactory), + yexception, + "Exceeded the range" + ); + } + + Y_UNIT_TEST(FloatFromInt) { + TTestContext ctx; + TStringStream json; + json << "1766718243"; + auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<float>::Id, ctx.TypeEnv), ctx.HolderFactory); + UNIT_ASSERT_VALUES_EQUAL(value.Get<float>(), 1766718243.0f); + } + + Y_UNIT_TEST(double) { + TTestContext ctx; + TStringStream json; + json << "7773.13"; + auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<double>::Id, ctx.TypeEnv), ctx.HolderFactory); + UNIT_ASSERT_VALUES_EQUAL(value.Get<double>(), 7773.13); + } + + Y_UNIT_TEST(DoubleFromInt) { + TTestContext ctx; + TStringStream json; + json << "-667319001"; + auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<double>::Id, ctx.TypeEnv), ctx.HolderFactory); + UNIT_ASSERT_VALUES_EQUAL(value.Get<double>(), -667319001.0l); + } + +} + +Y_UNIT_TEST_SUITE(SerializeStringTypes) { + Y_UNIT_TEST(Utf8) { + TTestContext ctx; + auto value = ctx.Vb.NewString("aaaaabbbbbcccccc"); + auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<NUdf::TUtf8>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(json, "\"aaaaabbbbbcccccc\""); + } + + Y_UNIT_TEST(String) { + TTestContext ctx; + + auto type = TDataType::Create(NUdf::TDataType<char*>::Id, ctx.TypeEnv); + + auto value1 = ctx.Vb.NewString("aaaaabbbbbcccccc"); + auto json1 = WriteValueToFuncJsonStr(value1, type); + UNIT_ASSERT_VALUES_EQUAL(json1, "\"YWFhYWFiYmJiYmNjY2NjYw==\""); + + auto value2 = ctx.Vb.NewString("абсёЙabc"); + auto json2 = WriteValueToFuncJsonStr(value2, type); + UNIT_ASSERT_VALUES_EQUAL(json2, "\"0LDQsdGB0ZHQmWFiYw==\""); + } +} + +Y_UNIT_TEST_SUITE(DeserializeStringTypes) { + Y_UNIT_TEST(String) { + TTestContext ctx; + TStringStream json; + json << "\"fffaaae423\""; + auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TUtf8>::Id, ctx.TypeEnv), ctx.HolderFactory); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("fffaaae423"), TStringBuf(value.AsStringRef())); + } +} + +Y_UNIT_TEST_SUITE(SerializeDateTypes) { + Y_UNIT_TEST(Date) { + TTestContext ctx; + ui16 date; + ctx.Vb.MakeDate(2022, 2, 9, date); + auto json = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(date), TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(json, "\"2022-02-09\""); + } + + Y_UNIT_TEST(Datetime) { + TTestContext ctx; + ui32 datetime; + ctx.Vb.MakeDatetime(2021, 1, 1, 14, 5, 43, datetime); + auto json = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(datetime), TDataType::Create(NUdf::TDataType<NUdf::TDatetime>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(json, "\"2021-01-01T14:05:43Z\""); + } + + Y_UNIT_TEST(Timestamp) { + TTestContext ctx; + auto value = ui64(1644755212879622); + auto json = WriteValueToFuncJsonStr(NUdf::TUnboxedValuePod(value), TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(json, "\"2022-02-13T12:26:52.879622Z\""); + } + + Y_UNIT_TEST(TzDate) { + TTestContext ctx; + ui16 rawDate; + ctx.Vb.MakeDate(2023, 4, 14, rawDate); + auto date = NUdf::TUnboxedValuePod(rawDate); + date.SetTimezoneId(530); + auto json = WriteValueToFuncJsonStr(date, TDataType::Create(NUdf::TDataType<NUdf::TTzDate>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(json, "\"2023-04-14,Pacific/Easter\""); + } + + Y_UNIT_TEST(TzDateTime) { + TTestContext ctx; + ui16 timeZone = 530; + ui32 rawDate; + ctx.Vb.MakeDatetime(2023, 4, 14, 0, 15, 0, rawDate, timeZone); + auto date = NUdf::TUnboxedValuePod(rawDate); + date.SetTimezoneId(timeZone); + auto json = WriteValueToFuncJsonStr(date, TDataType::Create(NUdf::TDataType<NUdf::TTzDatetime>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(json, "\"2023-04-14T00:15:00,Pacific/Easter\""); + } + + Y_UNIT_TEST(TzTimestamp) { + TTestContext ctx; + auto value = NUdf::TUnboxedValuePod(ui64(1644755564087924)); + value.SetTimezoneId(327); + auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<NUdf::TTzTimestamp>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(json, "\"2022-02-13T19:32:44.087924,Asia/Vientiane\""); + } + + Y_UNIT_TEST(Interval) { + TTestContext ctx; + // 2 days 2 hours in ms + auto value = NUdf::TUnboxedValuePod(i64(180000000000)); + auto json = WriteValueToFuncJsonStr(value, TDataType::Create(NUdf::TDataType<NUdf::TInterval>::Id, ctx.TypeEnv)); + UNIT_ASSERT_VALUES_EQUAL(json, "\"P2DT2H\""); + } +} + +Y_UNIT_TEST_SUITE(DeserializeDateTypes) { + + Y_UNIT_TEST(Date) { + TTestContext ctx; + TStringStream json; + json << "\"2020-09-11\""; + auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv), ctx.HolderFactory); + ui16 date; + ctx.Vb.MakeDate(2020, 9, 11, date); + UNIT_ASSERT_VALUES_EQUAL(value.Get<ui16>(), date); + } + + Y_UNIT_TEST(Datetime) { + TTestContext ctx; + TStringStream json; + json << "\"2021-07-14T00:00:43Z\""; + auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TDatetime>::Id, ctx.TypeEnv), ctx.HolderFactory); + ui32 datetime; + ctx.Vb.MakeDatetime(2021, 7, 14, 0, 0, 43, datetime); + UNIT_ASSERT_VALUES_EQUAL(value.Get<ui32>(), datetime); + } + + Y_UNIT_TEST(TzDate) { + TTestContext ctx; + TStringStream json; + json << "\"2020-09-11,Europe/Moscow\""; // timeZoneId == 1 + auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TTzDate>::Id, ctx.TypeEnv), ctx.HolderFactory); + ui16 date; + ctx.Vb.MakeDate(2020, 9, 10, date); + UNIT_ASSERT_VALUES_EQUAL(value.GetTimezoneId(), 1); + UNIT_ASSERT_VALUES_EQUAL(value.Get<ui16>(), date); + } + + Y_UNIT_TEST(TzDatetime) { + TTestContext ctx; + TStringStream json; + json << "\"2020-09-11T01:11:05,Europe/Moscow\""; // timeZoneId == 1 + auto value = ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TTzDatetime>::Id, ctx.TypeEnv), ctx.HolderFactory); + ui32 datetime; + ctx.Vb.MakeDatetime(2020, 9, 11, 1, 11, 5, datetime, 1); + UNIT_ASSERT_VALUES_EQUAL(value.GetTimezoneId(), 1); + UNIT_ASSERT_VALUES_EQUAL(value.Get<ui32>(), datetime); + } + + Y_UNIT_TEST(WrongTypeEx) { + TTestContext ctx; + TStringStream json; + json << "[\"2020-22-12\"]"; + UNIT_ASSERT_EXCEPTION_CONTAINS( + ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv), ctx.HolderFactory), + yexception, + "Unexpected json type (expected string" + ); + } + + Y_UNIT_TEST(WrongFormatEx) { + TTestContext ctx; + TStringStream json; + json << "\"2020-18-43\""; + UNIT_ASSERT_EXCEPTION_CONTAINS( + ReadJsonValue(&json, TDataType::Create(NUdf::TDataType<NUdf::TDate>::Id, ctx.TypeEnv), ctx.HolderFactory), + yexception, + "Invalid date format" + ); + } +} + +} // namespace +} // namespace + |