diff options
author | elshiko <elshiko@yandex-team.ru> | 2022-02-10 16:49:20 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:20 +0300 |
commit | dbb5d0d21c8cfc3d59e0f6d3b6f4ab06b58561f1 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 /library | |
parent | 455ab915435b70e433ed91eee2ca84f03122d6cb (diff) | |
download | ydb-dbb5d0d21c8cfc3d59e0f6d3b6f4ab06b58561f1.tar.gz |
Restoring authorship annotation for <elshiko@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library')
-rw-r--r-- | library/cpp/json/yson/json2yson.cpp | 184 | ||||
-rw-r--r-- | library/cpp/json/yson/json2yson.h | 328 | ||||
-rw-r--r-- | library/cpp/json/yson/json2yson_ut.cpp | 198 | ||||
-rw-r--r-- | library/cpp/json/yson/ut/ya.make | 40 | ||||
-rw-r--r-- | library/cpp/json/yson/ya.make | 30 | ||||
-rw-r--r-- | library/cpp/protobuf/util/pb_io.cpp | 6 | ||||
-rw-r--r-- | library/cpp/protobuf/util/pb_io.h | 8 | ||||
-rw-r--r-- | library/cpp/protobuf/util/pb_io_ut.cpp | 20 | ||||
-rw-r--r-- | library/cpp/retry/retry.h | 34 | ||||
-rw-r--r-- | library/cpp/retry/retry_ut.cpp | 80 | ||||
-rw-r--r-- | library/cpp/yson/json/yson2json_adapter.cpp | 126 | ||||
-rw-r--r-- | library/cpp/yson/json/yson2json_adapter.h | 56 | ||||
-rw-r--r-- | library/cpp/yson/node/node_builder.h | 2 |
13 files changed, 556 insertions, 556 deletions
diff --git a/library/cpp/json/yson/json2yson.cpp b/library/cpp/json/yson/json2yson.cpp index 237fe1a60f..f72cb7a9ef 100644 --- a/library/cpp/json/yson/json2yson.cpp +++ b/library/cpp/json/yson/json2yson.cpp @@ -1,108 +1,108 @@ -#include "json2yson.h" - +#include "json2yson.h" + #include <library/cpp/yson/parser.h> #include <library/cpp/yson/json/json_writer.h> #include <library/cpp/yson/json/yson2json_adapter.h> - -namespace NJson2Yson { - static void WriteJsonValue(const NJson::TJsonValue& jsonValue, NYT::TYson2JsonCallbacksAdapter* adapter) { - switch (jsonValue.GetType()) { - default: - case NJson::JSON_NULL: - adapter->OnNull(); - break; - case NJson::JSON_BOOLEAN: - adapter->OnBoolean(jsonValue.GetBoolean()); - break; - case NJson::JSON_DOUBLE: - adapter->OnDouble(jsonValue.GetDouble()); - break; - case NJson::JSON_INTEGER: - adapter->OnInteger(jsonValue.GetInteger()); - break; - case NJson::JSON_UINTEGER: - adapter->OnUInteger(jsonValue.GetUInteger()); - break; - case NJson::JSON_STRING: - adapter->OnString(jsonValue.GetString()); - break; - case NJson::JSON_ARRAY: { - adapter->OnOpenArray(); - const NJson::TJsonValue::TArray& arr = jsonValue.GetArray(); - for (const auto& it : arr) - WriteJsonValue(it, adapter); - adapter->OnCloseArray(); - break; - } - case NJson::JSON_MAP: { - adapter->OnOpenMap(); + +namespace NJson2Yson { + static void WriteJsonValue(const NJson::TJsonValue& jsonValue, NYT::TYson2JsonCallbacksAdapter* adapter) { + switch (jsonValue.GetType()) { + default: + case NJson::JSON_NULL: + adapter->OnNull(); + break; + case NJson::JSON_BOOLEAN: + adapter->OnBoolean(jsonValue.GetBoolean()); + break; + case NJson::JSON_DOUBLE: + adapter->OnDouble(jsonValue.GetDouble()); + break; + case NJson::JSON_INTEGER: + adapter->OnInteger(jsonValue.GetInteger()); + break; + case NJson::JSON_UINTEGER: + adapter->OnUInteger(jsonValue.GetUInteger()); + break; + case NJson::JSON_STRING: + adapter->OnString(jsonValue.GetString()); + break; + case NJson::JSON_ARRAY: { + adapter->OnOpenArray(); + const NJson::TJsonValue::TArray& arr = jsonValue.GetArray(); + for (const auto& it : arr) + WriteJsonValue(it, adapter); + adapter->OnCloseArray(); + break; + } + case NJson::JSON_MAP: { + adapter->OnOpenMap(); const NJson::TJsonValue::TMapType& map = jsonValue.GetMap(); - for (const auto& it : map) { - adapter->OnMapKey(it.first); - WriteJsonValue(it.second, adapter); - } - adapter->OnCloseMap(); - break; - } - } - } - + for (const auto& it : map) { + adapter->OnMapKey(it.first); + WriteJsonValue(it.second, adapter); + } + adapter->OnCloseMap(); + break; + } + } + } + void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, NYson::TYsonWriter* ysonWriter) { NYT::TYson2JsonCallbacksAdapter adapter(ysonWriter); WriteJsonValue(inputValue, &adapter); } - void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, IOutputStream* outputStream) { + void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, IOutputStream* outputStream) { NYson::TYsonWriter ysonWriter(outputStream, NYson::EYsonFormat::Binary, ::NYson::EYsonType::Node, false); SerializeJsonValueAsYson(inputValue, &ysonWriter); - } - - void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, TString& result) { - TStringOutput resultStream(result); - SerializeJsonValueAsYson(inputValue, &resultStream); - } - - TString SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue) { - TString result; - SerializeJsonValueAsYson(inputValue, result); - return result; - } - - bool DeserializeYsonAsJsonValue(IInputStream* inputStream, NJson::TJsonValue* outputValue, bool throwOnError) { - NJson::TParserCallbacks parser(*outputValue); - NJson2Yson::TJsonBuilder consumer(&parser); + } + + void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, TString& result) { + TStringOutput resultStream(result); + SerializeJsonValueAsYson(inputValue, &resultStream); + } + + TString SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue) { + TString result; + SerializeJsonValueAsYson(inputValue, result); + return result; + } + + bool DeserializeYsonAsJsonValue(IInputStream* inputStream, NJson::TJsonValue* outputValue, bool throwOnError) { + NJson::TParserCallbacks parser(*outputValue); + NJson2Yson::TJsonBuilder consumer(&parser); NYson::TYsonParser ysonParser(&consumer, inputStream, ::NYson::EYsonType::Node); - try { - ysonParser.Parse(); - } catch (...) { - if (throwOnError) { - throw; - } - return false; - } - return true; - } - - bool DeserializeYsonAsJsonValue(TStringBuf str, NJson::TJsonValue* outputValue, bool throwOnError) { - TMemoryInput inputStream(str); - return DeserializeYsonAsJsonValue(&inputStream, outputValue, throwOnError); - } - - void ConvertYson2Json(IInputStream* inputStream, IOutputStream* outputStream) { + try { + ysonParser.Parse(); + } catch (...) { + if (throwOnError) { + throw; + } + return false; + } + return true; + } + + bool DeserializeYsonAsJsonValue(TStringBuf str, NJson::TJsonValue* outputValue, bool throwOnError) { + TMemoryInput inputStream(str); + return DeserializeYsonAsJsonValue(&inputStream, outputValue, throwOnError); + } + + void ConvertYson2Json(IInputStream* inputStream, IOutputStream* outputStream) { NYT::TJsonWriter writer(outputStream, ::NYson::EYsonType::Node, NYT::JF_TEXT, NYT::JAM_ON_DEMAND, NYT::SBF_BOOLEAN); NYson::TYsonParser ysonParser(&writer, inputStream, ::NYson::EYsonType::Node); - ysonParser.Parse(); - } + ysonParser.Parse(); + } + + void ConvertYson2Json(TStringBuf yson, IOutputStream* outputStream) { + TMemoryInput inputStream(yson); + ConvertYson2Json(&inputStream, outputStream); + } - void ConvertYson2Json(TStringBuf yson, IOutputStream* outputStream) { - TMemoryInput inputStream(yson); - ConvertYson2Json(&inputStream, outputStream); + TString ConvertYson2Json(TStringBuf yson) { + TString json; + TStringOutput outputStream(json); + ConvertYson2Json(yson, &outputStream); + return json; } - - TString ConvertYson2Json(TStringBuf yson) { - TString json; - TStringOutput outputStream(json); - ConvertYson2Json(yson, &outputStream); - return json; - } -} +} diff --git a/library/cpp/json/yson/json2yson.h b/library/cpp/json/yson/json2yson.h index 6d2ef4246f..758eb6d0cf 100644 --- a/library/cpp/json/yson/json2yson.h +++ b/library/cpp/json/yson/json2yson.h @@ -1,179 +1,179 @@ -#pragma once - +#pragma once + #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_value.h> #include <library/cpp/yson/writer.h> - -namespace NJson2Yson { + +namespace NJson2Yson { class TJsonBuilderImpl: public NYson::TYsonConsumerBase { - public: + public: TJsonBuilderImpl(NJson::TJsonCallbacks* parserCallbacks) - : ParserCallbacks_(parserCallbacks) - { - } - + : ParserCallbacks_(parserCallbacks) + { + } + void OnStringScalar(TStringBuf value) override { - ParserCallbacks_->OnString(value); - } - - void OnInt64Scalar(i64 value) override { - ParserCallbacks_->OnInteger(value); - } - - void OnUint64Scalar(ui64 value) override { - ParserCallbacks_->OnUInteger(value); - } - - void OnDoubleScalar(double value) override { - ParserCallbacks_->OnDouble(value); - } - - void OnBooleanScalar(bool value) override { - ParserCallbacks_->OnBoolean(value); - } - - void OnEntity() override { - ParserCallbacks_->OnNull(); - } - - void OnBeginList() override { - ParserCallbacks_->OnOpenArray(); - } - - void OnListItem() override { - } - - void OnEndList() override { - ParserCallbacks_->OnCloseArray(); - } - - void OnBeginMap() override { - ParserCallbacks_->OnOpenMap(); - } - + ParserCallbacks_->OnString(value); + } + + void OnInt64Scalar(i64 value) override { + ParserCallbacks_->OnInteger(value); + } + + void OnUint64Scalar(ui64 value) override { + ParserCallbacks_->OnUInteger(value); + } + + void OnDoubleScalar(double value) override { + ParserCallbacks_->OnDouble(value); + } + + void OnBooleanScalar(bool value) override { + ParserCallbacks_->OnBoolean(value); + } + + void OnEntity() override { + ParserCallbacks_->OnNull(); + } + + void OnBeginList() override { + ParserCallbacks_->OnOpenArray(); + } + + void OnListItem() override { + } + + void OnEndList() override { + ParserCallbacks_->OnCloseArray(); + } + + void OnBeginMap() override { + ParserCallbacks_->OnOpenMap(); + } + void OnKeyedItem(TStringBuf key) override { - ParserCallbacks_->OnMapKey(key); - } - - void OnEndMap() override { - ParserCallbacks_->OnCloseMap(); - } - - void OnBeginAttributes() override { - } - - void OnEndAttributes() override { - } - - private: + ParserCallbacks_->OnMapKey(key); + } + + void OnEndMap() override { + ParserCallbacks_->OnCloseMap(); + } + + void OnBeginAttributes() override { + } + + void OnEndAttributes() override { + } + + private: NJson::TJsonCallbacks* ParserCallbacks_; - }; - - template <typename TBase> - class TSkipAttributesProxy: public TBase { - public: + }; + + template <typename TBase> + class TSkipAttributesProxy: public TBase { + public: template <typename... TArgs> TSkipAttributesProxy<TBase>(TArgs&&... args) : TBase(std::forward<TArgs>(args)...) - { - } - + { + } + void OnStringScalar(TStringBuf value) override { - if (AttributesDepth == 0) { - TBase::OnStringScalar(value); - } - } - - void OnInt64Scalar(i64 value) override { - if (AttributesDepth == 0) { - TBase::OnInt64Scalar(value); - } - } - - void OnUint64Scalar(ui64 value) override { - if (AttributesDepth == 0) { - TBase::OnUint64Scalar(value); - } - } - - void OnDoubleScalar(double value) override { - if (AttributesDepth == 0) { - TBase::OnDoubleScalar(value); - } - } - - void OnBooleanScalar(bool value) override { - if (AttributesDepth == 0) { - TBase::OnBooleanScalar(value); - } - } - - void OnEntity() override { - if (AttributesDepth == 0) { - TBase::OnEntity(); - } - } - - void OnBeginList() override { - if (AttributesDepth == 0) { - TBase::OnBeginList(); - } - } - - void OnListItem() override { - if (AttributesDepth == 0) { - TBase::OnListItem(); - } - } - - void OnEndList() override { - if (AttributesDepth == 0) { - TBase::OnEndList(); - } - } - - void OnBeginMap() override { - if (AttributesDepth == 0) { - TBase::OnBeginMap(); - } - } - + if (AttributesDepth == 0) { + TBase::OnStringScalar(value); + } + } + + void OnInt64Scalar(i64 value) override { + if (AttributesDepth == 0) { + TBase::OnInt64Scalar(value); + } + } + + void OnUint64Scalar(ui64 value) override { + if (AttributesDepth == 0) { + TBase::OnUint64Scalar(value); + } + } + + void OnDoubleScalar(double value) override { + if (AttributesDepth == 0) { + TBase::OnDoubleScalar(value); + } + } + + void OnBooleanScalar(bool value) override { + if (AttributesDepth == 0) { + TBase::OnBooleanScalar(value); + } + } + + void OnEntity() override { + if (AttributesDepth == 0) { + TBase::OnEntity(); + } + } + + void OnBeginList() override { + if (AttributesDepth == 0) { + TBase::OnBeginList(); + } + } + + void OnListItem() override { + if (AttributesDepth == 0) { + TBase::OnListItem(); + } + } + + void OnEndList() override { + if (AttributesDepth == 0) { + TBase::OnEndList(); + } + } + + void OnBeginMap() override { + if (AttributesDepth == 0) { + TBase::OnBeginMap(); + } + } + void OnKeyedItem(TStringBuf key) override { - if (AttributesDepth == 0) { - TBase::OnKeyedItem(key); - } - } - - void OnEndMap() override { - if (AttributesDepth == 0) { - TBase::OnEndMap(); - } - } - - void OnBeginAttributes() override { - ++AttributesDepth; - } - - void OnEndAttributes() override { - --AttributesDepth; + if (AttributesDepth == 0) { + TBase::OnKeyedItem(key); + } + } + + void OnEndMap() override { + if (AttributesDepth == 0) { + TBase::OnEndMap(); + } + } + + void OnBeginAttributes() override { + ++AttributesDepth; + } + + void OnEndAttributes() override { + --AttributesDepth; Y_ASSERT(AttributesDepth >= 0); - } - - private: - int AttributesDepth = 0; - }; - - using TJsonBuilder = TSkipAttributesProxy<TJsonBuilderImpl>; - - void ConvertYson2Json(IInputStream* inputStream, IOutputStream* outputStream); - void ConvertYson2Json(TStringBuf yson, IOutputStream* outputStream); - TString ConvertYson2Json(TStringBuf yson); - - bool DeserializeYsonAsJsonValue(IInputStream* inputStream, NJson::TJsonValue* outputValue, bool throwOnError = false); - bool DeserializeYsonAsJsonValue(TStringBuf str, NJson::TJsonValue* outputValue, bool throwOnError = false); - + } + + private: + int AttributesDepth = 0; + }; + + using TJsonBuilder = TSkipAttributesProxy<TJsonBuilderImpl>; + + void ConvertYson2Json(IInputStream* inputStream, IOutputStream* outputStream); + void ConvertYson2Json(TStringBuf yson, IOutputStream* outputStream); + TString ConvertYson2Json(TStringBuf yson); + + bool DeserializeYsonAsJsonValue(IInputStream* inputStream, NJson::TJsonValue* outputValue, bool throwOnError = false); + bool DeserializeYsonAsJsonValue(TStringBuf str, NJson::TJsonValue* outputValue, bool throwOnError = false); + void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, NYson::TYsonWriter* ysonWriter); - void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, IOutputStream* outputStream); - void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, TString& result); - TString SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue); -} + void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, IOutputStream* outputStream); + void SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue, TString& result); + TString SerializeJsonValueAsYson(const NJson::TJsonValue& inputValue); +} diff --git a/library/cpp/json/yson/json2yson_ut.cpp b/library/cpp/json/yson/json2yson_ut.cpp index e57fac97ad..9eb23354cf 100644 --- a/library/cpp/json/yson/json2yson_ut.cpp +++ b/library/cpp/json/yson/json2yson_ut.cpp @@ -1,107 +1,107 @@ #include "library/cpp/json/yson/json2yson.h" - + #include <library/cpp/blockcodecs/codecs.h> #include <library/cpp/histogram/simple/histogram.h> #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -#include <util/datetime/cputimer.h> -#include <util/stream/file.h> - -template <typename TCallBack> -ui64 Run(TCallBack&& callBack) { - TSimpleTimer timer; - callBack(); - return timer.Get().MicroSeconds(); -} - -static TString GetRequestsWithDecoding(const TString& inputPath, const NBlockCodecs::ICodec* codec) { - TIFStream inputFileStream(inputPath); - TString encodedRequests = inputFileStream.ReadAll(); - TString requests; - codec->Decode(encodedRequests, requests); - return requests; -} - + +#include <util/datetime/cputimer.h> +#include <util/stream/file.h> + +template <typename TCallBack> +ui64 Run(TCallBack&& callBack) { + TSimpleTimer timer; + callBack(); + return timer.Get().MicroSeconds(); +} + +static TString GetRequestsWithDecoding(const TString& inputPath, const NBlockCodecs::ICodec* codec) { + TIFStream inputFileStream(inputPath); + TString encodedRequests = inputFileStream.ReadAll(); + TString requests; + codec->Decode(encodedRequests, requests); + return requests; +} + Y_UNIT_TEST_SUITE(Json2Yson) { Y_UNIT_TEST(NOAPACHE_REQUESTS) { - const ui32 warmUpRetries = 5; + const ui32 warmUpRetries = 5; const TVector<double> percentiles = {0.25, 0.5, 0.6, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 0.97, 0.99, 1.0}; - - NSimpleHistogram::TMultiHistogramCalcer<ui64> calcer; - - TString requests = GetRequestsWithDecoding(GetWorkPath() + "/noapache_requests_sample_lz4", NBlockCodecs::Codec("lz4")); - TStringInput inputStream(requests); - - for (TString jsonRequest, jsonString, ysonString; inputStream.ReadLine(jsonRequest);) { - TStringInput jsonInput(jsonRequest); - NJson::TJsonValue readedJson; - NJson::ReadJsonTree(&jsonInput, &readedJson, true); - jsonRequest.clear(); - - ui64 writeTime = Max<ui64>(); - ui64 readTime = Max<ui64>(); - - for (ui32 i = 0; i < warmUpRetries; ++i) { - NJson::TJsonValue Json2Json; - TStringOutput jsonWriteOutput(jsonString); - NJsonWriter::TBuf jsonBuf(NJsonWriter::HEM_UNSAFE, &jsonWriteOutput); - - writeTime = Min(writeTime, Run([&]() { - jsonBuf.WriteJsonValue(&readedJson); - })); - - TStringInput jsonInput(jsonString); - NJson::TJsonReaderConfig config; - config.DontValidateUtf8 = true; - readTime = Min(readTime, Run([&]() { - NJson::ReadJsonTree(&jsonInput, &config, &Json2Json, true); - })); - - UNIT_ASSERT_VALUES_EQUAL( - NJsonWriter::TBuf().WriteJsonValue(&readedJson, true).Str(), - NJsonWriter::TBuf().WriteJsonValue(&Json2Json, true).Str()); - - jsonString.clear(); - } - - calcer.RecordValue("read_json", readTime); - calcer.RecordValue("write_json", writeTime); - calcer.RecordValue("read_and_write_json", readTime + writeTime); - - writeTime = Max<ui64>(); - readTime = Max<ui64>(); - - for (ui32 i = 0; i < warmUpRetries; ++i) { - NJson::TJsonValue convertedJson; - TStringOutput ysonOutput(ysonString); - - writeTime = Min(writeTime, Run([&]() { - NJson2Yson::SerializeJsonValueAsYson(readedJson, &ysonOutput); - })); - - TStringInput ysonInput(ysonString); - readTime = Min(readTime, Run([&]() { - NJson2Yson::DeserializeYsonAsJsonValue(&ysonInput, &convertedJson); - })); - - UNIT_ASSERT_VALUES_EQUAL( - NJsonWriter::TBuf().WriteJsonValue(&convertedJson, true).Str(), - NJsonWriter::TBuf().WriteJsonValue(&readedJson, true).Str()); - - ysonString.clear(); - } - - calcer.RecordValue("read_yson", readTime); - calcer.RecordValue("write_yson", writeTime); - calcer.RecordValue("read_and_write_yson", readTime + writeTime); - } - - NJson::TJsonValue histogramJson = NSimpleHistogram::ToJson(calcer.Calc(), percentiles); - for (const auto& it : histogramJson.GetMap()) { - for (const auto& percentileValue : it.second.GetMap()) { - UNIT_ADD_METRIC(it.first + "_" + percentileValue.first, percentileValue.second.GetUInteger() / 1000.0); - } - } - } -} + + NSimpleHistogram::TMultiHistogramCalcer<ui64> calcer; + + TString requests = GetRequestsWithDecoding(GetWorkPath() + "/noapache_requests_sample_lz4", NBlockCodecs::Codec("lz4")); + TStringInput inputStream(requests); + + for (TString jsonRequest, jsonString, ysonString; inputStream.ReadLine(jsonRequest);) { + TStringInput jsonInput(jsonRequest); + NJson::TJsonValue readedJson; + NJson::ReadJsonTree(&jsonInput, &readedJson, true); + jsonRequest.clear(); + + ui64 writeTime = Max<ui64>(); + ui64 readTime = Max<ui64>(); + + for (ui32 i = 0; i < warmUpRetries; ++i) { + NJson::TJsonValue Json2Json; + TStringOutput jsonWriteOutput(jsonString); + NJsonWriter::TBuf jsonBuf(NJsonWriter::HEM_UNSAFE, &jsonWriteOutput); + + writeTime = Min(writeTime, Run([&]() { + jsonBuf.WriteJsonValue(&readedJson); + })); + + TStringInput jsonInput(jsonString); + NJson::TJsonReaderConfig config; + config.DontValidateUtf8 = true; + readTime = Min(readTime, Run([&]() { + NJson::ReadJsonTree(&jsonInput, &config, &Json2Json, true); + })); + + UNIT_ASSERT_VALUES_EQUAL( + NJsonWriter::TBuf().WriteJsonValue(&readedJson, true).Str(), + NJsonWriter::TBuf().WriteJsonValue(&Json2Json, true).Str()); + + jsonString.clear(); + } + + calcer.RecordValue("read_json", readTime); + calcer.RecordValue("write_json", writeTime); + calcer.RecordValue("read_and_write_json", readTime + writeTime); + + writeTime = Max<ui64>(); + readTime = Max<ui64>(); + + for (ui32 i = 0; i < warmUpRetries; ++i) { + NJson::TJsonValue convertedJson; + TStringOutput ysonOutput(ysonString); + + writeTime = Min(writeTime, Run([&]() { + NJson2Yson::SerializeJsonValueAsYson(readedJson, &ysonOutput); + })); + + TStringInput ysonInput(ysonString); + readTime = Min(readTime, Run([&]() { + NJson2Yson::DeserializeYsonAsJsonValue(&ysonInput, &convertedJson); + })); + + UNIT_ASSERT_VALUES_EQUAL( + NJsonWriter::TBuf().WriteJsonValue(&convertedJson, true).Str(), + NJsonWriter::TBuf().WriteJsonValue(&readedJson, true).Str()); + + ysonString.clear(); + } + + calcer.RecordValue("read_yson", readTime); + calcer.RecordValue("write_yson", writeTime); + calcer.RecordValue("read_and_write_yson", readTime + writeTime); + } + + NJson::TJsonValue histogramJson = NSimpleHistogram::ToJson(calcer.Calc(), percentiles); + for (const auto& it : histogramJson.GetMap()) { + for (const auto& percentileValue : it.second.GetMap()) { + UNIT_ADD_METRIC(it.first + "_" + percentileValue.first, percentileValue.second.GetUInteger() / 1000.0); + } + } + } +} diff --git a/library/cpp/json/yson/ut/ya.make b/library/cpp/json/yson/ut/ya.make index 1ba86b354b..4ceb65b279 100644 --- a/library/cpp/json/yson/ut/ya.make +++ b/library/cpp/json/yson/ut/ya.make @@ -1,28 +1,28 @@ -OWNER( - avitella - elshiko -) - +OWNER( + avitella + elshiko +) + UNITTEST_FOR(library/cpp/json/yson) - -ALLOCATOR(LF) - + +ALLOCATOR(LF) + DATA(sbr://363537653) - -PEERDIR( + +PEERDIR( library/cpp/blockcodecs library/cpp/histogram/simple library/cpp/testing/unittest -) - +) + SIZE(LARGE) - + TAG(ya:fat) -TIMEOUT(600) - -SRCS( - json2yson_ut.cpp -) - -END() +TIMEOUT(600) + +SRCS( + json2yson_ut.cpp +) + +END() diff --git a/library/cpp/json/yson/ya.make b/library/cpp/json/yson/ya.make index c61c924e93..9b289d674f 100644 --- a/library/cpp/json/yson/ya.make +++ b/library/cpp/json/yson/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - avitella - elshiko -) - -PEERDIR( +LIBRARY() + +OWNER( + avitella + elshiko +) + +PEERDIR( library/cpp/json library/cpp/yson library/cpp/yson/json -) - -SRCS( - json2yson.cpp -) - -END() +) + +SRCS( + json2yson.cpp +) + +END() diff --git a/library/cpp/protobuf/util/pb_io.cpp b/library/cpp/protobuf/util/pb_io.cpp index d0d9b65d89..6270ee0624 100644 --- a/library/cpp/protobuf/util/pb_io.cpp +++ b/library/cpp/protobuf/util/pb_io.cpp @@ -21,15 +21,15 @@ namespace NProtoBuf { } }; - void ParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven) { + void ParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven) { if (!m.ParseFromString(allowUneven ? Base64DecodeUneven(dataBase64) : Base64StrictDecode(dataBase64))) { ythrow yexception() << "can't parse " << m.GetTypeName() << " from base64-encoded string"; } } - bool TryParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven) { + bool TryParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven) { try { - ParseFromBase64String(dataBase64, m, allowUneven); + ParseFromBase64String(dataBase64, m, allowUneven); return true; } catch (const std::exception&) { return false; diff --git a/library/cpp/protobuf/util/pb_io.h b/library/cpp/protobuf/util/pb_io.h index 4ca101b28a..493c84cb5f 100644 --- a/library/cpp/protobuf/util/pb_io.h +++ b/library/cpp/protobuf/util/pb_io.h @@ -21,12 +21,12 @@ class IOutputStream; namespace NProtoBuf { /* Parse base64 URL encoded serialized message from string. */ - void ParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven = false); - bool TryParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven = false); + void ParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven = false); + bool TryParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven = false); template <typename T> - static T ParseFromBase64String(const TStringBuf& dataBase64, bool allowUneven = false) { + static T ParseFromBase64String(const TStringBuf& dataBase64, bool allowUneven = false) { T m; - ParseFromBase64String(dataBase64, m, allowUneven); + ParseFromBase64String(dataBase64, m, allowUneven); return m; } diff --git a/library/cpp/protobuf/util/pb_io_ut.cpp b/library/cpp/protobuf/util/pb_io_ut.cpp index cb08001997..875d6dc602 100644 --- a/library/cpp/protobuf/util/pb_io_ut.cpp +++ b/library/cpp/protobuf/util/pb_io_ut.cpp @@ -45,8 +45,8 @@ static const TString INCORRECT_ENUM_ID_MESSAGE = static const TString CORRECT_BASE64_MESSAGE = "CCo,"; -static const TString CORRECT_UNEVEN_BASE64_MESSAGE = "CCo"; - +static const TString CORRECT_UNEVEN_BASE64_MESSAGE = "CCo"; + static const TString INCORRECT_BASE64_MESSAGE = "CC"; Y_UNIT_TEST_SUITE(TTestProtoBufIO) { @@ -60,14 +60,14 @@ Y_UNIT_TEST_SUITE(TTestProtoBufIO) { UNIT_ASSERT(!NProtoBuf::TryParseFromBase64String(INCORRECT_BASE64_MESSAGE, message)); } { - NProtobufUtilUt::TTextTest message; - UNIT_ASSERT(NProtoBuf::TryParseFromBase64String(CORRECT_UNEVEN_BASE64_MESSAGE , message, true)); - } - { - NProtobufUtilUt::TTextTest message; - UNIT_ASSERT(!NProtoBuf::TryParseFromBase64String(CORRECT_UNEVEN_BASE64_MESSAGE , message, false)); - } - { + NProtobufUtilUt::TTextTest message; + UNIT_ASSERT(NProtoBuf::TryParseFromBase64String(CORRECT_UNEVEN_BASE64_MESSAGE , message, true)); + } + { + NProtobufUtilUt::TTextTest message; + UNIT_ASSERT(!NProtoBuf::TryParseFromBase64String(CORRECT_UNEVEN_BASE64_MESSAGE , message, false)); + } + { UNIT_ASSERT_VALUES_EQUAL(CORRECT_BASE64_MESSAGE, NProtoBuf::SerializeToBase64String(GetCorrectMessage())); } { diff --git a/library/cpp/retry/retry.h b/library/cpp/retry/retry.h index 478eb48197..c47ff5070f 100644 --- a/library/cpp/retry/retry.h +++ b/library/cpp/retry/retry.h @@ -82,12 +82,12 @@ struct TRetryOptions { }; template <typename TResult, typename TException = yexception> -TMaybe<TResult> DoWithRetry(std::function<TResult()> func, std::function<void(const TException&)> onFail, TRetryOptions retryOptions, bool throwLast) { +TMaybe<TResult> DoWithRetry(std::function<TResult()> func, std::function<void(const TException&)> onFail, TRetryOptions retryOptions, bool throwLast) { for (ui32 attempt = 0; attempt <= retryOptions.RetryCount; ++attempt) { try { return func(); - } catch (TException& ex) { - onFail(ex); + } catch (TException& ex) { + onFail(ex); if (attempt == retryOptions.RetryCount) { if (throwLast) { throw; @@ -101,27 +101,27 @@ TMaybe<TResult> DoWithRetry(std::function<TResult()> func, std::function<void(co return Nothing(); } -template <typename TResult, typename TException = yexception> -TMaybe<TResult> DoWithRetry(std::function<TResult()> func, TRetryOptions retryOptions, bool throwLast) { - return DoWithRetry<TResult, TException>(func, [](const TException&){}, retryOptions, throwLast); -} - +template <typename TResult, typename TException = yexception> +TMaybe<TResult> DoWithRetry(std::function<TResult()> func, TRetryOptions retryOptions, bool throwLast) { + return DoWithRetry<TResult, TException>(func, [](const TException&){}, retryOptions, throwLast); +} + template <typename TException = yexception> bool DoWithRetry(std::function<void()> func, std::function<void(const TException&)> onFail, TRetryOptions retryOptions, bool throwLast) { - auto f = [&]() { - func(); - return nullptr; - }; - return DoWithRetry<void*, TException>(f, onFail, retryOptions, throwLast).Defined(); -} - -template <typename TException = yexception> + auto f = [&]() { + func(); + return nullptr; + }; + return DoWithRetry<void*, TException>(f, onFail, retryOptions, throwLast).Defined(); +} + +template <typename TException = yexception> bool DoWithRetry(std::function<void()> func, TRetryOptions retryOptions, bool throwLast) { auto f = [&]() { func(); return nullptr; }; - return DoWithRetry<void*, TException>(f, [](const TException&){}, retryOptions, throwLast).Defined(); + return DoWithRetry<void*, TException>(f, [](const TException&){}, retryOptions, throwLast).Defined(); } void DoWithRetry(std::function<void()> func, TRetryOptions retryOptions); diff --git a/library/cpp/retry/retry_ut.cpp b/library/cpp/retry/retry_ut.cpp index 0d50176bc2..92153e987e 100644 --- a/library/cpp/retry/retry_ut.cpp +++ b/library/cpp/retry/retry_ut.cpp @@ -31,67 +31,67 @@ Y_UNIT_TEST_SUITE(Retry) { Y_UNIT_TEST(RetryOnExceptionSuccess) { UNIT_ASSERT_NO_EXCEPTION(DoWithRetry(TDoOnSecondOrThrow{}, TRetryOptions(1, TDuration::Zero()))); } - Y_UNIT_TEST(RetryOnExceptionSuccessWithOnFail) { - ui32 value = 0; - std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; - UNIT_ASSERT_NO_EXCEPTION(DoWithRetry<ui32>(TDoOnSecondOrThrow{}, cb, TRetryOptions(1, TDuration::Zero()), true)); - UNIT_ASSERT_EQUAL(value, 1); - } + Y_UNIT_TEST(RetryOnExceptionSuccessWithOnFail) { + ui32 value = 0; + std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT_NO_EXCEPTION(DoWithRetry<ui32>(TDoOnSecondOrThrow{}, cb, TRetryOptions(1, TDuration::Zero()), true)); + UNIT_ASSERT_EQUAL(value, 1); + } Y_UNIT_TEST(RetryOnExceptionFail) { UNIT_ASSERT_EXCEPTION(DoWithRetry(TDoOnSecondOrThrow{}, TRetryOptions(0, TDuration::Zero())), yexception); } - Y_UNIT_TEST(RetryOnExceptionFailWithOnFail) { - ui32 value = 0; - std::function<void(const yexception&)> cb = [&value](const yexception&) { value += 1; }; - UNIT_ASSERT_EXCEPTION(DoWithRetry<ui32>(TDoOnSecondOrThrow{}, cb, TRetryOptions(0, TDuration::Zero()), true), yexception); - UNIT_ASSERT_EQUAL(value, 1); - } + Y_UNIT_TEST(RetryOnExceptionFailWithOnFail) { + ui32 value = 0; + std::function<void(const yexception&)> cb = [&value](const yexception&) { value += 1; }; + UNIT_ASSERT_EXCEPTION(DoWithRetry<ui32>(TDoOnSecondOrThrow{}, cb, TRetryOptions(0, TDuration::Zero()), true), yexception); + UNIT_ASSERT_EQUAL(value, 1); + } Y_UNIT_TEST(RetryOnExceptionSuccessWithValue) { std::function<ui32()> f = TDoOnSecondOrThrow{}; UNIT_ASSERT(42 == *DoWithRetry<ui32>(f, TRetryOptions(1, TDuration::Zero()), false)); } - Y_UNIT_TEST(RetryOnExceptionSuccessWithValueWithOnFail) { - ui32 value = 0; - std::function<ui32()> f = TDoOnSecondOrThrow{}; - std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; - UNIT_ASSERT(42 == *DoWithRetry<ui32>(f, cb, TRetryOptions(1, TDuration::Zero()), false)); - UNIT_ASSERT_EQUAL(value, 1); - } + Y_UNIT_TEST(RetryOnExceptionSuccessWithValueWithOnFail) { + ui32 value = 0; + std::function<ui32()> f = TDoOnSecondOrThrow{}; + std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT(42 == *DoWithRetry<ui32>(f, cb, TRetryOptions(1, TDuration::Zero()), false)); + UNIT_ASSERT_EQUAL(value, 1); + } Y_UNIT_TEST(RetryOnExceptionFailWithValue) { std::function<ui32()> f = TDoOnSecondOrThrow{}; UNIT_ASSERT(!DoWithRetry<ui32>(f, TRetryOptions(0, TDuration::Zero()), false).Defined()); } - Y_UNIT_TEST(RetryOnExceptionFailWithValueWithOnFail) { - ui32 value = 0; - std::function<ui32()> f = TDoOnSecondOrThrow{}; - std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; - UNIT_ASSERT(!DoWithRetry<ui32>(f, cb, TRetryOptions(0, TDuration::Zero()), false).Defined()); - UNIT_ASSERT_EQUAL(value, 1); - } + Y_UNIT_TEST(RetryOnExceptionFailWithValueWithOnFail) { + ui32 value = 0; + std::function<ui32()> f = TDoOnSecondOrThrow{}; + std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT(!DoWithRetry<ui32>(f, cb, TRetryOptions(0, TDuration::Zero()), false).Defined()); + UNIT_ASSERT_EQUAL(value, 1); + } Y_UNIT_TEST(RetryOnExceptionSuccessWithValueAndRethrow) { std::function<ui32()> f = TDoOnSecondOrThrow{}; UNIT_ASSERT(42 == *DoWithRetry<ui32>(f, TRetryOptions(1, TDuration::Zero()), true)); } - Y_UNIT_TEST(RetryOnExceptionSuccessWithValueAndRethrowWithOnFail) { - ui32 value = 0; - std::function<ui32()> f = TDoOnSecondOrThrow{}; - std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; - UNIT_ASSERT(42 == *DoWithRetry<ui32>(f, cb, TRetryOptions(1, TDuration::Zero()), true)); - UNIT_ASSERT_EQUAL(value, 1); - } + Y_UNIT_TEST(RetryOnExceptionSuccessWithValueAndRethrowWithOnFail) { + ui32 value = 0; + std::function<ui32()> f = TDoOnSecondOrThrow{}; + std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT(42 == *DoWithRetry<ui32>(f, cb, TRetryOptions(1, TDuration::Zero()), true)); + UNIT_ASSERT_EQUAL(value, 1); + } Y_UNIT_TEST(RetryOnExceptionFailWithValueAndRethrow) { std::function<ui32()> f = TDoOnSecondOrThrow{}; UNIT_ASSERT_EXCEPTION(DoWithRetry<ui32>(f, TRetryOptions(0, TDuration::Zero()), true), yexception); } - Y_UNIT_TEST(RetryOnExceptionFailWithValueAndRethrowWithOnFail) { - ui32 value = 0; - std::function<ui32()> f = TDoOnSecondOrThrow{}; - std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; - UNIT_ASSERT_EXCEPTION(42 == *DoWithRetry<ui32>(f, cb, TRetryOptions(0, TDuration::Zero()), true), yexception); - UNIT_ASSERT_EQUAL(value, 1); - } + Y_UNIT_TEST(RetryOnExceptionFailWithValueAndRethrowWithOnFail) { + ui32 value = 0; + std::function<ui32()> f = TDoOnSecondOrThrow{}; + std::function<void(const yexception&)> cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT_EXCEPTION(42 == *DoWithRetry<ui32>(f, cb, TRetryOptions(0, TDuration::Zero()), true), yexception); + UNIT_ASSERT_EQUAL(value, 1); + } Y_UNIT_TEST(RetryOnRetCodeSuccess) { UNIT_ASSERT(true == DoWithRetryOnRetCode(TDoOnSecondOrFail{}, TRetryOptions(1, TDuration::Zero()))); diff --git a/library/cpp/yson/json/yson2json_adapter.cpp b/library/cpp/yson/json/yson2json_adapter.cpp index 58d9bf7fdb..b5e7c49d4d 100644 --- a/library/cpp/yson/json/yson2json_adapter.cpp +++ b/library/cpp/yson/json/yson2json_adapter.cpp @@ -1,82 +1,82 @@ -#include "yson2json_adapter.h" - -namespace NYT { +#include "yson2json_adapter.h" + +namespace NYT { TYson2JsonCallbacksAdapter::TYson2JsonCallbacksAdapter(::NYson::TYsonConsumerBase* impl, bool throwException) - : NJson::TJsonCallbacks(throwException) + : NJson::TJsonCallbacks(throwException) , Impl_(impl) - { - } - - bool TYson2JsonCallbacksAdapter::OnNull() { - WrapIfListItem(); + { + } + + bool TYson2JsonCallbacksAdapter::OnNull() { + WrapIfListItem(); Impl_->OnEntity(); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnBoolean(bool val) { - WrapIfListItem(); + return true; + } + + bool TYson2JsonCallbacksAdapter::OnBoolean(bool val) { + WrapIfListItem(); Impl_->OnBooleanScalar(val); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnInteger(long long val) { - WrapIfListItem(); + return true; + } + + bool TYson2JsonCallbacksAdapter::OnInteger(long long val) { + WrapIfListItem(); Impl_->OnInt64Scalar(val); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnUInteger(unsigned long long val) { - WrapIfListItem(); + return true; + } + + bool TYson2JsonCallbacksAdapter::OnUInteger(unsigned long long val) { + WrapIfListItem(); Impl_->OnUint64Scalar(val); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnString(const TStringBuf& val) { - WrapIfListItem(); + return true; + } + + bool TYson2JsonCallbacksAdapter::OnString(const TStringBuf& val) { + WrapIfListItem(); Impl_->OnStringScalar(val); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnDouble(double val) { - WrapIfListItem(); + return true; + } + + bool TYson2JsonCallbacksAdapter::OnDouble(double val) { + WrapIfListItem(); Impl_->OnDoubleScalar(val); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnOpenArray() { - WrapIfListItem(); + return true; + } + + bool TYson2JsonCallbacksAdapter::OnOpenArray() { + WrapIfListItem(); State_.ContextStack.push(true); Impl_->OnBeginList(); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnCloseArray() { + return true; + } + + bool TYson2JsonCallbacksAdapter::OnCloseArray() { State_.ContextStack.pop(); Impl_->OnEndList(); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnOpenMap() { - WrapIfListItem(); + return true; + } + + bool TYson2JsonCallbacksAdapter::OnOpenMap() { + WrapIfListItem(); State_.ContextStack.push(false); Impl_->OnBeginMap(); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnCloseMap() { + return true; + } + + bool TYson2JsonCallbacksAdapter::OnCloseMap() { State_.ContextStack.pop(); Impl_->OnEndMap(); - return true; - } - - bool TYson2JsonCallbacksAdapter::OnMapKey(const TStringBuf& val) { + return true; + } + + bool TYson2JsonCallbacksAdapter::OnMapKey(const TStringBuf& val) { Impl_->OnKeyedItem(val); - return true; - } - - void TYson2JsonCallbacksAdapter::WrapIfListItem() { + return true; + } + + void TYson2JsonCallbacksAdapter::WrapIfListItem() { if (!State_.ContextStack.empty() && State_.ContextStack.top()) { Impl_->OnListItem(); - } - } -} + } + } +} diff --git a/library/cpp/yson/json/yson2json_adapter.h b/library/cpp/yson/json/yson2json_adapter.h index 9b42386732..da1bf5ba70 100644 --- a/library/cpp/yson/json/yson2json_adapter.h +++ b/library/cpp/yson/json/yson2json_adapter.h @@ -1,15 +1,15 @@ -#pragma once - +#pragma once + #include <library/cpp/yson/consumer.h> - + #include <library/cpp/json/json_reader.h> - -#include <util/generic/stack.h> - -namespace NYT { - class TYson2JsonCallbacksAdapter - : public NJson::TJsonCallbacks { - public: + +#include <util/generic/stack.h> + +namespace NYT { + class TYson2JsonCallbacksAdapter + : public NJson::TJsonCallbacks { + public: class TState { private: // Stores current context stack @@ -22,19 +22,19 @@ namespace NYT { public: TYson2JsonCallbacksAdapter(::NYson::TYsonConsumerBase* impl, bool throwException = false); - - bool OnNull() override; - bool OnBoolean(bool val) override; - bool OnInteger(long long val) override; - bool OnUInteger(unsigned long long val) override; - bool OnString(const TStringBuf& val) override; - bool OnDouble(double val) override; - bool OnOpenArray() override; - bool OnCloseArray() override; - bool OnOpenMap() override; - bool OnCloseMap() override; - bool OnMapKey(const TStringBuf& val) override; - + + bool OnNull() override; + bool OnBoolean(bool val) override; + bool OnInteger(long long val) override; + bool OnUInteger(unsigned long long val) override; + bool OnString(const TStringBuf& val) override; + bool OnDouble(double val) override; + bool OnOpenArray() override; + bool OnCloseArray() override; + bool OnOpenMap() override; + bool OnCloseMap() override; + bool OnMapKey(const TStringBuf& val) override; + TState State() const { return State_; } @@ -43,11 +43,11 @@ namespace NYT { State_ = state; } - private: - void WrapIfListItem(); - - private: + private: + void WrapIfListItem(); + + private: ::NYson::TYsonConsumerBase* Impl_; TState State_; - }; + }; } diff --git a/library/cpp/yson/node/node_builder.h b/library/cpp/yson/node/node_builder.h index 01c749cee4..69800016e0 100644 --- a/library/cpp/yson/node/node_builder.h +++ b/library/cpp/yson/node/node_builder.h @@ -5,7 +5,7 @@ #include <library/cpp/json/json_reader.h> #include <library/cpp/yson/consumer.h> - + #include <util/generic/stack.h> namespace NYT { |