diff options
| author | monster <[email protected]> | 2022-02-10 16:47:19 +0300 |
|---|---|---|
| committer | Daniil Cherednik <[email protected]> | 2022-02-10 16:47:19 +0300 |
| commit | dd76ae1f6213d065375ab296699f764faafbe5bd (patch) | |
| tree | 075a4553d0011d8f92752c0891794febb15912cf /library/cpp/yson | |
| parent | f02a874a7290593efbe4b3aeae69a04b46c1cc86 (diff) | |
Restoring authorship annotation for <[email protected]>. Commit 1 of 2.
Diffstat (limited to 'library/cpp/yson')
30 files changed, 1094 insertions, 1094 deletions
diff --git a/library/cpp/yson/consumer.cpp b/library/cpp/yson/consumer.cpp index 40ae452978c..33e9ee63271 100644 --- a/library/cpp/yson/consumer.cpp +++ b/library/cpp/yson/consumer.cpp @@ -1,15 +1,15 @@ -#include "consumer.h" +#include "consumer.h" #include "string.h" -#include "parser.h" - +#include "parser.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + void TYsonConsumerBase::OnRaw(TStringBuf str, NYT::NYson::EYsonType type) { ParseYsonStringBuffer(str, this, type); } - + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/consumer.h b/library/cpp/yson/consumer.h index d5a9d663355..fe1902c8f06 100644 --- a/library/cpp/yson/consumer.h +++ b/library/cpp/yson/consumer.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <library/cpp/yt/yson/consumer.h> - -#include <util/generic/strbuf.h> -#include <util/system/defaults.h> - + +#include <util/generic/strbuf.h> +#include <util/system/defaults.h> + namespace NYson { struct TYsonConsumerBase : public virtual NYT::NYson::IYsonConsumer { diff --git a/library/cpp/yson/detail.h b/library/cpp/yson/detail.h index 27f5e8ffff2..d2a4a6308ef 100644 --- a/library/cpp/yson/detail.h +++ b/library/cpp/yson/detail.h @@ -1,47 +1,47 @@ -#pragma once - -#include "public.h" -#include "zigzag.h" - -#include <util/generic/vector.h> -#include <util/generic/maybe.h> -#include <util/generic/buffer.h> -#include <util/string/escape.h> -#include <util/string/cast.h> -#include <util/stream/input.h> - +#pragma once + +#include "public.h" +#include "zigzag.h" + +#include <util/generic/vector.h> +#include <util/generic/maybe.h> +#include <util/generic/buffer.h> +#include <util/string/escape.h> +#include <util/string/cast.h> +#include <util/stream/input.h> + namespace NYson { namespace NDetail { //////////////////////////////////////////////////////////////////////////////// - + //! Indicates the beginning of a list. const char BeginListSymbol = '['; //! Indicates the end of a list. const char EndListSymbol = ']'; - + //! Indicates the beginning of a map. const char BeginMapSymbol = '{'; //! Indicates the end of a map. const char EndMapSymbol = '}'; - + //! Indicates the beginning of an attribute map. const char BeginAttributesSymbol = '<'; //! Indicates the end of an attribute map. const char EndAttributesSymbol = '>'; - + //! Separates items in lists. const char ListItemSeparatorSymbol = ';'; //! Separates items in maps, attributes. const char KeyedItemSeparatorSymbol = ';'; //! Separates keys from values in maps. const char KeyValueSeparatorSymbol = '='; - + //! Indicates an entity. const char EntitySymbol = '#'; - + //! Indicates end of stream. const char EndSymbol = '\0'; - + //! Marks the beginning of a binary string literal. const char StringMarker = '\x01'; //! Marks the beginning of a binary i64 literal. @@ -53,19 +53,19 @@ namespace NYson { const char TrueMarker = '\x05'; //! Marks the beginning of a binary ui64 literal. const char Uint64Marker = '\x06'; - + //////////////////////////////////////////////////////////////////////////////// - + template <bool EnableLinePositionInfo> class TPositionInfo; - + template <> class TPositionInfo<true> { private: int Offset; int Line; int Column; - + public: TPositionInfo() : Offset(0) @@ -73,7 +73,7 @@ namespace NYson { , Column(1) { } - + void OnRangeConsumed(const char* begin, const char* end) { Offset += end - begin; for (auto current = begin; current != end; ++current) { @@ -85,7 +85,7 @@ namespace NYson { } } }; - + template <> class TPositionInfo<false> { private: @@ -95,13 +95,13 @@ namespace NYson { TPositionInfo() : Offset(0) { - } - + } + void OnRangeConsumed(const char* begin, const char* end) { Offset += end - begin; } }; - + template <class TBlockStream, class TPositionBase> class TCharStream : public TBlockStream, @@ -111,11 +111,11 @@ namespace NYson { : TBlockStream(blockStream) { } - + bool IsEmpty() const { return TBlockStream::Begin() == TBlockStream::End(); } - + template <bool AllowFinish> void Refresh() { while (IsEmpty() && !TBlockStream::IsFinished()) { @@ -125,46 +125,46 @@ namespace NYson { ythrow TYsonException() << "Premature end of yson stream"; } } - + void Refresh() { return Refresh<false>(); } - + template <bool AllowFinish> char GetChar() { Refresh<AllowFinish>(); return !IsEmpty() ? *TBlockStream::Begin() : '\0'; } - + char GetChar() { return GetChar<false>(); } - + void Advance(size_t bytes) { TPositionBase::OnRangeConsumed(TBlockStream::Begin(), TBlockStream::Begin() + bytes); TBlockStream::Advance(bytes); } - + size_t Length() const { return TBlockStream::End() - TBlockStream::Begin(); } }; - + template <class TBaseStream> class TCodedStream : public TBaseStream { private: static const int MaxVarintBytes = 10; static const int MaxVarint32Bytes = 5; - + const ui8* BeginByte() const { return reinterpret_cast<const ui8*>(TBaseStream::Begin()); } - + const ui8* EndByte() const { return reinterpret_cast<const ui8*>(TBaseStream::End()); } - + // Following functions is an adaptation Protobuf code from coded_stream.cc bool ReadVarint32FromArray(ui32* value) { // Fast path: We have enough bytes left in the buffer to guarantee that @@ -172,7 +172,7 @@ namespace NYson { const ui8* ptr = BeginByte(); ui32 b; ui32 result; - + b = *(ptr++); result = (b & 0x7F); if (!(b & 0x80)) @@ -193,26 +193,26 @@ namespace NYson { result |= b << 28; if (!(b & 0x80)) goto done; - + // If the input is larger than 32 bits, we still need to read it all // and discard the high-order bits. - + for (int i = 0; i < MaxVarintBytes - MaxVarint32Bytes; i++) { b = *(ptr++); if (!(b & 0x80)) goto done; } - + // We have overrun the maximum size of a Varint (10 bytes). Assume // the data is corrupt. return false; - + done: TBaseStream::Advance(ptr - BeginByte()); *value = result; return true; } - + bool ReadVarint32Fallback(ui32* value) { if (BeginByte() + MaxVarint32Bytes <= EndByte() || // Optimization: If the Varint ends at exactly the end of the buffer, @@ -227,7 +227,7 @@ namespace NYson { return ReadVarint32Slow(value); } } - + bool ReadVarint32Slow(ui32* value) { ui64 result; // Directly invoke ReadVarint64Fallback, since we already tried to optimize @@ -239,15 +239,15 @@ namespace NYson { return false; } } - + bool ReadVarint64Slow(ui64* value) { // Slow path: This read might cross the end of the buffer, so we // need to check and refresh the buffer if and when it does. - + ui64 result = 0; int count = 0; ui32 b; - + do { if (count == MaxVarintBytes) { return false; @@ -260,11 +260,11 @@ namespace NYson { TBaseStream::Advance(1); ++count; } while (b & 0x80); - + *value = result; return true; - } - + } + bool ReadVarint64Fallback(ui64* value) { if (BeginByte() + MaxVarintBytes <= EndByte() || // Optimization: If the Varint ends at exactly the end of the buffer, @@ -273,14 +273,14 @@ namespace NYson { { // Fast path: We have enough bytes left in the buffer to guarantee that // this read won't cross the end, so we can skip the checks. - + const ui8* ptr = BeginByte(); ui32 b; - + // Splitting into 32-bit pieces gives better performance on 32-bit // processors. ui32 part0 = 0, part1 = 0, part2 = 0; - + b = *(ptr++); part0 = (b & 0x7F); if (!(b & 0x80)) @@ -321,11 +321,11 @@ namespace NYson { part2 |= (b & 0x7F) << 7; if (!(b & 0x80)) goto done; - + // We have overrun the maximum size of a Varint (10 bytes). The data // must be corrupt. return false; - + done: TBaseStream::Advance(ptr - BeginByte()); *value = (static_cast<ui64>(part0)) | @@ -336,13 +336,13 @@ namespace NYson { return ReadVarint64Slow(value); } } - + public: TCodedStream(const TBaseStream& baseStream) : TBaseStream(baseStream) { } - + bool ReadVarint64(ui64* value) { if (BeginByte() < EndByte() && *BeginByte() < 0x80) { *value = *BeginByte(); @@ -352,7 +352,7 @@ namespace NYson { return ReadVarint64Fallback(value); } } - + bool ReadVarint32(ui32* value) { if (BeginByte() < EndByte() && *BeginByte() < 0x80) { *value = *BeginByte(); @@ -363,13 +363,13 @@ namespace NYson { } } }; - + enum ENumericResult { Int64 = 0, Uint64 = 1, Double = 2 }; - + template <class TBlockStream, bool EnableLinePositionInfo> class TLexerBase : public TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>> { @@ -377,7 +377,7 @@ namespace NYson { using TBaseStream = TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>>; TVector<char> Buffer_; TMaybe<ui64> MemoryLimit_; - + void CheckMemoryLimit() { if (MemoryLimit_ && Buffer_.capacity() > *MemoryLimit_) { ythrow TYsonException() @@ -385,17 +385,17 @@ namespace NYson { << Buffer_.capacity() << ", limit " << (*MemoryLimit_); } } - + public: TLexerBase(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) : TBaseStream(blockStream) , MemoryLimit_(memoryLimit) { } - + protected: /// Lexer routines - + template <bool AllowFinish> ENumericResult ReadNumeric(TStringBuf* value) { Buffer_.clear(); @@ -418,18 +418,18 @@ namespace NYson { CheckMemoryLimit(); TBaseStream::Advance(1); } - + *value = TStringBuf(Buffer_.data(), Buffer_.size()); return result; - } - + } + template <bool AllowFinish> double ReadNanOrInf() { static const TStringBuf nanString = "nan"; static const TStringBuf infString = "inf"; static const TStringBuf plusInfString = "+inf"; static const TStringBuf minusInfString = "-inf"; - + TStringBuf expectedString; double expectedValue; char ch = TBaseStream::template GetChar<AllowFinish>(); @@ -515,25 +515,25 @@ namespace NYson { } CheckMemoryLimit(); TBaseStream::Advance(1); - } + } *value = TStringBuf(Buffer_.data(), Buffer_.size()); - } - + } + void ReadUnquotedString(TStringBuf* value) { return ReadUnquotedString<false>(value); - } - + } + void ReadBinaryString(TStringBuf* value) { ui32 ulength = 0; if (!TBaseStream::ReadVarint32(&ulength)) { ythrow TYsonException() << "Error parsing varint value"; } - + i32 length = ZigZagDecode32(ulength); if (length < 0) { ythrow TYsonException() << "Negative binary string literal length " << length; } - + if (TBaseStream::Begin() + length <= TBaseStream::End()) { *value = TStringBuf(TBaseStream::Begin(), length); TBaseStream::Advance(length); @@ -546,29 +546,29 @@ namespace NYson { continue; } size_t readingBytes = Min(needToRead, TBaseStream::Length()); - + Buffer_.insert(Buffer_.end(), TBaseStream::Begin(), TBaseStream::Begin() + readingBytes); CheckMemoryLimit(); needToRead -= readingBytes; TBaseStream::Advance(readingBytes); } *value = TStringBuf(Buffer_.data(), Buffer_.size()); - } - } - + } + } + template <bool AllowFinish> bool ReadBoolean() { Buffer_.clear(); - + static TStringBuf trueString = "true"; static TStringBuf falseString = "false"; - + auto throwIncorrectBoolean = [&]() { ythrow TYsonException() << "Incorrect boolean string " << TString(Buffer_.data(), Buffer_.size()); }; - - Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); - TBaseStream::Advance(1); + + Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); + TBaseStream::Advance(1); if (Buffer_[0] == trueString[0]) { for (size_t i = 1; i < trueString.size(); ++i) { Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); @@ -588,21 +588,21 @@ namespace NYson { } return false; } else { - throwIncorrectBoolean(); - } + throwIncorrectBoolean(); + } Y_FAIL("unreachable"); ; - } + } void ReadBinaryInt64(i64* result) { ui64 uvalue; if (!TBaseStream::ReadVarint64(&uvalue)) { ythrow TYsonException() << "Error parsing varint value"; - } + } *result = ZigZagDecode64(uvalue); - } - + } + void ReadBinaryUint64(ui64* result) { ui64 uvalue; if (!TBaseStream::ReadVarint64(&uvalue)) { @@ -610,16 +610,16 @@ namespace NYson { } *result = uvalue; } - + void ReadBinaryDouble(double* value) { size_t needToRead = sizeof(double); - + while (needToRead != 0) { if (TBaseStream::IsEmpty()) { TBaseStream::Refresh(); continue; } - + size_t chunkSize = Min(needToRead, TBaseStream::Length()); if (chunkSize == 0) { ythrow TYsonException() << "Error parsing binary double literal"; @@ -631,8 +631,8 @@ namespace NYson { needToRead -= chunkSize; TBaseStream::Advance(chunkSize); } - } - + } + /// Helpers void SkipCharToken(char symbol) { char ch = SkipSpaceAndGetChar(); @@ -641,8 +641,8 @@ namespace NYson { } TBaseStream::Advance(1); - } - + } + static bool IsSpaceFast(char ch) { static const ui8 lookupTable[] = { @@ -650,24 +650,24 @@ namespace NYson { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; return lookupTable[static_cast<ui8>(ch)]; } - + template <bool AllowFinish> char SkipSpaceAndGetChar() { if (!TBaseStream::IsEmpty()) { @@ -678,11 +678,11 @@ namespace NYson { } return SkipSpaceAndGetCharFallback<AllowFinish>(); } - + char SkipSpaceAndGetChar() { return SkipSpaceAndGetChar<false>(); - } - + } + template <bool AllowFinish> char SkipSpaceAndGetCharFallback() { while (true) { @@ -697,63 +697,63 @@ namespace NYson { break; } TBaseStream::Advance(1); - } + } return TBaseStream::template GetChar<AllowFinish>(); - } + } }; - + //////////////////////////////////////////////////////////////////////////////// - + } - + //////////////////////////////////////////////////////////////////////////////// - + class TStringReader { private: const char* BeginPtr; const char* EndPtr; - + public: TStringReader() : BeginPtr(nullptr) , EndPtr(nullptr) { } - + TStringReader(const char* begin, const char* end) : BeginPtr(begin) , EndPtr(end) { } - + const char* Begin() const { return BeginPtr; } - + const char* End() const { return EndPtr; } - + void RefreshBlock() { Y_FAIL("unreachable"); } - + void Advance(size_t bytes) { BeginPtr += bytes; } - + bool IsFinished() const { return true; } - + void SetBuffer(const char* begin, const char* end) { BeginPtr = begin; EndPtr = end; } }; - + //////////////////////////////////////////////////////////////////////////////// - + class TStreamReader { public: TStreamReader( @@ -767,40 +767,40 @@ namespace NYson { BeginPtr = EndPtr = Buffer; FinishFlag = false; } - + const char* Begin() const { return BeginPtr; } - + const char* End() const { return EndPtr; } - + void RefreshBlock() { size_t bytes = Stream->Read(Buffer, BufferSize); BeginPtr = Buffer; EndPtr = Buffer + bytes; FinishFlag = (bytes == 0); } - + void Advance(size_t bytes) { BeginPtr += bytes; } - + bool IsFinished() const { return FinishFlag; } - + private: IInputStream* Stream; char* Buffer; size_t BufferSize; - + const char* BeginPtr; const char* EndPtr; bool FinishFlag; }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/format.h b/library/cpp/yson/format.h index 2ff6dc9f6e2..c2bad91a6a6 100644 --- a/library/cpp/yson/format.h +++ b/library/cpp/yson/format.h @@ -1,25 +1,25 @@ -#pragma once - -#include "token.h" - +#pragma once + +#include "token.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + const ETokenType BeginListToken = LeftBracket; const ETokenType EndListToken = RightBracket; - + const ETokenType BeginMapToken = LeftBrace; const ETokenType EndMapToken = RightBrace; - + const ETokenType BeginAttributesToken = LeftAngle; const ETokenType EndAttributesToken = RightAngle; - + const ETokenType ListItemSeparatorToken = Semicolon; const ETokenType KeyedItemSeparatorToken = Semicolon; const ETokenType KeyValueSeparatorToken = Equals; - + const ETokenType EntityToken = Hash; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/json/json_writer.cpp b/library/cpp/yson/json/json_writer.cpp index 87481256ecd..2c961a02fa8 100644 --- a/library/cpp/yson/json/json_writer.cpp +++ b/library/cpp/yson/json/json_writer.cpp @@ -1,16 +1,16 @@ -#include "json_writer.h" - +#include "json_writer.h" + #include <library/cpp/json/json_writer.h> - -namespace NYT { + +namespace NYT { //////////////////////////////////////////////////////////////////////////////// - + static bool IsSpecialJsonKey(const TStringBuf& key) { return key.size() > 0 && key[0] == '$'; } - + //////////////////////////////////////////////////////////////////////////////// - + TJsonWriter::TJsonWriter( IOutputStream* output, ::NYson::EYsonType type, @@ -41,15 +41,15 @@ namespace NYT { if (Type == ::NYson::EYsonType::MapFragment) { ythrow ::NYson::TYsonException() << ("Map fragments are not supported by Json"); } - + UnderlyingJsonWriter.Reset(new NJson::TJsonWriter( output, config)); JsonWriter = UnderlyingJsonWriter.Get(); HasAttributes = false; InAttributesBalance = 0; - } - + } + void TJsonWriter::EnterNode() { if (AttributesMode == JAM_NEVER) { HasAttributes = false; @@ -65,15 +65,15 @@ namespace NYT { HasAttributes = true; } HasUnfoldedStructureStack.push_back(HasAttributes); - + if (HasAttributes) { JsonWriter->Write("$value"); HasAttributes = false; - } - + } + Depth += 1; - } - + } + void TJsonWriter::LeaveNode() { Y_ASSERT(!HasUnfoldedStructureStack.empty()); if (HasUnfoldedStructureStack.back()) { @@ -81,54 +81,54 @@ namespace NYT { JsonWriter->CloseMap(); } HasUnfoldedStructureStack.pop_back(); - + Depth -= 1; if (Depth == 0 && Type == ::NYson::EYsonType::ListFragment && InAttributesBalance == 0) { JsonWriter->Flush(); Output->Write("\n"); } - } - + } + bool TJsonWriter::IsWriteAllowed() { if (AttributesMode == JAM_NEVER) { return InAttributesBalance == 0; } return true; - } - + } + void TJsonWriter::OnStringScalar(TStringBuf value) { if (IsWriteAllowed()) { EnterNode(); WriteStringScalar(value); LeaveNode(); } - } - + } + void TJsonWriter::OnInt64Scalar(i64 value) { if (IsWriteAllowed()) { EnterNode(); JsonWriter->Write(value); LeaveNode(); } - } - + } + void TJsonWriter::OnUint64Scalar(ui64 value) { if (IsWriteAllowed()) { EnterNode(); JsonWriter->Write(value); LeaveNode(); } - } - + } + void TJsonWriter::OnDoubleScalar(double value) { if (IsWriteAllowed()) { EnterNode(); JsonWriter->Write(value); LeaveNode(); } - } - + } + void TJsonWriter::OnBooleanScalar(bool value) { if (IsWriteAllowed()) { if (BooleanFormat == SBF_STRING) { @@ -139,40 +139,40 @@ namespace NYT { LeaveNode(); } } - } - + } + void TJsonWriter::OnEntity() { if (IsWriteAllowed()) { EnterNode(); JsonWriter->WriteNull(); LeaveNode(); } - } - + } + void TJsonWriter::OnBeginList() { if (IsWriteAllowed()) { EnterNode(); JsonWriter->OpenArray(); } - } - + } + void TJsonWriter::OnListItem() { - } - + } + void TJsonWriter::OnEndList() { if (IsWriteAllowed()) { JsonWriter->CloseArray(); LeaveNode(); } } - + void TJsonWriter::OnBeginMap() { if (IsWriteAllowed()) { EnterNode(); JsonWriter->OpenMap(); } - } - + } + void TJsonWriter::OnKeyedItem(TStringBuf name) { if (IsWriteAllowed()) { if (IsSpecialJsonKey(name)) { @@ -181,15 +181,15 @@ namespace NYT { WriteStringScalar(name); } } - } - + } + void TJsonWriter::OnEndMap() { if (IsWriteAllowed()) { JsonWriter->CloseMap(); LeaveNode(); - } - } - + } + } + void TJsonWriter::OnBeginAttributes() { InAttributesBalance += 1; if (AttributesMode != JAM_NEVER) { @@ -197,24 +197,24 @@ namespace NYT { JsonWriter->Write("$attributes"); JsonWriter->OpenMap(); } - } - + } + void TJsonWriter::OnEndAttributes() { InAttributesBalance -= 1; if (AttributesMode != JAM_NEVER) { HasAttributes = true; JsonWriter->CloseMap(); } - } - + } + void TJsonWriter::WriteStringScalar(const TStringBuf& value) { JsonWriter->Write(value); - } - + } + void TJsonWriter::Flush() { JsonWriter->Flush(); } - + //////////////////////////////////////////////////////////////////////////////// -} +} diff --git a/library/cpp/yson/json/json_writer.h b/library/cpp/yson/json/json_writer.h index d84ac0de530..43e5293b0d4 100644 --- a/library/cpp/yson/json/json_writer.h +++ b/library/cpp/yson/json/json_writer.h @@ -1,31 +1,31 @@ -#pragma once - +#pragma once + #include <library/cpp/yson/public.h> #include <library/cpp/yson/consumer.h> - + #include <library/cpp/json/json_writer.h> - -#include <util/generic/vector.h> - -namespace NYT { + +#include <util/generic/vector.h> + +namespace NYT { //////////////////////////////////////////////////////////////////////////////// - + enum EJsonFormat { JF_TEXT, JF_PRETTY }; - + enum EJsonAttributesMode { JAM_NEVER, JAM_ON_DEMAND, JAM_ALWAYS }; - + enum ESerializedBoolFormat { SBF_BOOLEAN, SBF_STRING }; - + class TJsonWriter : public ::NYson::TYsonConsumerBase { public: @@ -44,26 +44,26 @@ namespace NYT { ESerializedBoolFormat booleanFormat = SBF_STRING); void Flush(); - + void OnStringScalar(TStringBuf value) override; void OnInt64Scalar(i64 value) override; void OnUint64Scalar(ui64 value) override; void OnDoubleScalar(double value) override; void OnBooleanScalar(bool value) override; - + void OnEntity() override; - + void OnBeginList() override; void OnListItem() override; void OnEndList() override; - + void OnBeginMap() override; void OnKeyedItem(TStringBuf key) override; void OnEndMap() override; - + void OnBeginAttributes() override; void OnEndAttributes() override; - + private: THolder<NJson::TJsonWriter> UnderlyingJsonWriter; NJson::TJsonWriter* JsonWriter; @@ -71,19 +71,19 @@ namespace NYT { ::NYson::EYsonType Type; EJsonAttributesMode AttributesMode; ESerializedBoolFormat BooleanFormat; - + void WriteStringScalar(const TStringBuf& value); - + void EnterNode(); void LeaveNode(); bool IsWriteAllowed(); - + TVector<bool> HasUnfoldedStructureStack; int InAttributesBalance; bool HasAttributes; int Depth; }; - + //////////////////////////////////////////////////////////////////////////////// - + } diff --git a/library/cpp/yson/lexer.cpp b/library/cpp/yson/lexer.cpp index 5eae94273bf..6864991c248 100644 --- a/library/cpp/yson/lexer.cpp +++ b/library/cpp/yson/lexer.cpp @@ -1,16 +1,16 @@ -#include "lexer.h" -#include "lexer_detail.h" -#include "token.h" - -#include <util/generic/ptr.h> - +#include "lexer.h" +#include "lexer_detail.h" +#include "token.h" + +#include <util/generic/ptr.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TStatelessLexer::TImpl { private: THolder<TStatelessYsonLexerImplBase> Impl; - + public: TImpl(bool enableLinePositionInfo = false) : Impl(enableLinePositionInfo @@ -18,26 +18,26 @@ namespace NYson { : static_cast<TStatelessYsonLexerImplBase*>(new TStatelesYsonLexerImpl<false>())) { } - + size_t GetToken(const TStringBuf& data, TToken* token) { return Impl->GetToken(data, token); } }; - + //////////////////////////////////////////////////////////////////////////////// TStatelessLexer::TStatelessLexer() : Impl(new TImpl()) - { - } - + { + } + TStatelessLexer::~TStatelessLexer() { } - + size_t TStatelessLexer::GetToken(const TStringBuf& data, TToken* token) { return Impl->GetToken(data, token); } - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/lexer.h b/library/cpp/yson/lexer.h index d9d701874d4..17cb87ec6dd 100644 --- a/library/cpp/yson/lexer.h +++ b/library/cpp/yson/lexer.h @@ -1,26 +1,26 @@ -#pragma once - -#include "public.h" -#include "token.h" - -#include <util/generic/ptr.h> - +#pragma once + +#include "public.h" +#include "token.h" + +#include <util/generic/ptr.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TStatelessLexer { public: TStatelessLexer(); - + ~TStatelessLexer(); - + size_t GetToken(const TStringBuf& data, TToken* token); - + private: class TImpl; THolder<TImpl> Impl; }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/lexer_detail.h b/library/cpp/yson/lexer_detail.h index 0bba30acdd2..5bb5dccc065 100644 --- a/library/cpp/yson/lexer_detail.h +++ b/library/cpp/yson/lexer_detail.h @@ -1,15 +1,15 @@ -#pragma once - -#include "detail.h" -#include "token.h" - +#pragma once + +#include "detail.h" +#include "token.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + namespace NDetail { /*! \internal */ //////////////////////////////////////////////////////////////////////////////// - + // EReadStartCase tree representation: // Root = xb // BinaryStringOrOtherSpecialToken = x0b @@ -29,17 +29,17 @@ namespace NYson { // Plus = 10011b // None = 10111b // Percent = 11011b - + enum EReadStartCase : unsigned { BinaryString = 0, // = 00b OtherSpecialToken = 2, // = 10b - + BinaryInt64 = 1, // = 001b BinaryDouble = 5, // = 101b BinaryFalse = 9, // = 1001b BinaryTrue = 13, // = 1101b BinaryUint64 = 17, // = 10001b - + Quote = 3, // = 00011b DigitOrMinus = 7, // = 00111b String = 11, // = 01011b @@ -48,34 +48,34 @@ namespace NYson { None = 23, // = 10111b Percent = 27 // = 11011b }; - + template <class TBlockStream, bool EnableLinePositionInfo> class TLexer : public TLexerBase<TBlockStream, EnableLinePositionInfo> { private: using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; - + static EReadStartCase GetStartState(char ch) { -#define NN EReadStartCase::None -#define BS EReadStartCase::BinaryString -#define BI EReadStartCase::BinaryInt64 -#define BD EReadStartCase::BinaryDouble -#define BF EReadStartCase::BinaryFalse -#define BT EReadStartCase::BinaryTrue -#define BU EReadStartCase::BinaryUint64 -#define SP NN // EReadStartCase::Space -#define DM EReadStartCase::DigitOrMinus -#define ST EReadStartCase::String -#define PL EReadStartCase::Plus -#define QU EReadStartCase::Quote -#define PC EReadStartCase::Percent -#define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken) - +#define NN EReadStartCase::None +#define BS EReadStartCase::BinaryString +#define BI EReadStartCase::BinaryInt64 +#define BD EReadStartCase::BinaryDouble +#define BF EReadStartCase::BinaryFalse +#define BT EReadStartCase::BinaryTrue +#define BU EReadStartCase::BinaryUint64 +#define SP NN // EReadStartCase::Space +#define DM EReadStartCase::DigitOrMinus +#define ST EReadStartCase::String +#define PL EReadStartCase::Plus +#define QU EReadStartCase::Quote +#define PC EReadStartCase::Percent +#define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken) + static const ui8 lookupTable[] = { NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - + // 32 SP, // ' ' NN, // '!' @@ -93,7 +93,7 @@ namespace NYson { DM, // '-' NN, // '.' NN, // '/' - + // 48 DM, DM, DM, DM, DM, DM, DM, DM, DM, DM, // '0' - '9' TT(Colon), // ':' @@ -102,7 +102,7 @@ namespace NYson { TT(Equals), // '=' TT(RightAngle), // '>' NN, // '?' - + // 64 NN, // '@' ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M' @@ -112,10 +112,10 @@ namespace NYson { TT(RightBracket), // ']' NN, // '^' ST, // '_' - + // 96 NN, // '`' - + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm' ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z' TT(LeftBrace), // '{' @@ -128,41 +128,41 @@ namespace NYson { NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN}; - -#undef NN -#undef BS -#undef BI -#undef BD -#undef SP -#undef DM -#undef ST -#undef PL -#undef QU -#undef TT + +#undef NN +#undef BS +#undef BI +#undef BD +#undef SP +#undef DM +#undef ST +#undef PL +#undef QU +#undef TT return static_cast<EReadStartCase>(lookupTable[static_cast<ui8>(ch)]); } - + public: TLexer(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) : TBase(blockStream, memoryLimit) { } - + void GetToken(TToken* token) { char ch1 = TBase::SkipSpaceAndGetChar(); auto state = GetStartState(ch1); auto stateBits = static_cast<unsigned>(state); - + if (ch1 == '\0') { *token = TToken::EndOfStream; return; } - + if (stateBits & 1) { // Other = x1b if (stateBits & 1 << 1) { // Other = xxx11b if (state == EReadStartCase::Quote) { @@ -174,9 +174,9 @@ namespace NYson { ReadNumeric<true>(token); } else if (state == EReadStartCase::Plus) { TBase::Advance(1); - + char ch2 = TBase::template GetChar<true>(); - + if (!isdigit(ch2)) { *token = TToken(ETokenType::Plus); } else { @@ -219,9 +219,9 @@ namespace NYson { } else { Y_FAIL("unreachable"); } - } + } } else { // BinaryStringOrOtherSpecialToken = x0b - TBase::Advance(1); + TBase::Advance(1); if (stateBits & 1 << 1) { // OtherSpecialToken = 10b Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::OtherSpecialToken)); *token = TToken(ETokenType(stateBits >> 2)); @@ -231,14 +231,14 @@ namespace NYson { TBase::ReadBinaryString(&value); *token = TToken(value); } - } - } - + } + } + template <bool AllowFinish> void ReadNumeric(TToken* token) { TStringBuf valueBuffer; ENumericResult numericResult = TBase::template ReadNumeric<AllowFinish>(&valueBuffer); - + if (numericResult == ENumericResult::Double) { try { *token = TToken(FromString<double>(valueBuffer)); @@ -258,39 +258,39 @@ namespace NYson { ythrow TYsonException() << "Error parsing uint64 literal " << valueBuffer; } } - } + } }; //////////////////////////////////////////////////////////////////////////////// /*! \endinternal */ - } - + } + class TStatelessYsonLexerImplBase { public: virtual size_t GetToken(const TStringBuf& data, TToken* token) = 0; - + virtual ~TStatelessYsonLexerImplBase() { } }; - + template <bool EnableLinePositionInfo> class TStatelesYsonLexerImpl: public TStatelessYsonLexerImplBase { private: using TLexer = NDetail::TLexer<TStringReader, EnableLinePositionInfo>; TLexer Lexer; - + public: TStatelesYsonLexerImpl() : Lexer(TStringReader(), Nothing()) { } - + size_t GetToken(const TStringBuf& data, TToken* token) override { Lexer.SetBuffer(data.begin(), data.end()); Lexer.GetToken(token); return Lexer.Begin() - data.begin(); } }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/node/node.cpp b/library/cpp/yson/node/node.cpp index b39e0707187..a533e34fa62 100644 --- a/library/cpp/yson/node/node.cpp +++ b/library/cpp/yson/node/node.cpp @@ -1,5 +1,5 @@ #include "node.h" - + #include "node_io.h" #include <library/cpp/yson/writer.h> @@ -13,15 +13,15 @@ namespace NYT { bool TNode::TNull::operator==(const TNull&) const { return true; } - -//////////////////////////////////////////////////////////////////////////////// - + +//////////////////////////////////////////////////////////////////////////////// + bool TNode::TUndefined::operator==(const TUndefined&) const { return true; } - + //////////////////////////////////////////////////////////////////////////////// - + namespace NNodeCmp { bool IsComparableType(const TNode::EType type) { @@ -94,15 +94,15 @@ bool operator>=(const TNode& lhs, const TNode& rhs) TNode::TNode() : Value_(TUndefined{}) { } - + TNode::TNode(const char* s) : Value_(TString(s)) { } - + TNode::TNode(TStringBuf s) : Value_(TString(s)) { } - + TNode::TNode(std::string_view s) : Value_(TString(s)) { } @@ -114,12 +114,12 @@ TNode::TNode(const std::string& s) TNode::TNode(TString s) : Value_(std::move(s)) { } - + TNode::TNode(int i) : Value_(static_cast<i64>(i)) { } - - + + TNode::TNode(unsigned int ui) : Value_(static_cast<ui64>(ui)) { } @@ -127,27 +127,27 @@ TNode::TNode(unsigned int ui) TNode::TNode(long i) : Value_(static_cast<i64>(i)) { } - + TNode::TNode(unsigned long ui) : Value_(static_cast<ui64>(ui)) { } - + TNode::TNode(long long i) : Value_(static_cast<i64>(i)) { } - + TNode::TNode(unsigned long long ui) : Value_(static_cast<ui64>(ui)) { } - + TNode::TNode(double d) : Value_(d) { } - + TNode::TNode(bool b) : Value_(b) { } - + TNode::TNode(TMapType map) : Value_(std::move(map)) { } @@ -161,39 +161,39 @@ TNode::TNode(const TNode& rhs) } Value_ = rhs.Value_; } - + TNode& TNode::operator=(const TNode& rhs) { if (this != &rhs) { TNode tmp = rhs; Move(std::move(tmp)); - } + } return *this; } - + TNode::TNode(TNode&& rhs) noexcept : TNode() { Move(std::move(rhs)); } - + TNode& TNode::operator=(TNode&& rhs) noexcept { if (this != &rhs) { TNode tmp = std::move(rhs); Move(std::move(tmp)); - } + } return *this; } - + TNode::~TNode() = default; - + void TNode::Clear() { ClearAttributes(); Value_ = TUndefined(); } - + bool TNode::IsString() const { return std::holds_alternative<TString>(Value_); @@ -203,37 +203,37 @@ bool TNode::IsInt64() const { return std::holds_alternative<i64>(Value_); } - + bool TNode::IsUint64() const { return std::holds_alternative<ui64>(Value_); } - + bool TNode::IsDouble() const { return std::holds_alternative<double>(Value_); } - + bool TNode::IsBool() const { return std::holds_alternative<bool>(Value_); } - + bool TNode::IsList() const { return std::holds_alternative<TListType>(Value_); } - + bool TNode::IsMap() const { return std::holds_alternative<TMapType>(Value_); } - + bool TNode::IsEntity() const { return IsNull(); } - + bool TNode::IsNull() const { return std::holds_alternative<TNull>(Value_); @@ -260,9 +260,9 @@ bool TNode::Empty() const return std::get<TMapType>(Value_).empty(); default: ythrow TTypeError() << "Empty() called for type " << GetType(); - } + } } - + size_t TNode::Size() const { switch (GetType()) { @@ -274,9 +274,9 @@ size_t TNode::Size() const return std::get<TMapType>(Value_).size(); default: ythrow TTypeError() << "Size() called for type " << GetType(); - } + } } - + TNode::EType TNode::GetType() const { return std::visit(TOverloaded{ @@ -291,61 +291,61 @@ TNode::EType TNode::GetType() const [](const TNull&) { return Null; } }, Value_); } - + const TString& TNode::AsString() const { CheckType(String); return std::get<TString>(Value_); } - + i64 TNode::AsInt64() const { CheckType(Int64); return std::get<i64>(Value_); } - + ui64 TNode::AsUint64() const { CheckType(Uint64); return std::get<ui64>(Value_); } - + double TNode::AsDouble() const { CheckType(Double); return std::get<double>(Value_); } - + bool TNode::AsBool() const { CheckType(Bool); return std::get<bool>(Value_); } - + const TNode::TListType& TNode::AsList() const { CheckType(List); return std::get<TListType>(Value_); } - + const TNode::TMapType& TNode::AsMap() const { CheckType(Map); return std::get<TMapType>(Value_); } - + TNode::TListType& TNode::AsList() { CheckType(List); return std::get<TListType>(Value_); } - + TNode::TMapType& TNode::AsMap() { CheckType(Map); return std::get<TMapType>(Value_); } - + const TString& TNode::UncheckedAsString() const noexcept { return std::get<TString>(Value_); @@ -397,7 +397,7 @@ TNode TNode::CreateList() node.Value_ = TListType{}; return node; } - + TNode TNode::CreateList(TListType list) { TNode node; @@ -411,7 +411,7 @@ TNode TNode::CreateMap() node.Value_ = TMapType{}; return node; } - + TNode TNode::CreateMap(TMapType map) { TNode node; @@ -425,19 +425,19 @@ TNode TNode::CreateEntity() node.Value_ = TNull{}; return node; } - + const TNode& TNode::operator[](size_t index) const { CheckType(List); return std::get<TListType>(Value_)[index]; } - + TNode& TNode::operator[](size_t index) { CheckType(List); return std::get<TListType>(Value_)[index]; } - + const TNode& TNode::At(size_t index) const { CheckType(List); const auto& list = std::get<TListType>(Value_); @@ -461,7 +461,7 @@ TNode& TNode::Add() & AssureList(); return std::get<TListType>(Value_).emplace_back(); } - + TNode TNode::Add() && { return std::move(Add()); @@ -473,7 +473,7 @@ TNode& TNode::Add(const TNode& node) & std::get<TListType>(Value_).emplace_back(node); return *this; } - + TNode TNode::Add(const TNode& node) && { return std::move(Add(node)); @@ -485,7 +485,7 @@ TNode& TNode::Add(TNode&& node) & std::get<TListType>(Value_).emplace_back(std::move(node)); return *this; } - + TNode TNode::Add(TNode&& node) && { return std::move(Add(std::move(node))); @@ -496,14 +496,14 @@ bool TNode::HasKey(const TStringBuf key) const CheckType(Map); return std::get<TMapType>(Value_).contains(key); } - + TNode& TNode::operator()(const TString& key, const TNode& value) & { AssureMap(); std::get<TMapType>(Value_)[key] = value; return *this; } - + TNode TNode::operator()(const TString& key, const TNode& value) && { return std::move(operator()(key, value)); @@ -515,7 +515,7 @@ TNode& TNode::operator()(const TString& key, TNode&& value) & std::get<TMapType>(Value_)[key] = std::move(value); return *this; } - + TNode TNode::operator()(const TString& key, TNode&& value) && { return std::move(operator()(key, std::move(value))); @@ -531,15 +531,15 @@ const TNode& TNode::operator[](const TStringBuf key) const return notFound; } else { return i->second; - } + } } - + TNode& TNode::operator[](const TStringBuf key) { AssureMap(); return std::get<TMapType>(Value_)[key]; } - + const TNode& TNode::At(const TStringBuf key) const { CheckType(Map); const auto& map = std::get<TMapType>(Value_); @@ -782,68 +782,68 @@ bool TNode::HasAttributes() const { return Attributes_ && !Attributes_->Empty(); } - + void TNode::ClearAttributes() { if (Attributes_) { Attributes_.Destroy(); - } + } } - + const TNode& TNode::GetAttributes() const { static TNode notFound = TNode::CreateMap(); if (!Attributes_) { return notFound; - } + } return *Attributes_; } - + TNode& TNode::Attributes() { if (!Attributes_) { CreateAttributes(); - } + } return *Attributes_; } - + void TNode::MoveWithoutAttributes(TNode&& rhs) { Value_ = std::move(rhs.Value_); rhs.Clear(); } - + void TNode::Move(TNode&& rhs) { Value_ = std::move(rhs.Value_); Attributes_ = std::move(rhs.Attributes_); } - + void TNode::CheckType(EType type) const { Y_ENSURE_EX(GetType() == type, TTypeError() << "TNode type " << type << " expected, actual type " << GetType(); ); } - + void TNode::AssureMap() { if (std::holds_alternative<TUndefined>(Value_)) { Value_ = TMapType(); } else { CheckType(Map); - } + } } - + void TNode::AssureList() { if (std::holds_alternative<TUndefined>(Value_)) { Value_ = TListType(); } else { CheckType(List); - } + } } - + void TNode::CreateAttributes() { Attributes_ = MakeHolder<TNode>(); @@ -862,54 +862,54 @@ void TNode::Load(IInputStream* in) } //////////////////////////////////////////////////////////////////////////////// - + bool operator==(const TNode& lhs, const TNode& rhs) { if (std::holds_alternative<TNode::TUndefined>(lhs.Value_) || std::holds_alternative<TNode::TUndefined>(rhs.Value_)) - { + { // TODO: should try to remove this behaviour if nobody uses it. return false; - } - + } + if (lhs.GetType() != rhs.GetType()) { - return false; - } - - if (lhs.Attributes_) { - if (rhs.Attributes_) { + return false; + } + + if (lhs.Attributes_) { + if (rhs.Attributes_) { if (*lhs.Attributes_ != *rhs.Attributes_) { return false; } - } else { - return false; - } - } else { - if (rhs.Attributes_) { - return false; - } - } - + } else { + return false; + } + } else { + if (rhs.Attributes_) { + return false; + } + } + return rhs.Value_ == lhs.Value_; -} - +} + bool operator!=(const TNode& lhs, const TNode& rhs) -{ - return !(lhs == rhs); -} - +{ + return !(lhs == rhs); +} + bool GetBool(const TNode& node) -{ - if (node.IsBool()) { - return node.AsBool(); - } else if (node.IsString()) { - return node.AsString() == "true"; - } else { +{ + if (node.IsBool()) { + return node.AsBool(); + } else if (node.IsString()) { + return node.AsString() == "true"; + } else { ythrow TNode::TTypeError() << "GetBool(): not a boolean or string type"; - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node.h b/library/cpp/yson/node/node.h index 5f90f95df07..bc7609e6da9 100644 --- a/library/cpp/yson/node/node.h +++ b/library/cpp/yson/node/node.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <util/generic/bt_exception.h> #include <util/generic/cast.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/variant.h> -#include <util/generic/vector.h> -#include <util/generic/yexception.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> #include <util/generic/ylimits.h> #include <util/string/cast.h> @@ -14,23 +14,23 @@ class IInputStream; class IOutputStream; - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TNode -{ -public: + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TNode +{ +public: class TLookupError : public TWithBackTrace<yexception> { }; - class TTypeError + class TTypeError : public TWithBackTrace<yexception> - { }; - - enum EType { + { }; + + enum EType { Undefined = 0 /*"undefined"*/, // NOTE: string representation of all node types @@ -43,20 +43,20 @@ public: List = 6 /*"list_node"*/, Map = 7 /*"map_node"*/, Null = 8 /*"null"*/, - }; - + }; + using TListType = TVector<TNode>; using TMapType = THashMap<TString, TNode>; - + private: struct TNull { bool operator==(const TNull&) const; }; - + struct TUndefined { bool operator==(const TUndefined&) const; }; - + using TValue = std::variant< bool, i64, @@ -68,9 +68,9 @@ private: TNull, TUndefined >; - + public: - + TNode(); TNode(const char* s); TNode(TStringBuf s); @@ -78,7 +78,7 @@ public: explicit TNode(const std::string& s); TNode(TString s); TNode(int i); - + //this case made speccially for prevent mess cast of EType into TNode through TNode(int) constructor //usual case of error SomeNode == TNode::Undefined <-- SomeNode indeed will be compared with TNode(0) without this method //correct way is SomeNode.GetType() == TNode::Undefined @@ -102,17 +102,17 @@ public: TNode(double d); TNode(bool b); TNode(TMapType map); - + TNode(const TNode& rhs); TNode& operator=(const TNode& rhs); - + TNode(TNode&& rhs) noexcept; TNode& operator=(TNode&& rhs) noexcept; - + ~TNode(); - + void Clear(); - + bool IsString() const; bool IsInt64() const; bool IsUint64() const; @@ -127,7 +127,7 @@ public: bool IsUndefined() const; // Returns true if TNode is neither Null, nor Undefined bool HasValue() const; - + template<typename T> bool IsOfType() const noexcept; @@ -136,9 +136,9 @@ public: bool Empty() const; size_t Size() const; - + EType GetType() const; - + const TString& AsString() const; i64 AsInt64() const; ui64 AsUint64() const; @@ -148,7 +148,7 @@ public: const TMapType& AsMap() const; TListType& AsList(); TMapType& AsMap(); - + const TString& UncheckedAsString() const noexcept; i64 UncheckedAsInt64() const noexcept; ui64 UncheckedAsUint64() const noexcept; @@ -180,31 +180,31 @@ public: static TNode CreateMap(); static TNode CreateMap(TMapType map); static TNode CreateEntity(); - + const TNode& operator[](size_t index) const; TNode& operator[](size_t index); const TNode& At(size_t index) const; TNode& At(size_t index); - + TNode& Add() &; TNode Add() &&; TNode& Add(const TNode& node) &; TNode Add(const TNode& node) &&; TNode& Add(TNode&& node) &; TNode Add(TNode&& node) &&; - + bool HasKey(const TStringBuf key) const; TNode& operator()(const TString& key, const TNode& value) &; TNode operator()(const TString& key, const TNode& value) &&; TNode& operator()(const TString& key, TNode&& value) &; TNode operator()(const TString& key, TNode&& value) &&; - + const TNode& operator[](const TStringBuf key) const; TNode& operator[](const TStringBuf key); const TNode& At(const TStringBuf key) const; TNode& At(const TStringBuf key); - + // map getters // works the same way like simple getters const TString& ChildAsString(const TStringBuf key) const; @@ -254,42 +254,42 @@ public: T& ChildAs(size_t index); - // attributes + // attributes bool HasAttributes() const; void ClearAttributes(); const TNode& GetAttributes() const; TNode& Attributes(); - + void MoveWithoutAttributes(TNode&& rhs); - + // Serialize TNode using binary yson format. // Methods for ysaveload. void Save(IOutputStream* output) const; void Load(IInputStream* input); -private: +private: void Move(TNode&& rhs); - + void CheckType(EType type) const; - + void AssureMap(); void AssureList(); - + void CreateAttributes(); - -private: + +private: TValue Value_; THolder<TNode> Attributes_; - - friend bool operator==(const TNode& lhs, const TNode& rhs); - friend bool operator!=(const TNode& lhs, const TNode& rhs); -}; - + + friend bool operator==(const TNode& lhs, const TNode& rhs); + friend bool operator!=(const TNode& lhs, const TNode& rhs); +}; + bool operator==(const TNode& lhs, const TNode& rhs); bool operator!=(const TNode& lhs, const TNode& rhs); - + bool GetBool(const TNode& node); - + inline bool TNode::IsArithmetic() const { return IsInt64() || IsUint64() || IsDouble() || IsBool(); } @@ -508,8 +508,8 @@ inline const T& TNode::As() const { return std::get<T>(Value_); } -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + namespace NNodeCmp { bool operator<(const TNode& lhs, const TNode& rhs); bool operator<=(const TNode& lhs, const TNode& rhs); @@ -520,4 +520,4 @@ namespace NNodeCmp { //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT +} // namespace NYT diff --git a/library/cpp/yson/node/node_builder.cpp b/library/cpp/yson/node/node_builder.cpp index b4431bc77af..b21bc4502e9 100644 --- a/library/cpp/yson/node/node_builder.cpp +++ b/library/cpp/yson/node/node_builder.cpp @@ -1,96 +1,96 @@ -#include "node_builder.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -TNodeBuilder::TNodeBuilder(TNode* node) -{ - Stack_.push(node); -} - +#include "node_builder.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +TNodeBuilder::TNodeBuilder(TNode* node) +{ + Stack_.push(node); +} + void TNodeBuilder::OnStringScalar(TStringBuf value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnInt64Scalar(i64 value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnUint64Scalar(ui64 value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnDoubleScalar(double value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnBooleanScalar(bool value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnEntity() -{ - AddNode(TNode::CreateEntity(), true); -} - -void TNodeBuilder::OnBeginList() -{ - AddNode(TNode::CreateList(), false); -} - -void TNodeBuilder::OnListItem() -{ - Stack_.push(&Stack_.top()->Add()); -} - -void TNodeBuilder::OnEndList() -{ - Stack_.pop(); -} - -void TNodeBuilder::OnBeginMap() -{ - AddNode(TNode::CreateMap(), false); -} - +{ + AddNode(value, true); +} + +void TNodeBuilder::OnInt64Scalar(i64 value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnUint64Scalar(ui64 value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnDoubleScalar(double value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnBooleanScalar(bool value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnEntity() +{ + AddNode(TNode::CreateEntity(), true); +} + +void TNodeBuilder::OnBeginList() +{ + AddNode(TNode::CreateList(), false); +} + +void TNodeBuilder::OnListItem() +{ + Stack_.push(&Stack_.top()->Add()); +} + +void TNodeBuilder::OnEndList() +{ + Stack_.pop(); +} + +void TNodeBuilder::OnBeginMap() +{ + AddNode(TNode::CreateMap(), false); +} + void TNodeBuilder::OnKeyedItem(TStringBuf key) -{ +{ Stack_.push(&(*Stack_.top())[TString(key)]); -} - -void TNodeBuilder::OnEndMap() -{ - Stack_.pop(); -} - -void TNodeBuilder::OnBeginAttributes() -{ - Stack_.push(&Stack_.top()->Attributes()); -} - -void TNodeBuilder::OnEndAttributes() -{ - Stack_.pop(); -} - +} + +void TNodeBuilder::OnEndMap() +{ + Stack_.pop(); +} + +void TNodeBuilder::OnBeginAttributes() +{ + Stack_.push(&Stack_.top()->Attributes()); +} + +void TNodeBuilder::OnEndAttributes() +{ + Stack_.pop(); +} + void TNodeBuilder::OnNode(TNode node) { AddNode(std::move(node), true); } -void TNodeBuilder::AddNode(TNode value, bool pop) -{ +void TNodeBuilder::AddNode(TNode value, bool pop) +{ Stack_.top()->MoveWithoutAttributes(std::move(value)); - if (pop) - Stack_.pop(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + if (pop) + Stack_.pop(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_builder.h b/library/cpp/yson/node/node_builder.h index 69800016e09..35bb256a5e4 100644 --- a/library/cpp/yson/node/node_builder.h +++ b/library/cpp/yson/node/node_builder.h @@ -1,23 +1,23 @@ -#pragma once - +#pragma once + #include "node.h" - + #include <library/cpp/json/json_reader.h> #include <library/cpp/yson/consumer.h> -#include <util/generic/stack.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TNodeBuilder +#include <util/generic/stack.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TNodeBuilder : public ::NYson::TYsonConsumerBase -{ -public: - TNodeBuilder(TNode* node); - +{ +public: + TNodeBuilder(TNode* node); + void OnStringScalar(TStringBuf) override; void OnInt64Scalar(i64) override; void OnUint64Scalar(ui64) override; @@ -33,14 +33,14 @@ public: void OnBeginAttributes() override; void OnEndAttributes() override; void OnNode(TNode node); - -private: + +private: TStack<TNode*> Stack_; - -private: - inline void AddNode(TNode node, bool pop); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + +private: + inline void AddNode(TNode node, bool pop); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_io.cpp b/library/cpp/yson/node/node_io.cpp index 294a7f7217f..ea53e4d8b7f 100644 --- a/library/cpp/yson/node/node_io.cpp +++ b/library/cpp/yson/node/node_io.cpp @@ -1,8 +1,8 @@ #include "node_io.h" - -#include "node_builder.h" -#include "node_visitor.h" - + +#include "node_builder.h" +#include "node_visitor.h" + #include <library/cpp/yson/json/json_writer.h> #include <library/cpp/yson/parser.h> #include <library/cpp/yson/writer.h> @@ -11,13 +11,13 @@ #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_value.h> -#include <util/stream/input.h> -#include <util/stream/output.h> -#include <util/stream/str.h> +#include <util/stream/input.h> +#include <util/stream/output.h> +#include <util/stream/str.h> #include <util/stream/mem.h> - -namespace NYT { - + +namespace NYT { + static void WalkJsonTree(const NJson::TJsonValue& jsonValue, NJson::TJsonCallbacks* callbacks) { using namespace NJson; @@ -68,20 +68,20 @@ static void WalkJsonTree(const NJson::TJsonValue& jsonValue, NJson::TJsonCallbac static TNode CreateEmptyNodeByType(::NYson::EYsonType type) { - TNode result; - switch (type) { + TNode result; + switch (type) { case ::NYson::EYsonType::ListFragment: - result = TNode::CreateList(); - break; + result = TNode::CreateList(); + break; case ::NYson::EYsonType::MapFragment: - result = TNode::CreateMap(); - break; - default: - break; - } + result = TNode::CreateMap(); + break; + default: + break; + } return result; } - + TNode NodeFromYsonString(const TStringBuf input, ::NYson::EYsonType type) { TMemoryInput stream(input); @@ -106,19 +106,19 @@ TNode NodeFromYsonStream(IInputStream* input, ::NYson::EYsonType type) { TNode result = CreateEmptyNodeByType(type); - TNodeBuilder builder(&result); + TNodeBuilder builder(&result); ::NYson::TYsonParser parser(&builder, input, type); - parser.Parse(); - return result; -} - + parser.Parse(); + return result; +} + void NodeToYsonStream(const TNode& node, IOutputStream* output, NYson::EYsonFormat format) -{ +{ ::NYson::TYsonWriter writer(output, format); - TNodeVisitor visitor(&writer); - visitor.Visit(node); -} - + TNodeVisitor visitor(&writer); + visitor.Visit(node); +} + void NodeToCanonicalYsonStream(const TNode& node, IOutputStream* output, NYson::EYsonFormat format) { ::NYson::TYsonWriter writer(output, format); @@ -149,6 +149,6 @@ TNode NodeFromJsonValue(const NJson::TJsonValue& input) return result; } -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_io.h b/library/cpp/yson/node/node_io.h index 2ad23b658f2..5b632433175 100644 --- a/library/cpp/yson/node/node_io.h +++ b/library/cpp/yson/node/node_io.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include "node.h" #include <library/cpp/yson/public.h> - + namespace NJson { class TJsonValue; } // namespace NJson -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + // Parse TNode from string in YSON format TNode NodeFromYsonString(const TStringBuf input, ::NYson::EYsonType type = ::NYson::EYsonType::Node); // Serialize TNode to string in one of YSON formats with random order of maps' keys (don't use in tests) TString NodeToYsonString(const TNode& node, ::NYson::EYsonFormat format = ::NYson::EYsonFormat::Text); - + // Same as the latter, but maps' keys are sorted lexicographically (to be used in tests) TString NodeToCanonicalYsonString(const TNode& node, ::NYson::EYsonFormat format = ::NYson::EYsonFormat::Text); @@ -35,6 +35,6 @@ TNode NodeFromJsonString(const TStringBuf input); // Convert TJsonValue to TNode TNode NodeFromJsonValue(const NJson::TJsonValue& input); -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_visitor.cpp b/library/cpp/yson/node/node_visitor.cpp index 899fbfa02aa..824d9348677 100644 --- a/library/cpp/yson/node/node_visitor.cpp +++ b/library/cpp/yson/node/node_visitor.cpp @@ -1,12 +1,12 @@ -#include "node_visitor.h" - +#include "node_visitor.h" + #include <util/generic/algorithm.h> #include <util/string/printf.h> -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + namespace { template <typename Fun> @@ -31,122 +31,122 @@ void Iterate(const TNode::TMapType& nodeMap, bool sortByKey, Fun action) //////////////////////////////////////////////////////////////////////////////// TNodeVisitor::TNodeVisitor(NYson::IYsonConsumer* consumer, bool sortMapKeys) - : Consumer_(consumer) + : Consumer_(consumer) , SortMapKeys_(sortMapKeys) -{ } - -void TNodeVisitor::Visit(const TNode& node) -{ - VisitAny(node); -} - -void TNodeVisitor::VisitAny(const TNode& node) -{ - if (node.HasAttributes()) { - Consumer_->OnBeginAttributes(); +{ } + +void TNodeVisitor::Visit(const TNode& node) +{ + VisitAny(node); +} + +void TNodeVisitor::VisitAny(const TNode& node) +{ + if (node.HasAttributes()) { + Consumer_->OnBeginAttributes(); Iterate(node.GetAttributes().AsMap(), SortMapKeys_, [&](const std::pair<TString, TNode>& item) { - Consumer_->OnKeyedItem(item.first); + Consumer_->OnKeyedItem(item.first); if (item.second.IsUndefined()) { ythrow TNode::TTypeError() << "unable to visit attribute value of type " << TNode::EType::Undefined << "; attribute name: `" << item.first << '\'' ; } - VisitAny(item.second); + VisitAny(item.second); }); - Consumer_->OnEndAttributes(); - } - - switch (node.GetType()) { + Consumer_->OnEndAttributes(); + } + + switch (node.GetType()) { case TNode::String: - VisitString(node); - break; + VisitString(node); + break; case TNode::Int64: - VisitInt64(node); - break; + VisitInt64(node); + break; case TNode::Uint64: - VisitUint64(node); - break; + VisitUint64(node); + break; case TNode::Double: - VisitDouble(node); - break; + VisitDouble(node); + break; case TNode::Bool: - VisitBool(node); - break; + VisitBool(node); + break; case TNode::List: VisitList(node.AsList()); - break; + break; case TNode::Map: VisitMap(node.AsMap()); - break; + break; case TNode::Null: - VisitEntity(); - break; + VisitEntity(); + break; case TNode::Undefined: ythrow TNode::TTypeError() << "unable to visit TNode of type " << node.GetType(); default: Y_FAIL("Unexpected type: %d", node.GetType()); - } -} - -void TNodeVisitor::VisitString(const TNode& node) -{ - Consumer_->OnStringScalar(node.AsString()); -} - -void TNodeVisitor::VisitInt64(const TNode& node) -{ - Consumer_->OnInt64Scalar(node.AsInt64()); -} - -void TNodeVisitor::VisitUint64(const TNode& node) -{ - Consumer_->OnUint64Scalar(node.AsUint64()); -} - -void TNodeVisitor::VisitDouble(const TNode& node) -{ - Consumer_->OnDoubleScalar(node.AsDouble()); -} - -void TNodeVisitor::VisitBool(const TNode& node) -{ - Consumer_->OnBooleanScalar(node.AsBool()); -} - + } +} + +void TNodeVisitor::VisitString(const TNode& node) +{ + Consumer_->OnStringScalar(node.AsString()); +} + +void TNodeVisitor::VisitInt64(const TNode& node) +{ + Consumer_->OnInt64Scalar(node.AsInt64()); +} + +void TNodeVisitor::VisitUint64(const TNode& node) +{ + Consumer_->OnUint64Scalar(node.AsUint64()); +} + +void TNodeVisitor::VisitDouble(const TNode& node) +{ + Consumer_->OnDoubleScalar(node.AsDouble()); +} + +void TNodeVisitor::VisitBool(const TNode& node) +{ + Consumer_->OnBooleanScalar(node.AsBool()); +} + void TNodeVisitor::VisitList(const TNode::TListType& nodeList) -{ - Consumer_->OnBeginList(); +{ + Consumer_->OnBeginList(); size_t index = 0; for (const auto& item : nodeList) { - Consumer_->OnListItem(); + Consumer_->OnListItem(); if (item.IsUndefined()) { ythrow TNode::TTypeError() << "unable to visit list node child of type " << TNode::EType::Undefined << "; list index: " << index; } - VisitAny(item); + VisitAny(item); ++index; - } - Consumer_->OnEndList(); -} - + } + Consumer_->OnEndList(); +} + void TNodeVisitor::VisitMap(const TNode::TMapType& nodeMap) -{ - Consumer_->OnBeginMap(); +{ + Consumer_->OnBeginMap(); Iterate(nodeMap, SortMapKeys_, [&](const std::pair<TString, TNode>& item) { - Consumer_->OnKeyedItem(item.first); + Consumer_->OnKeyedItem(item.first); if (item.second.IsUndefined()) { ythrow TNode::TTypeError() << "unable to visit map node child of type " << TNode::EType::Undefined << "; map key: `" << item.first << '\'' ; } - VisitAny(item.second); + VisitAny(item.second); }); - Consumer_->OnEndMap(); -} - -void TNodeVisitor::VisitEntity() -{ - Consumer_->OnEntity(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + Consumer_->OnEndMap(); +} + +void TNodeVisitor::VisitEntity() +{ + Consumer_->OnEntity(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_visitor.h b/library/cpp/yson/node/node_visitor.h index db258323092..d4eb8e56675 100644 --- a/library/cpp/yson/node/node_visitor.h +++ b/library/cpp/yson/node/node_visitor.h @@ -1,37 +1,37 @@ -#pragma once - +#pragma once + #include "node.h" #include <library/cpp/yson/consumer.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TNodeVisitor -{ -public: + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TNodeVisitor +{ +public: TNodeVisitor(NYson::IYsonConsumer* consumer, bool sortMapKeys = false); - - void Visit(const TNode& node); + + void Visit(const TNode& node); void VisitMap(const TNode::TMapType& nodeMap); void VisitList(const TNode::TListType& nodeMap); - -private: + +private: NYson::IYsonConsumer* Consumer_; bool SortMapKeys_; - -private: - void VisitAny(const TNode& node); - - void VisitString(const TNode& node); - void VisitInt64(const TNode& node); - void VisitUint64(const TNode& node); - void VisitDouble(const TNode& node); - void VisitBool(const TNode& node); - void VisitEntity(); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + +private: + void VisitAny(const TNode& node); + + void VisitString(const TNode& node); + void VisitInt64(const TNode& node); + void VisitUint64(const TNode& node); + void VisitDouble(const TNode& node); + void VisitBool(const TNode& node); + void VisitEntity(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/parser.cpp b/library/cpp/yson/parser.cpp index 783f9b90479..3af222ed4e7 100644 --- a/library/cpp/yson/parser.cpp +++ b/library/cpp/yson/parser.cpp @@ -1,14 +1,14 @@ -#include "parser.h" -#include "consumer.h" -#include "format.h" -#include "parser_detail.h" - -#include <util/stream/input.h> -#include <util/generic/buffer.h> - +#include "parser.h" +#include "consumer.h" +#include "format.h" +#include "parser_detail.h" + +#include <util/stream/input.h> +#include <util/generic/buffer.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TYsonParser::TImpl { public: TImpl( @@ -24,7 +24,7 @@ namespace NYson { , MemoryLimit_(memoryLimit) { } - + void Parse() { TBuffer buffer(64 << 10); ParseYsonStreamImpl<NYT::NYson::IYsonConsumer, TStreamReader>( @@ -48,26 +48,26 @@ namespace NYson { TYsonParser::TYsonParser( NYT::NYson::IYsonConsumer* consumer, IInputStream* stream, - EYsonType type, - bool enableLinePositionInfo, + EYsonType type, + bool enableLinePositionInfo, TMaybe<ui64> memoryLimit) : Impl(new TImpl(consumer, stream, type, enableLinePositionInfo, memoryLimit)) - { - } - + { + } + TYsonParser::~TYsonParser() { } - + void TYsonParser::Parse() { Impl->Parse(); } - + //////////////////////////////////////////////////////////////////////////////// - + class TStatelessYsonParser::TImpl { private: THolder<TStatelessYsonParserImplBase> Impl; - + public: TImpl( NYT::NYson::IYsonConsumer* consumer, @@ -79,31 +79,31 @@ namespace NYson { : static_cast<TStatelessYsonParserImplBase*>(new TStatelessYsonParserImpl<NYT::NYson::IYsonConsumer, false>(consumer, memoryLimit))) { } - + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) { Impl->Parse(data, type); } }; - + //////////////////////////////////////////////////////////////////////////////// - + TStatelessYsonParser::TStatelessYsonParser( NYT::NYson::IYsonConsumer* consumer, - bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit) + bool enableLinePositionInfo, + TMaybe<ui64> memoryLimit) : Impl(new TImpl(consumer, enableLinePositionInfo, memoryLimit)) { } - + TStatelessYsonParser::~TStatelessYsonParser() { } void TStatelessYsonParser::Parse(const TStringBuf& data, EYsonType type) { - Impl->Parse(data, type); - } - + Impl->Parse(data, type); + } + //////////////////////////////////////////////////////////////////////////////// - + void ParseYsonStringBuffer( const TStringBuf& buffer, NYT::NYson::IYsonConsumer* consumer, @@ -117,9 +117,9 @@ namespace NYson { enableLinePositionInfo, memoryLimit); } - + //////////////////////////////////////////////////////////////////////////////// - + class TYsonListParser::TImpl { public: TImpl( @@ -135,7 +135,7 @@ namespace NYson { , Reader_(Stream_, Buffer_.Data(), Buffer_.Capacity()) { } - + bool Parse() { if (!Impl_) { Impl_.Reset( @@ -145,7 +145,7 @@ namespace NYson { } return Impl_->Parse(); } - + private: NYT::NYson::IYsonConsumer* Consumer_; IInputStream* Stream_; @@ -155,9 +155,9 @@ namespace NYson { TStreamReader Reader_; THolder<TYsonListParserImplBase> Impl_; }; - + //////////////////////////////////////////////////////////////////////////////// - + TYsonListParser::TYsonListParser( NYT::NYson::IYsonConsumer* consumer, IInputStream* stream, diff --git a/library/cpp/yson/parser.h b/library/cpp/yson/parser.h index dce35a8cd40..715341d5b57 100644 --- a/library/cpp/yson/parser.h +++ b/library/cpp/yson/parser.h @@ -1,19 +1,19 @@ -#pragma once - -#include "public.h" - -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> - +#pragma once + +#include "public.h" + +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> + class IInputStream; - + namespace NYT::NYson { struct IYsonConsumer; } // namespace NYT::NYson namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TYsonParser { public: TYsonParser( @@ -22,36 +22,36 @@ namespace NYson { EYsonType type = ::NYson::EYsonType::Node, bool enableLinePositionInfo = false, TMaybe<ui64> memoryLimit = Nothing()); - + ~TYsonParser(); - + void Parse(); - + private: class TImpl; THolder<TImpl> Impl; }; - + //////////////////////////////////////////////////////////////////////////////// - + class TStatelessYsonParser { public: TStatelessYsonParser( NYT::NYson::IYsonConsumer* consumer, bool enableLinePositionInfo = false, TMaybe<ui64> memoryLimit = Nothing()); - + ~TStatelessYsonParser(); - + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node); - + private: class TImpl; THolder<TImpl> Impl; }; - + //////////////////////////////////////////////////////////////////////////////// - + class TYsonListParser { public: TYsonListParser( @@ -59,7 +59,7 @@ namespace NYson { IInputStream* stream, bool enableLinePositionInfo = false, TMaybe<ui64> memoryLimit = Nothing()); - + ~TYsonListParser(); bool Parse(); // Returns false, if there is no more list items @@ -79,5 +79,5 @@ namespace NYson { TMaybe<ui64> memoryLimit = Nothing()); //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/parser_detail.h b/library/cpp/yson/parser_detail.h index 44223caf125..b6eee05664a 100644 --- a/library/cpp/yson/parser_detail.h +++ b/library/cpp/yson/parser_detail.h @@ -1,43 +1,43 @@ -#pragma once - -#include "detail.h" - +#pragma once + +#include "detail.h" + namespace NYson { namespace NDetail { //////////////////////////////////////////////////////////////////////////////// - + template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> class TParser : public TLexerBase<TBlockStream, EnableLinePositionInfo> { private: using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; TConsumer* Consumer; - + public: TParser(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) : TBase(blockStream, memoryLimit) , Consumer(consumer) { } - + void DoParse(EYsonType ysonType) { switch (ysonType) { case ::NYson::EYsonType::Node: ParseNode<true>(); break; - + case ::NYson::EYsonType::ListFragment: ParseListFragment<true>(EndSymbol); break; - + case ::NYson::EYsonType::MapFragment: ParseMapFragment<true>(EndSymbol); break; - + default: Y_FAIL("unreachable"); } - + while (!(TBase::IsFinished() && TBase::IsEmpty())) { if (TBase::template SkipSpaceAndGetChar<true>() != EndSymbol) { ythrow TYsonException() << "Stray '" << (*TBase::Begin()) << "' found"; @@ -46,12 +46,12 @@ namespace NYson { } } } - + bool DoParseListFragment(bool first) { bool ret = first ? first : ParseListSeparator<true>(EndSymbol); return ret && ParseListItem<true>(EndSymbol); - } - + } + void ParseAttributes() { Consumer->OnBeginAttributes(); ParseMapFragment(EndAttributesSymbol); @@ -65,19 +65,19 @@ namespace NYson { TBase::SkipCharToken(EndMapSymbol); Consumer->OnEndMap(); } - + void ParseList() { Consumer->OnBeginList(); ParseListFragment(EndListSymbol); TBase::SkipCharToken(EndListSymbol); Consumer->OnEndList(); } - + template <bool AllowFinish> void ParseNode() { return ParseNode<AllowFinish>(TBase::SkipSpaceAndGetChar()); } - + template <bool AllowFinish> void ParseNode(char ch) { if (ch == BeginAttributesSymbol) { @@ -85,18 +85,18 @@ namespace NYson { ParseAttributes(); ch = TBase::SkipSpaceAndGetChar(); } - + switch (ch) { case BeginMapSymbol: TBase::Advance(1); ParseMap(); break; - + case BeginListSymbol: TBase::Advance(1); ParseList(); break; - + case '"': { TBase::Advance(1); TStringBuf value; @@ -146,7 +146,7 @@ namespace NYson { TBase::Advance(1); Consumer->OnEntity(); break; - + default: { if (isdigit((unsigned char)ch) || ch == '-' || ch == '+') { // case of '+' is handled in AfterPlus state ReadNumeric<AllowFinish>(); @@ -167,11 +167,11 @@ namespace NYson { } } } - } + } void ParseKey() { return ParseKey(TBase::SkipSpaceAndGetChar()); - } + } void ParseKey(char ch) { switch (ch) { @@ -199,8 +199,8 @@ namespace NYson { } } } - } - + } + template <bool AllowFinish> void ParseMapFragment(char endSymbol) { char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); @@ -221,13 +221,13 @@ namespace NYson { ythrow TYsonException() << "Expected '" << KeyedItemSeparatorSymbol << "' or '" << endSymbol << "' but '" << ch << "' found"; } - } - } - + } + } + void ParseMapFragment(char endSymbol) { ParseMapFragment<false>(endSymbol); } - + template <bool AllowFinish> bool ParseListItem(char endSymbol) { char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); @@ -237,7 +237,7 @@ namespace NYson { return true; } return false; - } + } template <bool AllowFinish> bool ParseListSeparator(char endSymbol) { @@ -250,17 +250,17 @@ namespace NYson { << "' or '" << endSymbol << "' but '" << ch << "' found"; } return false; - } + } template <bool AllowFinish> void ParseListFragment(char endSymbol) { while (ParseListItem<AllowFinish>(endSymbol) && ParseListSeparator<AllowFinish>(endSymbol)) { - } - } - + } + } + void ParseListFragment(char endSymbol) { ParseListFragment<false>(endSymbol); - } + } template <bool AllowFinish> void ReadNumeric() { @@ -295,13 +295,13 @@ namespace NYson { } Consumer->OnUint64Scalar(value); } - } + } }; - + //////////////////////////////////////////////////////////////////////////////// - - } - + + } + template <class TConsumer, class TBlockStream> void ParseYsonStreamImpl( const TBlockStream& blockStream, @@ -317,9 +317,9 @@ namespace NYson { using TImpl = NDetail::TParser<TConsumer, TBlockStream, false>; TImpl impl(blockStream, consumer, memoryLimit); impl.DoParse(parsingMode); - } - } - + } + } + class TStatelessYsonParserImplBase { public: virtual void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) = 0; @@ -334,27 +334,27 @@ namespace NYson { private: using TParser = NDetail::TParser<TConsumer, TStringReader, EnableLinePositionInfo>; TParser Parser; - + public: TStatelessYsonParserImpl(TConsumer* consumer, TMaybe<ui64> memoryLimit) : Parser(TStringReader(), consumer, memoryLimit) { } - + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) override { Parser.SetBuffer(data.begin(), data.end()); Parser.DoParse(type); - } + } }; - + class TYsonListParserImplBase { public: virtual bool Parse() = 0; - + virtual ~TYsonListParserImplBase() { } }; - + template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> class TYsonListParserImpl : public TYsonListParserImplBase { @@ -362,20 +362,20 @@ namespace NYson { using TParser = NDetail::TParser<TConsumer, TBlockStream, EnableLinePositionInfo>; TParser Parser; bool First = true; - + public: TYsonListParserImpl(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) : Parser(blockStream, consumer, memoryLimit) { } - + bool Parse() override { bool ret = Parser.DoParseListFragment(First); First = false; return ret; } }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/public.h b/library/cpp/yson/public.h index 1ed793592ba..217fff9c8bc 100644 --- a/library/cpp/yson/public.h +++ b/library/cpp/yson/public.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <library/cpp/yt/misc/enum.h> -#include <util/generic/yexception.h> - +#include <util/generic/yexception.h> + #include <library/cpp/yt/yson_string/public.h> #include <library/cpp/yt/yson/public.h> namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + using NYT::NYson::EYsonFormat; using NYT::NYson::EYsonType; - + class TYsonStringBuf; struct TYsonConsumerBase; - + class TYsonWriter; class TYsonParser; class TStatelessYsonParser; @@ -24,7 +24,7 @@ namespace NYson { class TYsonException : public yexception {}; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/token.cpp b/library/cpp/yson/token.cpp index c8584c8c2ea..ae9fd741b80 100644 --- a/library/cpp/yson/token.cpp +++ b/library/cpp/yson/token.cpp @@ -1,11 +1,11 @@ -#include "token.h" - -#include <util/string/vector.h> -#include <util/string/printf.h> - +#include "token.h" + +#include <util/string/vector.h> +#include <util/string/printf.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + ETokenType CharToTokenType(char ch) { switch (ch) { case ';': @@ -40,7 +40,7 @@ namespace NYson { return ETokenType::EndOfStream; } } - + char TokenTypeToChar(ETokenType type) { switch (type) { case ETokenType::Semicolon: @@ -74,16 +74,16 @@ namespace NYson { default: Y_FAIL("unreachable"); } - } - + } + TString TokenTypeToString(ETokenType type) { return TString(1, TokenTypeToChar(type)); - } - + } + //////////////////////////////////////////////////////////////////////////////// - + const TToken TToken::EndOfStream; - + TToken::TToken() : Type_(ETokenType::EndOfStream) , Int64Value(0) @@ -92,7 +92,7 @@ namespace NYson { , BooleanValue(false) { } - + TToken::TToken(ETokenType type) : Type_(type) , Int64Value(0) @@ -111,7 +111,7 @@ namespace NYson { break; } } - + TToken::TToken(const TStringBuf& stringValue) : Type_(ETokenType::String) , StringValue(stringValue) @@ -120,8 +120,8 @@ namespace NYson { , DoubleValue(0.0) , BooleanValue(false) { - } - + } + TToken::TToken(i64 int64Value) : Type_(ETokenType::Int64) , Int64Value(int64Value) @@ -129,7 +129,7 @@ namespace NYson { , DoubleValue(0.0) { } - + TToken::TToken(ui64 uint64Value) : Type_(ETokenType::Uint64) , Int64Value(0) @@ -138,7 +138,7 @@ namespace NYson { , BooleanValue(false) { } - + TToken::TToken(double doubleValue) : Type_(ETokenType::Double) , Int64Value(0) @@ -147,7 +147,7 @@ namespace NYson { , BooleanValue(false) { } - + TToken::TToken(bool booleanValue) : Type_(ETokenType::Boolean) , Int64Value(0) @@ -155,36 +155,36 @@ namespace NYson { , BooleanValue(booleanValue) { } - + bool TToken::IsEmpty() const { return Type_ == ETokenType::EndOfStream; } - + const TStringBuf& TToken::GetStringValue() const { CheckType(ETokenType::String); return StringValue; } - + i64 TToken::GetInt64Value() const { CheckType(ETokenType::Int64); return Int64Value; } - + ui64 TToken::GetUint64Value() const { CheckType(ETokenType::Uint64); return Uint64Value; } - + double TToken::GetDoubleValue() const { CheckType(ETokenType::Double); return DoubleValue; } - + bool TToken::GetBooleanValue() const { CheckType(ETokenType::Boolean); return BooleanValue; } - + void TToken::CheckType(ETokenType expectedType) const { if (Type_ != expectedType) { if (Type_ == ETokenType::EndOfStream) { @@ -194,9 +194,9 @@ namespace NYson { << "', Type: " << TokenTypeToString(Type_) << ", ExpectedType: " << TokenTypeToString(expectedType) << ")"; } - } - } - + } + } + void TToken::Reset() { Type_ = ETokenType::EndOfStream; Int64Value = 0; @@ -205,32 +205,32 @@ namespace NYson { StringValue = TStringBuf(); BooleanValue = false; } - + TString ToString(const TToken& token) { switch (token.GetType()) { case ETokenType::EndOfStream: return TString(); - + case ETokenType::String: return TString(token.GetStringValue()); - + case ETokenType::Int64: return ::ToString(token.GetInt64Value()); - + case ETokenType::Uint64: return ::ToString(token.GetUint64Value()); - + case ETokenType::Double: return ::ToString(token.GetDoubleValue()); - + case ETokenType::Boolean: return token.GetBooleanValue() ? "true" : "false"; - + default: return TokenTypeToString(token.GetType()); } - } - + } + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/token.h b/library/cpp/yson/token.h index 7283e569504..ee455a29ca6 100644 --- a/library/cpp/yson/token.h +++ b/library/cpp/yson/token.h @@ -1,21 +1,21 @@ -#pragma once - -#include "public.h" - -#include <util/generic/strbuf.h> - +#pragma once + +#include "public.h" + +#include <util/generic/strbuf.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + enum ETokenType { EndOfStream, - + String, Int64, Uint64, Double, Boolean, - + // Special values: // YSON Semicolon, // ; @@ -27,7 +27,7 @@ namespace NYson { RightBrace, // } LeftAngle, // < RightAngle, // > - + // Table ranges LeftParenthesis, // ( RightParenthesis, // ) @@ -35,23 +35,23 @@ namespace NYson { Colon, // : Comma, // , }; - + //////////////////////////////////////////////////////////////////////////////// - + ETokenType CharToTokenType(char ch); char TokenTypeToChar(ETokenType type); TString TokenTypeToString(ETokenType type); - + //////////////////////////////////////////////////////////////////////////////// - + class TLexerImpl; - + //////////////////////////////////////////////////////////////////////////////// - + class TToken { public: static const TToken EndOfStream; - + TToken(); TToken(ETokenType type); explicit TToken(const TStringBuf& stringValue); @@ -59,35 +59,35 @@ namespace NYson { explicit TToken(ui64 int64Value); explicit TToken(double doubleValue); explicit TToken(bool booleanValue); - + ETokenType GetType() const { return Type_; } - + bool IsEmpty() const; const TStringBuf& GetStringValue() const; i64 GetInt64Value() const; ui64 GetUint64Value() const; double GetDoubleValue() const; bool GetBooleanValue() const; - + void CheckType(ETokenType expectedType) const; void Reset(); - + private: friend class TLexerImpl; - + ETokenType Type_; - + TStringBuf StringValue; i64 Int64Value; ui64 Uint64Value; double DoubleValue; bool BooleanValue; }; - + TString ToString(const TToken& token); - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/tokenizer.cpp b/library/cpp/yson/tokenizer.cpp index 06760170d48..595700e62f7 100644 --- a/library/cpp/yson/tokenizer.cpp +++ b/library/cpp/yson/tokenizer.cpp @@ -1,37 +1,37 @@ -#include "tokenizer.h" - +#include "tokenizer.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + TTokenizer::TTokenizer(const TStringBuf& input) : Input(input) , Parsed(0) { } - + bool TTokenizer::ParseNext() { Input = Input.Tail(Parsed); Token.Reset(); Parsed = Lexer.GetToken(Input, &Token); return !CurrentToken().IsEmpty(); } - + const TToken& TTokenizer::CurrentToken() const { return Token; } - + ETokenType TTokenizer::GetCurrentType() const { return CurrentToken().GetType(); } - + TStringBuf TTokenizer::GetCurrentSuffix() const { return Input.Tail(Parsed); } - + const TStringBuf& TTokenizer::CurrentInput() const { return Input; } - + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/tokenizer.h b/library/cpp/yson/tokenizer.h index 0576aace95c..25e42ab9fc3 100644 --- a/library/cpp/yson/tokenizer.h +++ b/library/cpp/yson/tokenizer.h @@ -1,28 +1,28 @@ -#pragma once - -#include "public.h" -#include "lexer.h" - +#pragma once + +#include "public.h" +#include "lexer.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TTokenizer { public: explicit TTokenizer(const TStringBuf& input); - + bool ParseNext(); const TToken& CurrentToken() const; ETokenType GetCurrentType() const; TStringBuf GetCurrentSuffix() const; const TStringBuf& CurrentInput() const; - + private: TStringBuf Input; TToken Token; TStatelessLexer Lexer; size_t Parsed; }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/varint.cpp b/library/cpp/yson/varint.cpp index d538ee3cffa..b962dcede7f 100644 --- a/library/cpp/yson/varint.cpp +++ b/library/cpp/yson/varint.cpp @@ -1,12 +1,12 @@ #include "varint.h" -#include "zigzag.h" - +#include "zigzag.h" + #include <util/generic/yexception.h> - + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + int WriteVarUInt64(IOutputStream* output, ui64 value) { bool stop = false; int bytesWritten = 0; @@ -19,22 +19,22 @@ namespace NYson { byte &= 0x7F; } output->Write(byte); - } + } return bytesWritten; - } - + } + int WriteVarInt32(IOutputStream* output, i32 value) { return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode32(value))); } - + int WriteVarInt64(IOutputStream* output, i64 value) { return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode64(value))); } - + int ReadVarUInt64(IInputStream* input, ui64* value) { size_t count = 0; ui64 result = 0; - + ui8 byte = 0; do { if (7 * count > 8 * sizeof(ui64)) { @@ -56,16 +56,16 @@ namespace NYson { int bytesRead = ReadVarUInt64(input, &varInt); if (varInt > Max<ui32>()) { ythrow yexception() << "The data is too long to read ui64"; - } + } *value = ZigZagDecode32(static_cast<ui32>(varInt)); return bytesRead; } - + int ReadVarInt64(IInputStream* input, i64* value) { ui64 varInt; int bytesRead = ReadVarUInt64(input, &varInt); *value = ZigZagDecode64(varInt); return bytesRead; - } - + } + } // namespace NYson diff --git a/library/cpp/yson/writer.cpp b/library/cpp/yson/writer.cpp index 054459f9f5f..9f0386e213d 100644 --- a/library/cpp/yson/writer.cpp +++ b/library/cpp/yson/writer.cpp @@ -1,18 +1,18 @@ -#include "writer.h" +#include "writer.h" #include "detail.h" -#include "format.h" +#include "format.h" #include "parser.h" #include "varint.h" -#include "zigzag.h" - -#include <util/string/cast.h> - +#include "zigzag.h" + +#include <util/string/cast.h> + #include <cmath> namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + // Copied from <util/string/escape.cpp> namespace { inline char HexDigit(char value) { @@ -22,26 +22,26 @@ namespace NYson { else return 'A' + value - 10; } - + inline char OctDigit(char value) { Y_ASSERT(value < 8); return '0' + value; } - + inline bool IsPrintable(char c) { return c >= 32 && c <= 126; } - + inline bool IsHexDigit(char c) { return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); } - + inline bool IsOctDigit(char c) { return c >= '0' && c <= '7'; } - + const size_t ESCAPE_C_BUFFER_SIZE = 4; - + inline size_t EscapeC(unsigned char c, char next, char r[ESCAPE_C_BUFFER_SIZE]) { // (1) Printable characters go as-is, except backslash and double quote. // (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible). @@ -87,33 +87,33 @@ namespace NYson { return 4; } } - + void EscapeC(const char* str, size_t len, IOutputStream& output) { char buffer[ESCAPE_C_BUFFER_SIZE]; - + size_t i, j; for (i = 0, j = 0; i < len; ++i) { size_t rlen = EscapeC(str[i], (i + 1 < len ? str[i + 1] : 0), buffer); - + if (rlen > 1) { output.Write(str + j, i - j); j = i + 1; output.Write(buffer, rlen); } } - + if (j > 0) { output.Write(str + j, len - j); } else { output.Write(str, len); } - } - + } + TString FloatToStringWithNanInf(double value) { if (std::isfinite(value)) { return ::ToString(value); } - + static const TStringBuf nanLiteral = "%nan"; static const TStringBuf infLiteral = "%inf"; static const TStringBuf negativeInfLiteral = "%-inf"; @@ -130,7 +130,7 @@ namespace NYson { } } - + //////////////////////////////////////////////////////////////////////////////// TYsonWriter::TYsonWriter( @@ -147,17 +147,17 @@ namespace NYson { { Y_ASSERT(stream); } - + void TYsonWriter::WriteIndent() { for (int i = 0; i < IndentSize * Depth; ++i) { Stream->Write(' '); } } - + bool TYsonWriter::IsTopLevelFragmentContext() const { return Depth == 0 && (Type == ::NYson::EYsonType::ListFragment || Type == ::NYson::EYsonType::MapFragment); - } - + } + void TYsonWriter::EndNode() { if (IsTopLevelFragmentContext()) { ETokenType separatorToken = @@ -168,15 +168,15 @@ namespace NYson { if (Format == EYsonFormat::Text || Format == EYsonFormat::Pretty) { Stream->Write('\n'); } - } - } - + } + } + void TYsonWriter::BeginCollection(ETokenType beginToken) { Stream->Write(TokenTypeToChar(beginToken)); ++Depth; BeforeFirstItem = true; } - + void TYsonWriter::CollectionItem(ETokenType separatorToken) { if (!IsTopLevelFragmentContext()) { if (!BeforeFirstItem) { @@ -187,21 +187,21 @@ namespace NYson { Stream->Write('\n'); WriteIndent(); } - } - + } + BeforeFirstItem = false; } void TYsonWriter::EndCollection(ETokenType endToken) { --Depth; if (Format == EYsonFormat::Pretty && !BeforeFirstItem) { - Stream->Write('\n'); - WriteIndent(); - } + Stream->Write('\n'); + WriteIndent(); + } Stream->Write(TokenTypeToChar(endToken)); BeforeFirstItem = false; - } - + } + void TYsonWriter::WriteStringScalar(const TStringBuf& value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::StringMarker); @@ -213,12 +213,12 @@ namespace NYson { Stream->Write('"'); } } - + void TYsonWriter::OnStringScalar(TStringBuf value) { WriteStringScalar(value); EndNode(); - } - + } + void TYsonWriter::OnInt64Scalar(i64 value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::Int64Marker); @@ -227,8 +227,8 @@ namespace NYson { Stream->Write(::ToString(value)); } EndNode(); - } - + } + void TYsonWriter::OnUint64Scalar(ui64 value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::Uint64Marker); @@ -238,8 +238,8 @@ namespace NYson { Stream->Write("u"); } EndNode(); - } - + } + void TYsonWriter::OnDoubleScalar(double value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::DoubleMarker); @@ -252,44 +252,44 @@ namespace NYson { } } EndNode(); - } - + } + void TYsonWriter::OnBooleanScalar(bool value) { if (Format == EYsonFormat::Binary) { Stream->Write(value ? NDetail::TrueMarker : NDetail::FalseMarker); } else { Stream->Write(value ? "%true" : "%false"); - } + } EndNode(); - } - + } + void TYsonWriter::OnEntity() { Stream->Write(TokenTypeToChar(EntityToken)); EndNode(); - } - + } + void TYsonWriter::OnBeginList() { BeginCollection(BeginListToken); } - + void TYsonWriter::OnListItem() { CollectionItem(ListItemSeparatorToken); } - + void TYsonWriter::OnEndList() { EndCollection(EndListToken); EndNode(); } - + void TYsonWriter::OnBeginMap() { BeginCollection(BeginMapToken); } - + void TYsonWriter::OnKeyedItem(TStringBuf key) { CollectionItem(KeyedItemSeparatorToken); - + WriteStringScalar(key); - + if (Format == NYson::EYsonFormat::Pretty) { Stream->Write(' '); } @@ -297,26 +297,26 @@ namespace NYson { if (Format == NYson::EYsonFormat::Pretty) { Stream->Write(' '); } - + BeforeFirstItem = false; - } + } void TYsonWriter::OnEndMap() { EndCollection(EndMapToken); EndNode(); - } - + } + void TYsonWriter::OnBeginAttributes() { BeginCollection(BeginAttributesToken); } - + void TYsonWriter::OnEndAttributes() { EndCollection(EndAttributesToken); if (Format == NYson::EYsonFormat::Pretty) { Stream->Write(' '); } } - + void TYsonWriter::OnRaw(TStringBuf yson, EYsonType type) { if (EnableRaw) { Stream->Write(yson); @@ -325,19 +325,19 @@ namespace NYson { TYsonConsumerBase::OnRaw(yson, type); } } - + TYsonWriter::TState TYsonWriter::State() const { TState state; state.Depth = Depth; state.BeforeFirstItem = BeforeFirstItem; return state; - } - + } + void TYsonWriter::Reset(const TState& state) { Depth = state.Depth; BeforeFirstItem = state.BeforeFirstItem; - } - + } + //////////////////////////////////////////////////////////////////////////////// void ReformatYsonStream( @@ -351,5 +351,5 @@ namespace NYson { } //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/writer.h b/library/cpp/yson/writer.h index 40f5d7d5014..0ae04be6e84 100644 --- a/library/cpp/yson/writer.h +++ b/library/cpp/yson/writer.h @@ -1,17 +1,17 @@ -#pragma once - -#include "public.h" -#include "token.h" -#include "consumer.h" - -#include <util/generic/noncopyable.h> - +#pragma once + +#include "public.h" +#include "token.h" +#include "consumer.h" + +#include <util/generic/noncopyable.h> + class IOutputStream; class IZeroCopyInput; - + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TYsonWriter : public TYsonConsumerBase, private TNonCopyable { @@ -20,7 +20,7 @@ namespace NYson { private: int Depth; bool BeforeFirstItem; - + friend class TYsonWriter; }; @@ -37,23 +37,23 @@ namespace NYson { void OnDoubleScalar(double value) override; void OnBooleanScalar(bool value) override; void OnEntity() override; - + void OnBeginList() override; void OnListItem() override; void OnEndList() override; - + void OnBeginMap() override; void OnKeyedItem(TStringBuf key) override; void OnEndMap() override; - + void OnBeginAttributes() override; void OnEndAttributes() override; - + void OnRaw(TStringBuf yson, EYsonType type = ::NYson::EYsonType::Node) override; - + TState State() const; void Reset(const TState& state); - + protected: IOutputStream* Stream; EYsonFormat Format; @@ -62,28 +62,28 @@ namespace NYson { int Depth; bool BeforeFirstItem; - + static const int IndentSize = 4; - + void WriteIndent(); void WriteStringScalar(const TStringBuf& value); - + void BeginCollection(ETokenType beginToken); void CollectionItem(ETokenType separatorToken); void EndCollection(ETokenType endToken); - + bool IsTopLevelFragmentContext() const; void EndNode(); }; - + //////////////////////////////////////////////////////////////////////////////// - + void ReformatYsonStream( IInputStream* input, IOutputStream* output, EYsonFormat format = EYsonFormat::Binary, EYsonType type = ::NYson::EYsonType::Node); - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/ya.make b/library/cpp/yson/ya.make index c55a189b105..b30109a31fa 100644 --- a/library/cpp/yson/ya.make +++ b/library/cpp/yson/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - +LIBRARY() + OWNER( ermolovd g:yt ) - + PEERDIR( library/cpp/yt/misc library/cpp/yt/yson ) -SRCS( +SRCS( consumer.cpp - lexer.cpp + lexer.cpp parser.cpp - token.cpp - tokenizer.cpp + token.cpp + tokenizer.cpp varint.cpp - writer.cpp -) - -END() + writer.cpp +) + +END() diff --git a/library/cpp/yson/zigzag.h b/library/cpp/yson/zigzag.h index 2f1190508fb..dc130a74109 100644 --- a/library/cpp/yson/zigzag.h +++ b/library/cpp/yson/zigzag.h @@ -1,31 +1,31 @@ -#pragma once - -#include <util/system/defaults.h> - +#pragma once + +#include <util/system/defaults.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + //! Functions that provide coding of integers with property: 0 <= f(x) <= 2 * |x| //! Actually taken 'as is' from protobuf/wire_format_lite.h - + inline ui32 ZigZagEncode32(i32 n) { // Note: the right-shift must be arithmetic return (ui32(n) << 1) ^ (n >> 31); } - + inline i32 ZigZagDecode32(ui32 n) { return (n >> 1) ^ -static_cast<i32>(n & 1); } - + inline ui64 ZigZagEncode64(i64 n) { // Note: the right-shift must be arithmetic return (ui64(n) << 1) ^ (n >> 63); } - + inline i64 ZigZagDecode64(ui64 n) { return (n >> 1) ^ -static_cast<i64>(n & 1); } - + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson |