diff options
author | Mikhail Borisov <borisov.mikhail@gmail.com> | 2022-02-10 16:45:40 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:40 +0300 |
commit | 5d50718e66d9c037dc587a0211110b7d25a66185 (patch) | |
tree | e98df59de24d2ef7c77baed9f41e4875a2fef972 /library/cpp/yson_pull/detail | |
parent | a6a92afe03e02795227d2641b49819b687f088f8 (diff) | |
download | ydb-5d50718e66d9c037dc587a0211110b7d25a66185.tar.gz |
Restoring authorship annotation for Mikhail Borisov <borisov.mikhail@gmail.com>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/yson_pull/detail')
23 files changed, 816 insertions, 816 deletions
diff --git a/library/cpp/yson_pull/detail/byte_reader.h b/library/cpp/yson_pull/detail/byte_reader.h index 36ffe8446b..7cea50d323 100644 --- a/library/cpp/yson_pull/detail/byte_reader.h +++ b/library/cpp/yson_pull/detail/byte_reader.h @@ -1,29 +1,29 @@ -#pragma once - -#include "cescape.h" -#include "fail.h" -#include "stream_counter.h" - +#pragma once + +#include "cescape.h" +#include "fail.h" +#include "stream_counter.h" + #include <library/cpp/yson_pull/input.h> - + namespace NYsonPull { namespace NDetail { template <class StreamCounter> class byte_reader { NYsonPull::NInput::IStream& stream_; StreamCounter stream_counter_; - + public: byte_reader(NYsonPull::NInput::IStream& stream) : stream_(stream) { } - + // const-ness added to prevent direct stream mutation const NYsonPull::NInput::IStream& stream() { return stream_; } - + template <typename... Args> ATTRIBUTE(noinline, cold) void fail[[noreturn]](const char* msg, Args&&... args) { @@ -32,23 +32,23 @@ namespace NYsonPull { msg, std::forward<Args>(args)...); } - + template <bool AllowFinish> void fill_buffer() { stream_.fill_buffer(); - + if (!AllowFinish) { auto& buf = stream_.buffer(); - if (Y_UNLIKELY(buf.is_empty() && stream_.at_end())) { + if (Y_UNLIKELY(buf.is_empty() && stream_.at_end())) { fail("Premature end of stream"); } } } - + void fill_buffer() { return fill_buffer<true>(); - } - + } + template <bool AllowFinish> ui8 get_byte() { fill_buffer<AllowFinish>(); @@ -57,11 +57,11 @@ namespace NYsonPull { ? *buf.pos() : ui8{'\0'}; } - + ui8 get_byte() { return get_byte<true>(); } - + void advance(size_t bytes) { auto& buf = stream_.buffer(); stream_counter_.update( @@ -70,5 +70,5 @@ namespace NYsonPull { buf.advance(bytes); } }; - } + } } diff --git a/library/cpp/yson_pull/detail/byte_writer.h b/library/cpp/yson_pull/detail/byte_writer.h index 71ffe6230b..dc1d4b4b96 100644 --- a/library/cpp/yson_pull/detail/byte_writer.h +++ b/library/cpp/yson_pull/detail/byte_writer.h @@ -1,27 +1,27 @@ -#pragma once - -#include "macros.h" - +#pragma once + +#include "macros.h" + #include <library/cpp/yson_pull/output.h> - -#include <util/system/types.h> - -#include <cstddef> -#include <cstring> - + +#include <util/system/types.h> + +#include <cstddef> +#include <cstring> + namespace NYsonPull { namespace NDetail { template <class StreamCounter> class byte_writer { NYsonPull::NOutput::IStream& stream_; StreamCounter stream_counter_; - + public: byte_writer(NYsonPull::NOutput::IStream& stream) : stream_(stream) { } - + // const-ness added to prevent direct stream mutation const NYsonPull::NOutput::IStream& stream() { return stream_; @@ -29,11 +29,11 @@ namespace NYsonPull { const StreamCounter& counter() { return stream_counter_; } - + void flush_buffer() { stream_.flush_buffer(); } - + void advance(size_t bytes) { auto& buf = stream_.buffer(); stream_counter_.update( @@ -41,10 +41,10 @@ namespace NYsonPull { buf.pos() + bytes); buf.advance(bytes); } - + void write(ui8 c) { auto& buf = stream_.buffer(); - if (Y_LIKELY(!buf.is_full())) { + if (Y_LIKELY(!buf.is_full())) { *buf.pos() = c; advance(1); } else { @@ -53,11 +53,11 @@ namespace NYsonPull { stream_.flush_buffer({ptr, 1}); } } - + void write(const ui8* data, size_t size) { auto& buf = stream_.buffer(); auto free_buf = buf.available(); - if (Y_LIKELY(size < free_buf)) { + if (Y_LIKELY(size < free_buf)) { ::memcpy(buf.pos(), data, size); advance(size); } else { @@ -71,7 +71,7 @@ namespace NYsonPull { stream_.flush_buffer({reinterpret_cast<const char*>(data), size}); } - } + } }; - } + } } diff --git a/library/cpp/yson_pull/detail/cescape.h b/library/cpp/yson_pull/detail/cescape.h index 2e468fc0f3..1ea150e69a 100644 --- a/library/cpp/yson_pull/detail/cescape.h +++ b/library/cpp/yson_pull/detail/cescape.h @@ -1,40 +1,40 @@ -#pragma once - -#include "byte_writer.h" -#include "cescape_decode.h" -#include "cescape_encode.h" -#include "macros.h" - -#include <util/generic/strbuf.h> +#pragma once + +#include "byte_writer.h" +#include "cescape_decode.h" +#include "cescape_encode.h" +#include "macros.h" + +#include <util/generic/strbuf.h> #include <util/generic/string.h> -#include <util/generic/vector.h> - -/* REFERENCES FOR ESCAPE SEQUENCE INTERPRETATION: - * C99 p. 6.4.3 Universal character names. - * C99 p. 6.4.4.4 Character constants. - * - * <simple-escape-sequence> ::= { - * \' , \" , \? , \\ , - * \a , \b , \f , \n , \r , \t , \v - * } - * - * <octal-escape-sequence> ::= \ <octal-digit> {1, 3} - * <hexadecimal-escape-sequence> ::= \x <hexadecimal-digit> + - * <universal-character-name> ::= \u <hexadecimal-digit> {4} - * || \U <hexadecimal-digit> {8} - * - * NOTE (6.4.4.4.7): - * Each octal or hexadecimal escape sequence is the longest sequence of characters that can - * constitute the escape sequence. - * - * THEREFORE: - * - Octal escape sequence spans until rightmost non-octal-digit character. - * - Octal escape sequence always terminates after three octal digits. - * - Hexadecimal escape sequence spans until rightmost non-hexadecimal-digit character. - * - Universal character name consists of exactly 4 or 8 hexadecimal digit. - * - */ - +#include <util/generic/vector.h> + +/* REFERENCES FOR ESCAPE SEQUENCE INTERPRETATION: + * C99 p. 6.4.3 Universal character names. + * C99 p. 6.4.4.4 Character constants. + * + * <simple-escape-sequence> ::= { + * \' , \" , \? , \\ , + * \a , \b , \f , \n , \r , \t , \v + * } + * + * <octal-escape-sequence> ::= \ <octal-digit> {1, 3} + * <hexadecimal-escape-sequence> ::= \x <hexadecimal-digit> + + * <universal-character-name> ::= \u <hexadecimal-digit> {4} + * || \U <hexadecimal-digit> {8} + * + * NOTE (6.4.4.4.7): + * Each octal or hexadecimal escape sequence is the longest sequence of characters that can + * constitute the escape sequence. + * + * THEREFORE: + * - Octal escape sequence spans until rightmost non-octal-digit character. + * - Octal escape sequence always terminates after three octal digits. + * - Hexadecimal escape sequence spans until rightmost non-hexadecimal-digit character. + * - Universal character name consists of exactly 4 or 8 hexadecimal digit. + * + */ + namespace NYsonPull { namespace NDetail { namespace NCEscape { @@ -48,7 +48,7 @@ namespace NYsonPull { size); }); } - + // dest must have at least 4*data.size() bytes available inline size_t encode(ui8* dest, TStringBuf data) { auto* dest_begin = dest; @@ -61,11 +61,11 @@ namespace NYsonPull { }); return dest - dest_begin; } - + template <typename U> void encode(byte_writer<U>& dest, TStringBuf data) { auto& buffer = dest.stream().buffer(); - if (Y_LIKELY(buffer.available() >= data.size() * 4)) { + if (Y_LIKELY(buffer.available() >= data.size() * 4)) { auto size = encode(buffer.pos(), data); dest.advance(size); } else { @@ -77,14 +77,14 @@ namespace NYsonPull { }); } } - + inline TString encode(TStringBuf data) { TString result; result.reserve(data.size()); encode(result, data); return result; - } - + } + inline void decode(TString& dest, TStringBuf data) { NImpl::unescape_impl( reinterpret_cast<const ui8*>(data.begin()), @@ -96,7 +96,7 @@ namespace NYsonPull { dest.append(reinterpret_cast<const char*>(p), len); }); } - + inline void decode_inplace(TVector<ui8>& data) { auto* out = static_cast<ui8*>( ::memchr(data.data(), '\\', data.size())); @@ -115,14 +115,14 @@ namespace NYsonPull { }); data.resize(out - &data[0]); } - + inline TString decode(TStringBuf data) { TString result; result.reserve(data.size()); decode(result, data); return result; } - + ATTRIBUTE(noinline, cold) inline TString quote(TStringBuf str) { TString result; @@ -132,7 +132,7 @@ namespace NYsonPull { result += '"'; return result; } - + ATTRIBUTE(noinline, cold) inline TString quote(ui8 ch) { char c = ch; diff --git a/library/cpp/yson_pull/detail/cescape_decode.h b/library/cpp/yson_pull/detail/cescape_decode.h index 377112924b..2ee5dd9500 100644 --- a/library/cpp/yson_pull/detail/cescape_decode.h +++ b/library/cpp/yson_pull/detail/cescape_decode.h @@ -1,10 +1,10 @@ -#pragma once - -#include <util/system/types.h> - -#include <algorithm> -#include <cstring> - +#pragma once + +#include <util/system/types.h> + +#include <algorithm> +#include <cstring> + namespace NYsonPull { namespace NDetail { namespace NCEscape { @@ -12,7 +12,7 @@ namespace NYsonPull { inline ui8 as_digit(ui8 c) { return c - ui8{'0'}; } - + inline ui8 as_hexdigit(ui8 c) { static constexpr ui8 hex_decode_map[256] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, @@ -37,10 +37,10 @@ namespace NYsonPull { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}; - + return hex_decode_map[c]; } - + inline const ui8* read_oct(ui8& result, const ui8* p, ui8 n) { auto digit = ui8{0}; while (n-- && (digit = as_digit(*p)) < 8) { @@ -49,7 +49,7 @@ namespace NYsonPull { } return p; } - + inline const ui8* read_hex(ui8& result, const ui8* p, ui8 n) { auto digit = ui8{0}; while (n-- && (digit = as_hexdigit(*p)) < 16) { @@ -58,7 +58,7 @@ namespace NYsonPull { } return p; } - + inline const ui8* unescape_char_and_advance( ui8& result, const ui8* p, @@ -88,7 +88,7 @@ namespace NYsonPull { result = '\t'; ++p; break; - + case 'x': { ++p; result = 0; @@ -101,7 +101,7 @@ namespace NYsonPull { result = 'x'; } } break; - + case '0': case '1': case '2': @@ -111,7 +111,7 @@ namespace NYsonPull { result, p, std::min<ptrdiff_t>(3, end - p)); break; - + case '4': case '5': case '6': @@ -124,7 +124,7 @@ namespace NYsonPull { } return p; } - + template <typename T, typename U> inline void unescape_impl( const ui8* p, @@ -148,7 +148,7 @@ namespace NYsonPull { } } } - } + } } // namespace NCEscape } // namespace NDetail } diff --git a/library/cpp/yson_pull/detail/cescape_encode.h b/library/cpp/yson_pull/detail/cescape_encode.h index bf9a44ba5f..bf5765f1d9 100644 --- a/library/cpp/yson_pull/detail/cescape_encode.h +++ b/library/cpp/yson_pull/detail/cescape_encode.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <util/system/types.h> - -// Whether to ensure strict ASCII compatibility -// Turns UTF-8 strings into unreadable garbage for no known reason -//#define CESCAPE_STRICT_ASCII - + +// Whether to ensure strict ASCII compatibility +// Turns UTF-8 strings into unreadable garbage for no known reason +//#define CESCAPE_STRICT_ASCII + namespace NYsonPull { namespace NDetail { namespace NCEscape { @@ -14,29 +14,29 @@ namespace NYsonPull { constexpr ui8 hex_digits[] = "0123456789ABCDEF"; return hex_digits[value]; } - + inline ui8 oct_digit(ui8 value) { return '0' + value; } - + inline bool is_printable(ui8 c) { -#ifdef CESCAPE_STRICT_ASCII +#ifdef CESCAPE_STRICT_ASCII return c >= 32 && c <= 126; -#else +#else return c >= 32; -#endif +#endif } - + inline bool is_hex_digit(ui8 c) { return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); } - + inline bool is_oct_digit(ui8 c) { return c >= '0' && c <= '7'; } - + constexpr size_t ESCAPE_C_BUFFER_SIZE = 4; - + inline size_t escape_char( ui8 c, ui8 next, @@ -85,16 +85,16 @@ namespace NYsonPull { return 4; } } - + template <typename T> inline void escape_impl(const ui8* str, size_t len, T&& consume) { ui8 buffer[ESCAPE_C_BUFFER_SIZE]; - + size_t i, j; for (i = 0, j = 0; i < len; ++i) { auto next_char = i + 1 < len ? str[i + 1] : 0; size_t rlen = escape_char(str[i], next_char, buffer); - + if (rlen > 1) { consume(str + j, i - j); j = i + 1; diff --git a/library/cpp/yson_pull/detail/fail.h b/library/cpp/yson_pull/detail/fail.h index a140f34c91..6937612d0b 100644 --- a/library/cpp/yson_pull/detail/fail.h +++ b/library/cpp/yson_pull/detail/fail.h @@ -1,11 +1,11 @@ -#pragma once - -#include "format_string.h" -#include "macros.h" - +#pragma once + +#include "format_string.h" +#include "macros.h" + #include <library/cpp/yson_pull/exceptions.h> #include <library/cpp/yson_pull/position_info.h> - + namespace NYsonPull { namespace NDetail { template <typename... Args> diff --git a/library/cpp/yson_pull/detail/format_string.h b/library/cpp/yson_pull/detail/format_string.h index 7413beb6f1..683fd1bf36 100644 --- a/library/cpp/yson_pull/detail/format_string.h +++ b/library/cpp/yson_pull/detail/format_string.h @@ -1,26 +1,26 @@ -#pragma once - -#include <util/generic/strbuf.h> +#pragma once + +#include <util/generic/strbuf.h> #include <util/generic/string.h> -#include <util/string/builder.h> - +#include <util/string/builder.h> + namespace NYsonPull { namespace NDetail { namespace NImpl { inline void apply_args(TStringBuilder&) { } - + template <typename T, typename... Args> inline void apply_args(TStringBuilder& builder, T&& arg, Args&&... args) { apply_args(builder << arg, std::forward<Args>(args)...); } } - + template <typename... Args> TString format_string(Args&&... args) { TStringBuilder builder; NImpl::apply_args(builder, std::forward<Args>(args)...); return TString(std::move(builder)); } - } + } } diff --git a/library/cpp/yson_pull/detail/input/buffered.h b/library/cpp/yson_pull/detail/input/buffered.h index fef577df8b..9b1482577f 100644 --- a/library/cpp/yson_pull/detail/input/buffered.h +++ b/library/cpp/yson_pull/detail/input/buffered.h @@ -1,31 +1,31 @@ -#pragma once - +#pragma once + #include <library/cpp/yson_pull/detail/macros.h> - + #include <library/cpp/yson_pull/exceptions.h> #include <library/cpp/yson_pull/input.h> - -#include <cstdio> -#include <memory> - + +#include <cstdio> +#include <memory> + namespace NYsonPull { namespace NDetail { namespace NInput { class TBuffered: public NYsonPull::NInput::IStream { TArrayHolder<ui8> buffer_; size_t size_; - + public: explicit TBuffered(size_t buffer_size) : buffer_{new ui8[buffer_size]} , size_{buffer_size} { } - + protected: ui8* buffer_data() const { return buffer_.Get(); } - + size_t buffer_size() const { return size_; } diff --git a/library/cpp/yson_pull/detail/input/stdio_file.h b/library/cpp/yson_pull/detail/input/stdio_file.h index 328931d435..c412b7e59b 100644 --- a/library/cpp/yson_pull/detail/input/stdio_file.h +++ b/library/cpp/yson_pull/detail/input/stdio_file.h @@ -1,35 +1,35 @@ -#pragma once - -#include "buffered.h" - +#pragma once + +#include "buffered.h" + #include <library/cpp/yson_pull/detail/macros.h> - + #include <library/cpp/yson_pull/exceptions.h> #include <library/cpp/yson_pull/input.h> - -#include <cstdio> -#include <memory> - + +#include <cstdio> +#include <memory> + namespace NYsonPull { namespace NDetail { namespace NInput { class TStdioFile: public TBuffered { FILE* file_; - + public: TStdioFile(FILE* file, size_t buffer_size) : TBuffered(buffer_size) , file_{file} { } - + protected: result do_fill_buffer() override { auto nread = ::fread(buffer_data(), 1, buffer_size(), file_); - if (Y_UNLIKELY(nread == 0)) { - if (ferror(file_)) { + if (Y_UNLIKELY(nread == 0)) { + if (ferror(file_)) { throw NException::TSystemError(); } - if (feof(file_)) { + if (feof(file_)) { return result::at_end; } } @@ -37,6 +37,6 @@ namespace NYsonPull { return result::have_more_data; } }; - } + } } // namespace NDetail } diff --git a/library/cpp/yson_pull/detail/input/stream.h b/library/cpp/yson_pull/detail/input/stream.h index 5ed993f606..791cd5a3f5 100644 --- a/library/cpp/yson_pull/detail/input/stream.h +++ b/library/cpp/yson_pull/detail/input/stream.h @@ -1,69 +1,69 @@ -#pragma once - +#pragma once + #include <library/cpp/yson_pull/detail/macros.h> - + #include <library/cpp/yson_pull/input.h> - -#include <util/stream/buffered.h> -#include <util/stream/file.h> -#include <util/stream/zerocopy.h> -#include <util/system/file.h> - -namespace NYsonPull { - namespace NDetail { - namespace NInput { - class TStreamBase: public NYsonPull::NInput::IStream { - protected: + +#include <util/stream/buffered.h> +#include <util/stream/file.h> +#include <util/stream/zerocopy.h> +#include <util/system/file.h> + +namespace NYsonPull { + namespace NDetail { + namespace NInput { + class TStreamBase: public NYsonPull::NInput::IStream { + protected: result DoFillBufferFrom(IZeroCopyInput& input) { - void* ptr = nullptr; - size_t size = input.Next(&ptr); - if (Y_UNLIKELY(size == 0)) { - return result::at_end; - } - buffer().reset(static_cast<ui8*>(ptr), static_cast<ui8*>(ptr) + size); - return result::have_more_data; - } - }; - - class TZeroCopy: public TStreamBase { + void* ptr = nullptr; + size_t size = input.Next(&ptr); + if (Y_UNLIKELY(size == 0)) { + return result::at_end; + } + buffer().reset(static_cast<ui8*>(ptr), static_cast<ui8*>(ptr) + size); + return result::have_more_data; + } + }; + + class TZeroCopy: public TStreamBase { IZeroCopyInput* Input; - - public: + + public: explicit TZeroCopy(IZeroCopyInput* input) - : Input(input) - { - } - - protected: - result do_fill_buffer() override { - return DoFillBufferFrom(*Input); - } - }; - + : Input(input) + { + } + + protected: + result do_fill_buffer() override { + return DoFillBufferFrom(*Input); + } + }; + template <typename TBuffered> - class TOwned: public TStreamBase { - TBuffered Input; - - public: + class TOwned: public TStreamBase { + TBuffered Input; + + public: template <typename... Args> - explicit TOwned(Args&&... args) - : Input(std::forward<Args>(args)...) - { - } - - protected: - result do_fill_buffer() override { - return DoFillBufferFrom(Input); - } - }; - + explicit TOwned(Args&&... args) + : Input(std::forward<Args>(args)...) + { + } + + protected: + result do_fill_buffer() override { + return DoFillBufferFrom(Input); + } + }; + class TFHandle: public TOwned<TFileInput> { - public: - TFHandle(int fd, size_t buffer_size) + public: + TFHandle(int fd, size_t buffer_size) : TOwned<TFileInput>(Duplicate(fd), buffer_size) - { - } - }; - } + { + } + }; + } } // namespace NDetail } diff --git a/library/cpp/yson_pull/detail/lexer_base.h b/library/cpp/yson_pull/detail/lexer_base.h index 20919e496a..572bdb3d18 100644 --- a/library/cpp/yson_pull/detail/lexer_base.h +++ b/library/cpp/yson_pull/detail/lexer_base.h @@ -1,27 +1,27 @@ -#pragma once - -#include "byte_reader.h" -#include "cescape.h" -#include "macros.h" -#include "number.h" +#pragma once + +#include "byte_reader.h" +#include "cescape.h" +#include "macros.h" +#include "number.h" #include "percent_scalar.h" -#include "stream_counter.h" -#include "varint.h" - -#include <util/generic/maybe.h> -#include <util/generic/vector.h> -#include <util/string/cast.h> - +#include "stream_counter.h" +#include "varint.h" + +#include <util/generic/maybe.h> +#include <util/generic/vector.h> +#include <util/string/cast.h> + namespace NYsonPull { namespace NDetail { template <bool EnableLinePositionInfo> class lexer_base: public byte_reader<stream_counter<EnableLinePositionInfo>> { using Base = byte_reader< stream_counter<EnableLinePositionInfo>>; - + TVector<ui8> token_buffer_; TMaybe<size_t> memory_limit_; - + public: lexer_base( NYsonPull::NInput::IStream& buffer, @@ -29,28 +29,28 @@ namespace NYsonPull { : Base(buffer) , memory_limit_{memory_limit} { } - + ATTRIBUTE(noinline, hot) ui8 skip_space_and_get_byte() { auto& buf = Base::stream().buffer(); - if (Y_LIKELY(!buf.is_empty())) { + if (Y_LIKELY(!buf.is_empty())) { auto ch = *buf.pos(); - if (Y_LIKELY(!is_space(ch))) { + if (Y_LIKELY(!is_space(ch))) { return ch; } } return skip_space_and_get_byte_fallback(); } - + ATTRIBUTE(hot) ui8 get_byte() { auto& buf = Base::stream().buffer(); - if (Y_LIKELY(!buf.is_empty())) { + if (Y_LIKELY(!buf.is_empty())) { return *buf.pos(); } return Base::get_byte(); - } - + } + number read_numeric() { token_buffer_.clear(); auto type = number_type::int64; @@ -64,7 +64,7 @@ namespace NYsonPull { } else if (ch == 'u') { token_buffer_.push_back(ch); type = number_type::uint64; - } else if (Y_UNLIKELY(isalpha(ch))) { + } else if (Y_UNLIKELY(isalpha(ch))) { COLD_BLOCK_BYVALUE Base::fail("Unexpected ", NCEscape::quote(ch), " in numeric literal"); COLD_BLOCK_END @@ -74,7 +74,7 @@ namespace NYsonPull { check_memory_limit(); Base::advance(1); } - + auto str = token_buffer(); try { switch (type) { @@ -86,12 +86,12 @@ namespace NYsonPull { str.Chop(1); // 'u' suffix return FromString<ui64>(str); } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } catch (const std::exception& err) { Base::fail(err.what()); } - } - + } + TStringBuf read_quoted_string() { auto count_trailing_slashes = [](ui8* begin, ui8* end) { auto count = size_t{0}; @@ -102,7 +102,7 @@ namespace NYsonPull { } return count; }; - + token_buffer_.clear(); auto& buf = Base::stream().buffer(); while (true) { @@ -135,12 +135,12 @@ namespace NYsonPull { token_buffer_.push_back('"'); } check_memory_limit(); - } + } NCEscape::decode_inplace(token_buffer_); return token_buffer(); - } - + } + TStringBuf read_unquoted_string() { token_buffer_.clear(); while (true) { @@ -155,20 +155,20 @@ namespace NYsonPull { Base::advance(1); } return token_buffer(); - } - + } + ATTRIBUTE(noinline, hot) TStringBuf read_binary_string() { auto slength = NVarInt::read<i32>(*this); - if (Y_UNLIKELY(slength < 0)) { + if (Y_UNLIKELY(slength < 0)) { COLD_BLOCK_BYVALUE Base::fail("Negative binary string literal length ", slength); COLD_BLOCK_END } auto length = static_cast<ui32>(slength); - + auto& buf = Base::stream().buffer(); - if (Y_LIKELY(buf.available() >= length)) { + if (Y_LIKELY(buf.available() >= length)) { auto result = TStringBuf{ reinterpret_cast<const char*>(buf.pos()), length}; @@ -177,8 +177,8 @@ namespace NYsonPull { } else { // reading in Buffer return read_binary_string_fallback(length); } - } - + } + ATTRIBUTE(noinline) TStringBuf read_binary_string_fallback(size_t length) { auto& buf = Base::stream().buffer(); @@ -187,7 +187,7 @@ namespace NYsonPull { while (needToRead) { this->Base::template fill_buffer<false>(); auto chunk_size = std::min(needToRead, buf.available()); - + token_buffer_.insert( token_buffer_.end(), buf.pos(), @@ -197,26 +197,26 @@ namespace NYsonPull { Base::advance(chunk_size); } return token_buffer(); - } - + } + percent_scalar read_percent_scalar() { auto throw_incorrect_percent_scalar = [&]() { Base::fail("Incorrect %-literal prefix ", NCEscape::quote(token_buffer())); }; - + auto assert_literal = [&](TStringBuf literal) -> void { for (size_t i = 2; i < literal.size(); ++i) { token_buffer_.push_back(this->Base::template get_byte<false>()); Base::advance(1); - if (Y_UNLIKELY(token_buffer_.back() != literal[i])) { + if (Y_UNLIKELY(token_buffer_.back() != literal[i])) { throw_incorrect_percent_scalar(); } } }; - + token_buffer_.clear(); token_buffer_.push_back(this->Base::template get_byte<false>()); - Base::advance(1); + Base::advance(1); switch (token_buffer_[0]) { case 't': @@ -236,32 +236,32 @@ namespace NYsonPull { return percent_scalar(-std::numeric_limits<double>::infinity()); default: throw_incorrect_percent_scalar(); - } + } + + Y_UNREACHABLE(); + } - Y_UNREACHABLE(); - } - i64 read_binary_int64() { return NVarInt::read<i64>(*this); } - + ui64 read_binary_uint64() { return NVarInt::read<ui64>(*this); } - + double read_binary_double() { union { double as_double; ui8 as_bytes[sizeof(double)]; } data; static_assert(sizeof(data) == sizeof(double), "bad union size"); - + auto needToRead = sizeof(double); - + auto& buf = Base::stream().buffer(); while (needToRead != 0) { Base::fill_buffer(); - + auto chunk_size = std::min(needToRead, buf.available()); if (chunk_size == 0) { Base::fail("Error parsing binary double literal"); @@ -274,8 +274,8 @@ namespace NYsonPull { Base::advance(chunk_size); } return data.as_double; - } - + } + private: static bool is_space(ui8 ch) { static const ui8 lookupTable[] = @@ -284,24 +284,24 @@ namespace NYsonPull { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; return lookupTable[ch]; } - + ATTRIBUTE(noinline, cold) ui8 skip_space_and_get_byte_fallback() { auto& buf = Base::stream().buffer(); @@ -318,12 +318,12 @@ namespace NYsonPull { } Base::advance(1); } - } + } return Base::get_byte(); } void check_memory_limit() { - if (Y_UNLIKELY(memory_limit_ && token_buffer_.capacity() > *memory_limit_)) { + if (Y_UNLIKELY(memory_limit_ && token_buffer_.capacity() > *memory_limit_)) { COLD_BLOCK_BYVALUE Base::fail( "Memory limit exceeded while parsing YSON stream: " @@ -331,13 +331,13 @@ namespace NYsonPull { token_buffer_.capacity(), ", limit ", *memory_limit_); COLD_BLOCK_END - } - } - + } + } + TStringBuf token_buffer() const { auto* begin = reinterpret_cast<const char*>(token_buffer_.data()); return {begin, token_buffer_.size()}; } }; - } + } } diff --git a/library/cpp/yson_pull/detail/macros.h b/library/cpp/yson_pull/detail/macros.h index 8d9707c351..7243f9cfe1 100644 --- a/library/cpp/yson_pull/detail/macros.h +++ b/library/cpp/yson_pull/detail/macros.h @@ -1,24 +1,24 @@ -#pragma once - -#include <util/system/compiler.h> - -#if defined(__GNUC__) -#define ATTRIBUTE(args...) __attribute__((args)) -#else -#define ATTRIBUTE(...) -#endif - -#if defined(__GNUC__) && !defined(__clang__) +#pragma once + +#include <util/system/compiler.h> + +#if defined(__GNUC__) +#define ATTRIBUTE(args...) __attribute__((args)) +#else +#define ATTRIBUTE(...) +#endif + +#if defined(__GNUC__) && !defined(__clang__) #define COLD_BLOCK_BYVALUE [=]() ATTRIBUTE(noinline, cold) { #define COLD_BLOCK_BYREF [&]() ATTRIBUTE(noinline, cold) { #define COLD_BLOCK_END \ } \ (); -#else -// Clang does not support gnu-style attributes on lambda functions yet +#else +// Clang does not support gnu-style attributes on lambda functions yet #define COLD_BLOCK_BYVALUE [=]() { #define COLD_BLOCK_BYREF [&]() { #define COLD_BLOCK_END \ } \ (); -#endif +#endif diff --git a/library/cpp/yson_pull/detail/number.h b/library/cpp/yson_pull/detail/number.h index 0dc442e3d2..5595f55e05 100644 --- a/library/cpp/yson_pull/detail/number.h +++ b/library/cpp/yson_pull/detail/number.h @@ -1,7 +1,7 @@ -#pragma once - +#pragma once + #include <util/system/types.h> - + namespace NYsonPull { namespace NDetail { enum class number_type { @@ -9,7 +9,7 @@ namespace NYsonPull { uint64, int64 }; - + struct number { number_type type; union { @@ -17,21 +17,21 @@ namespace NYsonPull { ui64 as_uint64; i64 as_int64; } value; - + number(double v) { type = number_type::float64; value.as_float64 = v; } - + number(i64 v) { type = number_type::int64; value.as_int64 = v; } - + number(ui64 v) { type = number_type::uint64; value.as_uint64 = v; } }; - } + } } diff --git a/library/cpp/yson_pull/detail/output/buffered.h b/library/cpp/yson_pull/detail/output/buffered.h index fbb18b5467..475cf34785 100644 --- a/library/cpp/yson_pull/detail/output/buffered.h +++ b/library/cpp/yson_pull/detail/output/buffered.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <library/cpp/yson_pull/detail/macros.h> - + #include <library/cpp/yson_pull/output.h> - -#include <util/generic/strbuf.h> - + +#include <util/generic/strbuf.h> + namespace NYsonPull { namespace NDetail { namespace NOutput { @@ -13,29 +13,29 @@ namespace NYsonPull { class TBuffered: public NYsonPull::NOutput::IStream { TArrayHolder<ui8> buffer_; size_t size_; - + public: TBuffered(size_t buffer_size) : buffer_{new ui8[buffer_size]} , size_{buffer_size} { reset_buffer(); } - + protected: void do_flush_buffer(TStringBuf extra) override { auto& buf = buffer(); if (!buf.is_empty()) { - do_write({reinterpret_cast<const char*>(buf.begin()), buf.used()}); + do_write({reinterpret_cast<const char*>(buf.begin()), buf.used()}); reset_buffer(); } if (extra.size() >= buf.available()) { do_write(extra); - } else if (extra.size() > 0) { + } else if (extra.size() > 0) { ::memcpy(buf.pos(), extra.data(), extra.size()); buf.advance(extra.size()); } } - + private: void do_write(TStringBuf data) { // CRTP dispatch @@ -46,6 +46,6 @@ namespace NYsonPull { buffer().reset(buffer_.Get(), buffer_.Get() + size_); } }; - } + } } // namespace NDetail } diff --git a/library/cpp/yson_pull/detail/output/stdio_file.h b/library/cpp/yson_pull/detail/output/stdio_file.h index d59ea6bfb9..03f2b40dc5 100644 --- a/library/cpp/yson_pull/detail/output/stdio_file.h +++ b/library/cpp/yson_pull/detail/output/stdio_file.h @@ -1,33 +1,33 @@ -#pragma once - -#include "buffered.h" - +#pragma once + +#include "buffered.h" + #include <library/cpp/yson_pull/detail/macros.h> - + #include <library/cpp/yson_pull/exceptions.h> - -#include <cstdio> - + +#include <cstdio> + namespace NYsonPull { namespace NDetail { namespace NOutput { class TStdioFile: public TBuffered<TStdioFile> { FILE* file_; - + public: TStdioFile(FILE* file, size_t buffer_size) : TBuffered<TStdioFile>(buffer_size) , file_(file) { } - + void write(TStringBuf data) { auto nwritten = ::fwrite(data.data(), 1, data.size(), file_); - if (Y_UNLIKELY(static_cast<size_t>(nwritten) != data.size())) { + if (Y_UNLIKELY(static_cast<size_t>(nwritten) != data.size())) { throw NException::TSystemError(); } } }; - } + } } // namespace NDetail } diff --git a/library/cpp/yson_pull/detail/output/stream.h b/library/cpp/yson_pull/detail/output/stream.h index 98ab7d9555..d4810f3353 100644 --- a/library/cpp/yson_pull/detail/output/stream.h +++ b/library/cpp/yson_pull/detail/output/stream.h @@ -1,56 +1,56 @@ -#pragma once - -#include "buffered.h" - +#pragma once + +#include "buffered.h" + #include <library/cpp/yson_pull/detail/macros.h> #include <library/cpp/yson_pull/exceptions.h> - -#include <util/stream/output.h> -#include <util/stream/file.h> -#include <util/system/file.h> - -namespace NYsonPull { - namespace NDetail { - namespace NOutput { - class TStream: public TBuffered<TStream> { + +#include <util/stream/output.h> +#include <util/stream/file.h> +#include <util/system/file.h> + +namespace NYsonPull { + namespace NDetail { + namespace NOutput { + class TStream: public TBuffered<TStream> { IOutputStream* Output; - - public: + + public: TStream(IOutputStream* output, size_t buffer_size) - : TBuffered<TStream>(buffer_size) - , Output(output) - { - } - - void write(TStringBuf data) { - Output->Write(data); - } - }; - + : TBuffered<TStream>(buffer_size) + , Output(output) + { + } + + void write(TStringBuf data) { + Output->Write(data); + } + }; + template <typename TOutput> - class TOwned: public TBuffered<TOwned<TOutput>> { - TOutput Output; - - public: + class TOwned: public TBuffered<TOwned<TOutput>> { + TOutput Output; + + public: template <typename... Args> - TOwned(size_t buffer_size, Args&&... args) - : TBuffered<TOwned>(buffer_size) - , Output(std::forward<Args>(args)...) - { - } - - void write(TStringBuf data) { - Output.Write(data); - } - }; - + TOwned(size_t buffer_size, Args&&... args) + : TBuffered<TOwned>(buffer_size) + , Output(std::forward<Args>(args)...) + { + } + + void write(TStringBuf data) { + Output.Write(data); + } + }; + class TFHandle: public TOwned<TUnbufferedFileOutput> { - public: - TFHandle(int fd, size_t buffer_size) + public: + TFHandle(int fd, size_t buffer_size) : TOwned<TUnbufferedFileOutput>(buffer_size, Duplicate(fd)) - { - } - }; - } + { + } + }; + } } // namespace NDetail } diff --git a/library/cpp/yson_pull/detail/reader.h b/library/cpp/yson_pull/detail/reader.h index 4aa99e88e0..0e02396358 100644 --- a/library/cpp/yson_pull/detail/reader.h +++ b/library/cpp/yson_pull/detail/reader.h @@ -1,18 +1,18 @@ -#pragma once - -#include "lexer_base.h" -#include "symbols.h" - +#pragma once + +#include "lexer_base.h" +#include "symbols.h" + #include <library/cpp/yson_pull/reader.h> - -#include <util/generic/maybe.h> -#include <util/generic/vector.h> - + +#include <util/generic/maybe.h> +#include <util/generic/vector.h> + namespace NYsonPull { namespace NDetail { /*! \internal */ //////////////////////////////////////////////////////////////////////////////// - + enum class special_token : ui8 { // Special values: // YSON @@ -26,7 +26,7 @@ namespace NYsonPull { left_angle = 7, // < right_angle = 8, // > }; - + // char_class tree representation: // Root = xb // BinaryStringOrOtherSpecialToken = x0b @@ -48,7 +48,7 @@ namespace NYsonPull { // Percent = 11011b enum class char_class : ui8 { binary_string = 0, // = 00b - + special_token_mask = 2, // = 10b semicolon = 2 + (0 << 2), equals = 2 + (1 << 2), @@ -59,14 +59,14 @@ namespace NYsonPull { right_brace = 2 + (6 << 2), left_angle = 2 + (7 << 2), right_angle = 2 + (8 << 2), - + binary_scalar_mask = 1, binary_int64 = 1 + (0 << 2), // = 001b binary_double = 1 + (1 << 2), // = 101b binary_false = 1 + (2 << 2), // = 1001b binary_true = 1 + (3 << 2), // = 1101b binary_uint64 = 1 + (4 << 2), // = 10001b - + other_mask = 3, quote = 3 + (0 << 2), // = 00011b number = 3 + (1 << 2), // = 00111b @@ -74,30 +74,30 @@ namespace NYsonPull { percent = 3 + (6 << 2), // = 11011b none = 3 + (5 << 2), // = 10111b }; - + #define CHAR_SUBCLASS(x) (static_cast<ui8>(x) >> 2) - + inline char_class get_char_class(ui8 ch) { -#define NN char_class::none -#define BS char_class::binary_string -#define BI char_class::binary_int64 -#define BD char_class::binary_double -#define BF char_class::binary_false -#define BT char_class::binary_true -#define BU char_class::binary_uint64 -#define SP NN // char_class::space -#define NB char_class::number -#define ST char_class::string -#define QU char_class::quote -#define PC char_class::percent -#define TT(name) (static_cast<char_class>( \ +#define NN char_class::none +#define BS char_class::binary_string +#define BI char_class::binary_int64 +#define BD char_class::binary_double +#define BF char_class::binary_false +#define BT char_class::binary_true +#define BU char_class::binary_uint64 +#define SP NN // char_class::space +#define NB char_class::number +#define ST char_class::string +#define QU char_class::quote +#define PC char_class::percent +#define TT(name) (static_cast<char_class>( \ (static_cast<ui8>(special_token::name) << 2) | static_cast<ui8>(char_class::special_token_mask))) - + static constexpr char_class lookup[256] = { NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - + // 32 SP, // ' ' NN, // '!' @@ -115,7 +115,7 @@ namespace NYsonPull { NB, // '-' NN, // '.' NN, // '/' - + // 48 NB, NB, NB, NB, NB, NB, NB, NB, NB, NB, // '0' - '9' NN, // ':' @@ -124,7 +124,7 @@ namespace NYsonPull { TT(equals), // '=' TT(right_angle), // '>' NN, // '?' - + // 64 NN, // '@' ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M' @@ -134,10 +134,10 @@ namespace NYsonPull { TT(right_bracket), // ']' NN, // '^' ST, // '_' - + // 96 NN, // '`' - + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm' ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z' TT(left_brace), // '{' @@ -150,24 +150,24 @@ namespace NYsonPull { NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN}; - -#undef NN -#undef BS -#undef BI -#undef BD -#undef SP -#undef NB -#undef ST -#undef QU -#undef TT + +#undef NN +#undef BS +#undef BI +#undef BD +#undef SP +#undef NB +#undef ST +#undef QU +#undef TT return lookup[ch]; } - + template <bool EnableLinePositionInfo> class gen_reader_impl { enum class state { @@ -177,20 +177,20 @@ namespace NYsonPull { equals = 3, //! expecting '=' (followed by value) value = 4, //! expecting a value value_noattr = 5, //! expecting a value w/o attrs (after attrs) - + // by design, rare states have numbers starting from first_rare_state first_rare_state = 6, before_begin = first_rare_state, //! before started reading the stream before_end = first_rare_state + 1, //! Expecting end of stream after_end = first_rare_state + 2, //! after end of stream }; - + lexer_base<EnableLinePositionInfo> lexer_; state state_; TEvent event_; TVector<EEventType> stack_; EStreamType mode_; - + public: gen_reader_impl( NYsonPull::NInput::IStream& buffer, @@ -200,14 +200,14 @@ namespace NYsonPull { , state_{state::before_begin} , mode_{mode} { } - + const TEvent& last_event() const { return event_; } - + ATTRIBUTE(hot) const TEvent& next_event() { - if (Y_LIKELY(state_ < state::first_rare_state)) { + if (Y_LIKELY(state_ < state::first_rare_state)) { // 'hot' handler for in-stream events next_event_hot(); } else { @@ -216,15 +216,15 @@ namespace NYsonPull { } return event_; } - + private: ATTRIBUTE(hot) void next_event_hot() { auto ch = lexer_.get_byte(); auto cls = get_char_class(ch); - if (Y_UNLIKELY(cls == char_class::none)) { + if (Y_UNLIKELY(cls == char_class::none)) { ch = lexer_.skip_space_and_get_byte(); - if (Y_UNLIKELY(ch == NSymbol::eof)) { + if (Y_UNLIKELY(ch == NSymbol::eof)) { handle_eof(); return; } @@ -253,10 +253,10 @@ namespace NYsonPull { state_delimiter(ch, cls); break; default: - Y_UNREACHABLE(); + Y_UNREACHABLE(); } - } - + } + ATTRIBUTE(noinline, cold) void next_event_cold() { switch (state_) { @@ -269,22 +269,22 @@ namespace NYsonPull { state_before_end(); break; default: - Y_UNREACHABLE(); + Y_UNREACHABLE(); } } - + //! Present a scalar value for caller template <typename T> void yield(T value) { event_ = TEvent{TScalar{value}}; } - + //! Present a scalar value with non-scalar tag (i.e. key) template <typename T> void yield(EEventType type, T value) { event_ = TEvent{type, TScalar{value}}; } - + //! Present a value from number variant void yield(const number& value) { switch (value.type) { @@ -299,7 +299,7 @@ namespace NYsonPull { break; } } - + //! Present a value from %-literal variant void yield(const percent_scalar& value) { switch (value.type) { @@ -316,47 +316,47 @@ namespace NYsonPull { void yield(EEventType type) { event_ = TEvent{type}; } - + //! Push the opening of a paired event void push(EEventType type) { stack_.push_back(type); } - + //! Close the paired_event, verify that delimiters are well-formed void pop(EEventType first, EEventType last) { - if (Y_UNLIKELY(stack_.empty() || stack_.back() != first)) { + if (Y_UNLIKELY(stack_.empty() || stack_.back() != first)) { pop_fail(first, last); return; } stack_.pop_back(); - + yield(last); switch (first) { - case EEventType::BeginList: + case EEventType::BeginList: next(state::delimiter); break; - - case EEventType::BeginMap: + + case EEventType::BeginMap: next(state::delimiter); break; - - case EEventType::BeginAttributes: + + case EEventType::BeginAttributes: next(state::value_noattr); break; - - case EEventType::BeginStream: + + case EEventType::BeginStream: next(state::after_end); break; - + default: - Y_UNREACHABLE(); + Y_UNREACHABLE(); } - - if (Y_UNLIKELY(mode_ == EStreamType::Node && stack_.size() == 1 && state_ == state::delimiter)) { + + if (Y_UNLIKELY(mode_ == EStreamType::Node && stack_.size() == 1 && state_ == state::delimiter)) { next(state::before_end); } } - + ATTRIBUTE(noinline, cold) void pop_fail(EEventType first, EEventType last) { if (stack_.empty()) { @@ -365,16 +365,16 @@ namespace NYsonPull { lexer_.fail("Unpaired events: expected opening '", first, "' for '", last, "', but '", stack_.back(), "' is found."); } } - + //! Transition to new_state void next(state new_state) { state_ = new_state; } - + bool in_map() { - return (stack_.back() == EEventType::BeginMap) || (stack_.back() == EEventType::BeginAttributes) || (stack_.back() == EEventType::BeginStream && mode_ == EStreamType::MapFragment); + return (stack_.back() == EEventType::BeginMap) || (stack_.back() == EEventType::BeginAttributes) || (stack_.back() == EEventType::BeginStream && mode_ == EStreamType::MapFragment); } - + ATTRIBUTE(noinline, cold) void handle_eof() { switch (state_) { @@ -382,18 +382,18 @@ namespace NYsonPull { case state::maybe_key: case state::delimiter: case state::before_end: - pop(EEventType::BeginStream, EEventType::EndStream); + pop(EEventType::BeginStream, EEventType::EndStream); return; - + default: lexer_.fail("Unexpected end of stream"); } } - + ATTRIBUTE(noinline, cold) void state_before_begin() { - push(EEventType::BeginStream); - yield(EEventType::BeginStream); + push(EEventType::BeginStream); + yield(EEventType::BeginStream); switch (mode_) { case EStreamType::Node: next(state::value); @@ -405,10 +405,10 @@ namespace NYsonPull { next(state::maybe_key); break; default: - Y_UNREACHABLE(); + Y_UNREACHABLE(); } } - + ATTRIBUTE(noinline, cold) void state_before_end() { auto ch = lexer_.skip_space_and_get_byte(); @@ -418,10 +418,10 @@ namespace NYsonPull { lexer_.fail("Expected stream end, but found ", NCEscape::quote(ch)); } } - + ATTRIBUTE(hot) void state_delimiter(ui8 ch, char_class cls) { - if (Y_LIKELY(ch == NSymbol::item_separator)) { + if (Y_LIKELY(ch == NSymbol::item_separator)) { lexer_.advance(1); next(in_map() ? state::maybe_key : state::maybe_value); // immediately read next value @@ -430,7 +430,7 @@ namespace NYsonPull { } state_delimiter_fallback(ch, cls); } - + ATTRIBUTE(noinline, hot) void state_delimiter_fallback(ui8 ch, char_class cls) { auto cls_bits = static_cast<ui8>(cls); @@ -439,24 +439,24 @@ namespace NYsonPull { lexer_.advance(1); switch (token) { /* // handled in the fast track - case special_token::semicolon: - next(in_map()? state::maybe_key : state::maybe_value); - // immediately read next value - return next_event(); - */ - + case special_token::semicolon: + next(in_map()? state::maybe_key : state::maybe_value); + // immediately read next value + return next_event(); + */ + case special_token::right_bracket: - pop(EEventType::BeginList, EEventType::EndList); + pop(EEventType::BeginList, EEventType::EndList); return; - + case special_token::right_brace: - pop(EEventType::BeginMap, EEventType::EndMap); + pop(EEventType::BeginMap, EEventType::EndMap); return; - + case special_token::right_angle: - pop(EEventType::BeginAttributes, EEventType::EndAttributes); + pop(EEventType::BeginAttributes, EEventType::EndAttributes); return; - + default: break; } @@ -470,14 +470,14 @@ namespace NYsonPull { NCEscape::quote(NSymbol::end_map), ", ", NCEscape::quote(NSymbol::end_attributes)); COLD_BLOCK_END - } - + } + ATTRIBUTE(noinline, hot) void state_maybe_key(ui8 ch, char_class cls) { auto key = TStringBuf{}; // Keys are always strings, put binary-string key into fast lane - if (Y_LIKELY(ch == NSymbol::string_marker)) { - lexer_.advance(1); + if (Y_LIKELY(ch == NSymbol::string_marker)) { + lexer_.advance(1); key = lexer_.read_binary_string(); } else { switch (cls) { @@ -485,21 +485,21 @@ namespace NYsonPull { lexer_.advance(1); key = lexer_.read_quoted_string(); break; - + case char_class::string: key = lexer_.read_unquoted_string(); break; - + case char_class::right_brace: lexer_.advance(1); - pop(EEventType::BeginMap, EEventType::EndMap); + pop(EEventType::BeginMap, EEventType::EndMap); return; - + case char_class::right_angle: lexer_.advance(1); - pop(EEventType::BeginAttributes, EEventType::EndAttributes); + pop(EEventType::BeginAttributes, EEventType::EndAttributes); return; - + default: COLD_BLOCK_BYVALUE lexer_.fail("Unexpected ", NCEscape::quote(ch), ", expected key string"); @@ -507,25 +507,25 @@ namespace NYsonPull { } } - yield(EEventType::Key, key); + yield(EEventType::Key, key); next(state::equals); } ATTRIBUTE(hot) void state_equals(ui8 ch) { // skip '=' - if (Y_UNLIKELY(ch != NSymbol::key_value_separator)) { - COLD_BLOCK_BYVALUE + if (Y_UNLIKELY(ch != NSymbol::key_value_separator)) { + COLD_BLOCK_BYVALUE lexer_.fail("Unexpected ", NCEscape::quote(ch), ", expected ", NCEscape::quote(NSymbol::key_value_separator)); - COLD_BLOCK_END + COLD_BLOCK_END } lexer_.advance(1); next(state::value); // immediately read the following value // (this symbol yields no result) next_event_hot(); - } - + } + ATTRIBUTE(noinline, hot) void state_value(ui8 ch, char_class cls) { auto cls_bits = static_cast<ui8>(cls); @@ -549,7 +549,7 @@ namespace NYsonPull { } } } - + ATTRIBUTE(noinline) void state_value_special(special_token token, ui8 ch) { // Value starters are always accepted values @@ -558,37 +558,37 @@ namespace NYsonPull { yield(TScalar{}); next(state::delimiter); return; - + case special_token::left_bracket: - push(EEventType::BeginList); - yield(EEventType::BeginList); + push(EEventType::BeginList); + yield(EEventType::BeginList); next(state::maybe_value); return; - + case special_token::left_brace: - push(EEventType::BeginMap); - yield(EEventType::BeginMap); + push(EEventType::BeginMap); + yield(EEventType::BeginMap); next(state::maybe_key); return; - + default: break; } - + // ...closing-chars are only allowed in maybe_value state if (state_ == state::maybe_value) { switch (token) { case special_token::right_bracket: - pop(EEventType::BeginList, EEventType::EndList); + pop(EEventType::BeginList, EEventType::EndList); return; - + case special_token::right_brace: - pop(EEventType::BeginMap, EEventType::EndMap); + pop(EEventType::BeginMap, EEventType::EndMap); return; - + // right_angle is impossible in maybe_value state // (only in delimiter, maybe_key) - + default: break; } @@ -596,17 +596,17 @@ namespace NYsonPull { // attributes are not allowed after attributes (thus, value_noattr state) if (state_ != state::value_noattr && token == special_token::left_angle) { - push(EEventType::BeginAttributes); - yield(EEventType::BeginAttributes); + push(EEventType::BeginAttributes); + yield(EEventType::BeginAttributes); next(state::maybe_key); - return; + return; } - + COLD_BLOCK_BYVALUE lexer_.fail("Unexpected ", NCEscape::quote(ch)); COLD_BLOCK_END - } - + } + ATTRIBUTE(hot) void state_value_binary_scalar(char_class cls) { lexer_.advance(1); @@ -614,28 +614,28 @@ namespace NYsonPull { case char_class::binary_double: yield(lexer_.read_binary_double()); break; - + case char_class::binary_int64: yield(lexer_.read_binary_int64()); break; - + case char_class::binary_uint64: yield(lexer_.read_binary_uint64()); break; - + case char_class::binary_false: yield(false); break; - + case char_class::binary_true: yield(true); break; - + default: - Y_UNREACHABLE(); + Y_UNREACHABLE(); } } - + ATTRIBUTE(noinline) void state_value_text_scalar(char_class cls) { switch (cls) { @@ -643,20 +643,20 @@ namespace NYsonPull { lexer_.advance(1); yield(lexer_.read_quoted_string()); break; - + case char_class::number: yield(lexer_.read_numeric()); break; - + case char_class::string: yield(lexer_.read_unquoted_string()); break; - + case char_class::percent: lexer_.advance(1); yield(lexer_.read_percent_scalar()); break; - + case char_class::none: COLD_BLOCK_BYVALUE lexer_.fail("Invalid yson value."); @@ -664,14 +664,14 @@ namespace NYsonPull { break; default: - Y_UNREACHABLE(); + Y_UNREACHABLE(); } } }; - + class reader_impl: public gen_reader_impl<false> { public: using gen_reader_impl<false>::gen_reader_impl; }; - } + } } diff --git a/library/cpp/yson_pull/detail/stream_counter.h b/library/cpp/yson_pull/detail/stream_counter.h index c96698d091..3b41b27eb6 100644 --- a/library/cpp/yson_pull/detail/stream_counter.h +++ b/library/cpp/yson_pull/detail/stream_counter.h @@ -1,26 +1,26 @@ -#pragma once - +#pragma once + #include <library/cpp/yson_pull/position_info.h> - -#include <cstddef> - + +#include <cstddef> + namespace NYsonPull { namespace NDetail { template <bool EnableLinePositionInfo> class stream_counter; - + template <> class stream_counter<true> { private: size_t offset_ = 0; size_t line_ = 1; size_t column_ = 1; - + public: TPositionInfo info() const { return {offset_, line_, column_}; } - + void update(const ui8* begin, const ui8* end) { offset_ += end - begin; for (auto current = begin; current != end; ++current) { @@ -30,22 +30,22 @@ namespace NYsonPull { column_ = 1; } } - } + } }; - + template <> class stream_counter<false> { private: size_t offset_ = 0; - + public: TPositionInfo info() const { return {offset_, {}, {}}; } - + void update(const ui8* begin, const ui8* end) { offset_ += end - begin; } }; - } + } } diff --git a/library/cpp/yson_pull/detail/symbols.h b/library/cpp/yson_pull/detail/symbols.h index be49f80695..fe94bb9c41 100644 --- a/library/cpp/yson_pull/detail/symbols.h +++ b/library/cpp/yson_pull/detail/symbols.h @@ -1,38 +1,38 @@ -#pragma once - -#include <util/generic/strbuf.h> +#pragma once + +#include <util/generic/strbuf.h> #include <util/system/types.h> - + namespace NYsonPull { namespace NDetail { namespace NSymbol { #define SYM(name, value) constexpr ui8 name = value - + //! Indicates the beginning of a list. SYM(begin_list, '['); //! Indicates the end of a list. SYM(end_list, ']'); - + //! Indicates the beginning of a map. SYM(begin_map, '{'); //! Indicates the end of a map. SYM(end_map, '}'); - + //! Indicates the beginning of an attribute map. SYM(begin_attributes, '<'); //! Indicates the end of an attribute map. SYM(end_attributes, '>'); - + //! Separates items in lists and pairs in maps or attribute maps. SYM(item_separator, ';'); //! Separates keys from values in maps and attribute maps. SYM(key_value_separator, '='); - + //! Indicates an entity. SYM(entity, '#'); //! Indicates end of stream. SYM(eof, '\0'); - + //! Marks the beginning of a binary string literal. SYM(string_marker, '\x01'); //! Marks the beginning of a binary int64 literal. @@ -45,11 +45,11 @@ namespace NYsonPull { SYM(false_marker, '\x04'); //! Marks a binary `true' boolean value. SYM(true_marker, '\x05'); - + //! Text string quote symbol SYM(quote, '"'); - -#undef SYM + +#undef SYM } } } diff --git a/library/cpp/yson_pull/detail/traits.h b/library/cpp/yson_pull/detail/traits.h index 0d1ab6f660..869a3b9c44 100644 --- a/library/cpp/yson_pull/detail/traits.h +++ b/library/cpp/yson_pull/detail/traits.h @@ -1,7 +1,7 @@ -#pragma once - -#include <type_traits> - +#pragma once + +#include <type_traits> + namespace NYsonPull { namespace NDetail { namespace NTraits { @@ -9,17 +9,17 @@ namespace NYsonPull { using if_signed = typename std::enable_if< std::is_signed<T>::value, U>::type; - + template <typename T, typename U> using if_unsigned = typename std::enable_if< std::is_unsigned<T>::value, U>::type; - + template <typename T> using to_unsigned = typename std::enable_if< std::is_signed<T>::value, typename std::make_unsigned<T>::type>::type; - + template <typename T> using to_signed = typename std::enable_if< std::is_unsigned<T>::value, diff --git a/library/cpp/yson_pull/detail/varint.h b/library/cpp/yson_pull/detail/varint.h index 6426238d75..38bf45d925 100644 --- a/library/cpp/yson_pull/detail/varint.h +++ b/library/cpp/yson_pull/detail/varint.h @@ -1,15 +1,15 @@ -#pragma once - -#include "byte_reader.h" -#include "byte_writer.h" -#include "traits.h" -#include "zigzag.h" - -#include <util/system/types.h> - -#include <cstddef> -#include <type_traits> - +#pragma once + +#include "byte_reader.h" +#include "byte_writer.h" +#include "traits.h" +#include "zigzag.h" + +#include <util/system/types.h> + +#include <cstddef> +#include <type_traits> + namespace NYsonPull { namespace NDetail { namespace NVarInt { @@ -18,7 +18,7 @@ namespace NYsonPull { constexpr inline size_t max_size() { return (8 * sizeof(T) - 1) / 7 + 1; } - + template <typename T> inline size_t write(ui64 value, T&& consume) { auto stop = false; @@ -35,17 +35,17 @@ namespace NYsonPull { } return nwritten; } - + template <typename U> inline bool read_fast(byte_reader<U>& reader, ui64* value) { auto& buf = reader.stream().buffer(); auto* ptr = buf.pos(); ui32 b; - + // Splitting into 32-bit pieces gives better performance on 32-bit // processors. ui32 part0 = 0, part1 = 0, part2 = 0; - + b = *(ptr++); part0 = (b & 0x7F); if (!(b & 0x80)) @@ -86,17 +86,17 @@ namespace NYsonPull { part2 |= (b & 0x7F) << 7; if (!(b & 0x80)) goto done; - + // We have overrun the maximum size of a Varint (10 bytes). The data // must be corrupt. return false; - + done: reader.advance(ptr - buf.pos()); *value = (static_cast<ui64>(part0)) | (static_cast<ui64>(part1) << 28) | (static_cast<ui64>(part2) << 56); return true; } - + template <typename U> inline bool read_fast(byte_reader<U>& reader, ui32* value) { // Fast path: We have enough bytes left in the buffer to guarantee that @@ -105,7 +105,7 @@ namespace NYsonPull { auto* ptr = buf.pos(); ui32 b; ui32 result; - + b = *(ptr++); result = (b & 0x7F); if (!(b & 0x80)) @@ -126,37 +126,37 @@ namespace NYsonPull { result |= b << 28; if (!(b & 0x80)) goto done; - + // FIXME // If the input is larger than 32 bits, we still need to read it all // and discard the high-order bits. - + for (size_t i = 0; i < max_size<ui64>() - max_size<ui32>(); i++) { b = *(ptr++); if (!(b & 0x80)) goto done; } - + // We have overrun the maximum size of a Varint (10 bytes). Assume // the data is corrupt. return false; - + done: reader.advance(ptr - buf.pos()); *value = result; return true; } - + template <typename U> inline bool read_slow(byte_reader<U>& reader, ui64* value) { // Slow path: This read might cross the end of the buffer, so we // need to check and refresh the buffer if and when it does. - + auto& buf = reader.stream().buffer(); ui64 result = 0; int count = 0; ui32 b; - + do { if (count == max_size<ui64>()) { return false; @@ -170,11 +170,11 @@ namespace NYsonPull { reader.advance(1); ++count; } while (b & 0x80); - + *value = result; return true; } - + template <typename U> inline bool read_slow(byte_reader<U>& reader, ui32* value) { ui64 result; @@ -183,18 +183,18 @@ namespace NYsonPull { *value = static_cast<ui32>(result); return true; } - + return false; } - + // Following functions is an adaptation // of Protobuf code from coded_stream.cc template <typename T, typename U> inline bool read_dispatch(byte_reader<U>& reader, T* value) { auto& buf = reader.stream().buffer(); - // NOTE: checking for 64-bit max_size(), since 32-bit - // read_fast() might fallback to 64-bit reading - if (buf.available() >= max_size<ui64>() || + // NOTE: checking for 64-bit max_size(), since 32-bit + // read_fast() might fallback to 64-bit reading + if (buf.available() >= max_size<ui64>() || // Optimization: If the Varint ends at exactly the end of the buffer, // we can detect that and still use the fast path. (!buf.is_empty() && !(buf.end()[-1] & 0x80))) @@ -207,11 +207,11 @@ namespace NYsonPull { return read_slow(reader, value); } } - + } - + // Various functions to read/write varints. - + // Returns the number of bytes written. template <typename T> inline NTraits::if_unsigned<T, size_t> write(ui8* data, T value) { @@ -219,21 +219,21 @@ namespace NYsonPull { static_cast<ui64>(value), [&](ui8 byte) { *data++ = byte; }); } - + template <typename T> inline NTraits::if_signed<T, size_t> write(ui8* data, T value) { return NImpl::write( static_cast<ui64>(NZigZag::encode(value)), [&](ui8 byte) { *data++ = byte; }); } - + template <typename T, typename U> inline void write(byte_writer<U>& stream, T value) { ui8 data[NImpl::max_size<T>()]; auto size = write(data, value); stream.write(data, size); } - + template <typename T, typename U> inline NTraits::if_unsigned<T, T> read(byte_reader<U>& reader) { auto value = T{}; @@ -243,13 +243,13 @@ namespace NYsonPull { reader.advance(1); return value; } - - if (Y_UNLIKELY(!NImpl::read_dispatch(reader, &value))) { + + if (Y_UNLIKELY(!NImpl::read_dispatch(reader, &value))) { reader.fail("Error parsing varint value"); } return value; } - + template <typename T, typename U> inline NTraits::if_signed<T, T> read(byte_reader<U>& reader) { return NZigZag::decode( diff --git a/library/cpp/yson_pull/detail/writer.h b/library/cpp/yson_pull/detail/writer.h index ebd6b4edf2..b24b994292 100644 --- a/library/cpp/yson_pull/detail/writer.h +++ b/library/cpp/yson_pull/detail/writer.h @@ -1,23 +1,23 @@ -#pragma once - -#include "byte_writer.h" -#include "cescape.h" +#pragma once + +#include "byte_writer.h" +#include "cescape.h" #include "percent_scalar.h" -#include "stream_counter.h" -#include "symbols.h" -#include "varint.h" - +#include "stream_counter.h" +#include "symbols.h" +#include "varint.h" + #include <library/cpp/yson_pull/consumer.h> #include <library/cpp/yson_pull/event.h> #include <library/cpp/yson_pull/output.h> #include <library/cpp/yson_pull/stream_type.h> #include <library/cpp/yson_pull/writer.h> - -#include <util/generic/vector.h> -#include <util/system/yassert.h> - -#include <cmath> - + +#include <util/generic/vector.h> +#include <util/system/yassert.h> + +#include <cmath> + namespace NYsonPull { namespace NDetail { class writer: public IConsumer { @@ -30,91 +30,91 @@ namespace NYsonPull { before_end, after_end, }; - + byte_writer<stream_counter<false>> stream_; TVector<EEventType> stack_; bool need_item_separator_ = false; EStreamType mode_ = EStreamType::ListFragment; state state_ = state::before_begin; - + public: - void OnBeginStream() override { - update_state(EEventType::BeginStream); + void OnBeginStream() override { + update_state(EEventType::BeginStream); } - - void OnEndStream() override { - update_state(EEventType::EndStream); + + void OnEndStream() override { + update_state(EEventType::EndStream); stream_.flush_buffer(); } - - void OnBeginList() override { + + void OnBeginList() override { begin_node(); write(NSymbol::begin_list); - update_state(EEventType::BeginList); + update_state(EEventType::BeginList); begin_collection(collection_type::list); } - - void OnEndList() override { - update_state(EEventType::EndList); + + void OnEndList() override { + update_state(EEventType::EndList); end_collection(collection_type::list); write(NSymbol::end_list); end_node(); } - - void OnBeginMap() override { + + void OnBeginMap() override { begin_node(); write(NSymbol::begin_map); - update_state(EEventType::BeginMap); + update_state(EEventType::BeginMap); begin_collection(collection_type::map); } - - void OnEndMap() override { - update_state(EEventType::EndMap); + + void OnEndMap() override { + update_state(EEventType::EndMap); end_collection(collection_type::map); write(NSymbol::end_map); end_node(); } - - void OnBeginAttributes() override { + + void OnBeginAttributes() override { begin_node(); write(NSymbol::begin_attributes); - update_state(EEventType::BeginAttributes); + update_state(EEventType::BeginAttributes); begin_collection(collection_type::attributes); } - - void OnEndAttributes() override { - update_state(EEventType::EndAttributes); + + void OnEndAttributes() override { + update_state(EEventType::EndAttributes); end_collection(collection_type::attributes); write(NSymbol::end_attributes); // no end_node } - - void OnEntity() override { - begin_node(); - update_state(EEventType::Scalar); + + void OnEntity() override { + begin_node(); + update_state(EEventType::Scalar); write(NSymbol::entity); end_node(); } - + protected: enum class collection_type { list, map, attributes, }; - + writer(NYsonPull::NOutput::IStream& stream, EStreamType mode) : stream_(stream) , mode_{mode} { } - + bool need_item_separator() const { return need_item_separator_; } void need_item_separator(bool value) { need_item_separator_ = value; } - + size_t depth() const { Y_ASSERT(!stack_.empty()); if (mode_ == EStreamType::Node) { @@ -126,124 +126,124 @@ namespace NYsonPull { EStreamType mode() const { return mode_; } - + void write(ui8 c) { stream_.write(c); } - + void write(TStringBuf value) { write_raw(value.data(), value.size()); } - + void write_raw(const void* ptr, size_t len) { stream_.write(static_cast<const ui8*>(ptr), len); } - + template <typename T> void write_varint(T value) { NVarInt::write(stream_, value); } - + void write_escaped_string(TStringBuf value) { write(NSymbol::quote); NCEscape::encode(stream_, value); write(NSymbol::quote); } - + void push(EEventType type) { stack_.push_back(type); } - + void pop(EEventType type) { if (stack_.empty()) { fail("Unpaired events: empty event stack"); - } + } if (stack_.back() != type) { fail("Unpaired events: expected ", type, ", got ", stack_.back()); - } + } stack_.pop_back(); } - + void update_state(EEventType event) { switch (state_) { case state::before_begin: - if (event != EEventType::BeginStream) { + if (event != EEventType::BeginStream) { fail("Expected begin_stream, got ", event); } begin_stream(); return; - + case state::before_end: - if (event != EEventType::EndStream) { + if (event != EEventType::EndStream) { fail("Expected end_stream, got ", event); } - end_stream(); - return; - + end_stream(); + return; + case state::after_end: fail("Attempted write past stream end"); - + case state::maybe_key: - if (event == EEventType::Key) { + if (event == EEventType::Key) { state_ = state::value; return; } - + switch (event) { - case EEventType::EndStream: + case EEventType::EndStream: end_stream(); return; - - case EEventType::EndMap: - pop(EEventType::BeginMap); + + case EEventType::EndMap: + pop(EEventType::BeginMap); next_state(); return; - - case EEventType::EndAttributes: - pop(EEventType::BeginAttributes); + + case EEventType::EndAttributes: + pop(EEventType::BeginAttributes); state_ = state::value_noattr; return; - + default: fail("Unexpected event ", event, " in maybe_key"); } break; - + case state::maybe_value: switch (event) { - case EEventType::EndList: - pop(EEventType::BeginList); + case EEventType::EndList: + pop(EEventType::BeginList); next_state(); return; - - case EEventType::EndStream: + + case EEventType::EndStream: end_stream(); return; - + default: break; } [[fallthrough]]; case state::value: - if (event == EEventType::BeginAttributes) { - push(EEventType::BeginAttributes); + if (event == EEventType::BeginAttributes) { + push(EEventType::BeginAttributes); next_state(); return; } [[fallthrough]]; case state::value_noattr: switch (event) { - case EEventType::Scalar: + case EEventType::Scalar: next_state(); return; - case EEventType::BeginList: - push(EEventType::BeginList); + case EEventType::BeginList: + push(EEventType::BeginList); next_state(); return; - case EEventType::BeginMap: - push(EEventType::BeginMap); + case EEventType::BeginMap: + push(EEventType::BeginMap); next_state(); return; @@ -251,94 +251,94 @@ namespace NYsonPull { fail("Unexpected event ", event, " (in value_*)"); } break; - } + } } - + void next_state() { Y_ASSERT(!stack_.empty()); switch (stack_.back()) { - case EEventType::BeginMap: - case EEventType::BeginAttributes: + case EEventType::BeginMap: + case EEventType::BeginAttributes: state_ = state::maybe_key; break; - - case EEventType::BeginList: + + case EEventType::BeginList: state_ = state::maybe_value; break; - - case EEventType::BeginStream: + + case EEventType::BeginStream: state_ = state::before_end; break; - + default: - Y_UNREACHABLE(); + Y_UNREACHABLE(); } } - + void begin_stream() { - push(EEventType::BeginStream); + push(EEventType::BeginStream); switch (mode_) { case EStreamType::ListFragment: - push(EEventType::BeginList); + push(EEventType::BeginList); state_ = state::maybe_value; break; - + case EStreamType::MapFragment: - push(EEventType::BeginMap); + push(EEventType::BeginMap); state_ = state::maybe_key; break; - + case EStreamType::Node: state_ = state::value; break; } } - + void end_stream() { switch (mode_) { case EStreamType::ListFragment: - pop(EEventType::BeginList); + pop(EEventType::BeginList); break; - + case EStreamType::MapFragment: - pop(EEventType::BeginMap); + pop(EEventType::BeginMap); break; - + case EStreamType::Node: break; } - pop(EEventType::BeginStream); + pop(EEventType::BeginStream); state_ = state::after_end; } - + virtual void begin_node() { if (need_item_separator_) { write(NSymbol::item_separator); } } - + virtual void end_node() { need_item_separator_ = true; } - + virtual void begin_key() { begin_node(); } - + virtual void end_key() { need_item_separator_ = false; write(NSymbol::key_value_separator); } - + virtual void begin_collection(collection_type type) { Y_UNUSED(type); need_item_separator_ = false; } - + virtual void end_collection(collection_type type) { need_item_separator_ = (type != collection_type::attributes); } - + template <typename... Args> ATTRIBUTE(noinline, cold) void fail[[noreturn]](const char* msg, Args&&... args) { @@ -350,62 +350,62 @@ namespace NYsonPull { stream_.counter().info()); } }; - - class TBinaryWriterImpl final: public writer { + + class TBinaryWriterImpl final: public writer { public: - TBinaryWriterImpl(NYsonPull::NOutput::IStream& stream, EStreamType mode) + TBinaryWriterImpl(NYsonPull::NOutput::IStream& stream, EStreamType mode) : writer(stream, mode) { } - - void OnScalarBoolean(bool value) override { - update_state(EEventType::Scalar); - + + void OnScalarBoolean(bool value) override { + update_state(EEventType::Scalar); + begin_node(); write(value ? NSymbol::true_marker : NSymbol::false_marker); end_node(); } - - void OnScalarInt64(i64 value) override { - update_state(EEventType::Scalar); - + + void OnScalarInt64(i64 value) override { + update_state(EEventType::Scalar); + begin_node(); write(NSymbol::int64_marker); write_varint(value); end_node(); } - - void OnScalarUInt64(ui64 value) override { - update_state(EEventType::Scalar); - + + void OnScalarUInt64(ui64 value) override { + update_state(EEventType::Scalar); + begin_node(); write(NSymbol::uint64_marker); write_varint(value); end_node(); } - - void OnScalarFloat64(double value) override { + + void OnScalarFloat64(double value) override { update_state(EEventType::Scalar); - + begin_node(); write(NSymbol::double_marker); write_raw(&value, sizeof value); end_node(); } - - void OnScalarString(TStringBuf value) override { - update_state(EEventType::Scalar); - + + void OnScalarString(TStringBuf value) override { + update_state(EEventType::Scalar); + begin_node(); write(NSymbol::string_marker); write_varint(static_cast<i32>(value.size())); write_raw(value.data(), value.size()); end_node(); } - - void OnKey(TStringBuf name) override { - update_state(EEventType::Key); - + + void OnKey(TStringBuf name) override { + update_state(EEventType::Key); + begin_key(); write(NSymbol::string_marker); write_varint(static_cast<i32>(name.size())); @@ -413,50 +413,50 @@ namespace NYsonPull { end_key(); } }; - - class TTextWriterImpl: public writer { + + class TTextWriterImpl: public writer { public: - TTextWriterImpl(NYsonPull::NOutput::IStream& stream, EStreamType mode) + TTextWriterImpl(NYsonPull::NOutput::IStream& stream, EStreamType mode) : writer(stream, mode) { } - - void OnScalarBoolean(bool value) override { - update_state(EEventType::Scalar); - + + void OnScalarBoolean(bool value) override { + update_state(EEventType::Scalar); + begin_node(); write(value ? percent_scalar::true_literal : percent_scalar::false_literal); end_node(); } - - void OnScalarInt64(i64 value) override { - update_state(EEventType::Scalar); - + + void OnScalarInt64(i64 value) override { + update_state(EEventType::Scalar); + char buf[32]; - auto len = ::snprintf(buf, sizeof(buf), "%" PRIi64, value); - + auto len = ::snprintf(buf, sizeof(buf), "%" PRIi64, value); + begin_node(); write_raw(buf, len); end_node(); } - - void OnScalarUInt64(ui64 value) override { - update_state(EEventType::Scalar); - + + void OnScalarUInt64(ui64 value) override { + update_state(EEventType::Scalar); + char buf[32]; - auto len = ::snprintf(buf, sizeof(buf), "%" PRIu64, value); - + auto len = ::snprintf(buf, sizeof(buf), "%" PRIu64, value); + begin_node(); write_raw(buf, len); write('u'); end_node(); } - - void OnScalarFloat64(double value) override { - update_state(EEventType::Scalar); - + + void OnScalarFloat64(double value) override { + update_state(EEventType::Scalar); + begin_node(); - + if (std::isfinite(value)) { char buf[32]; auto len = ::snprintf(buf, sizeof(buf), "%#.17lg", value); @@ -467,27 +467,27 @@ namespace NYsonPull { write(percent_scalar::positive_inf_literal); } else { write(percent_scalar::negative_inf_literal); - } + } end_node(); } - - void OnScalarString(TStringBuf value) override { - update_state(EEventType::Scalar); - + + void OnScalarString(TStringBuf value) override { + update_state(EEventType::Scalar); + begin_node(); write_escaped_string(value); end_node(); } - - void OnKey(TStringBuf name) override { - update_state(EEventType::Key); - + + void OnKey(TStringBuf name) override { + update_state(EEventType::Key); + begin_key(); write_escaped_string(name); end_key(); } - + protected: void begin_node() override { if (need_item_separator()) { @@ -495,7 +495,7 @@ namespace NYsonPull { write(' '); } } - + void end_node() override { if (mode() != EStreamType::Node && depth() == 0) { write(NSymbol::item_separator); @@ -505,26 +505,26 @@ namespace NYsonPull { writer::end_node(); } } - + void end_key() override { write(' '); writer::end_key(); write(' '); } }; - - class TPrettyWriterImpl final: public TTextWriterImpl { + + class TPrettyWriterImpl final: public TTextWriterImpl { size_t indent_size_; - + public: - TPrettyWriterImpl( + TPrettyWriterImpl( NYsonPull::NOutput::IStream& stream, EStreamType mode, size_t indent_size) - : TTextWriterImpl(stream, mode) + : TTextWriterImpl(stream, mode) , indent_size_{indent_size} { } - + protected: void begin_node() override { if (need_item_separator()) { @@ -532,35 +532,35 @@ namespace NYsonPull { newline(); } } - + void begin_collection(collection_type type) override { - TTextWriterImpl::begin_collection(type); + TTextWriterImpl::begin_collection(type); newline(); } - + void end_collection(collection_type type) override { - TTextWriterImpl::end_collection(type); + TTextWriterImpl::end_collection(type); newline(); } - + void newline() { write('\n'); indent(depth()); } - + void indent(size_t count) { for (size_t i = 0; i < count * indent_size_; ++i) { write(' '); } } }; - + template <typename T, typename... Args> NYsonPull::TWriter make_writer( THolder<NYsonPull::NOutput::IStream> stream, Args&&... args) { auto impl = MakeHolder<T>(*stream, std::forward<Args>(args)...); return NYsonPull::TWriter(std::move(stream), std::move(impl)); - } - } + } + } } diff --git a/library/cpp/yson_pull/detail/zigzag.h b/library/cpp/yson_pull/detail/zigzag.h index 14917d11b8..98fcac0e9f 100644 --- a/library/cpp/yson_pull/detail/zigzag.h +++ b/library/cpp/yson_pull/detail/zigzag.h @@ -1,23 +1,23 @@ -#pragma once - -#include "traits.h" - +#pragma once + +#include "traits.h" + namespace NYsonPull { namespace NDetail { namespace NZigZag { //! Functions that provide coding of integers with property: 0 <= f(x) <= 2 * |x| - - template <typename TSigned> - inline NTraits::to_unsigned<TSigned> encode(TSigned x) { - using TUnsigned = NTraits::to_unsigned<TSigned>; - constexpr auto rshift = sizeof(TSigned) * 8 - 1; - return (static_cast<TUnsigned>(x) << 1) ^ static_cast<TUnsigned>(x >> rshift); + + template <typename TSigned> + inline NTraits::to_unsigned<TSigned> encode(TSigned x) { + using TUnsigned = NTraits::to_unsigned<TSigned>; + constexpr auto rshift = sizeof(TSigned) * 8 - 1; + return (static_cast<TUnsigned>(x) << 1) ^ static_cast<TUnsigned>(x >> rshift); } - - template <typename TUnsigned> - inline NTraits::to_signed<TUnsigned> decode(TUnsigned x) { - using TSigned = NTraits::to_signed<TUnsigned>; - return static_cast<TSigned>(x >> 1) ^ -static_cast<TSigned>(x & 1); + + template <typename TUnsigned> + inline NTraits::to_signed<TUnsigned> decode(TUnsigned x) { + using TSigned = NTraits::to_signed<TUnsigned>; + return static_cast<TSigned>(x >> 1) ^ -static_cast<TSigned>(x & 1); } } } // namespace NDetail |